test login button

README.md

@@ -7,6 +7,13 @@ sdk: gradio
 sdk_version: 4.7.1
 app_file: app.py
 pinned: false
+
+hf_oauth: true
+# optional, default duration is 8 hours/480 minutes. Max duration is 30 days/43200 minutes.
+hf_oauth_expiration_minutes: 480
+# optional, see "Scopes" below. "openid profile" is always included.
+hf_oauth_scopes:
+ - inference-api
 ---
 
 Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -5,7 +5,7 @@ import gradio as gr
 from app_debug import get_demo as get_demo_debug
 from app_leaderboard import get_demo as get_demo_leaderboard
 from app_text_classification import get_demo as get_demo_text_classification
-from utils.run_jobs import start_process_run_job, stop_thread
+from run_jobs import start_process_run_job, stop_thread
 
 try:
     with gr.Blocks(theme=gr.themes.Soft(primary_hue="green")) as demo:

app_debug.py

@@ -3,12 +3,12 @@ from os.path import isfile, join
 import html
 
 import gradio as gr
-
-import utils.pipe as pipe
-from utils.io_utils import get_logs_file
+import os
+import pipe
+from io_utils import get_logs_file
 
 LOG_PATH = "./tmp"
-CONFIG_PATH = "./cicd/configs/"
+CONFIG_PATH = "./cicd/configs/submitted/"
 MAX_FILES_NUM = 20
 
 
@@ -69,17 +69,19 @@ def get_queue_status():
 
 
 def get_demo():
+    if not os.path.exists(CONFIG_PATH):
+        os.makedirs(CONFIG_PATH)
     with gr.Row():
         gr.HTML(        
             value=get_queue_status,
             every=5,
         )
-    with gr.Accordion(label="Log Files", open=False):
-        with gr.Row():
-            gr.Files(value=get_log_files, label="Log Files", every=10)
+    with gr.Accordion(label="Log Files", open=True):
         with gr.Row():
             gr.Textbox(
                 value=get_logs_file, every=0.5, lines=10, visible=True, label="Current Log File"
             )
+        with gr.Row():
+            gr.Files(value=get_log_files, label="Log Files", every=10)
     with gr.Accordion(label="Config Files", open=False):
         gr.Files(value=get_config_files, label="Config Files", every=10)

app_leaderboard.py

@@ -5,10 +5,10 @@ import gradio as gr
 import pandas as pd
 import datetime
 
-from utils.fetch_utils import (check_dataset_and_get_config,
+from fetch_utils import (check_dataset_and_get_config,
                          check_dataset_and_get_split)
 
-import utils.leaderboard as leaderboard
+import leaderboard
 logger = logging.getLogger(__name__)
 global update_time 
 update_time = datetime.datetime.fromtimestamp(0)
@@ -88,11 +88,29 @@ def get_demo(leaderboard_tab):
     dataset_ids = get_dataset_ids(records)
 
     column_names = records.columns.tolist()
+    issue_columns = column_names[:11]
+    info_columns = column_names[15:]
     default_columns = ["model_id", "dataset_id", "total_issues", "report_link"]
     default_df = records[default_columns]  # extract columns selected
     types = get_types(default_df)
     display_df = get_display_df(default_df)  # the styled dataframe to display
 
+    with gr.Row():
+        with gr.Column():
+          issue_columns_select = gr.CheckboxGroup(
+              label="Issue Columns",
+              choices=issue_columns,
+              value=[],
+              interactive=True,
+          )
+        with gr.Column():
+          info_columns_select = gr.CheckboxGroup(
+              label="Info Columns",
+              choices=info_columns,
+              value=default_columns,
+              interactive=True,
+          )
+    
     with gr.Row():
         task_select = gr.Dropdown(
             label="Task",
@@ -110,42 +128,35 @@ def get_demo(leaderboard_tab):
             interactive=True,
         )
 
-    with gr.Row():
-        columns_select = gr.CheckboxGroup(
-            label="Show columns",
-            choices=column_names,
-            value=default_columns,
-            interactive=True,
-        )
-
     with gr.Row():
         leaderboard_df = gr.DataFrame(display_df, datatype=types, interactive=False)
 
-    def update_leaderboard_records(model_id, dataset_id, columns, task):
+    def update_leaderboard_records(model_id, dataset_id, issue_columns, info_columns, task):
         global update_time
         if datetime.datetime.now() - update_time < datetime.timedelta(minutes=10):
             return gr.update()
         update_time = datetime.datetime.now()
         logger.info("Updating leaderboard records")
         leaderboard.records = get_records_from_dataset_repo(leaderboard.LEADERBOARD)
-        return filter_table(model_id, dataset_id, columns, task)
+        return filter_table(model_id, dataset_id, issue_columns, info_columns, task)
 
     leaderboard_tab.select(
         fn=update_leaderboard_records, 
-        inputs=[model_select, dataset_select, columns_select, task_select], 
+        inputs=[model_select, dataset_select, issue_columns_select, info_columns_select, task_select], 
         outputs=[leaderboard_df])
 
     @gr.on(
         triggers=[
             model_select.change,
             dataset_select.change,
-            columns_select.change,
+            issue_columns_select.change,
+            info_columns_select.change,
             task_select.change,
         ],
-        inputs=[model_select, dataset_select, columns_select, task_select],
+        inputs=[model_select, dataset_select, issue_columns_select, info_columns_select, task_select],
         outputs=[leaderboard_df],
     )
-    def filter_table(model_id, dataset_id, columns, task):
+    def filter_table(model_id, dataset_id, issue_columns, info_columns, task):
         logger.info("Filtering leaderboard records")
         records = leaderboard.records
         # filter the table based on task
@@ -156,8 +167,9 @@ def get_demo(leaderboard_tab):
         if dataset_id and dataset_id != "Any":
             df = df[(df["dataset_id"] == dataset_id)]
 
-        # filter the table based on the columns
-        df = df[columns]
+        # filter the table based on the columns 
+        issue_columns.sort()
+        df = df[info_columns + issue_columns]
         types = get_types(df)
         display_df = get_display_df(df)
         return gr.update(value=display_df, datatype=types, interactive=False)

app_legacy.py

@@ -376,7 +376,7 @@ def get_demo():
         selected = read_scanners("./config.yaml")
         scan_config = selected + ["data_leakage"]
         scanners = gr.CheckboxGroup(
-            choices=scan_config, value=selected, label="Scan Settings", visible=True
+            choices=scan_config, value=selected, visible=True
         )
 
     with gr.Row():

app_text_classification.py

@@ -2,12 +2,12 @@ import uuid
 
 import gradio as gr
 
-from utils.io_utils import read_scanners, write_scanners
-from utils.ui_helpers import (
+from io_utils import read_scanners, write_scanners
+from text_classification_ui_helpers import (
     get_related_datasets_from_leaderboard,
     align_columns_and_show_prediction,
+    get_dataset_splits,
     check_dataset,
-    show_hf_token_info,
     precheck_model_ds_enable_example_btn,
     try_submit,
     empty_column_mapping,
@@ -16,12 +16,11 @@ from utils.ui_helpers import (
 )
 
 import logging
-from utils.wordings import (
+from wordings import (
   CONFIRM_MAPPING_DETAILS_MD, 
   INTRODUCTION_MD, 
-  USE_INFERENCE_API_TIP, 
+  LOG_IN_TIPS, 
   CHECK_LOG_SECTION_RAW,
-  HF_TOKEN_INVALID_STYLED
 )
 
 MAX_LABELS = 40
@@ -34,6 +33,8 @@ logger = logging.getLogger(__name__)
 def get_demo():
     with gr.Row():
         gr.Markdown(INTRODUCTION_MD)
+        gr.HTML(LOG_IN_TIPS)
+        gr.LoginButton()
         uid_label = gr.Textbox(
             label="Evaluation ID:", value=uuid.uuid4, visible=False, interactive=False
         )
@@ -58,7 +59,7 @@ def get_demo():
     with gr.Row():
         first_line_ds = gr.DataFrame(label="Dataset Preview", visible=False)
     with gr.Row():
-        loading_status = gr.HTML(visible=True)
+        loading_dataset_info = gr.HTML(visible=True)
     with gr.Row():
         example_btn = gr.Button(
             "Validate Model & Dataset",
@@ -66,11 +67,13 @@ def get_demo():
             variant="primary",
             interactive=False,
         )
-
     with gr.Row():
-        example_input = gr.HTML(visible=False)
+        loading_validation = gr.HTML(visible=True)
+    with gr.Row():
+        validation_result = gr.HTML(visible=False)
     with gr.Row():
-        example_prediction = gr.Label(label="Model Prediction Sample", visible=False)
+        example_input = gr.Textbox(label="Example Input", visible=False, interactive=False)
+        example_prediction = gr.Label(label="Model Sample Prediction", visible=False)
 
     with gr.Row():
         with gr.Accordion(
@@ -89,27 +92,8 @@ def get_demo():
                     for _ in range(MAX_LABELS, MAX_LABELS + MAX_FEATURES):
                         column_mappings.append(gr.Dropdown(visible=False))
 
-    with gr.Accordion(label="Model Wrap Advance Config", open=True):
-        gr.HTML(USE_INFERENCE_API_TIP)
-
-        run_inference = gr.Checkbox(value=True, label="Run with Inference API")
-        inference_token = gr.Textbox(
-            placeholder="hf-xxxxxxxxxxxxxxxxxxxx",
-            value="",
-            label="HF Token for Inference API",
-            visible=True,
-            interactive=True,
-        )
-        inference_token_info = gr.HTML(value=HF_TOKEN_INVALID_STYLED, visible=False)
-
-        inference_token.change(
-            fn=show_hf_token_info,
-            inputs=[inference_token],
-            outputs=[inference_token_info],
-        )
-
-    with gr.Accordion(label="Scanner Advance Config (optional)", open=False):
-        scanners = gr.CheckboxGroup(label="Scan Settings", visible=True)
+    with gr.Accordion(label="Scanner Advanced Config (optional)", open=False):
+        scanners = gr.CheckboxGroup(visible=True)
 
         @gr.on(triggers=[uid_label.change], inputs=[uid_label], outputs=[scanners])
         def get_scanners(uid):
@@ -117,7 +101,16 @@ def get_demo():
             # we remove data_leakage from the default scanners
             # Reason: data_leakage barely raises any issues and takes too many requests
             # when using inference API, causing rate limit error
-            scan_config = selected + ["data_leakage"]
+            scan_config = [
+                "ethical_bias", 
+                "text_perturbation", 
+                "robustness",
+                "performance",
+                "underconfidence",
+                "overconfidence",
+                "spurious_correlation",
+                "data_leakage",
+                ]
             return gr.update(
                 choices=scan_config, value=selected, label="Scan Settings", visible=True
             )
@@ -147,18 +140,20 @@ def get_demo():
         inputs=[model_id_input],
         outputs=[dataset_id_input],
     ).then(
-        fn=check_dataset, 
-        inputs=[dataset_id_input], 
-        outputs=[dataset_config_input, dataset_split_input, loading_status]
+        fn=check_dataset,
+        inputs=[dataset_id_input],
+        outputs=[dataset_config_input, dataset_split_input, loading_dataset_info],
     )
     
     gr.on(
-        triggers=[dataset_id_input.change],
+        triggers=[dataset_id_input.input, dataset_id_input.select],
         fn=check_dataset,
         inputs=[dataset_id_input],
-        outputs=[dataset_config_input, dataset_split_input, loading_status]
+        outputs=[dataset_config_input, dataset_split_input, loading_dataset_info]
     )
 
+    dataset_config_input.change(fn=get_dataset_splits, inputs=[dataset_id_input, dataset_config_input], outputs=[dataset_split_input])
+
     gr.on(
         triggers=[model_id_input.change, dataset_id_input.change, dataset_config_input.change],
         fn=empty_column_mapping,
@@ -187,6 +182,7 @@ def get_demo():
     gr.on(
         triggers=[
             model_id_input.change,
+            model_id_input.input,
             dataset_id_input.change,
             dataset_config_input.change,
             dataset_split_input.change,
@@ -198,7 +194,13 @@ def get_demo():
             dataset_config_input,
             dataset_split_input,
         ],
-        outputs=[example_btn, first_line_ds, loading_status],
+        outputs=[
+            example_btn, 
+            first_line_ds,
+            validation_result,
+            example_input,
+            example_prediction,
+            column_mapping_accordion,],
     )
 
     gr.on(
@@ -212,15 +214,14 @@ def get_demo():
             dataset_config_input,
             dataset_split_input,
             uid_label,
-            run_inference,
-            inference_token,
         ],
         outputs=[
+            validation_result,
             example_input,
             example_prediction,
             column_mapping_accordion,
             run_btn,
-            loading_status,
+            loading_validation,
             *column_mappings,
         ],
     )
@@ -235,24 +236,26 @@ def get_demo():
             dataset_id_input,
             dataset_config_input,
             dataset_split_input,
-            run_inference,
-            inference_token,
             uid_label,
         ],
-        outputs=[run_btn, logs, uid_label],
+        outputs=[
+            run_btn, 
+            logs, 
+            uid_label,            
+            validation_result,
+            example_input,
+            example_prediction,
+            column_mapping_accordion,
+          ],
     )
 
     gr.on(
         triggers=[
-            run_inference.input,
-            inference_token.input,
             scanners.input,
         ],
         fn=enable_run_btn,
         inputs=[
             uid_label,
-            run_inference, 
-            inference_token, 
             model_id_input, 
             dataset_id_input, 
             dataset_config_input, 
@@ -266,8 +269,6 @@ def get_demo():
         fn=enable_run_btn,
         inputs=[
             uid_label,
-            run_inference, 
-            inference_token, 
             model_id_input, 
             dataset_id_input, 
             dataset_config_input, 

utils/io_utils.py → io_utils.py

@@ -1,15 +1,25 @@
 import os
-
+import logging
 import yaml
 
 YAML_PATH = "./cicd/configs"
 LOG_FILE = "temp_log"
 
+logger = logging.getLogger(__name__)
 
 class Dumper(yaml.Dumper):
     def increase_indent(self, flow=False, *args, **kwargs):
         return super().increase_indent(flow=flow, indentless=False)
 
+def get_submitted_yaml_path(uid):
+    if not os.path.exists(f"{YAML_PATH}/submitted"):
+        os.makedirs(f"{YAML_PATH}/submitted")
+    if not os.path.exists(f"{YAML_PATH}/{uid}_config.yaml"):
+        logger.error(f"config.yaml does not exist for {uid}")
+        os.system(f"cp config.yaml {YAML_PATH}/{uid}_config.yaml")
+    if not os.path.exists(f"{YAML_PATH}/submitted/{uid}_config.yaml"):
+        os.system(f"cp {YAML_PATH}/{uid}_config.yaml {YAML_PATH}/submitted/{uid}_config.yaml")
+    return f"{YAML_PATH}/submitted/{uid}_config.yaml"
 
 def get_yaml_path(uid):
     if not os.path.exists(YAML_PATH):
@@ -72,6 +82,8 @@ def read_column_mapping(uid):
         config = yaml.load(f, Loader=yaml.FullLoader)
         if config:
             column_mapping = config.get("column_mapping", dict())
+    if column_mapping is None:
+        column_mapping = {}
     return column_mapping
 
 

isolated_env.py

@@ -1,7 +1,7 @@
 import os
 import subprocess
 
-from utils.io_utils import write_log_to_user_file
+from io_utils import write_log_to_user_file
 
 
 def prepare_venv(execution_id, deps):

requirements.txt

@@ -4,4 +4,6 @@ hf-transfer
 torch==2.0.1
 transformers
 datasets
+tabulate
+gradio[oauth]
 -e git+https://github.com/Giskard-AI/cicd.git#egg=giskard-cicd

utils/run_jobs.py → run_jobs.py

@@ -6,7 +6,7 @@ import threading
 import time
 from pathlib import Path
 
-import utils.pipe as pipe
+import pipe
 from app_env import (
     HF_GSK_HUB_HF_TOKEN,
     HF_GSK_HUB_KEY,
@@ -17,9 +17,9 @@ from app_env import (
     HF_SPACE_ID,
     HF_WRITE_TOKEN,
 )
-from utils.io_utils import LOG_FILE, get_yaml_path, write_log_to_user_file
+from io_utils import LOG_FILE, get_submitted_yaml_path, write_log_to_user_file
 from isolated_env import prepare_venv
-from utils.leaderboard import LEADERBOARD
+from leaderboard import LEADERBOARD
 
 is_running = False
 
@@ -50,7 +50,6 @@ def prepare_env_and_get_command(
     d_id,
     config,
     split,
-    inference,
     inference_token,
     uid,
     label_mapping,
@@ -60,10 +59,6 @@ def prepare_env_and_get_command(
     if os.environ.get("SPACE_ID") == "giskardai/giskard-evaluator":
         leaderboard_dataset = LEADERBOARD
 
-    inference_type = "hf_pipeline"
-    if inference and inference_token:
-        inference_type = "hf_inference_api"
-
     executable = "giskard_scanner"
     try:
         # Copy the current requirements (might be changed)
@@ -98,9 +93,9 @@ def prepare_env_and_get_command(
         "--label_mapping",
         json.dumps(label_mapping),
         "--scan_config",
-        get_yaml_path(uid),
+        get_submitted_yaml_path(uid),
         "--inference_type",
-        inference_type,
+        "hf_inference_api",
         "--inference_api_token",
         inference_token,
     ]

utils/text_classification.py → text_classification.py

@@ -1,15 +1,19 @@
+import json
 import logging
 
 import datasets
 import huggingface_hub
+import pandas as pd
+from transformers import pipeline
 import requests
 import os
-
 from app_env import HF_WRITE_TOKEN
 
 logger = logging.getLogger(__name__)
 AUTH_CHECK_URL = "https://huggingface.co/api/whoami-v2"
 
+logger = logging.getLogger(__file__)
+
 class HuggingFaceInferenceAPIResponse:
     def __init__(self, message):
         self.message = message
@@ -18,23 +22,28 @@ class HuggingFaceInferenceAPIResponse:
 def get_labels_and_features_from_dataset(ds):
     try:
         dataset_features = ds.features
-        label_keys = [i for i in dataset_features.keys() if i.startswith('label')]
+        label_keys = [i for i in dataset_features.keys() if i.startswith("label")]
+        features = [f for f in dataset_features.keys() if not f.startswith("label")]
+
         if len(label_keys) == 0: # no labels found
             # return everything for post processing
-            return list(dataset_features.keys()), list(dataset_features.keys())
+            return list(dataset_features.keys()), list(dataset_features.keys()), None
+        
+        labels = None
         if not isinstance(dataset_features[label_keys[0]], datasets.ClassLabel):
-            if hasattr(dataset_features[label_keys[0]], 'feature'):
+            if hasattr(dataset_features[label_keys[0]], "feature"):
                 label_feat = dataset_features[label_keys[0]].feature
                 labels = label_feat.names
+            else:
+                labels = ds.unique(label_keys[0])
         else:
             labels = dataset_features[label_keys[0]].names
-        features = [f for f in dataset_features.keys() if not f.startswith("label")]
-        return labels, features
+        return labels, features, label_keys
     except Exception as e:
         logging.warning(
             f"Get Labels/Features Failed for dataset: {e}"
         )
-        return None, None
+        return None, None, None
 
 def check_model_task(model_id):
     # check if model is valid on huggingface
@@ -78,11 +87,19 @@ def hf_inference_api(model_id, hf_token, payload):
     url = f"{hf_inference_api_endpoint}/models/{model_id}"
     headers = {"Authorization": f"Bearer {hf_token}"}
     response = requests.post(url, headers=headers, json=payload)
+
     if not hasattr(response, "status_code") or response.status_code != 200:
         logger.warning(f"Request to inference API returns {response}")
+  
     try:
+        output = response.json()
+        if "error" in output and "Input is too long" in output["error"]:
+          payload.update({"parameters": {"truncation": True, "max_length": 512}})
+          response = requests.post(url, headers=headers, json=payload)
+          if not hasattr(response, "status_code") or response.status_code != 200:
+              logger.warning(f"Request to inference API returns {response}")
         return response.json()
-    except Exception:
+    except Exception:                
         return {"error": response.content}
     
 def preload_hf_inference_api(model_id):
@@ -90,6 +107,165 @@ def preload_hf_inference_api(model_id):
     hf_token = os.environ.get(HF_WRITE_TOKEN, default="")
     hf_inference_api(model_id, hf_token, payload)
 
+def check_model_pipeline(model_id):
+    try:
+        task = huggingface_hub.model_info(model_id).pipeline_tag
+    except Exception:
+        return None
+
+    try:
+        ppl = pipeline(task=task, model=model_id)
+
+        return ppl
+    except Exception:
+        return None
+
+
+def text_classificaiton_match_label_case_unsensative(id2label_mapping, label):
+    for model_label in id2label_mapping.keys():
+        if model_label.upper() == label.upper():
+            return model_label, label
+    return None, label
+
+
+def text_classification_map_model_and_dataset_labels(id2label, dataset_features):
+    id2label_mapping = {id2label[k]: None for k in id2label.keys()}
+    dataset_labels = None
+    for feature in dataset_features.values():
+        if not isinstance(feature, datasets.ClassLabel):
+            continue
+        if len(feature.names) != len(id2label_mapping.keys()):
+            continue
+
+        dataset_labels = feature.names
+        # Try to match labels
+        for label in feature.names:
+            if label in id2label_mapping.keys():
+                model_label = label
+            else:
+                # Try to find case unsensative
+                model_label, label = text_classificaiton_match_label_case_unsensative(
+                    id2label_mapping, label
+                )
+            if model_label is not None:
+                id2label_mapping[model_label] = label
+            else:
+                print(f"Label {label} is not found in model labels")
+
+    return id2label_mapping, dataset_labels
+
+
+"""
+params:
+    column_mapping: dict
+    example: {
+        "text": "sentences",
+        "label": {
+            "label0": "LABEL_0",
+            "label1": "LABEL_1"
+        }
+    }
+    ppl: pipeline
+"""
+
+
+def check_column_mapping_keys_validity(column_mapping, ppl):
+    # get the element in all the list elements
+    column_mapping = json.loads(column_mapping)
+    if "data" not in column_mapping.keys():
+        return True
+    user_labels = set([pair[0] for pair in column_mapping["data"]])
+    model_labels = set([pair[1] for pair in column_mapping["data"]])
+
+    id2label = ppl.model.config.id2label
+    original_labels = set(id2label.values())
+
+    return user_labels == model_labels == original_labels
+
+
+"""
+params:
+    column_mapping: dict
+    dataset_features: dict
+    example: {
+        'text': Value(dtype='string', id=None), 
+        'label': ClassLabel(names=['negative', 'neutral', 'positive'], id=None)
+    }
+"""
+
+
+def infer_text_input_column(column_mapping, dataset_features):
+    # Check whether we need to infer the text input column
+    infer_text_input_column = True
+    feature_map_df = None
+
+    if "text" in column_mapping.keys():
+        dataset_text_column = column_mapping["text"]
+        if dataset_text_column in dataset_features.keys():
+            infer_text_input_column = False
+        else:
+            logging.warning(f"Provided {dataset_text_column} is not in Dataset columns")
+
+    if infer_text_input_column:
+        # Try to retrieve one
+        candidates = [
+            f for f in dataset_features if dataset_features[f].dtype == "string"
+        ]
+        feature_map_df = pd.DataFrame(
+            {"Dataset Features": [candidates[0]], "Model Input Features": ["text"]}
+        )
+        if len(candidates) > 0:
+            logging.debug(f"Candidates are {candidates}")
+            column_mapping["text"] = candidates[0]
+
+    return column_mapping, feature_map_df
+
+
+"""
+params:
+    column_mapping: dict
+    id2label_mapping: dict
+    example:
+    id2label_mapping: {
+        'negative': 'negative', 
+        'neutral': 'neutral', 
+        'positive': 'positive'
+        }
+"""
+
+
+def infer_output_label_column(
+    column_mapping, id2label_mapping, id2label, dataset_labels
+):
+    # Check whether we need to infer the output label column
+    if "data" in column_mapping.keys():
+        if isinstance(column_mapping["data"], list):
+            # Use the column mapping passed by user
+            for user_label, model_label in column_mapping["data"]:
+                id2label_mapping[model_label] = user_label
+    elif None in id2label_mapping.values():
+        column_mapping["label"] = {i: None for i in id2label.keys()}
+        return column_mapping, None
+
+    if "data" not in column_mapping.keys():
+        # Column mapping should contain original model labels
+        column_mapping["label"] = {
+            str(i): id2label_mapping[label]
+            for i, label in zip(id2label.keys(), dataset_labels)
+        }
+
+    id2label_df = pd.DataFrame(
+        {
+            "Dataset Labels": dataset_labels,
+            "Model Prediction Labels": [
+                id2label_mapping[label] for label in dataset_labels
+            ],
+        }
+    )
+
+    return column_mapping, id2label_df
+
+
 def check_dataset_features_validity(d_id, config, split):
     # We assume dataset is ok here
     ds = datasets.load_dataset(d_id, config, split=split, trust_remote_code=True)
@@ -173,9 +349,51 @@ def get_sample_prediction(ppl, df, column_mapping, id2label_mapping):
     return prediction_input, prediction_result
 
 
+def text_classification_fix_column_mapping(column_mapping, ppl, d_id, config, split):
+    # load dataset as pd DataFrame
+    # get features column from dataset
+    df, dataset_features = check_dataset_features_validity(d_id, config, split)
+
+    column_mapping, feature_map_df = infer_text_input_column(
+        column_mapping, dataset_features
+    )
+    if feature_map_df is None:
+        # dataset does not have any features
+        return None, None, None, None, None
+
+    # Retrieve all labels
+    id2label = ppl.model.config.id2label
+
+    # Infer labels
+    id2label_mapping, dataset_labels = text_classification_map_model_and_dataset_labels(
+        id2label, dataset_features
+    )
+    column_mapping, id2label_df = infer_output_label_column(
+        column_mapping, id2label_mapping, id2label, dataset_labels
+    )
+    if id2label_df is None:
+        # does not able to infer output label column
+        return column_mapping, None, None, None, feature_map_df
+
+    # Get a sample prediction
+    prediction_input, prediction_result = get_sample_prediction(
+        ppl, df, column_mapping, id2label_mapping
+    )
+    if prediction_result is None:
+        # does not able to get a sample prediction
+        return column_mapping, prediction_input, None, id2label_df, feature_map_df
+
+    return (
+        column_mapping,
+        prediction_input,
+        prediction_result,
+        id2label_df,
+        feature_map_df,
+    )
+
 def strip_model_id_from_url(model_id):
     if model_id.startswith("https://huggingface.co/"):
-        return "/".join(model_id.split("/")[-2])
+        return "/".join(model_id.split("/")[-2:])
     return model_id
 
 def check_hf_token_validity(hf_token):

utils/ui_helpers.py → text_classification_ui_helpers.py

@@ -7,10 +7,15 @@ import datasets
 import gradio as gr
 import pandas as pd
 
-import utils.leaderboard as leaderboard
-from utils.io_utils import read_column_mapping, write_column_mapping
-from utils.run_jobs import save_job_to_pipe
-from utils.text_classification import (
+import leaderboard
+from io_utils import (
+  read_column_mapping, 
+  write_column_mapping, 
+  read_scanners, 
+  write_scanners,
+)
+from run_jobs import save_job_to_pipe
+from text_classification import (
     strip_model_id_from_url,
     check_model_task,
     preload_hf_inference_api,
@@ -19,17 +24,18 @@ from utils.text_classification import (
     check_hf_token_validity,
     HuggingFaceInferenceAPIResponse,
 )
-from utils.wordings import (
+from wordings import (
     CHECK_CONFIG_OR_SPLIT_RAW,
     CONFIRM_MAPPING_DETAILS_FAIL_RAW,
     MAPPING_STYLED_ERROR_WARNING,
     NOT_TEXT_CLASSIFICATION_MODEL_RAW,
     UNMATCHED_MODEL_DATASET_STYLED_ERROR,
     CHECK_LOG_SECTION_RAW,
-    get_styled_input,
+    VALIDATED_MODEL_DATASET_STYLED,
     get_dataset_fetch_error_raw,
 )
 import os
+from app_env import HF_WRITE_TOKEN
 
 MAX_LABELS = 40
 MAX_FEATURES = 20
@@ -37,9 +43,6 @@ MAX_FEATURES = 20
 ds_dict = None
 ds_config = None
 
-logger = logging.getLogger(__file__)
-
-
 def get_related_datasets_from_leaderboard(model_id):
     records = leaderboard.records
     model_id = strip_model_id_from_url(model_id)
@@ -47,9 +50,20 @@ def get_related_datasets_from_leaderboard(model_id):
     datasets_unique = list(model_records["dataset_id"].unique())
 
     if len(datasets_unique) == 0:
-        return gr.update(choices=[], value="")
+        return gr.update(choices=[])
     
-    return gr.update(choices=datasets_unique, value="")
+    return gr.update(choices=datasets_unique)
+
+
+logger = logging.getLogger(__file__)
+
+def get_dataset_splits(dataset_id, dataset_config):
+    try:
+        splits = datasets.get_dataset_split_names(dataset_id, dataset_config, trust_remote_code=True)
+        return gr.update(choices=splits, value=splits[0], visible=True)
+    except Exception as e:
+        logger.warn(f"Check your dataset {dataset_id} and config {dataset_config}: {e}")
+        return gr.update(visible=False)
 
 def check_dataset(dataset_id):
     logger.info(f"Loading {dataset_id}")
@@ -61,9 +75,7 @@ def check_dataset(dataset_id):
                 gr.update(visible=False),
                 ""
             )
-        splits = datasets.get_dataset_split_names(
-                        dataset_id, configs[0], trust_remote_code=True
-                    )
+        splits = datasets.get_dataset_split_names(dataset_id, configs[0], trust_remote_code=True)
         return (
             gr.update(choices=configs, value=configs[0], visible=True),
             gr.update(choices=splits, value=splits[0], visible=True),
@@ -116,6 +128,7 @@ def export_mappings(all_mappings, key, subkeys, values):
             all_mappings[key][subkey] = values[i % len(values)]
     return all_mappings
 
+
 def list_labels_and_features_from_dataset(ds_labels, ds_features, model_labels, uid):
     all_mappings = read_column_mapping(uid)
     # For flattened raw datasets with no labels
@@ -125,7 +138,7 @@ def list_labels_and_features_from_dataset(ds_labels, ds_features, model_labels,
         ds_labels = list(shared_labels)
     if len(ds_labels) > MAX_LABELS:
         ds_labels = ds_labels[:MAX_LABELS]
-        gr.Warning(f"The number of labels is truncated to length {MAX_LABELS}")
+        gr.Warning(f"Too many labels to display for this spcae. We do not support more than {MAX_LABELS} in this space. You can use cli tool at https://github.com/Giskard-AI/cicd.")
 
     # sort labels to make sure the order is consistent
     # prediction gives the order based on probability
@@ -164,35 +177,70 @@ def list_labels_and_features_from_dataset(ds_labels, ds_features, model_labels,
 
     return lables + features
 
+
 def precheck_model_ds_enable_example_btn(
     model_id, dataset_id, dataset_config, dataset_split
-):
-    if model_id == "" or dataset_id == "":
-        return (gr.update(interactive=False), gr.update(visible=False), "")
+):  
     model_id = strip_model_id_from_url(model_id)
     model_task = check_model_task(model_id)
     preload_hf_inference_api(model_id)
-    if model_task is None or model_task != "text-classification":
-        gr.Warning(NOT_TEXT_CLASSIFICATION_MODEL_RAW)
-        return (gr.update(interactive=False), gr.update(visible=False), "")
-    
+
     if dataset_config is None or dataset_split is None or len(dataset_config) == 0:
-        return (gr.update(interactive=False), gr.update(visible=False), "")
-    
+        return (
+            gr.update(interactive=False), 
+            gr.update(visible=False),
+            gr.update(visible=False),
+            gr.update(visible=False),
+            gr.update(visible=False),
+            gr.update(visible=False),
+        )
+
     try:
         ds = datasets.load_dataset(dataset_id, dataset_config, trust_remote_code=True)
         df: pd.DataFrame = ds[dataset_split].to_pandas().head(5)
-        ds_labels, ds_features = get_labels_and_features_from_dataset(ds[dataset_split])
+        ds_labels, ds_features, _ = get_labels_and_features_from_dataset(ds[dataset_split])
+
+        if model_task is None or model_task != "text-classification":
+          gr.Warning(NOT_TEXT_CLASSIFICATION_MODEL_RAW)
+          return (
+              gr.update(interactive=False), 
+              gr.update(value=df, visible=True),
+              gr.update(visible=False),
+              gr.update(visible=False),
+              gr.update(visible=False),
+              gr.update(visible=False),
+          )
 
         if not isinstance(ds_labels, list) or not isinstance(ds_features, list):
             gr.Warning(CHECK_CONFIG_OR_SPLIT_RAW)
-            return (gr.update(interactive=False), gr.update(value=df, visible=True), "")
+            return (
+                gr.update(interactive=False), 
+                gr.update(value=df, visible=True),
+                gr.update(visible=False),
+                gr.update(visible=False),
+                gr.update(visible=False),
+                gr.update(visible=False),
+            )
 
-        return (gr.update(interactive=True), gr.update(value=df, visible=True), "")
+        return (
+                gr.update(interactive=True), 
+                gr.update(value=df, visible=True),
+                gr.update(visible=False),
+                gr.update(visible=False),
+                gr.update(visible=False),
+                gr.update(visible=False),
+            )
     except Exception as e:
         # Config or split wrong
         logger.warn(f"Check your dataset {dataset_id} and config {dataset_config} on split {dataset_split}: {e}")
-        return (gr.update(interactive=False), gr.update(value=pd.DataFrame(), visible=False), "")
+        return (
+            gr.update(interactive=False),
+            gr.update(visible=False),
+            gr.update(visible=False),
+            gr.update(visible=False),
+            gr.update(visible=False),
+            gr.update(visible=False),
+        )
 
 
 def align_columns_and_show_prediction(
@@ -201,8 +249,8 @@ def align_columns_and_show_prediction(
     dataset_config,
     dataset_split,
     uid,
-    run_inference,
-    inference_token,
+    profile: gr.OAuthProfile | None,
+    oauth_token: gr.OAuthToken | None,
 ):
     model_id = strip_model_id_from_url(model_id)
     model_task = check_model_task(model_id)
@@ -221,7 +269,7 @@ def align_columns_and_show_prediction(
         gr.Dropdown(visible=False) for _ in range(MAX_LABELS + MAX_FEATURES)
     ]
     
-    hf_token = os.environ.get("HF_WRITE_TOKEN", default="")
+    hf_token = os.environ.get(HF_WRITE_TOKEN, default="")
 
     prediction_input, prediction_response = get_example_prediction(
         model_id, dataset_id, dataset_config, dataset_split, hf_token
@@ -229,6 +277,7 @@ def align_columns_and_show_prediction(
 
     if prediction_input is None or prediction_response is None:
         return (
+            gr.update(visible=False),
             gr.update(visible=False),
             gr.update(visible=False),
             gr.update(visible=False, open=False),
@@ -239,6 +288,7 @@ def align_columns_and_show_prediction(
 
     if isinstance(prediction_response, HuggingFaceInferenceAPIResponse):
         return (
+            gr.update(visible=False),
             gr.update(visible=False),
             gr.update(visible=False),
             gr.update(visible=False, open=False),
@@ -250,12 +300,13 @@ def align_columns_and_show_prediction(
     model_labels = list(prediction_response.keys())
     
     ds = datasets.load_dataset(dataset_id, dataset_config, split=dataset_split, trust_remote_code=True)
-    ds_labels, ds_features = get_labels_and_features_from_dataset(ds)
+    ds_labels, ds_features, _ = get_labels_and_features_from_dataset(ds)
 
     # when dataset does not have labels or features
     if not isinstance(ds_labels, list) or not isinstance(ds_features, list):
         gr.Warning(CHECK_CONFIG_OR_SPLIT_RAW)
         return (
+            gr.update(visible=False),
             gr.update(visible=False),
             gr.update(visible=False),
             gr.update(visible=False, open=False),
@@ -268,6 +319,7 @@ def align_columns_and_show_prediction(
         return (
             gr.update(value=UNMATCHED_MODEL_DATASET_STYLED_ERROR, visible=True),
             gr.update(visible=False),
+            gr.update(visible=False),
             gr.update(visible=False, open=False),
             gr.update(interactive=False),
             "",
@@ -289,18 +341,20 @@ def align_columns_and_show_prediction(
     ):
         return (
             gr.update(value=MAPPING_STYLED_ERROR_WARNING, visible=True),
-            gr.update(visible=False),
+            gr.update(value=prediction_input, lines=min(len(prediction_input)//225 + 1, 5), visible=True),
+            gr.update(value=prediction_response, visible=True),
             gr.update(visible=True, open=True),
-            gr.update(interactive=(run_inference and inference_token != "")),
+            gr.update(interactive=(profile is not None and oauth_token is not None)),
             "",
             *column_mappings,
         )
 
     return (
-        gr.update(value=get_styled_input(prediction_input), visible=True),
+        gr.update(value=VALIDATED_MODEL_DATASET_STYLED, visible=True),
+        gr.update(value=prediction_input, lines=min(len(prediction_input)//225 + 1, 5), visible=True),
         gr.update(value=prediction_response, visible=True),
         gr.update(visible=True, open=False),
-        gr.update(interactive=(run_inference and inference_token != "")),
+        gr.update(interactive=(profile is not None and oauth_token is not None)),
         "",
         *column_mappings,
     )
@@ -308,18 +362,20 @@ def align_columns_and_show_prediction(
 
 def check_column_mapping_keys_validity(all_mappings):
     if all_mappings is None:
+        logger.warning("all_mapping is None")
         gr.Warning(CONFIRM_MAPPING_DETAILS_FAIL_RAW)
         return False
 
     if "labels" not in all_mappings.keys():
-        gr.Warning(CONFIRM_MAPPING_DETAILS_FAIL_RAW)
+        logger.warning(f"Label mapping is not valid, all_mappings: {all_mappings}")
         return False
 
     return True
 
-def enable_run_btn(uid, run_inference, inference_token, model_id, dataset_id, dataset_config, dataset_split):
-    if not run_inference or inference_token == "":
-        logger.warn("Inference API is not enabled")
+def enable_run_btn(uid, model_id, dataset_id, dataset_config, dataset_split, profile: gr.OAuthProfile | None, oath_token: gr.OAuthToken | None):
+    if profile is None:
+        return gr.update(interactive=False)
+    if oath_token is None:
         return gr.update(interactive=False)
     if model_id == "" or dataset_id == "" or dataset_config == "" or dataset_split == "":
         logger.warn("Model id or dataset id is not selected")
@@ -330,26 +386,27 @@ def enable_run_btn(uid, run_inference, inference_token, model_id, dataset_id, da
         logger.warn("Column mapping is not valid")
         return gr.update(interactive=False)
 
-    if not check_hf_token_validity(inference_token):
-        logger.warn("HF token is not valid")
-        return gr.update(interactive=False)
-    return gr.update(interactive=True)
-
-def construct_label_and_feature_mapping(all_mappings, ds_labels, ds_features):
+def construct_label_and_feature_mapping(all_mappings, ds_labels, ds_features, label_keys=None):
     label_mapping = {}
     if len(all_mappings["labels"].keys()) != len(ds_labels):
-        logger.warn("Label mapping corrupted: " + CONFIRM_MAPPING_DETAILS_FAIL_RAW)
+        logger.warn(f"""Label mapping corrupted: {CONFIRM_MAPPING_DETAILS_FAIL_RAW}. 
+                    \nall_mappings: {all_mappings}\nds_labels: {ds_labels}""")
       
     if len(all_mappings["features"].keys()) != len(ds_features):
-        logger.warn("Feature mapping corrupted: " + CONFIRM_MAPPING_DETAILS_FAIL_RAW)
+        logger.warn(f"""Feature mapping corrupted: {CONFIRM_MAPPING_DETAILS_FAIL_RAW}. 
+                    \nall_mappings: {all_mappings}\nds_features: {ds_features}""")
 
     for i, label in zip(range(len(ds_labels)),  ds_labels):
         # align the saved labels with dataset labels order
         label_mapping.update({str(i): all_mappings["labels"][label]})
 
     if "features" not in all_mappings.keys():
+        logger.warning("features not in all_mappings")
         gr.Warning(CONFIRM_MAPPING_DETAILS_FAIL_RAW)
+
     feature_mapping = all_mappings["features"]
+    if len(label_keys) > 0:
+      feature_mapping.update({"label": label_keys[0]})
     return label_mapping, feature_mapping
 
 def show_hf_token_info(token):
@@ -358,16 +415,18 @@ def show_hf_token_info(token):
         return gr.update(visible=True)
     return gr.update(visible=False)
 
-def try_submit(m_id, d_id, config, split, inference, inference_token, uid):
+def try_submit(m_id, d_id, config, split, uid, profile: gr.OAuthProfile | None, oath_token: gr.OAuthToken | None):
+    print(oath_token.token)
+    print(".>>>>>>>>>>>>>>>>>>>>>>")
     all_mappings = read_column_mapping(uid)
     if not check_column_mapping_keys_validity(all_mappings):
         return (gr.update(interactive=True), gr.update(visible=False))
 
     # get ds labels and features again for alignment
     ds = datasets.load_dataset(d_id, config, split=split, trust_remote_code=True)
-    ds_labels, ds_features = get_labels_and_features_from_dataset(ds)
-    label_mapping, feature_mapping = construct_label_and_feature_mapping(all_mappings, ds_labels, ds_features)
-
+    ds_labels, ds_features, label_keys = get_labels_and_features_from_dataset(ds)
+    label_mapping, feature_mapping = construct_label_and_feature_mapping(all_mappings, ds_labels, ds_features, label_keys)
+    
     eval_str = f"[{m_id}]<{d_id}({config}, {split} set)>"
     save_job_to_pipe(
         uid,
@@ -376,8 +435,7 @@ def try_submit(m_id, d_id, config, split, inference, inference_token, uid):
             d_id,
             config,
             split,
-            inference,
-            inference_token,
+            oath_token.token,
             uid,
             label_mapping,
             feature_mapping,
@@ -387,8 +445,16 @@ def try_submit(m_id, d_id, config, split, inference, inference_token, uid):
     )
     gr.Info("Your evaluation has been submitted")
 
+    new_uid = uuid.uuid4()
+    scanners = read_scanners(uid)
+    write_scanners(scanners, new_uid)
+
     return (
         gr.update(interactive=False),  # Submit button
         gr.update(value=f"{CHECK_LOG_SECTION_RAW}Your job id is: {uid}. ", lines=5, visible=True, interactive=False),
-        uuid.uuid4(),  # Allocate a new uuid
+        new_uid,  # Allocate a new uuid
+        gr.update(visible=False),
+        gr.update(visible=False),
+        gr.update(visible=False),
+        gr.update(visible=False),
     )

utils.py

@@ -0,0 +1,29 @@
+import sys
+
+import yaml
+
+
+# read scanners from yaml file
+# return a list of scanners
+def read_scanners(path):
+    scanners = []
+    with open(path, "r") as f:
+        config = yaml.load(f, Loader=yaml.FullLoader)
+        scanners = config.get("detectors", None)
+    return scanners
+
+
+# convert a list of scanners to yaml file
+def write_scanners(scanners):
+    with open("./scan_config.yaml", "w") as f:
+        # save scanners to detectors in yaml
+        yaml.dump({"detectors": scanners}, f)
+
+
+# convert column mapping dataframe to json
+def convert_column_mapping_to_json(df, label=""):
+    column_mapping = {}
+    column_mapping[label] = []
+    for _, row in df.iterrows():
+        column_mapping[label].append(row.tolist())
+    return column_mapping

utils/wordings.py → wordings.py

@@ -2,27 +2,28 @@ INTRODUCTION_MD = """
                 <h1 style="text-align: center;">
                 🐢Giskard Evaluator - Text Classification
                 </h1>
-                Welcome to the Giskard Evaluator Space! Get a model vulnerability report immediately by simply sharing your model and dataset id below.
+                Welcome to the Giskard Evaluator Space! Get a model vulnerability report immediately by simply sharing your model and dataset id below. 
+                You can also checkout our library documentation <a href="https://docs.giskard.ai/en/latest/getting_started/quickstart/index.html">here</a>.
                 """
 CONFIRM_MAPPING_DETAILS_MD = """
                             <h1 style="text-align: center;">
                             Confirm Pre-processing Details
                             </h1>
-                            Make sure the output variable's labels and the input variable's name are accurately mapped across both the dataset and the model.
+                            Make sure the output variable's labels and the input variable's name are accurately mapped across both the dataset and the model. You can select the output variable's labels from the dropdowns below.
                             """
 CONFIRM_MAPPING_DETAILS_FAIL_MD = """
                             <h1 style="text-align: center;">
                             Confirm Pre-processing Details
                             </h1>
-                            We're unable to automatically map the input variable's name and output variable's labels of your dataset with the model's. <b>Please manually check the mapping below.</b>
+                            We're unable to automatically map the input variable's name and output variable's labels of your dataset with the model's. Please manually check the mapping below.
                             """
 
 CONFIRM_MAPPING_DETAILS_FAIL_RAW = """
-                            We're unable to automatically map the input variable's name and output variable's labels of your dataset with the model's. <b>Please manually check the mapping below.</b>
+                            We're unable to automatically map the input variable's name and output variable's labels of your dataset with the model's. Please manually check the mapping below.
                             """
 
 CHECK_CONFIG_OR_SPLIT_RAW = """
-                            We're unanle to extract labels or features from your dataset. Please check your dataset config or split selection.
+                            Please check your dataset config or split.
                             """
 
 CHECK_LOG_SECTION_RAW = """
@@ -38,7 +39,7 @@ PREDICTION_SAMPLE_MD = """
 
 MAPPING_STYLED_ERROR_WARNING = """
                         <h3 style="text-align: center;color: orange; background-color: #fff0f3; border-radius: 8px; padding: 10px; ">
-                        ⚠️ We're unable to automatically map the input variable's name and output variable's labels of your dataset with the model's. <b>Please manually check the mapping below.</b>
+                        ⚠️ We're unable to automatically map the input variable's name and output variable's labels of your dataset with the model's. Please manually check the mapping below.
                         </h3>
                         """
 
@@ -57,7 +58,11 @@ USE_INFERENCE_API_TIP = """
                 <a href="https://huggingface.co/docs/api-inference/detailed_parameters#text-classification-task">
                     Hugging Face Inference API
                 </a>
-                . Please input your <a href="https://huggingface.co/settings/tokens">Hugging Face token</a> to do so.
+                . Please input your <a href="https://huggingface.co/settings/tokens">Hugging Face token</a> to do so. You can find it <a href="https://huggingface.co/settings/tokens">here</a>.
+            """
+
+LOG_IN_TIPS = """
+            To use the Hugging Face Inference API, you need to log in to your Hugging Face account.
             """
 
 HF_TOKEN_INVALID_STYLED= """
@@ -66,10 +71,10 @@ HF_TOKEN_INVALID_STYLED= """
                 </p>
                 """
 
+VALIDATED_MODEL_DATASET_STYLED = """
+            <h3 style="text-align: center;color: #4ca154; background-color: #e2fbe8; border-radius: 8px; padding: 10px; ">
+            Your model and dataset have been validated!
+            </h3>"""
+
 def get_dataset_fetch_error_raw(error):
     return f"""Sorry you cannot use this dataset because {error}. Contact HF team to support this dataset."""
-
-def get_styled_input(input):
-    return f"""<h3 style="text-align: center;color: #4ca154; background-color: #e2fbe8; border-radius: 8px; padding: 10px; ">
-            Your model and dataset have been validated! <br /> Sample input: {input}
-            </h3>"""