|
import os |
|
import time |
|
import traceback |
|
|
|
from config_store import ( |
|
get_process_config, |
|
get_inference_config, |
|
get_openvino_config, |
|
get_pytorch_config, |
|
) |
|
|
|
import gradio as gr |
|
from huggingface_hub import create_repo, whoami |
|
from optimum_benchmark.launchers.device_isolation_utils import * |
|
from optimum_benchmark.backends.openvino.utils import TASKS_TO_OVMODEL |
|
from optimum_benchmark.backends.transformers_utils import TASKS_TO_MODEL_LOADERS |
|
from optimum_benchmark import ( |
|
Benchmark, |
|
BenchmarkConfig, |
|
ProcessConfig, |
|
InferenceConfig, |
|
PyTorchConfig, |
|
OVConfig, |
|
) |
|
from optimum_benchmark.logging_utils import setup_logging |
|
|
|
|
|
DEVICE = "cpu" |
|
LAUNCHER = "process" |
|
SCENARIO = "inference" |
|
BACKENDS = ["pytorch", "openvino"] |
|
MODELS = [ |
|
"openai-community/gpt2", |
|
"google-bert/bert-base-uncased", |
|
"hf-internal-testing/tiny-random-LlamaForCausalLM", |
|
"hf-internal-testing/tiny-random-BertForSequenceClassification", |
|
] |
|
MODELS_TO_TASKS = { |
|
"openai-community/gpt2": "text-generation", |
|
"google-bert/bert-base-uncased": "text-classification", |
|
"hf-internal-testing/tiny-random-LlamaForCausalLM": "text-generation", |
|
"hf-internal-testing/tiny-random-BertForSequenceClassification": "text-classification", |
|
} |
|
TASKS = set(TASKS_TO_OVMODEL.keys()) & set(TASKS_TO_MODEL_LOADERS.keys()) |
|
|
|
|
|
def run_benchmark(kwargs, oauth_token: gr.OAuthToken): |
|
if oauth_token.token is None: |
|
gr.Error("Please login to be able to run the benchmark.") |
|
return tuple(None for _ in BACKENDS) |
|
|
|
timestamp = time.strftime("%Y-%m-%d-%H-%M-%S") |
|
username = whoami(oauth_token.token)["name"] |
|
repo_id = f"{username}/benchmarks" |
|
token = oauth_token.token |
|
|
|
create_repo(repo_id, token=token, repo_type="dataset", exist_ok=True) |
|
gr.Info(f'Created repository "{repo_id}" where results will be pushed.') |
|
|
|
configs = { |
|
"process": {}, |
|
"inference": {}, |
|
"pytorch": {}, |
|
"openvino": {}, |
|
} |
|
|
|
for key, value in kwargs.items(): |
|
if key.label == "model": |
|
model = value |
|
elif key.label == "task": |
|
task = value |
|
elif key.label == "backends": |
|
backends = value |
|
elif "." in key.label: |
|
backend, argument = key.label.split(".") |
|
configs[backend][argument] = value |
|
else: |
|
continue |
|
|
|
for key in configs.keys(): |
|
for k, v in configs[key].items(): |
|
if k in ["input_shapes", "generate_kwargs", "numactl_kwargs"]: |
|
configs[key][k] = eval(v) |
|
|
|
configs["process"] = ProcessConfig(**configs.pop("process")) |
|
configs["inference"] = InferenceConfig(**configs.pop("inference")) |
|
configs["pytorch"] = PyTorchConfig( |
|
task=task, |
|
model=model, |
|
device=DEVICE, |
|
**configs["pytorch"], |
|
) |
|
configs["openvino"] = OVConfig( |
|
task=task, |
|
model=model, |
|
device=DEVICE, |
|
**configs["openvino"], |
|
) |
|
|
|
outputs = { |
|
"pytorch": "Running benchmark for PyTorch backend", |
|
"openvino": "Running benchmark for OpenVINO backend", |
|
} |
|
|
|
yield tuple(outputs[b] for b in BACKENDS) |
|
|
|
for backend in backends: |
|
try: |
|
benchmark_name = f"{timestamp}/{backend}" |
|
benchmark_config = BenchmarkConfig( |
|
name=benchmark_name, |
|
backend=configs[backend], |
|
launcher=configs[LAUNCHER], |
|
scenario=configs[SCENARIO], |
|
) |
|
benchmark_config.push_to_hub( |
|
repo_id=repo_id, subfolder=benchmark_name, token=oauth_token.token |
|
) |
|
benchmark_report = Benchmark.launch(benchmark_config) |
|
benchmark_report.push_to_hub( |
|
repo_id=repo_id, subfolder=benchmark_name, token=oauth_token.token |
|
) |
|
benchmark = Benchmark(config=benchmark_config, report=benchmark_report) |
|
benchmark.push_to_hub( |
|
repo_id=repo_id, subfolder=benchmark_name, token=oauth_token.token |
|
) |
|
gr.Info(f"Pushed benchmark to {username}/benchmarks/{benchmark_name}") |
|
|
|
outputs[backend] = f"\n{benchmark_report.to_markdown_text()}" |
|
|
|
yield tuple(outputs[b] for b in BACKENDS) |
|
|
|
except Exception: |
|
gr.Error(f"Error while running benchmark for {backend}") |
|
|
|
outputs[backend] = f"\n```python\n{traceback.format_exc()}```" |
|
|
|
yield tuple(outputs[b] for b in BACKENDS) |
|
|
|
|
|
def build_demo(): |
|
with gr.Blocks() as demo: |
|
|
|
gr.LoginButton(min_width=250) |
|
|
|
|
|
gr.HTML( |
|
"""<img src="https://huggingface.co/spaces/optimum/optimum-benchmark-ui/resolve/main/huggy_bench.png" style="display: block; margin-left: auto; margin-right: auto; width: 30%;">""" |
|
"<h1 style='text-align: center'>🤗 Optimum-Benchmark Interface 🏋️</h1>" |
|
"<p style='text-align: center'>" |
|
"This Space uses <a href='https://github.com/huggingface/optimum-benchmark.git'>Optimum-Benchmark</a> to automatically benchmark a model from the Hub on different backends." |
|
"<br>The results (config and report) will be pushed under your namespace in a benchmark repository on the Hub." |
|
"</p>" |
|
) |
|
|
|
model = gr.Dropdown( |
|
label="model", |
|
choices=MODELS, |
|
value=MODELS[0], |
|
info="Model to run the benchmark on.", |
|
) |
|
task = gr.Dropdown( |
|
label="task", |
|
choices=TASKS, |
|
value="feature-extraction", |
|
info="Task to run the benchmark on.", |
|
) |
|
|
|
backends = gr.CheckboxGroup( |
|
interactive=True, |
|
label="backends", |
|
choices=BACKENDS, |
|
value=BACKENDS, |
|
info="Backends to run the benchmark on.", |
|
) |
|
|
|
with gr.Row(): |
|
with gr.Accordion(label="Process Config", open=False, visible=True): |
|
process_config = get_process_config() |
|
with gr.Row(): |
|
with gr.Accordion(label="Inference Config", open=False, visible=True): |
|
inference_config = get_inference_config() |
|
|
|
with gr.Row() as backend_configs: |
|
with gr.Accordion(label="PyTorch Config", open=False, visible=True): |
|
pytorch_config = get_pytorch_config() |
|
with gr.Accordion(label="OpenVINO Config", open=False, visible=True): |
|
openvino_config = get_openvino_config() |
|
|
|
with gr.Row(): |
|
button = gr.Button(value="Run Benchmark", variant="primary") |
|
|
|
with gr.Row() as markdown_outputs: |
|
with gr.Accordion(label="PyTorch Output", open=True, visible=True): |
|
pytorch_output = gr.Markdown() |
|
with gr.Accordion(label="OpenVINO Output", open=True, visible=True): |
|
openvino_output = gr.Markdown() |
|
|
|
model.change( |
|
inputs=model, outputs=task, fn=lambda value: MODELS_TO_TASKS[value] |
|
) |
|
|
|
backends.change( |
|
inputs=backends, |
|
outputs=backend_configs.children, |
|
fn=lambda values: [ |
|
gr.update(visible=value in values) for value in BACKENDS |
|
], |
|
) |
|
|
|
backends.change( |
|
inputs=backends, |
|
outputs=markdown_outputs.children, |
|
fn=lambda values: [ |
|
gr.update(visible=value in values) for value in BACKENDS |
|
], |
|
) |
|
|
|
button.click( |
|
fn=run_benchmark, |
|
inputs={ |
|
task, |
|
model, |
|
backends, |
|
*process_config.values(), |
|
*inference_config.values(), |
|
*pytorch_config.values(), |
|
*openvino_config.values(), |
|
}, |
|
outputs={ |
|
pytorch_output, |
|
openvino_output, |
|
}, |
|
concurrency_limit=1, |
|
) |
|
|
|
return demo |
|
|
|
|
|
demo = build_demo() |
|
|
|
if __name__ == "__main__": |
|
os.environ["LOG_TO_FILE"] = "0" |
|
os.environ["LOG_LEVEL"] = "INFO" |
|
setup_logging(level="INFO", prefix="MAIN-PROCESS") |
|
|
|
demo.queue(max_size=10).launch() |
|
|
