markdown

app.py

@@ -1,11 +1,13 @@
 import os
 import time
-from huggingface_hub import create_repo, whoami
+import traceback
 import gradio as gr
+from huggingface_hub import create_repo, whoami
+
+
 from config_store import (
     get_process_config,
     get_inference_config,
-    get_onnxruntime_config,
     get_openvino_config,
     get_pytorch_config,
     get_ipex_config,
@@ -13,13 +15,11 @@ from config_store import (
 from optimum_benchmark.launchers.base import Launcher  # noqa
 from optimum_benchmark.backends.openvino.utils import TASKS_TO_OVMODEL
 from optimum_benchmark.backends.transformers_utils import TASKS_TO_MODEL_LOADERS
-from optimum_benchmark.backends.onnxruntime.utils import TASKS_TO_ORTMODELS
 from optimum_benchmark.backends.ipex.utils import TASKS_TO_IPEXMODEL
 from optimum_benchmark import (
     BenchmarkConfig,
     PyTorchConfig,
     OVConfig,
-    ORTConfig,
     IPEXConfig,
     ProcessConfig,
     InferenceConfig,
@@ -31,15 +31,13 @@ from optimum_benchmark.logging_utils import setup_logging
 DEVICE = "cpu"
 LAUNCHER = "process"
 SCENARIO = "inference"
-BACKENDS = ["onnxruntime", "openvino", "pytorch", "ipex"]
+BACKENDS = ["openvino", "pytorch", "ipex"]
 MODELS = [
-    "hf-internal-testing/tiny-random-bert",
     "google-bert/bert-base-uncased",
     "openai-community/gpt2",
 ]
 TASKS = (
     set(TASKS_TO_OVMODEL.keys())
-    & set(TASKS_TO_ORTMODELS.keys())
     & set(TASKS_TO_IPEXMODEL.keys())
     & set(TASKS_TO_MODEL_LOADERS.keys())
 )
@@ -47,20 +45,19 @@ TASKS = (
 
 def run_benchmark(kwargs, oauth_token: gr.OAuthToken):
     if oauth_token.token is None:
-        return "You must be logged in to use this space"
+        raise gr.Error("Please login to be able to run the benchmark.")
 
+    timestamp = time.strftime("%Y-%m-%d-%H-%M-%S")
     username = whoami(oauth_token.token)["name"]
-    create_repo(
-        f"{username}/benchmarks",
-        token=oauth_token.token,
-        repo_type="dataset",
-        exist_ok=True,
-    )
+    repo_id = f"{username}/benchmarks"
+    token = oauth_token.token
+
+    create_repo(repo_id, token=token, repo_type="dataset", exist_ok=True)
+    gr.Info(f'Benchmark will be pushed to "{username}/benchmarks" on the Hub')
 
     configs = {
         "process": {},
         "inference": {},
-        "onnxruntime": {},
         "openvino": {},
         "pytorch": {},
         "ipex": {},
@@ -82,12 +79,6 @@ def run_benchmark(kwargs, oauth_token: gr.OAuthToken):
     configs["process"] = ProcessConfig(**configs.pop("process"))
     configs["inference"] = InferenceConfig(**configs.pop("inference"))
 
-    configs["onnxruntime"] = ORTConfig(
-        task=task,
-        model=model,
-        device=DEVICE,
-        **configs["onnxruntime"],
-    )
     configs["openvino"] = OVConfig(
         task=task,
         model=model,
@@ -107,18 +98,15 @@ def run_benchmark(kwargs, oauth_token: gr.OAuthToken):
         **configs["ipex"],
     )
 
-    md_output = (
-        f"<h3>Running benchmark for model {model} on task {task} with {backends}</h3>"
-    )
-
-    yield md_output
+    outputs = {
+        "openvino": "Running benchmark for OpenVINO backend",
+        "pytorch": "Running benchmark for PyTorch backend",
+        "ipex": "Running benchmark for IPEX backend",
+    }
 
-    timestamp = time.strftime("%Y-%m-%d-%H-%M-%S")
+    yield tuple(outputs[b] for b in BACKENDS)
 
     for backend in backends:
-        md_output += f"<br>🚀 Launching benchmark for {backend}"
-        yield md_output
-
         try:
             benchmark_name = f"{timestamp}/{backend}"
             benchmark_config = BenchmarkConfig(
@@ -128,32 +116,28 @@ def run_benchmark(kwargs, oauth_token: gr.OAuthToken):
                 scenario=configs[SCENARIO],
             )
             benchmark_config.push_to_hub(
-                repo_id=f"{username}/benchmarks",
-                subfolder=benchmark_name,
-                token=oauth_token.token,
+                repo_id=repo_id, subfolder=benchmark_name, token=oauth_token.token
             )
             benchmark_report = Benchmark.launch(benchmark_config)
             benchmark_report.push_to_hub(
-                repo_id=f"{username}/benchmarks",
-                subfolder=benchmark_name,
-                token=oauth_token.token,
+                repo_id=repo_id, subfolder=benchmark_name, token=oauth_token.token
             )
             benchmark = Benchmark(config=benchmark_config, report=benchmark_report)
             benchmark.push_to_hub(
-                repo_id=f"{username}/benchmarks",
-                subfolder=benchmark_name,
-                token=oauth_token.token,
+                repo_id=repo_id, subfolder=benchmark_name, token=oauth_token.token
             )
+            gr.Info(f"Pushed benchmark to {username}/benchmarks/{benchmark_name}")
 
-            md_output += (
-                f"<br>✅ Benchmark for {backend} backend completed successfully"
-            )
-            yield md_output
-        except Exception as e:
-            md_output += (
-                f"<br>❌ Error while running benchmark for {backend} backend: {e}"
-            )
-            yield md_output
+            outputs[backend] = f"\n{benchmark_report.to_markdown_text()}"
+
+            yield tuple(outputs[b] for b in BACKENDS)
+
+        except Exception:
+            gr.Error(f"Error while running benchmark for {backend}")
+
+            outputs[backend] = f"\n{traceback.format_exc()}"
+
+            yield tuple(outputs[b] for b in BACKENDS)
 
 
 def build_demo():
@@ -211,8 +195,6 @@ def build_demo():
                 inference_config = get_inference_config()
 
         with gr.Row() as backend_configs:
-            with gr.Accordion(label="OnnxRuntime Config", open=False, visible=True):
-                onnxruntime_config = get_onnxruntime_config()
             with gr.Accordion(label="OpenVINO Config", open=False, visible=True):
                 openvino_config = get_openvino_config()
             with gr.Accordion(label="PyTorch Config", open=False, visible=True):
@@ -231,8 +213,21 @@ def build_demo():
         with gr.Row():
             button = gr.Button(value="Run Benchmark", variant="primary")
 
-        with gr.Row():
-            md_output = gr.Markdown(label="Output", value="")
+        with gr.Row() as md_output:
+            with gr.Accordion(label="OpenVINO Output", open=True, visible=True):
+                openvino_output = gr.Markdown()
+            with gr.Accordion(label="PyTorch Output", open=True, visible=True):
+                pytorch_output = gr.Markdown()
+            with gr.Accordion(label="IPEX Output", open=True, visible=True):
+                ipex_output = gr.Markdown()
+
+        backends.change(
+            inputs=backends,
+            outputs=md_output.children,
+            fn=lambda values: [
+                gr.update(visible=value in values) for value in BACKENDS
+            ],
+        )
 
         button.click(
             fn=run_benchmark,
@@ -242,12 +237,15 @@ def build_demo():
                 backends,
                 *process_config.values(),
                 *inference_config.values(),
-                *onnxruntime_config.values(),
                 *openvino_config.values(),
                 *pytorch_config.values(),
                 *ipex_config.values(),
             },
-            outputs=[md_output],
+            outputs={
+                openvino_output,
+                pytorch_output,
+                ipex_output,
+            },
             concurrency_limit=1,
         )
 

config_store.py

@@ -16,6 +16,45 @@ def get_process_config():
     }
 
 
+def get_inference_config():
+    return {
+        "inference.warmup_runs": gr.Slider(
+            step=1,
+            value=10,
+            minimum=0,
+            maximum=10,
+            label="inference.warmup_runs",
+            info="Number of warmup runs",
+        ),
+        "inference.duration": gr.Slider(
+            step=1,
+            value=10,
+            minimum=0,
+            maximum=10,
+            label="inference.duration",
+            info="Minimum duration of the benchmark in seconds",
+        ),
+        "inference.iterations": gr.Slider(
+            step=1,
+            value=10,
+            minimum=0,
+            maximum=10,
+            label="inference.iterations",
+            info="Minimum number of iterations of the benchmark",
+        ),
+        "inference.latency": gr.Checkbox(
+            value=True,
+            label="inference.latency",
+            info="Measures the latency of the model",
+        ),
+        "inference.memory": gr.Checkbox(
+            value=False,
+            label="inference.memory",
+            info="Measures the peak memory consumption",
+        ),
+    }
+
+
 def get_pytorch_config():
     return {
         "pytorch.torch_dtype": gr.Dropdown(
@@ -90,42 +129,3 @@ def get_openvino_config():
 
 def get_ipex_config():
     return {}
-
-
-def get_inference_config():
-    return {
-        "inference.warmup_runs": gr.Slider(
-            step=1,
-            value=10,
-            minimum=0,
-            maximum=10,
-            label="inference.warmup_runs",
-            info="Number of warmup runs",
-        ),
-        "inference.duration": gr.Slider(
-            step=1,
-            value=10,
-            minimum=0,
-            maximum=10,
-            label="inference.duration",
-            info="Minimum duration of the benchmark in seconds",
-        ),
-        "inference.iterations": gr.Slider(
-            step=1,
-            value=10,
-            minimum=0,
-            maximum=10,
-            label="inference.iterations",
-            info="Minimum number of iterations of the benchmark",
-        ),
-        "inference.latency": gr.Checkbox(
-            value=True,
-            label="inference.latency",
-            info="Measures the latency of the model",
-        ),
-        "inference.memory": gr.Checkbox(
-            value=False,
-            label="inference.memory",
-            info="Measures the peak memory consumption",
-        ),
-    }

requirements.txt

@@ -1 +1 @@
-optimum-benchmark[openvino,onnxruntime,ipex]@git+https://github.com/huggingface/optimum-benchmark.git
+optimum-benchmark[openvino,onnxruntime,ipex]@git+https://github.com/huggingface/optimum-benchmark.git@markdown-report