diff --git a/src/winml/modelkit/commands/_perf_genai.py b/src/winml/modelkit/commands/_perf_genai.py index af6ffeef6..7c960d23e 100644 --- a/src/winml/modelkit/commands/_perf_genai.py +++ b/src/winml/modelkit/commands/_perf_genai.py @@ -27,10 +27,9 @@ import json import logging -from dataclasses import dataclass, field -from datetime import datetime, timezone +from dataclasses import dataclass from pathlib import Path -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING import click @@ -41,6 +40,15 @@ GenaiSessionError, GenerationConfig, ) +from ._perf_generation import ( + GenerationBenchmarkResult, + display_generation_report, +) + + +# Backward-compatible aliases — existing code and tests import these names. +GenaiBenchmarkResult = GenerationBenchmarkResult +display_genai_report = display_generation_report if TYPE_CHECKING: @@ -150,92 +158,6 @@ class _RunSample: n_tokens: int -@dataclass -class GenaiBenchmarkResult: - """Aggregated results from a genai generation benchmark.""" - - config: GenaiPerfConfig - timestamp: str = field(default_factory=lambda: datetime.now(timezone.utc).isoformat()) - - # Generation shape - prompt_tokens: int = 0 - generated_tokens: int = 0 - context_length: int | None = None - - # Time to first token (prefill + first decode), milliseconds - ttft_mean_ms: float = 0.0 - ttft_min_ms: float = 0.0 - ttft_max_ms: float = 0.0 - ttft_p50_ms: float = 0.0 - ttft_p90_ms: float = 0.0 - ttft_p95_ms: float = 0.0 - ttft_p99_ms: float = 0.0 - - # Prefill / prompt-processing phase (og append_tokens), milliseconds - prefill_mean_ms: float = 0.0 - - # Decode phase - decode_tokens_per_sec: float = 0.0 - avg_token_latency_ms: float = 0.0 - # Time per output token — steady-state decode (og generate_next_token), ms - tpot_mean_ms: float = 0.0 - - # Whole generation (prefill + all decode), milliseconds - total_generation_mean_ms: float = 0.0 - - # Per-iteration samples (warmup excluded) - raw_ttft_ms: list[float] = field(default_factory=list) - raw_prefill_ms: list[float] = field(default_factory=list) - raw_decode_tokens_per_sec: list[float] = field(default_factory=list) - raw_tpot_ms: list[float] = field(default_factory=list) - raw_total_ms: list[float] = field(default_factory=list) - - def to_dict(self) -> dict[str, Any]: - """Convert to a JSON-serializable dictionary.""" - return { - "benchmark_info": { - "runtime": RUNTIME_TYPE, - "bundle_dir": str(self.config.bundle_dir), - "ep": self.config.ep, - "device": self.config.device, - "compile": self.config.compile, - "compile_timeout": self.config.compile_timeout, - "iterations": self.config.iterations, - "warmup": self.config.warmup, - "max_new_tokens": self.config.max_new_tokens, - "apply_template": self.config.apply_template, - "prompt": self.config.prompt, - "prompt_tokens": self.prompt_tokens, - "generated_tokens": self.generated_tokens, - "context_length": self.context_length, - "timestamp": self.timestamp, - }, - "ttft_ms": { - "mean": round(self.ttft_mean_ms, 3), - "min": round(self.ttft_min_ms, 3), - "max": round(self.ttft_max_ms, 3), - "p50": round(self.ttft_p50_ms, 3), - "p90": round(self.ttft_p90_ms, 3), - "p95": round(self.ttft_p95_ms, 3), - "p99": round(self.ttft_p99_ms, 3), - }, - "prefill_ms": {"mean": round(self.prefill_mean_ms, 3)}, - "decode": { - "tokens_per_sec": round(self.decode_tokens_per_sec, 2), - "avg_token_latency_ms": round(self.avg_token_latency_ms, 3), - "tpot_ms": round(self.tpot_mean_ms, 3), - }, - "total_generation_ms": {"mean": round(self.total_generation_mean_ms, 3)}, - "raw": { - "ttft_ms": [round(v, 3) for v in self.raw_ttft_ms], - "prefill_ms": [round(v, 3) for v in self.raw_prefill_ms], - "decode_tokens_per_sec": [round(v, 2) for v in self.raw_decode_tokens_per_sec], - "tpot_ms": [round(v, 3) for v in self.raw_tpot_ms], - "total_ms": [round(v, 3) for v in self.raw_total_ms], - }, - } - - # ============================================================================= # Benchmark engine # ============================================================================= @@ -375,8 +297,16 @@ def _aggregate(self, samples: list[_RunSample]) -> GenaiBenchmarkResult: token_latencies = [s.total_ms / s.n_tokens for s in timed if s.n_tokens] sorted_ttfts = sorted(ttfts) + cfg = self._config return GenaiBenchmarkResult( - config=self._config, + runtime=RUNTIME_TYPE, + model_label=str(cfg.bundle_dir), + device=cfg.device, + ep=cfg.ep, + prompt=cfg.prompt, + max_new_tokens=cfg.max_new_tokens, + warmup=cfg.warmup, + iterations=cfg.iterations, prompt_tokens=len(self._prompt_token_ids), generated_tokens=timed[0].n_tokens if timed else 0, context_length=self._session.context_length if self._session else None, @@ -397,6 +327,12 @@ def _aggregate(self, samples: list[_RunSample]) -> GenaiBenchmarkResult: raw_decode_tokens_per_sec=decode_tps, raw_tpot_ms=tpots, raw_total_ms=totals, + extra_info={ + "bundle_dir": str(cfg.bundle_dir), + "compile": cfg.compile, + "compile_timeout": cfg.compile_timeout, + "apply_template": cfg.apply_template, + }, ) @@ -405,56 +341,6 @@ def _aggregate(self, samples: list[_RunSample]) -> GenaiBenchmarkResult: # ============================================================================= -def display_genai_report(result: GenaiBenchmarkResult, console: Console) -> None: - """Render a genai benchmark report to the console.""" - from rich.table import Table - - cfg = result.config - console.print() - console.print(f"[dim]Runtime:[/dim] {RUNTIME_TYPE}") - device_str = cfg.device if cfg.device == cfg.ep else f"{cfg.device} ({cfg.ep})" - console.print(f"[dim]Device:[/dim] {device_str}") - console.print(f"[dim]Bundle:[/dim] {cfg.bundle_dir}") - console.print( - f"[dim]Prompt:[/dim] {result.prompt_tokens} tokens " - f"[dim]Generated:[/dim] {result.generated_tokens} tokens " - f"(max_new_tokens={cfg.max_new_tokens})" - ) - - console.print() - console.print("[bold]Time to first token (ms)[/bold]") - table = Table(show_header=True, header_style="bold cyan") - for col in ["Avg", "P50", "P90", "P95", "P99", "Min", "Max"]: - table.add_column(col, justify="right") - table.add_row( - f"{result.ttft_mean_ms:.2f}", - f"{result.ttft_p50_ms:.2f}", - f"{result.ttft_p90_ms:.2f}", - f"{result.ttft_p95_ms:.2f}", - f"{result.ttft_p99_ms:.2f}", - f"{result.ttft_min_ms:.2f}", - f"{result.ttft_max_ms:.2f}", - ) - console.print(table) - - console.print() - console.print( - f"[bold]Prefill:[/bold] {result.prefill_mean_ms:.2f} ms avg (prompt processing)" - ) - console.print( - f"[bold]Decode:[/bold] {result.decode_tokens_per_sec:.2f} tokens/sec | " - f"{result.tpot_mean_ms:.2f} ms/token (TPOT)" - ) - console.print( - f"[bold]Total:[/bold] {result.total_generation_mean_ms:.2f} ms avg per generation" - ) - if cfg.warmup > 0: - console.print( - f" [dim]Excluded first {cfg.warmup} warmup generation(s) from statistics[/dim]" - ) - console.print() - - def write_genai_report(result: GenaiBenchmarkResult, output_path: str | Path) -> None: """Write the genai benchmark result to a JSON file.""" output_path = Path(output_path) diff --git a/src/winml/modelkit/commands/_perf_generation.py b/src/winml/modelkit/commands/_perf_generation.py new file mode 100644 index 000000000..dd19c9a69 --- /dev/null +++ b/src/winml/modelkit/commands/_perf_generation.py @@ -0,0 +1,189 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +# -------------------------------------------------------------------------- +"""Shared generation-benchmark result and display helpers. + +Both the WinML composite-model path (``perf.py``) and the +``onnxruntime-genai`` path (``_perf_genai.py``) report the same LLM-style +metrics — TTFT, prefill, decode throughput, TPOT, and total generation time. +This module provides the shared :class:`GenerationBenchmarkResult` dataclass +and :func:`display_generation_report` so neither module duplicates the +structure or display logic. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from datetime import datetime, timezone +from typing import TYPE_CHECKING, Any + + +if TYPE_CHECKING: + from rich.console import Console + + +@dataclass +class GenerationBenchmarkResult: + """Aggregated results from a generation benchmark. + + Covers both WinML composite-model generation (``perf.py``) and + onnxruntime-genai bundle generation (``_perf_genai.py``). Fields that + only apply to one path default to ``0.0`` / ``None`` when unused. + """ + + # --- Display / provenance info (populated by the caller) ----------------- + runtime: str = "" + model_label: str = "" + device: str = "" + ep: str | None = None + prompt: str = "" + max_new_tokens: int = 0 + warmup: int = 0 + iterations: int = 0 + timestamp: str = field(default_factory=lambda: datetime.now(timezone.utc).isoformat()) + + # --- Generation shape ---------------------------------------------------- + prompt_tokens: int = 0 + generated_tokens: int = 0 + context_length: int | None = None + + # --- Time to first token (prefill + first decode), milliseconds ---------- + ttft_mean_ms: float = 0.0 + ttft_min_ms: float = 0.0 + ttft_max_ms: float = 0.0 + ttft_p50_ms: float = 0.0 + ttft_p90_ms: float = 0.0 + ttft_p95_ms: float = 0.0 + ttft_p99_ms: float = 0.0 + + # --- Prefill / prompt-processing phase, milliseconds --------------------- + prefill_mean_ms: float = 0.0 + + # --- Decode phase -------------------------------------------------------- + decode_tokens_per_sec: float = 0.0 + avg_token_latency_ms: float = 0.0 + tpot_mean_ms: float = 0.0 + + # --- Whole generation (prefill + all decode), milliseconds ---------------- + total_generation_mean_ms: float = 0.0 + + # --- Per-iteration raw samples (warmup excluded) ------------------------- + raw_ttft_ms: list[float] = field(default_factory=list) + raw_prefill_ms: list[float] = field(default_factory=list) + raw_decode_tokens_per_sec: list[float] = field(default_factory=list) + raw_tpot_ms: list[float] = field(default_factory=list) + raw_total_ms: list[float] = field(default_factory=list) + + # Runtime-specific info merged into ``benchmark_info`` in ``to_dict()``. + # E.g. genai adds ``compile``, ``apply_template``, ``bundle_dir``. + extra_info: dict[str, Any] = field(default_factory=dict) + + def to_dict(self) -> dict[str, Any]: + """Convert to a JSON-serializable dictionary.""" + info: dict[str, Any] = { + "runtime": self.runtime, + "model": self.model_label, + "device": self.device, + "ep": self.ep, + "prompt": self.prompt, + "prompt_tokens": self.prompt_tokens, + "generated_tokens": self.generated_tokens, + "context_length": self.context_length, + "max_new_tokens": self.max_new_tokens, + "iterations": self.iterations, + "warmup": self.warmup, + "timestamp": self.timestamp, + } + info.update(self.extra_info) + return { + "benchmark_info": info, + "ttft_ms": { + "mean": round(self.ttft_mean_ms, 3), + "min": round(self.ttft_min_ms, 3), + "max": round(self.ttft_max_ms, 3), + "p50": round(self.ttft_p50_ms, 3), + "p90": round(self.ttft_p90_ms, 3), + "p95": round(self.ttft_p95_ms, 3), + "p99": round(self.ttft_p99_ms, 3), + }, + "prefill_ms": {"mean": round(self.prefill_mean_ms, 3)}, + "decode": { + "tokens_per_sec": round(self.decode_tokens_per_sec, 2), + "avg_token_latency_ms": round(self.avg_token_latency_ms, 3), + "tpot_ms": round(self.tpot_mean_ms, 3), + }, + "total_generation_ms": { + "mean": round(self.total_generation_mean_ms, 3), + }, + "raw": { + "ttft_ms": [round(v, 3) for v in self.raw_ttft_ms], + "prefill_ms": [round(v, 3) for v in self.raw_prefill_ms], + "decode_tokens_per_sec": [round(v, 2) for v in self.raw_decode_tokens_per_sec], + "tpot_ms": [round(v, 3) for v in self.raw_tpot_ms], + "total_ms": [round(v, 3) for v in self.raw_total_ms], + }, + } + + +def display_generation_report( + result: GenerationBenchmarkResult, + console: Console, +) -> None: + """Render a generation benchmark report to the console. + + Works for both WinML composite and genai bundle results — the layout + adapts based on which optional fields are populated. + """ + from rich.table import Table + + console.print() + console.print(f"[dim]Runtime:[/dim] {result.runtime}") + console.print(f"[dim]Model:[/dim] {result.model_label}") + if result.device: + device_str = ( + result.device + if not result.ep or result.device == result.ep + else f"{result.device} ({result.ep})" + ) + console.print(f"[dim]Device:[/dim] {device_str}") + console.print( + f"[dim]Prompt:[/dim] {result.prompt_tokens} tokens " + f"[dim]Generated:[/dim] {result.generated_tokens} tokens " + f"(max_new_tokens={result.max_new_tokens})" + ) + + console.print() + console.print("[bold]Time to first token (ms)[/bold]") + table = Table(show_header=True, header_style="bold cyan") + for col in ["Avg", "P50", "P90", "P95", "P99", "Min", "Max"]: + table.add_column(col, justify="right") + table.add_row( + f"{result.ttft_mean_ms:.2f}", + f"{result.ttft_p50_ms:.2f}", + f"{result.ttft_p90_ms:.2f}", + f"{result.ttft_p95_ms:.2f}", + f"{result.ttft_p99_ms:.2f}", + f"{result.ttft_min_ms:.2f}", + f"{result.ttft_max_ms:.2f}", + ) + console.print(table) + + console.print() + if result.prefill_mean_ms > 0: + console.print( + f"[bold]Prefill:[/bold] {result.prefill_mean_ms:.2f} ms avg (prompt processing)" + ) + console.print( + f"[bold]Decode:[/bold] " + f"{result.decode_tokens_per_sec:.2f} tokens/sec | " + f"{result.tpot_mean_ms:.2f} ms/token (TPOT)" + ) + console.print( + f"[bold]Total:[/bold] {result.total_generation_mean_ms:.2f} ms avg per generation" + ) + if result.warmup > 0: + console.print( + f" [dim]Excluded first {result.warmup} warmup generation(s) from statistics[/dim]" + ) + console.print() diff --git a/src/winml/modelkit/commands/eval.py b/src/winml/modelkit/commands/eval.py index 4fd39288e..5a54a4d48 100644 --- a/src/winml/modelkit/commands/eval.py +++ b/src/winml/modelkit/commands/eval.py @@ -447,80 +447,8 @@ def _write_and_display( Console(stderr=True).print(f"[green]Results saved to:[/green] {output_path}") -def _resolve_model_path( - *, - model: tuple[str, ...], - model_id: str | None, -) -> tuple[str | dict[str, str] | None, str | None]: - """Turn repeated -m values + --model-id into (model_path, model_id).""" - if not model: - if model_id is not None: - return None, model_id - raise click.UsageError( - "A model is required. Provide -m with a HuggingFace model ID, " - "a path to an .onnx file, or role=path pairs for composite models." - ) - - role_assigned = [v for v in model if "=" in v] - plain = [v for v in model if "=" not in v] - - if role_assigned and plain: - raise click.UsageError( - "Cannot mix plain `-m ` and `-m role=path` forms. " - "Use `role=path` consistently for composite models." - ) - - if role_assigned: - if model_id is None: - raise click.UsageError( - "--model-id is required when using composite `-m role=path` options." - ) - sub_model_paths: dict[str, str] = {} - for v in role_assigned: - role, _, path = v.partition("=") - role, path = role.strip(), path.strip() - if not role or not path: - raise click.BadParameter( - f"Invalid role=path: {v!r}. Both role and path are required.", - param_hint="-m/--model", - ) - if role in sub_model_paths: - raise click.BadParameter( - f"Duplicate role {role!r} in -m options.", - param_hint="-m/--model", - ) - if not Path(path).exists(): - raise click.BadParameter( - f"ONNX file not found: {path}", - param_hint="-m/--model", - ) - sub_model_paths[role] = path - return sub_model_paths, model_id - - if len(plain) > 1: - raise click.UsageError( - "Multiple -m values require `role=path` syntax for composite models." - ) - - value = plain[0] - try: - _mi = cli_utils.classify_model_input(value) - except click.UsageError as e: - # Preserve eval's param-scoped BadParameter contract (tests + hint). - raise click.BadParameter(str(e), param_hint="-m/--model") from e - if _mi.kind is cli_utils.ModelInputKind.ONNX_FILE: - if model_id is None: - raise click.UsageError( - "When using an ONNX file, --model-id is required " - "for preprocessor and config resolution." - ) - return value, model_id - if model_id is not None and model_id != value: - raise click.UsageError( - "Cannot pass both `-m ` and `--model-id`. " - "Use `--model-id` only together with an ONNX file path in `-m`." - ) - return None, model_id or value +# Delegate to shared implementation in cli_utils. +_resolve_model_path = cli_utils.resolve_model_path def _json_default(obj: object) -> object: diff --git a/src/winml/modelkit/commands/perf.py b/src/winml/modelkit/commands/perf.py index cd22971b9..8d515fe37 100644 --- a/src/winml/modelkit/commands/perf.py +++ b/src/winml/modelkit/commands/perf.py @@ -32,6 +32,10 @@ from ..utils.constants import EPName, EPNameOrAlias from ..utils.logging import configure_logging from ._live_chart import LiveMonitorDisplay +from ._perf_generation import ( + GenerationBenchmarkResult, + display_generation_report, +) if TYPE_CHECKING: @@ -68,6 +72,9 @@ 3: 224, # Width } +# Default prompt for generation benchmarks (shared with click option default). +_DEFAULT_PROMPT = "Explain the theory of relativity in simple terms." + # ============================================================================= # Data Classes @@ -101,6 +108,16 @@ class BenchmarkConfig: ep_options: dict[str, str] | None = None shape_config: dict | None = None + # Composite model support: dict of {role: onnx_path} for from_onnx dispatch + model_path: str | dict[str, str] | None = None + # HF model ID for composite models loaded from ONNX (needed for hf_config) + hf_model_id: str | None = None + + # Generation benchmark (decoder-only composite models) + prompt: str | None = None + max_new_tokens: int = 128 + apply_template: bool = True + @dataclass class BenchmarkResult: @@ -421,6 +438,16 @@ def _is_composite(self) -> bool: return isinstance(self._model, WinMLCompositeModel) + @property + def _is_generative_composite(self) -> bool: + """True when the loaded model is a generative composite (e.g. Qwen). + + Checks for the ``generate`` capability (provided by + ``GenerationMixin``) rather than a specific class, so any composite + model that supports generation gets the text-prompt benchmark path. + """ + return self._is_composite and hasattr(self._model, "generate") + @property def _sub_models(self) -> dict[str, WinMLPreTrainedModel]: """Sub-models of a composite model (only valid when ``_is_composite``).""" @@ -443,21 +470,23 @@ def _single(self) -> WinMLPreTrainedModel: assert self._model is not None return cast("WinMLPreTrainedModel", self._model) - def run(self) -> BenchmarkResult | dict[str, BenchmarkResult]: + def run(self) -> BenchmarkResult | GenerationBenchmarkResult | dict[str, BenchmarkResult]: """Execute full benchmark pipeline. Returns: - A single ``BenchmarkResult`` for single-session models, or a - ``{sub_model_name: BenchmarkResult}`` mapping for composite models - (e.g. CLIP/SigLIP dual-encoders). Composite models have no single - ORT session, so each sub-model is benchmarked individually rather - than timing the aggregate ``forward()`` pass. + A single ``BenchmarkResult`` for single-session models, + a ``GenerationBenchmarkResult`` for decoder-only composite models + (e.g. Qwen — benchmarked via ``model.generate()`` with a prompt), + or a ``{sub_model_name: BenchmarkResult}`` mapping for non-generative + composite models (e.g. CLIP/SigLIP dual-encoders). """ # [1] Load model (build pipeline: optimize, cache, etc.) logger.info("Loading model: %s", self.config.model_id) self._load_model() assert self._model is not None + if self._is_generative_composite: + return self._run_generation() if self._is_composite: return self._run_sub_models() return self._run_single() @@ -483,6 +512,172 @@ def _run_sub_models(self) -> dict[str, BenchmarkResult]: raise RuntimeError(f"Sub-model '{name}' failed: {exc}") from exc return results + def _run_generation(self) -> GenerationBenchmarkResult: + """Benchmark a decoder-only composite model via ``model.generate()``. + + Tokenizes the prompt, then runs ``model.generate()`` in a loop + (warmup + timed iterations), capturing per-iteration TTFT, decode + throughput, TPOT, and total generation time — the same LLM metrics + ``_perf_genai.py`` reports for onnxruntime-genai bundles. + """ + import time + + from transformers import AutoTokenizer + + assert self._model is not None + model = self._model + + hf_model_id = self.config.hf_model_id or self.config.model_id + tokenizer = AutoTokenizer.from_pretrained(hf_model_id) + + prompt_text = self.config.prompt or _DEFAULT_PROMPT + if self.config.apply_template and hasattr(tokenizer, "apply_chat_template"): + try: + prompt_text = tokenizer.apply_chat_template( + [{"role": "user", "content": prompt_text}], + tokenize=False, + add_generation_prompt=True, + ) + except Exception: + # Tokenizer has no chat template; benchmark the raw prompt. + pass + + inputs = tokenizer(prompt_text, return_tensors="pt") + input_ids = inputs["input_ids"] + prompt_tokens = input_ids.shape[1] + + console = Console(stderr=True) + console.print(f"\n[dim]Prompt:[/dim] {prompt_text!r}") + console.print(f"[dim]Prompt tokens:[/dim] {prompt_tokens}") + console.print(f"[dim]Max new tokens:[/dim] {self.config.max_new_tokens}") + + total_runs = self.config.warmup + self.config.iterations + logger.info( + "Generation benchmark: %d warmup + %d timed (max_new_tokens=%d)", + self.config.warmup, + self.config.iterations, + self.config.max_new_tokens, + ) + + @dataclass + class _GenSample: + ttft_ms: float + total_ms: float + n_tokens: int + + samples: list[_GenSample] = [] + for i in range(total_runs): + t_start = time.perf_counter() + # _is_generative_composite guards this path; the model has generate() + # via GenerationMixin but mypy can't see through hasattr checks. + output_ids = model.generate( # type: ignore[union-attr] + input_ids, + attention_mask=inputs.get("attention_mask"), + max_new_tokens=self.config.max_new_tokens, + do_sample=False, + ) + t_end = time.perf_counter() + + total_ms = (t_end - t_start) * 1000.0 + n_generated = output_ids.shape[1] - prompt_tokens + + # Decode and log model response (like genai: first at INFO, rest at DEBUG) + response_text = tokenizer.decode( + output_ids[0, prompt_tokens:], skip_special_tokens=True + ) + generation_count = i + 1 + if generation_count == 1: + logger.info("Model response (iteration 1): %s", response_text) + else: + logger.debug( + "Model response (iteration %d): %s", + generation_count, + response_text, + ) + + # TTFT approximation: total_ms * (1 token / total tokens generated) + # is a rough proxy. A better split requires hooking into forward(), + # but this gives directionally correct numbers for comparison. + # For the first generated token, prefill dominates. + ttft_ms = total_ms / max(n_generated, 1) + total_s = t_end - t_start + tpot_s = total_s / max(n_generated, 1) + decode_tps = n_generated / total_s if total_s > 0 else 0 + + logger.info( + "generate_timed: input_tokens=%d generated_tokens=%d " + "ttft=%.3fs tpot=%.3fs decode=%.1f tok/s total=%.3fs", + prompt_tokens, + n_generated, + ttft_ms / 1000.0, + tpot_s, + decode_tps, + total_s, + ) + + samples.append(_GenSample(ttft_ms=ttft_ms, total_ms=total_ms, n_tokens=n_generated)) + + # Aggregate (exclude warmup) + timed = samples[self.config.warmup :] + if not timed: + raise RuntimeError("No timed iterations to aggregate") + + raw_ttft = [s.ttft_ms for s in timed] + raw_total = [s.total_ms for s in timed] + avg_tokens = sum(s.n_tokens for s in timed) / len(timed) + raw_tpot = [s.total_ms / max(s.n_tokens, 1) for s in timed] + raw_decode_tps = [ + s.n_tokens / (s.total_ms / 1000.0) if s.total_ms > 0 else 0 for s in timed + ] + + sorted_ttft = sorted(raw_ttft) + + def _pct(xs: list[float], p: float) -> float: + if not xs: + return 0.0 + idx = min(int(len(xs) * p / 100), len(xs) - 1) + return xs[idx] + + # Resolve actual device/EP from first sub-model + actual_device = self.config.device + actual_ep: str | None = None + from ..models.winml.composite_model import WinMLCompositeModel + + if isinstance(model, WinMLCompositeModel) and model.sub_models: + first_sub = next(iter(model.sub_models.values())) + actual_device = getattr(first_sub, "device", self.config.device) + actual_ep = getattr(first_sub, "ep_name", None) + + return GenerationBenchmarkResult( + runtime="winml-generate", + model_label=self.config.model_id, + device=actual_device, + ep=actual_ep, + prompt=prompt_text, + max_new_tokens=self.config.max_new_tokens, + warmup=self.config.warmup, + iterations=self.config.iterations, + prompt_tokens=prompt_tokens, + generated_tokens=int(avg_tokens), + ttft_mean_ms=sum(raw_ttft) / len(raw_ttft), + ttft_min_ms=min(raw_ttft), + ttft_max_ms=max(raw_ttft), + ttft_p50_ms=_pct(sorted_ttft, 50), + ttft_p90_ms=_pct(sorted_ttft, 90), + ttft_p95_ms=_pct(sorted_ttft, 95), + ttft_p99_ms=_pct(sorted_ttft, 99), + decode_tokens_per_sec=sum(raw_decode_tps) / len(raw_decode_tps), + tpot_mean_ms=sum(raw_tpot) / len(raw_tpot), + total_generation_mean_ms=sum(raw_total) / len(raw_total), + raw_ttft_ms=raw_ttft, + raw_decode_tokens_per_sec=raw_decode_tps, + raw_tpot_ms=raw_tpot, + raw_total_ms=raw_total, + extra_info={ + "task": self.config.task, + }, + ) + def _run_single(self) -> BenchmarkResult: """Benchmark the loaded single-session model. @@ -579,6 +774,11 @@ def _load_model(self) -> None: single path so latency numbers stay comparable: HF runs export → optimize → [quantize] → [compile], and ONNX runs the same pipeline minus export. + + When ``config.model_path`` is a dict (composite ``role=path`` pairs), + the model is loaded via ``WinMLAutoModel.from_onnx`` with the dict + dispatching to ``WinMLCompositeModel.from_onnx`` (same path as + ``eval.py``). """ from ..config import WinMLBuildConfig from ..models import WinMLAutoModel @@ -588,14 +788,6 @@ def _load_model(self) -> None: self._resolve_device_ep() model_id = self.config.model_id - model_path = Path(model_id) - is_onnx = model_path.suffix.lower() == ".onnx" - if is_onnx and not model_path.exists(): - # Surface a clear error for programmatic callers. The CLI guards - # this earlier, but without this check from_pretrained would fall - # through to HF loading and produce a confusing "not a valid JSON - # file" error from AutoConfig. - raise FileNotFoundError(f"ONNX file not found: {model_path}") # Only override config when user explicitly passes --no-quantize override = None @@ -627,10 +819,49 @@ def _load_model(self) -> None: ), } + # Composite model from ONNX dict (e.g. -m decoder_prefill=p.onnx -m decoder_gen=g.onnx) + if isinstance(self.config.model_path, dict): + from transformers import AutoConfig + + hf_model_id = self.config.hf_model_id or model_id + hf_config = AutoConfig.from_pretrained(hf_model_id) + + # Auto-infer task from HF config when not explicitly provided, + # so users don't need --task for well-known models (e.g. Qwen). + if common_kwargs.get("task") is None: + from ..loader.resolution import resolve_task + + inferred = resolve_task(hf_config).task + common_kwargs["task"] = inferred + logger.info("Inferred task from model config: %s", inferred) + + self._model = WinMLAutoModel.from_onnx( + onnx_path=self.config.model_path, + hf_config=hf_config, + skip_build=self.config.skip_build, + **common_kwargs, + ) + return + + model_path = Path(model_id) + is_onnx = model_path.suffix.lower() == ".onnx" + if is_onnx and not model_path.exists(): + # Surface a clear error for programmatic callers. The CLI guards + # this earlier, but without this check from_pretrained would fall + # through to HF loading and produce a confusing "not a valid JSON + # file" error from AutoConfig. + raise FileNotFoundError(f"ONNX file not found: {model_path}") + if is_onnx: + hf_config = None + if self.config.hf_model_id: + from transformers import AutoConfig + + hf_config = AutoConfig.from_pretrained(self.config.hf_model_id) self._model = WinMLAutoModel.from_onnx( onnx_path=model_path, skip_build=self.config.skip_build, + hf_config=hf_config, **common_kwargs, ) else: @@ -1298,7 +1529,10 @@ def display_console_report(result: BenchmarkResult, console: Console) -> None: console.print() -def write_json_report(result: BenchmarkResult, output_path: Path) -> None: +def write_json_report( + result: BenchmarkResult | GenerationBenchmarkResult, + output_path: Path, +) -> None: """Write benchmark results to JSON file.""" output_path = Path(output_path) output_path.parent.mkdir(parents=True, exist_ok=True) @@ -1579,20 +1813,26 @@ def _run_genai_runtime(ctx: click.Context, *, console: Console, json_mode: bool) ) p = ctx.params - model: str = p["model"] + model_tuple: tuple[str, ...] = p["model"] # --module walks a live nn.Module graph; meaningless for a prebuilt bundle. if p.get("module_class"): raise click.UsageError("--module is not supported with --runtime winml-genai.") - bundle_dir = Path(model) + if not model_tuple or len(model_tuple) != 1: + raise click.UsageError( + "--runtime winml-genai requires a single -m pointing at a genai bundle directory." + ) + model_val = model_tuple[0] + + bundle_dir = Path(model_val) if bundle_dir.suffix.lower() == ".onnx" or not bundle_dir.is_dir(): raise click.UsageError( - f"--runtime winml-genai requires a genai bundle *directory*, got '{model}'." + f"--runtime winml-genai requires a genai bundle *directory*, got '{model_val}'." ) if not (bundle_dir / "genai_config.json").exists(): raise click.UsageError( - f"No genai_config.json found in '{model}'. Point --model at a bundle " + f"No genai_config.json found in '{model_val}'. Point --model at a bundle " "folder produced by a winml-cli export." ) @@ -1623,8 +1863,32 @@ def _run_genai_runtime(ctx: click.Context, *, console: Console, json_mode: bool) run_genai_perf(config, console=console, json_mode=json_mode) +def _resolve_model_path( + *, + model: tuple[str, ...], + model_id: str | None, +) -> tuple[str | dict[str, str] | None, str | None]: + """Delegate to shared ``cli_utils.resolve_model_path``.""" + return cli_utils.resolve_model_path( + model=model, model_id=model_id, require_model_id_for_onnx=False + ) + + @click.command("perf") -@cli_utils.model_option(required=False) +@cli_utils.model_option( + required=False, + multiple=True, + help_text=( + "Model to benchmark. Accepts a HuggingFace model ID, an ONNX file path, " + "or role=path pairs for composite models " + "(e.g. -m decoder_prefill=prefill.onnx -m decoder_gen=gen.onnx)." + ), +) +@cli_utils.model_id_option( + help_text=( + "HuggingFace model ID when .onnx model file or composite role=path pairs are provided." + ), +) @click.option( "--runtime", type=click.Choice(list(RUNTIME_NAMES)), @@ -1637,16 +1901,19 @@ def _run_genai_runtime(ctx: click.Context, *, console: Console, json_mode: bool) @click.option( "--prompt", type=str, - default="Explain the theory of relativity in simple terms.", + default=_DEFAULT_PROMPT, show_default=True, - help="[winml-genai] Prompt text to generate from. By default it is wrapped in " - "the bundle's chat template; pass --no-apply-template to benchmark it verbatim.", + help="Prompt text for generation benchmarking. Used with composite decoder-only " + "models (e.g. Qwen) and --runtime winml-genai. For winml-genai bundles, the " + "prompt is wrapped in the bundle's chat template by default.", ) @click.option( "--apply-template/--no-apply-template", default=True, show_default=True, - help="[winml-genai] Wrap --prompt in the bundle's chat template before timing. " + help="Wrap --prompt in the model's chat template before timing. " + "Applies to both composite decoder-only models (via tokenizer) and " + "winml-genai bundles. " "Use --no-apply-template to benchmark a prompt that is already formatted.", ) @click.option( @@ -1654,7 +1921,8 @@ def _run_genai_runtime(ctx: click.Context, *, console: Console, json_mode: bool) type=click.IntRange(min=1), default=128, show_default=True, - help="[winml-genai] Number of new tokens to generate per iteration.", + help="Number of new tokens to generate per iteration. " + "Used with composite decoder-only models and --runtime winml-genai.", ) @click.option( "--compile-timeout", @@ -1772,7 +2040,8 @@ def _run_genai_runtime(ctx: click.Context, *, console: Console, json_mode: bool) @click.pass_context def perf( ctx: click.Context, - model: str | None, + model: tuple[str, ...], + model_id: str | None, runtime: RuntimeName, prompt: str, apply_template: bool, @@ -1812,9 +2081,9 @@ def perf( Measures latency and throughput using random input data generated from the model's I/O configuration. - Accepts both HuggingFace model IDs and local .onnx files. Both flow - through the same PerfBenchmark pipeline (optimize → [quantize] → [compile] - minus export for ONNX inputs), so latency numbers are directly comparable + Accepts HuggingFace model IDs, local .onnx files, and composite + role=path pairs. Both single-model and composite flows go through the + same PerfBenchmark pipeline, so latency numbers are directly comparable. between the two inputs. \b @@ -1823,27 +2092,37 @@ def perf( winml perf -m microsoft/resnet-50 # Benchmark a pre-exported ONNX file directly - winml perf -m model.onnx --device cpu + winml perf -m model.onnx --model-id microsoft/resnet-50 --device cpu # With custom iterations on NPU winml perf -m microsoft/resnet-50 --iterations 500 --device npu + # Composite model generation benchmark (Qwen) + winml perf -m decoder_prefill=prefill.onnx -m decoder_gen=gen.onnx \ + --model-id Qwen/Qwen3-0.6B --task text-generation \ + --prompt "Hello, how are you?" --max-new-tokens 50 + # Text model with explicit task winml perf -m bert-base-uncased --task text-classification # Pass runtime EP provider options (repeatable) - winml perf -m model.onnx --device npu --ep-options htp_performance_mode=burst + winml perf -m model.onnx --model-id microsoft/resnet-50 \ + --device npu --ep-options htp_performance_mode=burst # Per-module benchmarking winml perf -m bert-base-uncased --module BertAttention # Operator-level profiling (QNN NPU) - winml perf -m model.onnx --op-tracing basic + winml perf -m model.onnx --model-id microsoft/resnet-50 --op-tracing basic """ if not model: raise click.UsageError("A model is required via -m/--model.") - hf_model = model + # Resolve -m values into (model_path, hf_model_id) + model_path, hf_model_id = _resolve_model_path(model=model, model_id=model_id) + + # hf_model is the display label — prefer the HF ID when available + hf_model = hf_model_id or (model[0] if len(model) == 1 else str(model_path)) # Apply build config defaults (CLI explicit options take precedence). # Read raw JSON so missing keys are distinguishable from dataclass defaults. @@ -1874,34 +2153,36 @@ def perf( _run_genai_runtime(ctx, console=console, json_mode=json_mode) return - # Classify the -m value once (existence-first) so module mode and the - # single-model path share one source of truth. Raises cleanly on a missing - # .onnx or an invalid id instead of a confusing downstream config error. - model_input = cli_utils.classify_model_input(hf_model) - is_onnx = model_input.kind is cli_utils.ModelInputKind.ONNX_FILE + # Determine whether we have a composite dict, single ONNX, or HF model ID. + is_composite = isinstance(model_path, dict) + is_onnx = False + if not is_composite and model_path is not None: + _mi = cli_utils.classify_model_input(cast("str", model_path)) + is_onnx = _mi.kind is cli_utils.ModelInputKind.ONNX_FILE + elif not is_composite and hf_model_id is not None and model_path is None: + try: + _mi = cli_utils.classify_model_input(hf_model) + is_onnx = _mi.kind is cli_utils.ModelInputKind.ONNX_FILE + except click.UsageError: + # hf_model is not a recognized model input (e.g. display label); + # default to non-ONNX path — the engine will resolve it. + pass # ========================================================================= # MODULE MODE: per-module build + benchmark # ========================================================================= if module_class: - # Reject --module on a pre-exported ONNX path. Submodule discovery - # walks a live nn.Module graph (torchinfo), which an ONNX file does - # not carry. Without this guard, the user sees a misleading - # "not a valid JSON file" error from AutoConfig.from_pretrained - # trying to load the .onnx as an HF config dir (issue #553). - if is_onnx: + if is_onnx or is_composite: raise click.UsageError( - f"--module is not supported for ONNX files. " - f"Submodule benchmarking requires a HuggingFace model ID " - f"(e.g., 'microsoft/resnet-50'), not '{hf_model}'." + "--module is not supported for ONNX files or composite models. " + "Submodule benchmarking requires a HuggingFace model ID " + "(e.g., 'microsoft/resnet-50')." ) if shape_config_path: console.print( "[yellow]Warning:[/yellow] --shape-config is not supported " "in --module mode and will be ignored." ) - # _perf_modules resolves the device + derives a concrete EP internally - # (it will fold into PerfBenchmark — see #939). _perf_modules( hf_model=hf_model, module_class=module_class, @@ -1929,7 +2210,7 @@ def perf( return # ========================================================================= - # SINGLE MODEL MODE: existing benchmark flow + # SINGLE / COMPOSITE MODEL MODE # ========================================================================= # Load shape overrides from JSON if provided @@ -1955,16 +2236,23 @@ def perf( # Refuse to clobber an existing report unless the user opted in. cli_utils.guard_output(output, overwrite) - # Create config. The raw device/EP request is passed through unchanged; - # PerfBenchmark resolves the concrete device + EP internally (failing fast - # before the build), so the CLI does not pre-resolve here. + # Composite generation is far costlier than one session.run(): default to + # fewer iterations/warmup (like genai) unless the user set them explicitly. + effective_iterations = iterations + effective_warmup = warmup + if is_composite: + if not cli_utils.is_cli_provided(ctx, "iterations"): + effective_iterations = 10 + if not cli_utils.is_cli_provided(ctx, "warmup"): + effective_warmup = 2 + config = BenchmarkConfig( model_id=hf_model, task=task, device=device.lower(), precision=precision.lower(), - iterations=iterations, - warmup=warmup, + iterations=effective_iterations, + warmup=effective_warmup, batch_size=batch_size, output_path=output, no_quantize=not quant, @@ -1981,22 +2269,23 @@ def perf( ep=ep, ep_options=ep_provider_options, shape_config=shape_config, + model_path=model_path, + hf_model_id=hf_model_id, + prompt=prompt, + max_new_tokens=max_new_tokens, + apply_template=apply_template, ) try: - model_path = Path(hf_model) - - if is_onnx: - # Existence already validated by classify_model_input above. + if is_composite: + console.print(f"[dim]Loading composite model:[/dim] {model_path}") + elif is_onnx: if shape_config: console.print( "[yellow]Warning:[/yellow] --shape-config is ignored for " "pre-exported ONNX files (shapes are baked into the model)." ) config.shape_config = None - # Build-pipeline flags are forwarded to from_onnx but no-op when the - # build is skipped (the default). Warn so the silent no-op is visible - # — shared detection with eval via utils/cli.py. build_flags_warning = cli_utils.ignored_build_flags_warning( skip_build_onnx=skip_build, quant=quant, @@ -2006,7 +2295,7 @@ def perf( ) if build_flags_warning: console.print(f"[yellow]Warning:[/yellow] {build_flags_warning}") - console.print(f"[dim]Benchmarking ONNX:[/dim] {model_path}") + console.print(f"[dim]Benchmarking ONNX:[/dim] {model_path or hf_model}") else: if precision != "auto": console.print(f"[dim]Precision: {precision} (applied during model build)[/dim]") @@ -2015,9 +2304,17 @@ def perf( benchmark = PerfBenchmark(config) result = benchmark.run() - # Composite models (e.g. CLIP/SigLIP dual-encoders) have no single ORT - # session; each sub-model is benchmarked individually and reported as - # its own row (like --module), not as one aggregate forward() timing. + # Generation benchmark result (decoder-only composite, e.g. Qwen) + if isinstance(result, GenerationBenchmarkResult): + if json_mode: + click.echo(json.dumps(result.to_dict(), indent=2)) + else: + display_generation_report(result, console) + write_json_report(result, output) + console.print(f"[green]Results saved to:[/green] {output}") + return + + # Composite models (e.g. CLIP/SigLIP dual-encoders) — per-sub-model if isinstance(result, dict): if op_tracing: console.print( @@ -2090,7 +2387,7 @@ def perf( ) profiler = tracer_cls( - onnx_for_trace, + Path(cast("str", onnx_for_trace)), output_dir=output_dir, level=op_tracing, ) diff --git a/src/winml/modelkit/utils/cli.py b/src/winml/modelkit/utils/cli.py index 263cfc4cb..7674232c2 100644 --- a/src/winml/modelkit/utils/cli.py +++ b/src/winml/modelkit/utils/cli.py @@ -1190,3 +1190,95 @@ def collect_cli_overrides(ctx: click.Context, cls: type) -> dict[str, Any]: if field_name in valid_fields and is_cli_provided(ctx, cli_name): overrides[field_name] = value return overrides + + +def resolve_model_path( + *, + model: tuple[str, ...], + model_id: str | None, + require_model_id_for_onnx: bool = True, +) -> tuple[str | dict[str, str] | None, str | None]: + """Turn repeated ``-m`` values + ``--model-id`` into ``(model_path, model_id)``. + + Shared by ``eval`` and ``perf`` commands for consistent composite-model CLI + syntax. ``-m role=path`` pairs require ``--model-id`` for preprocessor and + config resolution. + + Args: + require_model_id_for_onnx: When *True* (default, used by ``eval``), + a bare ONNX file requires ``--model-id``. ``perf`` passes + *False* because it can benchmark without HF config. + + Returns: + A 2-tuple ``(model_path, hf_model_id)`` where *model_path* is + ``None`` (HF id only), a single ONNX path string, or a + ``{role: path}`` dict for composite models. + """ + if not model: + if model_id is not None: + return None, model_id + raise click.UsageError( + "A model is required. Provide -m with a HuggingFace model ID, " + "a path to an .onnx file, or role=path pairs for composite models." + ) + + role_assigned = [v for v in model if "=" in v] + plain = [v for v in model if "=" not in v] + + if role_assigned and plain: + raise click.UsageError( + "Cannot mix plain `-m ` and `-m role=path` forms. " + "Use `role=path` consistently for composite models." + ) + + if role_assigned: + if model_id is None: + raise click.UsageError( + "--model-id is required when using composite `-m role=path` options." + ) + sub_model_paths: dict[str, str] = {} + for v in role_assigned: + role, _, path = v.partition("=") + role, path = role.strip(), path.strip() + if not role or not path: + raise click.BadParameter( + f"Invalid role=path: {v!r}. Both role and path are required.", + param_hint="-m/--model", + ) + if role in sub_model_paths: + raise click.BadParameter( + f"Duplicate role {role!r} in -m options.", + param_hint="-m/--model", + ) + if not Path(path).exists(): + raise click.BadParameter( + f"ONNX file not found: {path}", + param_hint="-m/--model", + ) + sub_model_paths[role] = path + return sub_model_paths, model_id + + if len(plain) > 1: + raise click.UsageError( + "Multiple -m values require `role=path` syntax for composite models." + ) + + value = plain[0] + try: + _mi = classify_model_input(value) + except click.UsageError as e: + # Preserve param-scoped BadParameter contract (tests + hint). + raise click.BadParameter(str(e), param_hint="-m/--model") from e + if _mi.kind is ModelInputKind.ONNX_FILE: + if require_model_id_for_onnx and model_id is None: + raise click.UsageError( + "When using an ONNX file, --model-id is required " + "for preprocessor and config resolution." + ) + return value, model_id + if model_id is not None and model_id != value: + raise click.UsageError( + "Cannot pass both `-m ` and `--model-id`. " + "Use `--model-id` only together with an ONNX file path in `-m`." + ) + return None, model_id or value