microsoft · zhenchaoni · Jul 24, 2026
@@ -659,6 +659,16 @@
     "elapsed": 1319.8,
     "command": "python.exe run_pytorch_baseline.py --model bert-base-uncased --task fill-mask --device cpu --num-samples 100 --dataset wikitext --split test --dataset-config wikitext-2-raw-v1 --columns-mapping {\"input_column\": \"text\"}"
   },
+  "Qwen/Qwen3-0.6B|text-generation|Salesforce/wikitext|wikitext-2-raw-v1|test|100": {
+    "status": "PASS",
+    "metric": {
+      "metric": "perplexity",
+      "value": 23.645425,
+      "num_samples": 100
+    },
+    "elapsed": 0.0,
+    "command": "python.exe run_pytorch_baseline.py --model Qwen/Qwen3-0.6B --task text-generation --device cpu --num-samples 100 --dataset Salesforce/wikitext --split test --dataset-config wikitext-2-raw-v1 --columns-mapping {\"input_column\": \"text\", \"num_tokens\": \"8192\", \"seqlen\": \"2048\"}"
+  },
   "distilbert/distilbert-base-uncased|fill-mask|Salesforce/wikitext|wikitext-2-raw-v1|test|100": {
     "status": "PASS",
     "metric": {

@@ -152,16 +152,28 @@ def _measure_pytorch_latency(task_evaluator: Any, warmup: int, iterations: int)
 def _load_pytorch_model(model_id: str, task: str, device_str: str):
     """Load a native PyTorch model with the task-appropriate AutoModel class."""
     import torch
+
+    device = torch.device(
+        device_str if device_str != "cuda" or torch.cuda.is_available() else "cpu"
+    )
+
+    # text-generation resolves to a composite (embeddings/ctx/iter/lm_head) that
+    # is not a runnable HF model. Wrap a plain causal LM in the perplexity
+    # scoring contract so the evaluator scores it through the same path as a
+    # genai bundle.
+    if task == "text-generation":
+        from winml.modelkit.models.winml import HFCausalLM
+
+        _out(f"Loading causal LM baseline adapter for {model_id} on {device}")
+        return HFCausalLM(model_id, device)
+
     from transformers import AutoConfig
 
     from winml.modelkit.loader import resolve_task
 
     config = AutoConfig.from_pretrained(model_id)
     cls = resolve_task(config, task=task).model_class
     _out(f"Loading {cls.__name__} for {model_id} on {device_str}")
-    device = torch.device(
-        device_str if device_str != "cuda" or torch.cuda.is_available() else "cpu"
-    )
     return cls.from_pretrained(model_id).to(device).eval()
 
 

@@ -1246,6 +1246,26 @@
       }
     }
   },
+  {
+    "hf_id": "Qwen/Qwen3-0.6B",
+    "task": "text-generation",
+    "model_type": "qwen3",
+    "group": "Top200",
+    "priority": "P1",
+    "dataset_config": {
+      "path": "Salesforce/wikitext",
+      "name": "wikitext-2-raw-v1",
+      "split": "test",
+      "samples": 100,
+      "metric": "perplexity",
+      "winml_metric_key": "perplexity",
+      "columns_mapping": {
+        "input_column": "text",
+        "num_tokens": "8192",
+        "seqlen": "2048"
+      }
+    }
+  },
   {
     "hf_id": "FacebookAI/xlm-roberta-base",
     "task": "fill-mask",

@@ -25,6 +25,8 @@
     # WinML-vs-baseline delta is small — pick a tighter threshold than default.
     "knn_top1_accuracy": ("delta_relative", 0.02, 0.05, True),
     "pseudo_perplexity": ("delta_relative", 0.05, 0.10, False),
+    # Perplexity (text-generation PPL): lower is better.
+    "perplexity": ("delta_relative", 0.05, 0.10, False),
     # CER (OCR error rate): lower is better.
     "cer": ("delta_relative", 0.05, 0.10, False),
     # AbsRel (depth-estimation error): lower is better.

@@ -533,6 +533,13 @@ def _resolve_model_path(
                 "for preprocessor and config resolution."
             )
         return value, model_id
+
+    # A local directory (e.g. an onnxruntime-genai bundle) is a model *path*,
+    # not a Hub model id. Route it to model_path so downstream loaders read the
+    # bundle from disk instead of treating the directory string as a Hub ref.
+    if Path(value).expanduser().is_dir():
+        return str(Path(value).expanduser()), model_id
+
     if model_id is not None and model_id != value:
         raise click.UsageError(
             "Cannot pass both `-m <hf_id>` and `--model-id`. "

@@ -39,6 +39,7 @@
     from .question_answering_evaluator import WinMLQuestionAnsweringEvaluator
     from .tensor_similarity_evaluator import TensorSimilarityEvaluator
     from .text_classification_evaluator import WinMLTextClassificationEvaluator
+    from .text_generation_evaluator import WinMLTextGenerationEvaluator
     from .token_classification_evaluator import WinMLTokenClassificationEvaluator
     from .zero_shot_classification_evaluator import WinMLZeroShotClassificationEvaluator
     from .zero_shot_image_classification_evaluator import WinMLZeroShotImageClassificationEvaluator
@@ -66,6 +67,8 @@
         ".question_answering_evaluator:WinMLQuestionAnsweringEvaluator",
     "WinMLTextClassificationEvaluator":
         ".text_classification_evaluator:WinMLTextClassificationEvaluator",
+    "WinMLTextGenerationEvaluator":
+        ".text_generation_evaluator:WinMLTextGenerationEvaluator",
     "WinMLTokenClassificationEvaluator":
         ".token_classification_evaluator:WinMLTokenClassificationEvaluator",
     "WinMLZeroShotClassificationEvaluator":
@@ -141,6 +144,7 @@ def __dir__() -> list[str]:
     "WinMLObjectDetectionEvaluator",
     "WinMLQuestionAnsweringEvaluator",
     "WinMLTextClassificationEvaluator",
+    "WinMLTextGenerationEvaluator",
     "WinMLTokenClassificationEvaluator",
     "WinMLZeroShotClassificationEvaluator",
     "WinMLZeroShotImageClassificationEvaluator",

@@ -69,6 +69,8 @@
         "winml.modelkit.eval.tensor_similarity_evaluator:TensorSimilarityEvaluator",
     "mask-generation":
         "winml.modelkit.eval.mask_generation_evaluator:WinMLMaskGenerationEvaluator",
+    "text-generation":
+        "winml.modelkit.eval.text_generation_evaluator:WinMLTextGenerationEvaluator",
 }
 # fmt: on
 
@@ -199,6 +201,14 @@ def get_evaluator_class(config: WinMLEvaluationConfig) -> type[WinMLEvaluator]:
         "path": "mattmdjaga/human_parsing_dataset",
         "split": "train",
     },
+    "text-generation": {
+        # Raw wikitext-2 test split scored token-by-token for perplexity;
+        # ``input_column`` names the text field the corpus is built from.
+        "path": "wikitext",
+        "name": "wikitext-2-raw-v1",
+        "split": "test",
+        "columns_mapping": {"input_column": "text"},
+    },
 }
 
 
@@ -231,6 +241,9 @@ def _load_model(config: WinMLEvaluationConfig) -> WinMLPreTrainedModel | WinMLCo
     from ..models import WinMLAutoModel
     from ..utils import cli as cli_utils
 
+    if config.task == "text-generation":
+        return _load_genai_causal_lm(config)
+
     quant_override: Any = None
     if not config.quant:
         from ..config import WinMLBuildConfig
@@ -282,6 +295,51 @@ def _load_model(config: WinMLEvaluationConfig) -> WinMLPreTrainedModel | WinMLCo
     )
 
 
+def _load_genai_causal_lm(config: WinMLEvaluationConfig) -> Any:
+    """Load a causal LM from an onnxruntime-genai bundle directory.
+
+    ``-m <bundle_dir>`` resolves to ``config.model_path`` (a local directory),
+    so the bundle directory is read from there. ``ep`` / ``device`` pass straight
+    through: an explicit ``--ep`` forces the whole decoder pipeline onto that EP,
+    while ``ep=None`` respects the bundle's ``genai_config.json`` routing. The
+    bundle is loaded as-is (``compile=False``); building or compiling a bundle is
+    ``winml build``'s job, not eval's.
+
+    Raises:
+        ValueError: no bundle directory was provided, the path is not a
+            directory, or it is missing the ``genai_config.json`` / ONNX files
+            that mark a genai bundle.
+    """
+    from pathlib import Path
+
+    from ..models.winml.genai_causal_lm import WinMLGenaiCausalLM
+
+    bundle_path = config.model_path
+    if not bundle_path or isinstance(bundle_path, dict):
+        raise ValueError(
+            "text-generation evaluation requires a genai bundle *directory* via "
+            "-m <bundle_dir>."
+        )
+
+    bundle_dir = Path(bundle_path).expanduser()
+    if not bundle_dir.is_dir():
+        raise ValueError(f"Genai bundle directory not found: {bundle_dir}")
+    if not (bundle_dir / "genai_config.json").is_file():
+        raise ValueError(
+            f"'{bundle_dir}' is not a genai bundle: no genai_config.json found. "
+            "Point -m at a bundle built with 'winml build ... --device npu --ep qnn'."
+        )
+    if not any(bundle_dir.glob("*.onnx")):
+        raise ValueError(f"'{bundle_dir}' contains no .onnx files; not a valid genai bundle.")
+
+    return WinMLGenaiCausalLM(
+        bundle_dir,
+        config.ep,
+        device=config.device,
+        compile=False,
+    )
+
+
 def _resolve_task(config: WinMLEvaluationConfig) -> str:
     """Resolve the eval task and validate it is supported.
 

@@ -0,0 +1,152 @@
+# -------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+# --------------------------------------------------------------------------
+
+"""Perplexity evaluator for causal-LM (text-generation) models.
+
+Does *not* go through HF's ``pipeline`` / ``evaluate`` libraries: perplexity is
+scored by teacher-forcing raw corpus tokens through the model's ``forward``, so
+the evaluator only needs a model honoring the causal-LM contract
+(``encode(text) -> list[int]`` and ``forward(ids).logits``).  The same code
+scores a WinML genai bundle or any object exposing that interface.
+
+Protocol: the dataset text column is concatenated and tokenized with the
+model's own tokenizer, capped at ``num_tokens``, then cut into contiguous
+non-overlapping ``seqlen``-token blocks (no detokenizer, no sliding window).
+Every token after the first in its block is scored once, giving
+``perplexity = exp(sum(NLL) / scored_positions)``.
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import TYPE_CHECKING, Any
+
+import numpy as np
+
+from .base_evaluator import WinMLEvaluator
+
+
+if TYPE_CHECKING:
+    from transformers.pipelines.base import Pipeline
+
+
+logger = logging.getLogger(__name__)
+
+__all__ = ["WinMLTextGenerationEvaluator"]
+
+
+class WinMLTextGenerationEvaluator(WinMLEvaluator):
+    """Evaluator computing disjoint fixed-length perplexity for causal LMs.
+
+    Constructor keeps the standard ``(config, model)`` signature so the registry
+    dispatch in :mod:`~winml.modelkit.eval.evaluate` works unmodified. ``model``
+    is a causal-LM inference object (e.g.
+    :class:`~winml.modelkit.models.winml.genai_causal_lm.WinMLGenaiCausalLM`).
+
+    Two scoring parameters are read from ``dataset.columns_mapping`` so they ride
+    the existing ``--column key=value`` CLI path (defaults come from the
+    text-generation schema in :mod:`~winml.modelkit.utils.eval_utils`):
+
+    * ``num_tokens`` -- total corpus tokens to score.
+    * ``seqlen`` -- non-overlapping block length.
+    """
+
+    _TASK = "text-generation"
+
+    def prepare_pipeline(self) -> Pipeline | None:  # type: ignore[override]
+        """No HF pipeline -- the model's ``forward`` is driven directly."""
+        return None
+
+    def prepare_data(self) -> list[list[int]]:
+        """Load, tokenize, and block the corpus into fixed-length token blocks.
+
+        Returns a list of token-ID blocks, each at least 2 tokens long (a block
+        needs a first token to condition on and at least one token to score).
+        """
+        from ..utils.eval_utils import get_default
+
+        mapping = self.config.dataset.columns_mapping
+        num_tokens = int(mapping.get("num_tokens", get_default(self._TASK, "num_tokens")))
+        seqlen = int(mapping.get("seqlen", get_default(self._TASK, "seqlen")))
+        if seqlen < 2:
+            raise ValueError(f"seqlen must be at least 2; got {seqlen}.")
+
+        ids = self._load_corpus_tokens(num_tokens)
+        blocks = [ids[i : i + seqlen] for i in range(0, len(ids), seqlen)]
+        blocks = [b for b in blocks if len(b) >= 2]
+        if not blocks:
+            raise ValueError(
+                f"Corpus produced no scorable blocks (got {len(ids)} tokens, "
+                f"seqlen={seqlen}). Increase num_tokens or lower seqlen."
+            )
+        self._seqlen = seqlen
+        logger.info(
+            "Perplexity corpus: %d tokens -> %d blocks (seqlen=%d)",
+            len(ids),
+            len(blocks),
+            seqlen,
+        )
+        return blocks
+
+    def compute(self) -> dict[str, Any]:
+        """Score every block and return perplexity plus corpus statistics."""
+        model = self.model
+        total_nll = 0.0
+        scored = 0
+        for block in self.data:
+            logits = model.forward(block).logits[0]
+            targets = np.asarray(block[1:], dtype=np.int64)
+            total_nll += _block_nll(logits, targets)
+            scored += len(targets)
+
+        if scored == 0:
+            raise RuntimeError("Perplexity evaluation scored 0 positions.")
+
+        return {
+            "perplexity": float(np.exp(total_nll / scored)),
+            "num_scored_positions": scored,
+            "num_blocks": len(self.data),
+            "seqlen": self._seqlen,
+        }
+
+    # ------------------------------------------------------------------
+    # Internals
+    # ------------------------------------------------------------------
+
+    def _load_corpus_tokens(self, num_tokens: int) -> list[int]:
+        """Concatenate the dataset text column and tokenize with the model.
+
+        Uses the model's own tokenizer (``model.encode``) so the token stream
+        matches the model under test exactly.
+        """
+        from datasets import load_dataset
+
+        from ..utils.eval_utils import get_default
+
+        ds_config = self.config.dataset
+        column = ds_config.columns_mapping.get(
+            "input_column", get_default(self._TASK, "input_column")
+        )
+        dataset = load_dataset(
+            ds_config.path,
+            name=ds_config.name,
+            split=ds_config.split,
+            revision=ds_config.revision,
+        )
+        if column not in dataset.column_names:
+            raise ValueError(
+                f"Dataset '{ds_config.path}' has no column '{column}'; "
+                f"available columns: {sorted(dataset.column_names)}. "
+                "Set it via --column input_column=<name>."
+            )
+        text = "\n\n".join(row for row in dataset[column] if row and row.strip())
+        return self.model.encode(text)[:num_tokens]
+
+
+def _block_nll(logits: np.ndarray, targets: np.ndarray) -> float:
+    """Sum of ``-log P(target)`` over positions, from raw (unnormalized) logits."""
+    x = logits.astype(np.float64)
+    logsumexp = x.max(axis=-1) + np.log(np.exp(x - x.max(axis=-1, keepdims=True)).sum(axis=-1))
+    return float((logsumexp - x[np.arange(len(targets)), targets]).sum())
@@ -198,6 +198,7 @@ def register_specialization(model_type: str, task: str, class_name: str) -> None
     register_genai_bundle,
     resolve_genai_bundle,
 )
+from .genai_causal_lm import CausalLMOutput, HFCausalLM, WinMLGenaiCausalLM
 from .image_classification import WinMLModelForImageClassification
 from .image_segmentation import (
     ImageSegmentationOutput,
@@ -219,15 +220,18 @@ def register_specialization(model_type: str, task: str, class_name: str) -> None
     "GENAI_BUNDLE_REGISTRY",
     "TASK_TO_WINML_CLASS",
     "WINML_MODEL_CLASS_MAPPING",
+    "CausalLMOutput",
     "GenaiBundleRecipe",
     "GenaiCompanionSpec",
     "GenaiTarget",
     "GenaiTransformerSpec",
+    "HFCausalLM",
     "ImageSegmentationOutput",
     "WinMLCache",
     "WinMLCompositeModel",
     "WinMLDecoderOnlyModel",
     "WinMLEncoderDecoderModel",
+    "WinMLGenaiCausalLM",
     "WinMLModelForDepthEstimation",
     "WinMLModelForFeatureExtraction",
     "WinMLModelForGenericTask",