feat(BACKEND-ROCM): verify Qwen capture provenance

.github/workflows/ci.yml

@@ -541,6 +541,8 @@ jobs:
         run: |
           sudo apt-get update -qq
           sudo DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends python3-numpy
+          python3 tests/scripts/test_qwen3_capture_tools.py
+          python3 tests/scripts/test_qwen3_capture_outputs.py
           python3 tests/scripts/test_ltx25_render_compare.py && python3 tests/scripts/test_ltx25_absolute_reference.py
           # The prompt-adherence half of the same tool (#2295, owning #1854's
           # first sub-question). 42 of its 47 cases need numpy only; the five

docs/USAGE.md

@@ -1220,6 +1220,57 @@ checkout at exactly the recorded byte counts, so those two sizes are locally
 confirmed. The distilled bf16 DiT is absent from the shared checkout, so its
 42,018,190,584 bytes rest on the tree listing and a range request alone.
 
+## Capture Qwen oracle evidence
+
+Use `scripts/qwen3-oracle-capture.py` for greedy continuations and
+`scripts/qwen3-neartie-gap.py` for gaps under the local engine's exact token
+prefix. Both accept `--kv-cache-dtype auto|bfloat16|fp8_e4m3`, `--seed`,
+`--max-tokens`, and `--runs`. `--repetitions` is an alias for `--runs`.
+Production execution is the default. `--execution-mode eager` and its
+`--enforce-eager` alias select diagnostics, which cannot be a denominator.
+
+Qwen3.5 captures require verified local model artifacts and at least 10
+deterministic repeats. Supply full model and vLLM revisions, the installed
+vLLM wheel archive, and a launcher JSON manifest. That manifest contains
+`vllm_revision`, `wheel_sha256`, and an immutable `sha256:` `image_digest`.
+Model files need matching HuggingFace download metadata or immutable cache
+snapshot paths. The tools hash the files and compare wheel members with the
+imported package. An installed `+g` version suffix verifies only its recorded
+VCS prefix. Image identity remains a launcher attestation.
+
+After supplying these paths and revisions, use an empty capture directory:
+
+```sh
+python3 scripts/qwen3-oracle-capture.py \
+  --model "$MODEL_DIR" --model-revision "$MODEL_REV" \
+  --vllm-revision "$VLLM_REV" --vllm-wheel "$VLLM_WHEEL" \
+  --runtime-manifest "$RUNTIME_MANIFEST" --kv-cache-dtype auto \
+  --execution-mode production --seed 0 --max-tokens 16 --runs 10 \
+  --per-prompt --out-dir "$CAPTURE_DIR"
+```
+
+Place the matching local gate's `our_ids.i32` dump in that directory before
+running `qwen3-neartie-gap.py`. Pass the same model, revisions, runtime inputs,
+cache mode, seed, and token count, replacing `--out-dir` with `--golden-dir`.
+The near-tie tool uses one request at a time, so its strict input capture must
+use `--per-prompt`. Each tool writes a provenance JSON file beside its NumPy
+outputs. `--provenance-out PATH` writes an identical additional copy. Strict
+captures refuse existing outputs and mismatched or incomplete evidence.
+Output and provenance paths cannot overwrite capture inputs, including symbolic
+links and hardlinks to those inputs.
+
+`sampling_normalized` records the supplied `SamplingParams` after constructor
+normalization. vLLM resolves engine requests on a clone. `sampling_resolved`
+remains null, and `sampling_resolution` records that observation limit.
+Teacher-forcing sampling carries the same qualification.
+
+Legacy Qwen3 distributional calls remain usable, including captures without
+complete provenance and near-tie inputs without manifests. Their manifests
+record missing provenance and observed nondeterminism. A legacy capture cannot
+supply a Qwen3.5 near-tie run. These tool checks do not establish GPU execution
+or accept new permanent goldens. Active-pin acceptance remains pending in
+[#2773](https://github.com/mudler/vllm.cpp/issues/2773).
+
 ## Look up interface details
 
 [Reference pages](reference/README.md) collect dense lookup material such as

scripts/agent-preflight.sh

@@ -438,6 +438,17 @@ echo "Mutation suites:"
 for suite in "${SUITES[@]}"; do
   run "$suite" python3 "tests/scripts/$suite.py"
 done
+# QWEN3-CAPTURE-TOOLS: begin
+if python3 -c 'import numpy' >/dev/null 2>&1; then
+  run "test_qwen3_capture_tools" python3 tests/scripts/test_qwen3_capture_tools.py
+  run "test_qwen3_capture_outputs" python3 tests/scripts/test_qwen3_capture_outputs.py
+else
+  for suite in test_qwen3_capture_tools test_qwen3_capture_outputs; do
+    skip "$suite" "PENDING: numpy is not importable. CI installs python3-numpy for these capture suites."
+  done
+fi
+# QWEN3-CAPTURE-TOOLS: end
+
 # THE ONE SUITE HERE WITH A THIRD-PARTY DEPENDENCY (#1612). It exercises
 # `scripts/ltx25-render-compare.py`, whose only import beyond the standard
 # library is numpy -- the tool reads PPM and WAV by hand precisely so that a

scripts/qwen3-neartie-gap.py

@@ -23,9 +23,20 @@
 #   PATH="${VLLM_ORACLE}/bin:$PATH" "${VLLM_ORACLE}/bin/python" \
 #     scripts/qwen3-neartie-gap.py --model Qwen/Qwen3-4B \
 #       --golden-dir tests/parity/goldens/qwen3_greedy_4b
+#
+# Qwen3.5 additionally requires a matching strict capture manifest and ten
+# identical raw-logprob repeats. See docs/USAGE.md for its provenance inputs.
 import argparse, os, sys
+import hashlib
+import io
+import math
+from pathlib import Path
+import re
 import numpy as np
 
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+import qwen3_oracle_common as common
+
 PROMPTS = [
     "The capital of France is", "Once upon a time,", "In the beginning God created",
     "The quick brown fox jumps over", "def fibonacci(n):",
@@ -49,80 +60,157 @@ def _parse_args(argv=None):
     # over-commits and REBOOTS the box. 0.40 is the safe ceiling (does not affect the
     # teacher-forced logprobs — only KV-cache capacity, unused here at max_tokens=1).
     ap.add_argument("--gpu-mem-util", type=float, default=0.40)
-    ap.add_argument("--enforce-eager", action="store_true",
-                    help="use eager execution for diagnostics instead of the "
-                         "production execution mode")
-    return ap.parse_args(argv)
+    ap.add_argument("--runs", "--repetitions", type=int, default=None)
+    common.add_options(ap)
+    return common.finish_args(ap, argv)
 
 
 def _llm_kwargs(args):
-    return {
-        "model": args.model,
-        "dtype": "bfloat16",
-        "enforce_eager": args.enforce_eager,
-        "gpu_memory_utilization": args.gpu_mem_util,
-    }
+    return common.llm_kwargs(args, args.gpu_mem_util)
 
 
 def _mode_narration(args):
-    eager = args.enforce_eager
-    mode = "eager diagnostic" if eager else "production"
-    return f"oracle execution mode: {mode} (enforce_eager={eager})"
+    return common.mode_narration(args)
+
+
+def validate_capture(context, capture, inputs):
+    common.require(capture.get("tool") == "qwen3-oracle-capture"
+                   and capture.get("regime") == "qwen3_5_strict"
+                   and capture.get("deterministic") is True
+                   and isinstance(capture.get("repetitions"), int) and capture["repetitions"] >= 10,
+                   "near-tie input is not a deterministic strict capture")
+    for key in ("provenance_status", "resolved_model_identity", "prompts_sha256", "sampling",
+                "sampling_normalized", "sampling_resolved", "sampling_resolution", "execution_mode", "batching"):
+        common.require(capture.get(key) == context[key], f"capture {key} differs")
+    for key in ("requested", "resolved"):
+        common.require(capture.get("cache", {}).get(key) == context["cache"][key], f"capture cache {key} differs")
+    for key in ("requested_revision", "identity", "files", "missing"):
+        common.require(capture.get("model", {}).get(key) == context["model"][key], f"capture model {key} differs")
+    for key in ("requested_revision", "package_files", "missing", "version"):
+        common.require(capture.get("runtime", {}).get(key) == context["runtime"][key], f"capture oracle {key} differs")
+    runtime = capture["runtime"]
+    revision = runtime.get("revision")
+    verification = runtime.get("revision_verification")
+    valid_revision = verification == "clean_git_source" and common.full_revision(revision)
+    if verification == "installed_version_vcs_prefix":
+        # A captured installed revision keeps its observed prefix, independently
+        # checked against the current verified full revision and package bytes.
+        match = re.search(r"(?:\+|\.)g([0-9a-f]{7,40})(?:[.+-]|$)", str(runtime.get("version")))
+        valid_revision = match is not None and revision == match.group(1)
+    common.require(valid_revision and context["runtime"]["requested_revision"].startswith(revision),
+                   "capture observed oracle revision is missing, unverified, or differs")
+    for key, field in (("wheel", "sha256"), ("image", "digest")):
+        common.require(capture.get("runtime", {}).get(key, {}).get(field) == context["runtime"][key][field],
+                       f"capture oracle {key} differs")
+    expected = {"greedy_ids.npy", "greedy_dist.npy", *(f"p{i}_prompt.i32" for i in range(len(PROMPTS)))}
+    outputs = capture.get("outputs", {})
+    common.require(set(outputs) == expected, "capture output manifest is incomplete")
+    common.require(capture.get("output_sha256") == hashlib.sha256(common.json_bytes(outputs)).hexdigest(),
+                   "capture output manifest hash differs")
+    for name in expected:
+        common.require(name in inputs and outputs[name] == {
+            "sha256": hashlib.sha256(inputs[name]).hexdigest(), "size": len(inputs[name])},
+            f"capture input hash differs: {name}")
 
 
 def main():
     args = _parse_args()
+    prompt_record = common.check_prompts(__file__, PROMPTS)
+    model = common.model_identity(args)
+    strict = common.is_qwen35(model["identity"])
+    if args.runs is None:
+        args.runs = 10 if strict else 1
+    common.strict_inputs(args, model)
+    import vllm
     from vllm import LLM, SamplingParams
 
-    N, T = len(PROMPTS), args.max_tokens
-    our = np.fromfile(os.path.join(args.golden_dir, "our_ids.i32"),
-                      dtype="<i4").reshape(N, T)
-    greedy = np.load(os.path.join(args.golden_dir, "greedy_ids.npy"))
-
+    runtime = common.runtime_identity(vllm, args, True) if strict else None
     print(_mode_narration(args), flush=True)
     llm = LLM(**_llm_kwargs(args))
-
-    gap_mnats = np.zeros((N, T), dtype="<i4")
-    print(f"=== teacher-forced near-tie gap: {args.model} (OUR prefix) ===")
-    max_gap = 0.0
-    worst = None
-    n_div = 0
+    runtime = runtime or common.runtime_identity(vllm, args, False)
+    context = common.resolved_context(args, llm, model, runtime, prompt_record, 1)
+    strict = context["regime"] == "qwen3_5_strict"
+    context["tool"] = "qwen3-neartie-gap"
+    directory = Path(args.golden_dir)
+    N, T = len(PROMPTS), args.max_tokens
+    common.record_sampling(context, SamplingParams(temperature=0.0, max_tokens=T, seed=args.seed or 0))
+    names = ["our_ids.i32", "greedy_ids.npy", *(f"p{i}_prompt.i32" for i in range(N))]
+    if strict:
+        names += ["greedy_dist.npy", "oracle-provenance.json"]
+    inputs = {name: (directory / name).read_bytes() for name in names}
+    if strict:
+        capture = common.read_json(directory / "oracle-provenance.json")
+        validate_capture(context, capture, inputs)
+    common.require(len(inputs["our_ids.i32"]) == N * T * 4, "local token array has the wrong size", "STRUCTURE_MISMATCH")
+    our = np.frombuffer(inputs["our_ids.i32"], dtype="<i4").reshape(N, T).copy()
+    greedy = np.load(io.BytesIO(inputs["greedy_ids.npy"]), allow_pickle=False)
+    common.require(greedy.shape == (N, T) and greedy.dtype == np.dtype("<i4"),
+                   "greedy token array has the wrong shape or dtype", "STRUCTURE_MISMATCH")
+    common.require(np.all(our >= 0), "teacher forcing needs a complete nonnegative token stream", "STRUCTURE_MISMATCH")
+    if strict:
+        dist = np.load(io.BytesIO(inputs["greedy_dist.npy"]), allow_pickle=False)
+        common.require(dist.shape == (N, T, capture["repetitions"]) and dist.dtype == np.dtype("<i4")
+                       and np.all(dist == greedy[:, :, None]), "capture distribution is not deterministic", "NONDETERMINISTIC")
+    prefixes = []
     for i in range(N):
-        # Use the EXACT prompt tokenization the oracle greedy capture used
-        # (p{i}_prompt.i32) so the teacher-forced prefix aligns bit-for-bit.
-        prompt_ids = np.fromfile(
-            os.path.join(args.golden_dir, f"p{i}_prompt.i32"), dtype="<i4").tolist()
-        our_ids = [int(x) for x in our[i]]
-        full = list(prompt_ids) + our_ids
-        sp = SamplingParams(temperature=0.0, max_tokens=1, prompt_logprobs=args.topk)
-        out = llm.generate({"prompt_token_ids": full}, sp)[0]
-        plp = out.prompt_logprobs
-        P = len(prompt_ids)
-        for j in range(T):
-            pos = P + j
-            d = plp[pos] or {}
-            arg_tid = max(d, key=lambda k: d[k].logprob) if d else -1
-            arg_lp = d[arg_tid].logprob if d else 0.0
-            our_tid = our_ids[j]
-            if our_tid in d:
-                gap = max(0.0, arg_lp - d[our_tid].logprob)
-                gap_mnats[i, j] = int(round(gap * 1000.0))
-                if gap > max_gap:
-                    max_gap, worst = gap, (i, j, gap)
+        data = inputs[f"p{i}_prompt.i32"]
+        common.require(data and len(data) % 4 == 0, "prompt token file is incomplete", "STRUCTURE_MISMATCH")
+        tokens = np.frombuffer(data, dtype="<i4").tolist()
+        common.require(all(token >= 0 for token in tokens), "prompt contains a negative token", "STRUCTURE_MISMATCH")
+        prefixes.append(tokens)
+    sp_args = {"temperature": 0.0, "max_tokens": 1, "prompt_logprobs": args.topk, "seed": args.seed or 0}
+    common.record_sampling(context, SamplingParams(**sp_args), key="teacher_forcing_sampling")
+    reference = []
+    gap_mnats = np.zeros((N, T), dtype="<i4")
+    deterministic = True
+    for repeat in range(args.runs):
+        for i in range(N):
+            full = prefixes[i] + our[i].tolist()
+            outputs = llm.generate({"prompt_token_ids": full}, SamplingParams(**sp_args))
+            common.require(len(outputs) == 1 and len(outputs[0].prompt_logprobs) == len(full),
+                           "teacher-forced logprobs do not cover the exact prefix", "STRUCTURE_MISMATCH")
+            logprobs = outputs[0].prompt_logprobs
+            observed = []
+            for j in range(T):
+                values = logprobs[len(prefixes[i]) + j]
+                common.require(isinstance(values, dict) and values, "missing teacher-forced logprobs", "STRUCTURE_MISMATCH")
+                row = sorted((int(token), float(value.logprob)) for token, value in values.items())
+                common.require(all(math.isfinite(value) for _, value in row), "teacher-forced logprob is not finite", "NONFINITE")
+                observed.append(row)
+                if repeat == 0:
+                    lookup = dict(row)
+                    token = int(our[i, j])
+                    gap = max(lookup.values()) - lookup[token] if token in lookup else None
+                    value = int(round(max(0.0, gap) * 1000.0)) if gap is not None else OUTSIDE_TOPK_MNATS
+                    common.require(value <= 2147483647, "near-tie gap overflows the output dtype", "STRUCTURE_MISMATCH")
+                    gap_mnats[i, j] = value
+            if repeat == 0:
+                reference.append(observed)
             else:
-                gap_mnats[i, j] = OUTSIDE_TOPK_MNATS
-                print(f"  p{i:2d} tok{j:2d}: OUR TOKEN {our_tid} OUTSIDE vLLM top-{args.topk}"
-                      f" (REAL divergence)")
-            if our_ids[j] != int(greedy[i, j]):
-                n_div += 1
-                print(f"  p{i:2d} tok{j:2d}: our={our_tid} vLLM_greedy={int(greedy[i,j])}"
-                      f" vLLM_argmax={arg_tid} gap={gap_mnats[i,j]/1000.0:.4f} nats")
+                deterministic &= observed == reference[i]
+    common.require(not strict or deterministic, "teacher-forced logprobs changed before millinat rounding", "NONDETERMINISTIC")
+    context["deterministic"] = bool(deterministic)
+    context["inputs"] = {name: {"sha256": hashlib.sha256(data).hexdigest(), "size": len(data)}
+                         for name, data in sorted(inputs.items())}
+    context["logprobs_sha256"] = hashlib.sha256(common.json_bytes(reference)).hexdigest()
+    for name, data in inputs.items():
+        common.require((directory / name).read_bytes() == data, f"input changed during teacher forcing: {name}")
+    common.confirm_inputs(args, vllm, context, __file__, PROMPTS)
+    payloads = {}
+    for name, array in (("our_ids.npy", our), ("neartie_gap_mnats.npy", gap_mnats)):
+        buffer = io.BytesIO()
+        np.save(buffer, array, allow_pickle=False)
+        payloads[name] = buffer.getvalue()
+    protected = common.capture_input_paths(args, vllm, context, __file__)
+    protected.update(directory / name for name in inputs)
+    common.publish(directory, payloads, context, "neartie-provenance.json", args.provenance_out,
+                   protected_inputs=protected)
+    print(f"wrote {directory}; teacher forcing uses OUR exact prefix; output_sha256={context['output_sha256']}")
 
-    np.save(os.path.join(args.golden_dir, "our_ids.npy"), our)
-    np.save(os.path.join(args.golden_dir, "neartie_gap_mnats.npy"), gap_mnats)
-    print(f"=== {n_div} token-divergent positions vs vLLM greedy; "
-          f"max near-tie gap {max_gap:.4f} nats (worst {worst}) ===")
-    print(f"wrote {args.golden_dir}/our_ids.npy + neartie_gap_mnats.npy {gap_mnats.shape}")
 
 if __name__ == "__main__":
-    main()
+    try:
+        main()
+    except (common.CaptureError, OSError, ValueError) as error:
+        print(str(error) if isinstance(error, common.CaptureError) else f"ARTIFACT_MISMATCH: {error}", file=sys.stderr)
+        raise SystemExit(1)