diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 9493a9e61..8fa74ca60 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -45,6 +45,7 @@ jobs: outputs: matrix: ${{ steps.set.outputs.matrix }} devices: ${{ steps.set.outputs.devices }} + excludes: ${{ steps.set.outputs.excludes }} steps: - uses: actions/checkout@v7 - id: set @@ -64,24 +65,35 @@ jobs: fi echo "devices=$devices" >> "$GITHUB_OUTPUT" + # Catalog-only rows (empty `devices`) are never dispatched. if [ -n "$PICK_MODEL" ]; then matrix=$(jq -c --arg s "$PICK_MODEL" ' ($s | split(",") | map(gsub("^\\s+|\\s+$"; "")) | map(select(length>0))) as $pick | map(select(.name as $n | $pick | index($n))) + | map(select(.devices != [])) ' "$MODELS_FILE") picked_n=$(jq 'length' <<<"$matrix") want_n=$(jq -n --arg s "$PICK_MODEL" \ '$s | split(",") | map(gsub("^\\s+|\\s+$"; "")) | map(select(length>0)) | length') if [ "$picked_n" != "$want_n" ]; then - echo "::error::model input has unknown name(s): $PICK_MODEL" - echo "::error::available: $(jq -r '.[].name' "$MODELS_FILE" | paste -sd, -)" + echo "::error::model input has unknown or catalog-only name(s): $PICK_MODEL" + echo "::error::available: $(jq -r '.[] | select(.devices != []) | .name' "$MODELS_FILE" | paste -sd, -)" exit 1 fi else - matrix=$(jq -c . "$MODELS_FILE") + matrix=$(jq -c '[.[] | select(.devices != [])]' "$MODELS_FILE") fi echo "matrix=$matrix" >> "$GITHUB_OUTPUT" + # Spec-decoding rows only run on SC8480XP; drop the other combos. + excludes=$(jq -c -n --argjson devices "$devices" --argjson models "$matrix" ' + [ $devices[] as $d | $models[] as $m + | select(($m.spec // null) != null and $d != "SC8480XP") + | {device: $d, model: {name: $m.name}} + ] + ') + echo "excludes=$excludes" >> "$GITHUB_OUTPUT" + bench: name: ${{ matrix.device }} · ${{ matrix.model.plugin }} · ${{ matrix.model.name }} needs: [build-sdk, load-models] @@ -93,6 +105,7 @@ jobs: matrix: device: ${{ fromJson(needs.load-models.outputs.devices) }} model: ${{ fromJson(needs.load-models.outputs.matrix) }} + exclude: ${{ fromJson(needs.load-models.outputs.excludes) }} env: QDC_API_KEY: ${{ secrets.QDC_API_KEY }} steps: diff --git a/sdk/benchmark/benchmark.c b/sdk/benchmark/benchmark.c index 247629636..51305078e 100644 --- a/sdk/benchmark/benchmark.c +++ b/sdk/benchmark/benchmark.c @@ -89,6 +89,7 @@ typedef struct { char* mm_model_path; char* mm_mmproj; char* mm_tokenizer; + char* mm_draft_model; bool force_vlm; /* run VLM path even without an mmproj (QAIRT bundles) */ bool mm_is_vlm; /* manager classified the resolved model as VLM (geniex_ModelType) */ const char* image_paths[MAX_PATHS]; @@ -514,6 +515,63 @@ static int resolve_via_mm(options_t* o, const char* id_in) { return 0; } +/* Without this the plugin gets a raw id like "org/repo:Q4_0", fails to + * open it as a file, and silently falls back to non-speculative decode. */ +static int resolve_draft_via_mm(options_t* o) { + if (!o->draft_model || looks_like_path(o->draft_model)) return 0; + if (!g_mm_inited) { + int32_t rc = geniex_model_init(o->mm_data_dir); + if (rc != GENIEX_SUCCESS) { + const char* m = geniex_model_last_error_message(); + fprintf(stderr, "ERROR: geniex_model_init: %s (%d)\n", m ? m : "?", rc); + return 1; + } + g_mm_inited = true; + } + size_t n = strlen(o->draft_model); + char* buf = (char*)malloc(n + 1); + if (!buf) return 1; + memcpy(buf, o->draft_model, n + 1); + const char* name; + const char* quant; + split_id(buf, &name, &quant); + geniex_ModelPaths paths; + memset(&paths, 0, sizeof(paths)); + int32_t rc = geniex_model_get_paths(o->draft_model, &paths); + if (rc != GENIEX_SUCCESS) { + geniex_ModelPullInput in; + memset(&in, 0, sizeof(in)); + in.struct_size = (uint32_t)sizeof(in); + in.model_name = name; + in.quant = quant; + in.hub = parse_hub(o->mm_hub); + in.chipset = o->mm_chipset; + in.model_type = GENIEX_MODEL_TYPE_AUTO; + fprintf(stderr, "[mm ] pulling draft %s%s%s ...\n", name, quant ? ":" : "", quant ? quant : ""); + rc = geniex_model_pull(&in); + if (rc != GENIEX_SUCCESS) { + const char* m = geniex_model_last_error_message(); + fprintf(stderr, "ERROR: geniex_model_pull(%s): %s (%d)\n", o->draft_model, m ? m : "?", rc); + free(buf); + return 1; + } + rc = geniex_model_get_paths(o->draft_model, &paths); + if (rc != GENIEX_SUCCESS) { + const char* m = geniex_model_last_error_message(); + fprintf(stderr, "ERROR: geniex_model_get_paths(%s): %s (%d)\n", o->draft_model, m ? m : "?", rc); + free(buf); + return 1; + } + } + free(buf); + o->mm_draft_model = paths.model_path; + paths.model_path = NULL; + geniex_model_paths_free(&paths); + o->draft_model = o->mm_draft_model; + fprintf(stderr, "[mm ] resolved draft %s -> %s\n", o->draft_model, o->mm_draft_model); + return 0; +} + /* If `path` is a directory, return a heap-allocated path to a regular file * inside it (preferring `tokenizer.json`, otherwise the lexicographically * first regular file). The SDK derives the model dir via `parent_path()`, @@ -658,6 +716,7 @@ static void parse_args(int argc, char** argv, options_t* o) { o->mm_model_path = NULL; o->mm_mmproj = NULL; o->mm_tokenizer = NULL; + o->mm_draft_model = NULL; o->force_vlm = false; o->mm_is_vlm = false; o->image_count = 0; @@ -1529,7 +1588,7 @@ static void write_json(const options_t* o, const char* device_id, int32_t ngl, i fprintf(f, " \"params\": {\n"); fprintf(f, " \"warmup\": %d, \"repetitions\": %d, \"n_prompt\": %d, \"n_gen\": %d,\n" - " \"temperature\": %.6f, \"seed\": %d, \"n_ctx\": %d, \"n_threads\": %d, \"n_gpu_layers\": %d\n", + " \"temperature\": %.6f, \"seed\": %d, \"n_ctx\": %d, \"n_threads\": %d, \"n_gpu_layers\": %d", o->warmup, o->repeat, o->n_prompt, @@ -1539,7 +1598,16 @@ static void write_json(const options_t* o, const char* device_id, int32_t ngl, i o->n_ctx, o->n_threads, ngl); - fprintf(f, " },\n"); + if (o->spec_type) { + fprintf(f, ",\n \"spec_type\": "); + json_write_quoted(f, o->spec_type); + if (o->draft_model) { + fprintf(f, ",\n \"draft_model\": "); + json_write_quoted(f, o->draft_model); + } + fprintf(f, ",\n \"draft_tokens\": %d", o->draft_tokens); + } + fprintf(f, "\n },\n"); fprintf(f, " \"runs\": [\n"); for (int i = 0; i < o->repeat; ++i) { const run_result_t* r = &runs[i]; @@ -1846,6 +1914,10 @@ static int run_one_cell(options_t* o) { o->force_vlm = true; } + if (resolve_draft_via_mm(o) != 0) { + return 1; + } + char* anchored = resolve_local_anchor(o->model_path); if (anchored) { fprintf(stderr, "[info] resolved model dir to anchor: %s\n", anchored); @@ -1912,6 +1984,10 @@ static int run_one_cell(options_t* o) { geniex_free(o->mm_tokenizer); o->mm_tokenizer = NULL; } + if (o->mm_draft_model) { + geniex_free(o->mm_draft_model); + o->mm_draft_model = NULL; + } return 0; } @@ -1955,6 +2031,10 @@ static int run_one_cell(options_t* o) { geniex_free(o->mm_tokenizer); o->mm_tokenizer = NULL; } + if (o->mm_draft_model) { + geniex_free(o->mm_draft_model); + o->mm_draft_model = NULL; + } return 0; } diff --git a/sdk/benchmark/qdc/bench-models.json b/sdk/benchmark/qdc/bench-models.json index 06a2d4769..af1599b5e 100644 --- a/sdk/benchmark/qdc/bench-models.json +++ b/sdk/benchmark/qdc/bench-models.json @@ -126,6 +126,28 @@ "vlm": true, "image": true }, + { + "name": "gemma-4-26B-A4B-it-assistant", + "plugin": "llama_cpp", + "devices": [], + "model_id": "RachidAR/gemma-4-26B-A4B-it-qat-assistant-q4_0-gguf:Q4_0", + "hub": "hf", + "url": "https://huggingface.co/RachidAR/gemma-4-26B-A4B-it-qat-assistant-q4_0-gguf/resolve/main/gemma-4-26b-A4B-it-assistant-Q4_0.gguf" + }, + { + "name": "gemma-4-26B-A4B-it-mtp", + "plugin": "llama_cpp", + "devices": ["npu", "hybrid"], + "model_id": "google/gemma-4-26B-A4B-it-qat-q4_0-gguf:Q4_0", + "hub": "hf", + "url": "https://huggingface.co/google/gemma-4-26B-A4B-it-qat-q4_0-gguf/resolve/main/gemma-4-26B_q4_0-it.gguf", + "ctx": [512, 1024, 4096, 8192], + "spec": { + "type": "draft-mtp", + "draft": "gemma-4-26B-A4B-it-assistant", + "n_max": 3 + } + }, { "name": "Qwen3.5-2B", "plugin": "llama_cpp", diff --git a/sdk/benchmark/qdc/linux/run_linux.sh b/sdk/benchmark/qdc/linux/run_linux.sh index 0d8fc165e..7f85b2d87 100644 --- a/sdk/benchmark/qdc/linux/run_linux.sh +++ b/sdk/benchmark/qdc/linux/run_linux.sh @@ -59,7 +59,7 @@ for ctx in "${CTX_ARR[@]}"; do : > "/data/local/tmp/matrix-qairt-${ctx}.tsv" done -while IFS='|' read -r name plugin devs model_id vlm image; do +while IFS='|' read -r name plugin devs model_id vlm image _spec_type _draft_id _draft_tokens; do [ -z "$name" ] && continue echo "=== plan $name id=$model_id ===" case "$plugin" in diff --git a/sdk/benchmark/qdc/run_qdc_jobs.py b/sdk/benchmark/qdc/run_qdc_jobs.py index d5cb541b5..a92067ae8 100644 --- a/sdk/benchmark/qdc/run_qdc_jobs.py +++ b/sdk/benchmark/qdc/run_qdc_jobs.py @@ -116,31 +116,43 @@ def resolve_model_url(m: dict, device: str) -> str | None: return m.get("url") +def _resolve_draft_model_id(models: list[dict], draft_name: str) -> str: + for m in models: + if m["name"] == draft_name: + return m["model_id"] + raise SystemExit(f"draft model {draft_name!r} not found in bench-models.json") + + def model_rows(models: list[dict], device: str) -> list[str]: """One pipe-delimited row per model, consumed by the device-side run scripts. Schema: name | plugin | csv_devices | model_id | vlm | image + | spec_type | draft_model_id | draft_tokens - The host passes the chipset slug as a single shared --chipset flag to - geniex-bench; the model-manager hub auto-routes "qualcomm/*" to - AI Hub and everything else to HuggingFace, so per-row hub overrides - aren't needed. mmproj/tokenizer paths come back from get_paths. - + Trailing three fields carry spec-decoding params or empty strings so + every script parses the same column count. Entries with empty + ``devices`` are catalog-only (e.g. spec draft models) and skipped. Rows for AI Hub models whose chipset isn't advertised are dropped upfront so the device doesn't waste time on a guaranteed-fail pull.""" rows = [] for m in models: if "model_id" not in m: raise SystemExit(f"{m['name']}: missing model_id in bench-models.json") + if not m["devices"]: + continue if m.get("hub") == "aihub" and not _aihub_chipset_supported(m, device): log.warning("no %s asset for %s, skipping", device, m["name"]) continue vlm = "1" if m.get("vlm") else "" image = "1" if m.get("image") else "" + spec = m.get("spec") or {} + spec_type = spec.get("type", "") + draft_id = _resolve_draft_model_id(models, spec["draft"]) if spec else "" + draft_tokens = str(spec.get("n_max", "")) if spec else "" rows.append( f"{m['name']}|{m['plugin']}|{','.join(m['devices'])}|{m['model_id']}" - f"|{vlm}|{image}" + f"|{vlm}|{image}|{spec_type}|{draft_id}|{draft_tokens}" ) return rows @@ -398,6 +410,76 @@ def _details_block( return lines +def _is_spec_cell(c: dict) -> bool: + return bool((c.get("params") or {}).get("spec_type")) + + +def _render_mtp_table(cells: list[dict], models: list[dict] | None) -> list[str]: + """Pair each spec cell with its no-spec baseline on (target model_id, + device, ctx). Emits nothing when no spec cells are present.""" + if not models: + return [] + spec_entries = [m for m in models if m.get("spec")] + if not spec_entries: + return [] + by_name_key: dict[str, dict[tuple[str, int], dict]] = {} + for c in cells: + by_name_key.setdefault(_model_label(c), {})[ + (c["device"], _ctx_from_cell(c)) + ] = c + rows: list[str] = [] + for spec_m in spec_entries: + baseline = next( + ( + m + for m in models + if m["model_id"] == spec_m["model_id"] + and not m.get("spec") + and m.get("devices") + ), + None, + ) + draft_m = next( + (m for m in models if m["name"] == spec_m["spec"]["draft"]), None + ) + spec_cells = by_name_key.get(spec_m["name"], {}) + base_cells = by_name_key.get(baseline["name"], {}) if baseline else {} + for (dev, ctx), sc in sorted(spec_cells.items()): + agg = sc.get("agg") or {} + s_dec = (agg.get("decode_tps") or {}).get("median") + bc = base_cells.get((dev, ctx)) + b_dec = ( + ((bc.get("agg") or {}).get("decode_tps") or {}).get("median") + if bc + else None + ) + uplift = f"{s_dec / b_dec:.2f}x" if s_dec and b_dec else "-" + p_med = (agg.get("prompt_tokens") or {}).get("median") + g_med = (agg.get("gen_tokens") or {}).get("median") + test = ( + f"pp{int(p_med)}+tg{int(g_med)}" + if p_med is not None and g_med is not None + else "-" + ) + rows.append( + f"| {spec_m['name']} | {draft_m['name'] if draft_m else '-'} | " + f"{dev} | {ctx} | {test} | " + f"{_fmt_med_sd(agg, 'decode_tps')} | " + f"{_fmt_med_sd((bc or {}).get('agg') or {}, 'decode_tps')} | " + f"{uplift} |" + ) + if not rows: + return [] + return [ + "", + "## MTP (speculative decoding)", + "", + "| Target | Draft | Device | Ctx | Test | Decode (mtp) | Decode (baseline) | Uplift |", + "|--------|-------|--------|----:|------|-------------:|------------------:|-------:|", + *rows, + ] + + def render( cells: list[dict], device: str, @@ -412,6 +494,8 @@ def render( ] sort_key = lambda c: (_model_label(c), c["plugin"], c["device"], _ctx_from_cell(c)) # noqa: E731 for c in sorted(cells, key=sort_key): + if _is_spec_cell(c): + continue agg = c.get("agg") or {} params = c.get("params") or {} model = _model_label(c) @@ -431,6 +515,7 @@ def render( f"{_fmt_med_sd(agg, 'ttft_ms')} | {_fmt_med_sd(agg, 'prefill_tps')} | " f"{_fmt_med_sd(agg, 'decode_tps')} |" ) + lines += _render_mtp_table(cells, models) return "\n".join(lines) + "\n" @@ -516,11 +601,21 @@ def main() -> int: if platform not in BUILDERS: raise SystemExit(f"{platform} not implemented yet") - models = json.loads(args.models_file.read_text()) + all_models = json.loads(args.models_file.read_text()) if args.model_name: - models = [m for m in models if m["name"] == args.model_name] + models = [m for m in all_models if m["name"] == args.model_name] if not models: raise SystemExit(f"model {args.model_name!r} not in {args.models_file}") + # Pull in every spec.draft dependency so _resolve_draft_model_id can + # still find it after --model-name has trimmed the list to one row. + needed = {m["spec"]["draft"] for m in models if m.get("spec")} + for name in needed - {m["name"] for m in models}: + entry = next((m for m in all_models if m["name"] == name), None) + if entry is None: + raise SystemExit(f"draft {name!r} not in {args.models_file}") + models.append(entry) + else: + models = all_models compute_pick = [c.strip() for c in args.compute.split(",") if c.strip()] if compute_pick: @@ -541,7 +636,12 @@ def main() -> int: raise SystemExit( f"no model in {args.models_file} runs any of --compute={compute_pick}" ) - ctx_list, pp_list, tg_list = resolve_sweep(args.ctx, args.pp, args.tg) + ctx_arg = args.ctx + if not ctx_arg: + active = [m for m in models if m.get("devices")] + if len(active) == 1 and active[0].get("ctx"): + ctx_arg = ",".join(str(x) for x in active[0]["ctx"]) + ctx_list, pp_list, tg_list = resolve_sweep(ctx_arg, args.pp, args.tg) log.info("sweep: ctx=%s pp=%s tg=%s", ctx_list, pp_list, tg_list) client = _qdc.make_client(api_key) @@ -564,6 +664,15 @@ def main() -> int: cells = download_cells( client, job_id, tmp, model_names=[m["name"] for m in models] ) + if not cells: + for name, data in _qdc.download_log_members( + client, job_id, tmp, lambda n: n.endswith((".log", ".stdout", ".txt")) + ): + print(f"===== QDC log: {name} =====") + try: + print(data.decode("utf-8", errors="replace")) + except Exception as e: + print(f"[decode failed: {e}]") if args.cells_out: args.cells_out.write_text(json.dumps(cells)) diff --git a/sdk/benchmark/qdc/tests/test_bench.py b/sdk/benchmark/qdc/tests/test_bench.py index 7eec38978..88c12e956 100644 --- a/sdk/benchmark/qdc/tests/test_bench.py +++ b/sdk/benchmark/qdc/tests/test_bench.py @@ -49,7 +49,7 @@ def test_bench(): (plugin, ctx): [] for plugin in ("llama_cpp", "qairt") for ctx in CTXS } for row in rows: - name, plugin, devs, model_id, vlm, image = row.split("|") + name, plugin, devs, model_id, vlm, image, *_spec = row.split("|") if plugin not in ("llama_cpp", "qairt"): continue if plugin != "qairt": diff --git a/sdk/benchmark/qdc/windows/run_windows.ps1 b/sdk/benchmark/qdc/windows/run_windows.ps1 index 171e32d53..b4dcb61aa 100644 --- a/sdk/benchmark/qdc/windows/run_windows.ps1 +++ b/sdk/benchmark/qdc/windows/run_windows.ps1 @@ -15,13 +15,14 @@ # >~7 GB GGUFs on X2 Elite). # # We sweep ctx in {512, 1024, 4096} per cell to align with test-llama.cpp's -# PERFORMANCE SESSION. Two prefill modes coexist: -# - llama_cpp cells use random-ids prefill (`-p N`, mirrors llama-bench -# `pp{N}`), so reported pp is exactly the ctx value; -# - qairt cells go through prompt_utf8 (the plugin doesn't accept -# pre-tokenized input_ids — see issue #1008), with a pre-trimmed -# `sample_prompt_${ctx}.txt` per ctx so prompt length is bounded. -# Each plugin gets its own per-ctx TSV so the two invocations don't mix. +# PERFORMANCE SESSION. Three buckets, each with its own per-ctx TSV so +# their invocations don't mix: +# - llama_cpp cells use random-ids prefill (`-p N`); +# - qairt cells go through prompt_utf8 with `sample_prompt_${ctx}.txt` +# because the plugin doesn't accept pre-tokenized input_ids (#1008); +# - spec (llama_cpp speculative-decoding) cells share random-ids +# prefill and additionally pass --spec-type/--draft-model/--draft-tokens +# as CLI-level flags per matrix invocation. $ErrorActionPreference = "Continue" @@ -63,17 +64,19 @@ $ctxList = @({CTX_LIST}) $ppList = @({PP_LIST}) $tgList = @({TG_LIST}) $tsvByPluginCtx = @{} -foreach ($plugin in @("llama", "qairt")) { +foreach ($plugin in @("llama", "qairt", "spec")) { foreach ($ctx in $ctxList) { $tsvByPluginCtx["$plugin-$ctx"] = "C:\Temp\matrix-$plugin-$ctx.tsv" Remove-Item $tsvByPluginCtx["$plugin-$ctx"] -ErrorAction SilentlyContinue } } +# Spec CLI params are per-invocation, not per-cell. Keyed by "$ctx". +$specParamsByCtx = @{} foreach ($row in $rows) { - $name, $plugin, $devs, $model_id, $vlm, $image = $row -split '\|' + $name, $plugin, $devs, $model_id, $vlm, $image, $spec_type, $draft_model_id, $draft_tokens = $row -split '\|' Write-Output "=== plan $name id=$model_id ===" - $bucket = if ($plugin -eq "qairt") { "qairt" } elseif ($plugin -eq "llama_cpp") { "llama" } else { "" } + $bucket = if ($spec_type) { "spec" } elseif ($plugin -eq "qairt") { "qairt" } elseif ($plugin -eq "llama_cpp") { "llama" } else { "" } if (-not $bucket) { Write-Output "WARN: unknown plugin $plugin in $name, skipping" continue @@ -87,6 +90,13 @@ foreach ($row in $rows) { "{0}-{1}-{2}-c{3}`t{1}`t{2}`t{4}`t`t`t{5}`t{6}" -f ` $name, $plugin, $d, $ctx, $model_id, $imgpath, $vlm ` | Add-Content $tsvByPluginCtx["$bucket-$ctx"] + if ($bucket -eq "spec") { + $specParamsByCtx["$ctx"] = @{ + type = $spec_type + draft = $draft_model_id + tokens = $draft_tokens + } + } } } } @@ -115,6 +125,26 @@ for ($i = 0; $i -lt $ctxList.Count; $i++) { --mm-data-dir $MM_CACHE --chipset "{CHIPSET}" Write-Output "rc=$LASTEXITCODE ($((Get-ChildItem $OUT).Count) cell json files so far)" } + + $specTsv = $tsvByPluginCtx["spec-$ctx"] + if ((Test-Path $specTsv) -and ((Get-Item $specTsv).Length -gt 0)) { + $sp = $specParamsByCtx["$ctx"] + # Spec-decoding needs `--draft-tokens` extra KV slots on the last + # decode step (target + draft), or llama.cpp trips + # "decode: failed to find a memory slot for batch of size N+1". + # Bench defaults pp+tg = ctx exactly, so trim tg by that margin. + $draftHeadroom = if ($sp.tokens) { [int]$sp.tokens + 1 } else { 4 } + $specTg = [Math]::Max(1, [int]$tg - $draftHeadroom) + Write-Output "=== matrix spec ctx=$ctx pp=$pp tg=$specTg type=$($sp.type) draft=$($sp.draft) n_max=$($sp.tokens) (random-ids prefill) ===" + Get-Content $specTsv + $extra = @() + if ($sp.tokens) { $extra += @("--draft-tokens", $sp.tokens) } + & "$BUNDLE\bin\geniex-bench.exe" --matrix-file $specTsv --output-json-dir "$OUT" -r 1 --no-warmup ` + -c $ctx -p $pp -n $specTg ` + --spec-type $sp.type --draft-model $sp.draft @extra ` + --mm-data-dir $MM_CACHE --chipset "{CHIPSET}" + Write-Output "rc=$LASTEXITCODE ($((Get-ChildItem $OUT).Count) cell json files so far)" + } } Write-Output "=== done ===" Stop-Transcript | Out-Null