From 2cc2f0df9ddcb1bb7fe60f7bd6934d2a9de1e4f2 Mon Sep 17 00:00:00 2001 From: Jason McCartney Date: Tue, 8 Sep 2026 15:58:22 -0700 Subject: [PATCH] Support the public SparkGLM appliance and long cold prefill Add the measured SparkGLM managed adapter and immutable-image recipe materializer for EXL3 and NVFP4. Keep lifecycle, admission, worker-first readiness and routing in LLooM. Apply the existing extended HTTP dispatcher to chat forwarding so long cold prefill does not hit Undici's independent five-minute timeout. Provenance: - Adapted existing LLooM SparkGLM integration developed at local source 75e08ca0923e7eb58f087b53032c82f88175ea3a, including its measured context settings. Existing Mia-derived MIT/Apache notices are retained in the launcher. No model weights or binaries are included. Verification: - Managed recipe test; delayed headers, streaming body and buffered transport regression; npm run check; npm test; format check; lint; interchange and package checks passed. Live SparkGLM uses this entrypoint hash and transport behavior; publishing source does not replace the installed gateway. --- backends/sparkglm/README.md | 115 ++++++++++ backends/sparkglm/entrypoint.sh | 166 +++++++++++++++ backends/sparkglm/materialize.mjs | 307 +++++++++++++++++++++++++++ docs/sparkglm.md | 13 ++ package.json | 2 +- src/server.mjs | 2 +- test/long-prefill-transport.test.mjs | 75 +++++++ test/sparkglm-managed.test.mjs | 229 ++++++++++++++++++++ 8 files changed, 907 insertions(+), 2 deletions(-) create mode 100644 backends/sparkglm/README.md create mode 100755 backends/sparkglm/entrypoint.sh create mode 100644 backends/sparkglm/materialize.mjs create mode 100644 docs/sparkglm.md create mode 100644 test/long-prefill-transport.test.mjs create mode 100644 test/sparkglm-managed.test.mjs diff --git a/backends/sparkglm/README.md b/backends/sparkglm/README.md new file mode 100644 index 0000000..19708b1 --- /dev/null +++ b/backends/sparkglm/README.md @@ -0,0 +1,115 @@ +# SparkGLM under LLooM + +Build and qualify SparkGLM separately. LLooM manages the resulting Docker +runtime: admission, worker-first TP2 startup, readiness, stop, routing and +telemetry. This adapter uses the patchers baked into the selected SparkGLM +image; it does not overlay the older Mia launcher implementation. + +Materialize a private, immutable-image recipe from the installed LLooM checkout: + +```sh +mkdir -p "$HOME/.lloom/sparkglm-recipes" +node backends/sparkglm/materialize.mjs \ + --image-id "$HEAD_IMAGE_ID" --worker-image-id "$WORKER_IMAGE_ID" \ + --source-revision "$SPARKGLM_COMMIT" \ + --output "$HOME/.lloom/sparkglm-recipes/linux-nvidia-dgx-spark-2x-sparkglm-exl3.json" +``` + +Image IDs must be complete `sha256:` identities present on their respective +nodes; source revision must be a complete commit. When the images are built +independently, verify their serving source and extension hashes match. Omit +`--worker-image-id` when both nodes have the same image. Keep the generated +recipe directory and this backend directory identical on both installed nodes. +Distributed setup invokes the same recipe on the worker. + +Review the ordinary setup plan, then apply and explicitly start: + +```sh +lloom setup --recipe linux-nvidia-dgx-spark-2x-sparkglm-exl3 \ + --recipes-root "$HOME/.lloom/sparkglm-recipes" --additive --no-auto-host --json +lloom setup --recipe linux-nvidia-dgx-spark-2x-sparkglm-exl3 \ + --recipes-root "$HOME/.lloom/sparkglm-recipes" --additive --no-auto-host --apply --yes --json +lloom runtime-start glm53-flash-exl3-cluster --json +lloom runtime-status glm53-flash-exl3-cluster --json +lloom runtime-stop glm53-flash-exl3-cluster --json +``` + +Installing the profile does not set keep-warm or resume suspended routes. +Independent Docker restart is disabled: LLooM owns distributed retries so a +failed head cannot repeatedly reload beside a stale NCCL worker. +Cluster and head readiness use `/v1/models` with the exact served-model ID. +The EXL3, NVFP4 and fixture profiles share a port; a generic `/health` response +from another active profile must not make a stopped runtime appear loaded. +Use `lloom route glm53f-local --json` to inspect the strict local canary alias. +Verify a streamed tool-call through that alias and confirm gateway metrics +attribute it to `glm-5.3-flash-exl3`, in addition to checking both rank image +identities and backend health. Run SparkGLM's documented shape warmup and +qualification harness; a successful LLooM health check does not qualify a +kernel or a release. + +`--e3` explicitly enables the unqualified E3 large-prefill experiment and must +only select an image containing its source-locked adapter. `--tiny` generates +an isolated `sparkglm-tiny` model/runtime with no production aliases. First +build the documented tinyGLM fixture on each host and expose its snapshot at +`${modelRoot}/sparkglm--tinyglm`. The launcher rejects dummy loading unless +the config identifies the synthetic `tinyglm-v1` fixture and speculation is +disabled. Stop the full runtime before starting the fixture. + +`--e3-policy concurrent` selects the newer experimental CPU-metadata policy +that keeps solo prefill on the reference path. It requires the corresponding +image helper; `--e3-trace` records initial selection decisions for qualification. + +`--nvfp4` generates the isolated `sparkglm-nvfp4` runtime for the separately +pinned current compressed-tensors checkpoint. It cannot be combined with +`--tiny` or `--e3`. This initial lane uses 8 GiB KV per rank and 65536 context; +The original 262144 context failed vLLM cache admission with this budget. +Use matched limits for diagnostic comparisons and independently tune finalists. Native kernel +execution and model quality require separate evidence. None of these options +changes the production Presence alias or promotes an experimental backend. +Use `--nvfp4-budget` when materializing that EXL3 comparison arm; it selects +the same explicit 8 GiB KV per rank, 65536 context, and admission reservation. +It can be combined with `--e3`, but cannot be used with a tiny fixture. + +`--nvfp4-tiny` selects the separate `sparkglm-tiny-nvfp4` fixture. Build its +reviewed dummy-loader image and metadata from SparkGLM's NVFP4 experiment +first. The image must contain the guarded deterministic FP4 initializer; +ordinary integer dummy weights are uninitialized in the underlying loader. +This option uses no production aliases and cannot enable E3 or real NVFP4 +loading. Finish model downloads before full-model cold starts: downloader +buffers count against GB10's shared memory budget. + +For performance experiments, `--mxfp8-draft` selects the separately pinned +MXFP8 DFlash2 checkpoint and requires an image with its quantized selector, +grouped-convolution and fused context K/V support. `--prefill-tokens N` varies prefill chunking. Draft TP must remain 2: the +inherited DFlash loader preserves the target parallel group even if vLLM accepts +a TP1 configuration flag. This adapter rejects that misleading request early. NVFP4 also +accepts an explicit `--moe-backend`; vLLM must still validate its compatibility, +including SwiGLU clamping and mixed quantization. These are experiment controls, +not recommended defaults. Final performance selection may use different tuned +settings for each target; the matched-budget option is a diagnostic control. + +`--context-tokens N` and `--kv-cache-gib N` explicitly tune context and per-rank +KV allocation. Context capability must be reported with results; vLLM retains +its cache admission check. Increasing KV requires actual host memory headroom. +Budgets above 8 GiB also increase LLooM's per-rank admission reservation; +smaller budgets preserve the existing conservative reservation. +MoE overrides also accept the guarded NVFP4 tiny fixture for backend validation. + +`--exl3-temp-rows 32|64|128` tunes the thin/fat expert threshold. The default +remains 128; a lower threshold requires operator and TP2 integration evidence. + +Large prefill chunks also increase the padded sliding-window draft cache +reservation. The observed 16K-chunk profile needed 15.2 GiB even for one 64K +request; smaller chunks are essential to explore on memory-limited NVFP4. + +Explicit `--context-tokens` also sets `maxPromptTokens: 0` on the gateway model. +This delegates token-budget validation to vLLM's exact rendered-token check; it +does not change or disable `max_model_len`. The gateway's character estimate and +98% rejection margin otherwise reject some valid near-window requests. The +backend still rejects prompt plus output exceeding its configured context. + +For explicit contexts above 524288 tokens, the managed no-output watchdog budget +is 30 minutes, matching the backend request deadline. The inherited ten-minute +budget interrupted a valid progressing 1M cold prefill. Watchdog recovery remains +enabled; this change permits the initial prefill silence and does not claim +endurance qualification or make a cold 1M request interactive. diff --git a/backends/sparkglm/entrypoint.sh b/backends/sparkglm/entrypoint.sh new file mode 100755 index 0000000..a100ae5 --- /dev/null +++ b/backends/sparkglm/entrypoint.sh @@ -0,0 +1,166 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: MIT AND Apache-2.0 +# Adapted from LLooM's Mia launcher; SparkGLM supplies the built runtime. +set -euo pipefail + +log() { printf '[sparkglm rank=%s] %s\n' "${NODE_RANK:-?}" "$*"; } + +: "${NODE_RANK:?NODE_RANK is required}" +: "${CLUSTER_NODE_COUNT:?CLUSTER_NODE_COUNT is required}" +: "${MASTER_ADDR:?MASTER_ADDR is required}" +: "${MODEL_DIR:?MODEL_DIR is required}" + +# The inherited DFlash loader preserves the target parallel configuration; +# accepting draft TP1 here would silently run TP2 and mislabel measurements. +if [[ "${SPEC_METHOD:-dflash}" == "dflash" && "${DFLASH_DRAFT_TP:-2}" != "${CLUSTER_NODE_COUNT}" ]]; then + log "independent DFlash draft TP is not implemented by this adapter; use target TP=${CLUSTER_NODE_COUNT}" + exit 1 +fi + +[[ -f "${MODEL_DIR}/config.json" ]] || { + log "missing target config: ${MODEL_DIR}/config.json" + exit 1 +} + +if [[ "${SPARKGLM_EXL3_E3:-0}" == "1" ]]; then + [[ -f /usr/local/lib/python3.12/dist-packages/sparkglm_e3.py && -f /usr/local/lib/python3.12/dist-packages/exl3_fat_moe_ext.so ]] || { + log "selected image does not contain the E3 adapter and extension"; exit 1; + } + if [[ "${SPARKGLM_EXL3_E3_POLICY:-large}" == "concurrent" ]]; then + [[ -f /usr/local/lib/python3.12/dist-packages/sparkglm_e3_policy.py ]] || { + log "selected image does not contain the concurrent E3 policy"; exit 1; + } + fi +fi +if [[ "${SPARKGLM_NVFP4_TINY:-0}" == "1" ]]; then + [[ -f /usr/local/lib/python3.12/dist-packages/sparkglm_nvfp4_tiny.py && -f /usr/local/lib/python3.12/dist-packages/sparkglm_nvfp4_tiny.pth ]] || { + log "selected image does not contain the guarded NVFP4 fixture initializer"; exit 1; + } +fi +if [[ "${SPARKGLM_MXFP8_DRAFT:-0}" == "1" ]]; then + grep -q '_fused_kv_weight_scale' /opt/glm53/patch_dflash2.py || { + log "selected image does not contain MXFP8 DFlash2 context projection support"; exit 1; + } +fi + +if [[ "${SPEC_METHOD:-dflash}" == "dflash" && ! -f "${DFLASH_MODEL_DIR:-}/config.json" ]]; then + log "missing DFlash2 config: ${DFLASH_MODEL_DIR:-unset}/config.json" + exit 1 +fi + +# Runtime patches come from the selected SparkGLM image, never the legacy +# Mia files installed alongside LLooM. Image identity is pinned by the recipe. +for patch in \ + patch_glm_video_placeholders.py \ + patch_suppress_stops_in_reasoning.py \ + patch_scheduler_decode_floor.py \ + patch_glm5_drafter_group.py \ + patch_hybrid_prefix_hit.py \ + patch_xgrammar_termination.py \ + patch_kpool_tail_slotmap.py \ + patch_spinwait.py \ + patch_indexer_workspace.py \ + patch_ablit.py; do + [[ -f "/opt/glm53/${patch}" ]] || { + log "missing SparkGLM runtime patch: /opt/glm53/${patch}" + exit 1 + } + # These source patchers use only the standard library. Avoid importing the + # serving stack through site .pth hooks ten times during each cold start. + python3 -S "/opt/glm53/${patch}" +done + +if [[ -z "${LIMIT_MM_PER_PROMPT:-}" ]]; then + LIMIT_MM_PER_PROMPT='{"image":4,"video":1}' +fi + +args=( + --served-model-name "${SERVED_MODEL_NAME:-glm-5.3-flash-exl3}" + --host "${VLLM_HOST:-0.0.0.0}" + --port "${VLLM_PORT:-8890}" + --tensor-parallel-size "${CLUSTER_NODE_COUNT}" + --nnodes "${CLUSTER_NODE_COUNT}" + --node-rank "${NODE_RANK}" + --master-addr "${MASTER_ADDR}" + --master-port "${MASTER_PORT:-29521}" + --distributed-executor-backend mp + --tool-call-parser glm47 + --enable-auto-tool-choice + --reasoning-parser glm45 + --enable-prefix-caching + --no-enable-flashinfer-autotune + --quantization "${QUANTIZATION:-exl3}" + --max-model-len "${MAX_MODEL_LEN:-1000000}" + --gpu-memory-utilization "${GPU_MEMORY_UTILIZATION:-0.87}" + --max-num-seqs "${MAX_NUM_SEQS:-4}" + --max-num-batched-tokens "${MAX_NUM_BATCHED_TOKENS:-7168}" + --kv-cache-dtype "${KV_CACHE_DTYPE:-fp8}" + --chat-template /opt/glm53/chat_template.jinja + --limit-mm-per-prompt "${LIMIT_MM_PER_PROMPT}" + --skip-mm-profiling +) + +if [[ "${SPARKGLM_TINY_DUMMY:-0}" == "1" ]]; then + python3 -S - "${MODEL_DIR}/config.json" <<'PYSAFE' +import json, sys +c = json.load(open(sys.argv[1])) +if c.get("quantization_config", {}).get("version") != "tinyglm-v1": + raise SystemExit("dummy loading requires the synthetic tinyGLM fixture") +PYSAFE + [[ "${SPEC_METHOD:-dflash}" == "none" ]] || { log "tinyGLM requires SPEC_METHOD=none"; exit 1; } + args+=(--load-format dummy --generation-config vllm) +fi +if [[ "${SPARKGLM_NVFP4_TINY:-0}" == "1" ]]; then + python3 -S - "${MODEL_DIR}/config.json" <<'PYSAFE' +import json, sys +if json.load(open(sys.argv[1])).get("_sparkglm_fixture") != "tinyglm-nvfp4-v1": + raise SystemExit("NVFP4 dummy loading requires the synthetic fixture") +PYSAFE + [[ "${SPEC_METHOD:-dflash}" == "none" && "${QUANTIZATION:-exl3}" == "compressed-tensors" ]] || { log "invalid NVFP4 fixture options"; exit 1; } + args+=(--load-format dummy --generation-config vllm) +fi +if [[ "${LANGUAGE_MODEL_ONLY:-0}" == "1" ]]; then + args+=(--language-model-only) +fi + +if [[ -n "${KV_CACHE_MEMORY_BYTES:-}" ]]; then + args+=(--kv-cache-memory-bytes "${KV_CACHE_MEMORY_BYTES}") +fi +if [[ -n "${MOE_BACKEND:-}" ]]; then + args+=(--moe-backend "${MOE_BACKEND}") +fi + +if [[ "${NODE_RANK}" != "0" ]]; then + args+=(--headless) +fi + +case "${SPEC_METHOD:-dflash}" in + dflash) + dflash_tokens="${DFLASH_TOKENS:-7}" + dflash_draft_tp="${DFLASH_DRAFT_TP:-2}" + [[ "${dflash_tokens}" =~ ^[0-9]+$ ]] || { log "invalid DFLASH_TOKENS=${dflash_tokens}"; exit 1; } + [[ "${dflash_draft_tp}" =~ ^[0-9]+$ ]] || { log "invalid DFLASH_DRAFT_TP=${dflash_draft_tp}"; exit 1; } + # Do not launch Python here: Mia's installed video .pth emits a status line + # on interpreter startup, which would contaminate command-substitution JSON. + printf -v spec '{"method":"dflash","model":"%s","num_speculative_tokens":%d,"kv_cache_dtype":"auto","draft_sample_method":"probabilistic","rejection_sample_method":"standard","draft_tensor_parallel_size":%d}' \ + "${DFLASH_MODEL_DIR}" "${dflash_tokens}" "${dflash_draft_tp}" + args+=(--speculative-config "${spec}") + ;; + mtp) + args+=(--speculative-config "{\"method\":\"mtp\",\"num_speculative_tokens\":${MTP_TOKENS:-2}}") + ;; + none) ;; + *) + log "unsupported SPEC_METHOD=${SPEC_METHOD}" + exit 1 + ;; +esac + +if [[ "${ENFORCE_EAGER:-0}" == "1" ]]; then + args+=(--enforce-eager) +else + args+=(--cudagraph-capture-sizes 1 2 4 8 16 24 32) +fi + +log "starting SparkGLM quant=${QUANTIZATION:-exl3} TP=${CLUSTER_NODE_COUNT}, spec=${SPEC_METHOD:-dflash}, draft-tp=${DFLASH_DRAFT_TP:-2}, mnbt=${MAX_NUM_BATCHED_TOKENS:-7168}, E2=${EXL3_FAT_KERNEL:-1}" +exec vllm serve "${MODEL_DIR}" "${args[@]}" diff --git a/backends/sparkglm/materialize.mjs b/backends/sparkglm/materialize.mjs new file mode 100644 index 0000000..7ce9021 --- /dev/null +++ b/backends/sparkglm/materialize.mjs @@ -0,0 +1,307 @@ +#!/usr/bin/env node +// SPDX-License-Identifier: Apache-2.0 +// Build locally; give LLooM an immutable image and ordinary distributed runtime. +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +export async function materialize({ + image, + workerImage = image, + sourceRevision, + e3 = false, + tiny = false, + nvfp4 = false, + nvfp4Tiny = false, + nvfp4Budget = false, + mxfp8Draft = false, + draftTp = 2, + prefillTokens = 7168, + moeBackend = 'auto', + e3Policy = 'large', + e3Trace = false, + contextTokens, + kvCacheGiB, + exl3TempRows = 128 +}) { + if ( + contextTokens !== undefined && + (!Number.isInteger(contextTokens) || contextTokens < 4096 || contextTokens > 1048576) + ) + throw new Error('Context token limit must be an integer in 4096..1048576'); + if (kvCacheGiB !== undefined && (!Number.isInteger(kvCacheGiB) || kvCacheGiB < 1 || kvCacheGiB > 32)) + throw new Error('KV cache budget must be an integer in 1..32 GiB'); + if (nvfp4Tiny && (nvfp4 || e3)) throw new Error('NVFP4 fixture cannot use real-model or E3 options'); + tiny = tiny || nvfp4Tiny; + if (!['large', 'concurrent'].includes(e3Policy)) throw new Error('Unknown E3 policy'); + if (!e3 && (e3Policy !== 'large' || e3Trace)) throw new Error('E3 policy controls require --e3'); + if (tiny && nvfp4Budget) throw new Error('Full-model comparison budget cannot use a tiny fixture'); + if (tiny && mxfp8Draft) throw new Error('Tiny fixtures have no speculative draft'); + if (draftTp !== 2) + throw new Error('This SparkGLM DFlash loader uses target TP2; independent draft TP1 is not implemented'); + if (!Number.isInteger(prefillTokens) || prefillTokens < 128 || prefillTokens > 32768) + throw new Error('Prefill token budget must be an integer in 128..32768'); + if (!['auto', 'flashinfer_cutlass', 'humming', 'marlin', 'flashinfer_b12x'].includes(moeBackend)) + throw new Error('Unsupported experimental MoE backend'); + if (moeBackend !== 'auto' && !nvfp4 && !nvfp4Tiny) throw new Error('MoE backend override is for NVFP4 experiments'); + if (!/^sha256:[0-9a-f]{64}$/.test(image || '')) throw new Error('full local image ID required'); + if (!/^sha256:[0-9a-f]{64}$/.test(workerImage || '')) throw new Error('full worker image ID required'); + if (!/^[0-9a-f]{40}$/.test(sourceRevision || '')) throw new Error('full SparkGLM source revision required'); + const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../..'); + const recipe = JSON.parse( + await fs.readFile(path.join(root, 'recipes/linux-nvidia-dgx-spark-2x-glm53-flash-exl3-vllm.json'), 'utf8') + ); + recipe.id = 'linux-nvidia-dgx-spark-2x-sparkglm-exl3'; + recipe.name = 'SparkGLM EXL3 TP2 managed runtime'; + recipe.version = 1; + recipe.summary = + 'Locally built SparkGLM image, managed by LLooM worker-first with ordinary admission, stop, routing and telemetry. Image must exist with this exact ID on both nodes.'; + recipe.provenance.source = `SparkGLM https://github.com/Enntity/sparkglm at ${sourceRevision}; local head image ${image}; worker image ${workerImage}. Runtime integration is unqualified until the managed gateway gates pass.`; + recipe.links = [ + { rel: 'upstream', href: `https://github.com/Enntity/sparkglm/tree/${sourceRevision}` }, + ...recipe.links.filter((v) => v.rel !== 'upstream') + ]; + recipe.backend.name = 'Locally built SparkGLM OpenAI server'; + recipe.hardware.notes = [ + 'TP2 worker-first lifecycle belongs to LLooM.', + 'No new kernel or capacity claim is implied by installing this profile.', + 'Preserve local image identity and source qualification on both ranks.' + ]; + // Explicit start for experiments; installing the profile must not reclaim GPUs. + recipe.models[0].settings.keepWarm = false; + for (const member of recipe.models[0].settings.placement.members) { + const boot = member.runtimeSettings.bootstrap; + boot.image = member.role === 'worker' ? workerImage : image; + boot.pull = false; + // Independent Docker retries cannot reconstruct the peer's NCCL session. + // LLooM owns retrying the distributed runtime as a pair. + boot.createArgs = boot.createArgs.map((v, index, args) => + String(index > 0 && args[index - 1] === '--restart' ? 'no' : v) + .replace('backends/glm53-exl3/entrypoint.sh', 'backends/sparkglm/entrypoint.sh') + .replace('GLM53_MIXED_PREFILL_CHUNK=skip', 'GLM53_MIXED_PREFILL_CHUNK=0') + .replace('GLM53_SPINWAIT_MS=stock', 'GLM53_SPINWAIT_MS=16') + ); + boot.createArgs.push( + '-e', + 'EXL3_FAT_TILE_M=64', + '-e', + 'EXL3_GROUPED_PREFILL_K4=1', + '-e', + 'EXL3_DECODE_COOP_K4=1', + '-e', + 'EXL3_DECODE_COOP_MAX_TOKENS=16', + '-e', + 'GLM53_MIXED_PREFILL_MAX_WAIT_MS=0' + ); + } + if (nvfp4 && (e3 || tiny)) throw new Error('NVFP4 cannot use the EXL3 experiment or dummy fixture'); + if (nvfp4) { + recipe.id = 'linux-nvidia-dgx-spark-2x-sparkglm-nvfp4'; + recipe.name = 'SparkGLM NVFP4 TP2 experimental runtime'; + const model = recipe.models[0]; + model.name = 'SparkGLM NVFP4 experimental'; + model.model = 'RedHatAI/GLM-5.3-Flash-NVFP4'; + model.gatewayModel = 'sparkglm-nvfp4'; + model.upstreamModel = 'sparkglm-nvfp4'; + model.backendConfig = 'sparkglm-nvfp4'; + model.runtime = 'sparkglm-nvfp4-cluster'; + model.aliases = []; + model.settings.contextWindow = 65536; + model.settings.memoryGb = 112; + const target = recipe.setup.steps.find((v) => v.id === 'download-target'); + Object.assign(target, { + title: 'Download pinned GLM-5.3 Flash NVFP4 target', + model: model.model, + revision: '240131d6a447c8d89acd428c5ddfc85598651744', + downloadSizeBytes: 197897969933 + }); + recipe.links = recipe.links.map((link) => + link.rel === 'model' ? { ...link, href: `https://huggingface.co/${model.model}/tree/${target.revision}` } : link + ); + recipe.keywords = recipe.keywords.filter((value) => !['exl3', 'tr3'].includes(value)); + recipe.keywords.push('nvfp4', 'compressed-tensors'); + for (const member of model.settings.placement.members) { + member.runtime = `sparkglm-nvfp4-${member.role}`; + member.resources.memoryGb = 112; + member.runtimeSettings.containerName = 'lloom-sparkglm-nvfp4-${nodeId}'; + member.runtimeSettings.bootstrap.createArgs = member.runtimeSettings.bootstrap.createArgs.map((v) => + String(v) + .replace(/^MODEL_DIR=.*/, 'MODEL_DIR=/models/RedHatAI--GLM-5.3-Flash-NVFP4') + .replace(/^SERVED_MODEL_NAME=.*/, 'SERVED_MODEL_NAME=sparkglm-nvfp4') + .replace(/^MAX_MODEL_LEN=.*/, 'MAX_MODEL_LEN=65536') + ); + member.runtimeSettings.bootstrap.createArgs.push( + '-e', + 'QUANTIZATION=compressed-tensors', + '-e', + 'KV_CACHE_MEMORY_BYTES=8589934592' + ); + } + } + if (nvfp4Budget && !nvfp4) { + const model = recipe.models[0]; + model.settings.contextWindow = 65536; + model.settings.memoryGb = 112; + for (const member of model.settings.placement.members) { + member.resources.memoryGb = 112; + member.runtimeSettings.bootstrap.createArgs = member.runtimeSettings.bootstrap.createArgs.map((v) => + String(v).replace(/^MAX_MODEL_LEN=.*/, 'MAX_MODEL_LEN=65536') + ); + member.runtimeSettings.bootstrap.createArgs.push('-e', 'KV_CACHE_MEMORY_BYTES=8589934592'); + } + } + if (tiny) { + const fixtureId = nvfp4Tiny ? 'sparkglm-tiny-nvfp4' : 'sparkglm-tiny'; + const fixtureDir = nvfp4Tiny ? 'sparkglm--tinyglm-nvfp4' : 'sparkglm--tinyglm'; + recipe.id = `linux-nvidia-dgx-spark-2x-${fixtureId}`; + recipe.name = 'SparkGLM synthetic TP2 integration fixture'; + recipe.setup.steps = recipe.setup.steps.filter((v) => v.id === 'check-docker'); + const model = recipe.models[0]; + model.name = 'tinyGLM synthetic integration fixture'; + model.model = nvfp4Tiny ? 'sparkglm/tinyglm-nvfp4' : 'sparkglm/tinyglm'; + model.gatewayModel = fixtureId; + model.upstreamModel = fixtureId; + model.backendConfig = fixtureId; + model.runtime = `${fixtureId}-cluster`; + model.aliases = []; + model.input = ['text']; + model.settings.contextWindow = 32768; + model.settings.maxOutputTokens = 1024; + model.settings.memoryGb = 24; + for (const member of model.settings.placement.members) { + member.runtime = `${fixtureId}-${member.role}`; + member.resources.memoryGb = 24; + member.runtimeSettings.containerName = `lloom-${fixtureId}-` + '${nodeId}'; + member.runtimeSettings.bootstrap.createArgs = member.runtimeSettings.bootstrap.createArgs.map((v) => + String(v) + .replace(/^MODEL_DIR=.*/, `MODEL_DIR=/models/${fixtureDir}`) + .replace(/^SERVED_MODEL_NAME=.*/, `SERVED_MODEL_NAME=${fixtureId}`) + .replace(/^SPEC_METHOD=.*/, 'SPEC_METHOD=none') + .replace(/^MAX_MODEL_LEN=.*/, 'MAX_MODEL_LEN=32768') + .replace(/^GPU_MEMORY_UTILIZATION=.*/, 'GPU_MEMORY_UTILIZATION=0.15') + ); + member.runtimeSettings.bootstrap.createArgs.push( + '-e', + nvfp4Tiny ? 'SPARKGLM_NVFP4_TINY=1' : 'SPARKGLM_TINY_DUMMY=1', + '-e', + 'LANGUAGE_MODEL_ONLY=1' + ); + if (nvfp4Tiny) member.runtimeSettings.bootstrap.createArgs.push('-e', 'QUANTIZATION=compressed-tensors'); + } + } + if (e3) + for (const member of recipe.models[0].settings.placement.members) { + member.runtimeSettings.bootstrap.createArgs.push('-e', 'SPARKGLM_EXL3_E3=1'); + if (e3Policy !== 'large') + member.runtimeSettings.bootstrap.createArgs.push('-e', `SPARKGLM_EXL3_E3_POLICY=${e3Policy}`); + if (e3Trace) member.runtimeSettings.bootstrap.createArgs.push('-e', 'SPARKGLM_EXL3_E3_TRACE=1'); + } + if (mxfp8Draft) { + const draft = recipe.setup.steps.find((step) => step.id === 'download-dflash2'); + Object.assign(draft, { + title: 'Download pinned GLM-5.3 Flash MXFP8 DFlash2 draft', + model: 'local-inference-lab/GLM-5.3-Flash-DFlash2-MXFP8', + revision: '610aa967a92bfeb97e3d848dcb8693553e8b6a55' + }); + // The original BF16 byte estimate must not describe the quantized checkpoint. + delete draft.downloadSizeBytes; + recipe.links = recipe.links.map((link) => + link.rel === 'draft-model' + ? { ...link, href: `https://huggingface.co/${draft.model}/tree/${draft.revision}` } + : link + ); + } + for (const model of recipe.models) { + if (![32, 64, 128].includes(exl3TempRows)) throw new Error('EXL3 temp rows must be 32, 64 or 128'); + if ((nvfp4 || nvfp4Tiny) && exl3TempRows !== 128) throw new Error('EXL3 temp rows do not apply to NVFP4'); + if (contextTokens !== undefined) { + model.settings.contextWindow = contextTokens; + // vLLM validates rendered token IDs against max_model_len. LLooM's + // character estimate otherwise rejects valid near-window requests. + model.settings.maxPromptTokens = 0; + // A full-window cold prefill can legitimately exceed ten minutes. + if (contextTokens > 524288) model.settings.watchdog.minNoProgressMs = 1800000; + } + // Explicit cache growth must also reach LLooM's admission planner. Keep the + // existing conservative reservation when a smaller cache is requested. + const extraCacheGiB = Math.max(0, (kvCacheGiB ?? 8) - 8); + model.settings.memoryGb += extraCacheGiB; + for (const member of model.settings.placement.members) { + member.resources.memoryGb += extraCacheGiB; + member.runtimeSettings.bootstrap.createArgs = member.runtimeSettings.bootstrap.createArgs.map((value) => { + let result = String(value) + .replace(/^EXL3_TEMP_ROWS_FUSED=.*/, `EXL3_TEMP_ROWS_FUSED=${exl3TempRows}`) + .replace(/^DFLASH_DRAFT_TP=.*/, `DFLASH_DRAFT_TP=${draftTp}`) + .replace(/^MAX_NUM_BATCHED_TOKENS=.*/, `MAX_NUM_BATCHED_TOKENS=${prefillTokens}`); + if (contextTokens !== undefined) result = result.replace(/^MAX_MODEL_LEN=.*/, `MAX_MODEL_LEN=${contextTokens}`); + if (mxfp8Draft) + result = result.replace( + /^DFLASH_MODEL_DIR=.*/, + 'DFLASH_MODEL_DIR=/models/local-inference-lab--GLM-5.3-Flash-DFlash2-MXFP8' + ); + return result; + }); + if (kvCacheGiB !== undefined) { + const args = member.runtimeSettings.bootstrap.createArgs; + const existing = args.findIndex((v) => String(v).startsWith('KV_CACHE_MEMORY_BYTES=')); + const setting = `KV_CACHE_MEMORY_BYTES=${kvCacheGiB * 1073741824}`; + if (existing >= 0) args[existing] = setting; + else args.push('-e', setting); + } + if (mxfp8Draft) member.runtimeSettings.bootstrap.createArgs.push('-e', 'SPARKGLM_MXFP8_DRAFT=1'); + if (moeBackend !== 'auto') member.runtimeSettings.bootstrap.createArgs.push('-e', `MOE_BACKEND=${moeBackend}`); + if (member.runtimeSettings.warmup?.body) { + member.runtimeSettings.warmup.body.model = model.upstreamModel; + } + } + } + // Profiles intentionally share the appliance port and run one at a time. + // A generic /health response must not make another registered model look loaded. + const served = recipe.models[0]; + served.settings.healthPath = '/v1/models'; + served.settings.healthModel = served.upstreamModel; + for (const member of served.settings.placement.members) { + if (member.role !== 'head') continue; + member.runtimeSettings.healthUrl = `http://${'${leaderAddress}'}:${served.settings.port}/v1/models`; + member.runtimeSettings.healthModel = served.upstreamModel; + } + return recipe; +} + +if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + const args = process.argv.slice(2); + const value = (k) => args[args.indexOf(k) + 1]; + try { + const recipe = await materialize({ + exl3TempRows: args.includes('--exl3-temp-rows') ? Number(value('--exl3-temp-rows')) : 128, + contextTokens: args.includes('--context-tokens') ? Number(value('--context-tokens')) : undefined, + kvCacheGiB: args.includes('--kv-cache-gib') ? Number(value('--kv-cache-gib')) : undefined, + image: value('--image-id'), + workerImage: args.includes('--worker-image-id') ? value('--worker-image-id') : undefined, + sourceRevision: value('--source-revision'), + e3: args.includes('--e3'), + tiny: args.includes('--tiny'), + nvfp4: args.includes('--nvfp4'), + nvfp4Tiny: args.includes('--nvfp4-tiny'), + nvfp4Budget: args.includes('--nvfp4-budget'), + mxfp8Draft: args.includes('--mxfp8-draft'), + draftTp: args.includes('--draft-tp') ? Number(value('--draft-tp')) : 2, + prefillTokens: args.includes('--prefill-tokens') ? Number(value('--prefill-tokens')) : 7168, + moeBackend: args.includes('--moe-backend') ? value('--moe-backend') : 'auto', + e3Policy: args.includes('--e3-policy') ? value('--e3-policy') : 'large', + e3Trace: args.includes('--e3-trace') + }); + if (!args.includes('--output')) throw new Error('--output required'); + await fs.writeFile(value('--output'), JSON.stringify(recipe, null, 2) + '\n'); + console.log( + JSON.stringify({ + recipe: recipe.id, + image: recipe.models[0].settings.placement.members[0].runtimeSettings.bootstrap.image, + output: value('--output') + }) + ); + } catch (e) { + console.error(e.message); + process.exitCode = 1; + } +} diff --git a/docs/sparkglm.md b/docs/sparkglm.md new file mode 100644 index 0000000..3a86852 --- /dev/null +++ b/docs/sparkglm.md @@ -0,0 +1,13 @@ +# Locally built SparkGLM under LLooM + +`backends/sparkglm` manages immutable locally built SparkGLM images on two +DGX Sparks. It preserves worker-first admission, readiness, stop and routing; +no independent Docker restart policy is enabled. See its README and the +SparkGLM public installer for pinned NVFP4 and EXL3 configurations. + +Cold long-context inference can exceed Undici's five-minute transport idle +deadline. Chat forwarding now uses the existing extended 30-minute dispatcher. +Configured backend deadlines, caller cancellation, stream-progress guards, +and explicit dispatcher overrides still apply. The regression covers delayed +headers, delayed streaming bodies, and buffered responses with a deliberately +short default transport deadline. This changes no model or alias by itself. diff --git a/package.json b/package.json index 508045f..e8cd1ac 100644 --- a/package.json +++ b/package.json @@ -73,7 +73,7 @@ "community:check": "node scripts/check-community-deploy.mjs", "package:check": "node scripts/check-package.mjs", "test:adoption": "node test/resource-fit.test.mjs && node test/model-acquisition.test.mjs && node test/dashboard-status.test.mjs", - "test:unit": "node test/protocol.test.mjs && node test/protocol-stream.test.mjs && node test/security.test.mjs && node test/community-host.test.mjs && node test/ds4fv-recipe.test.mjs && node test/dspark-hotfixes.test.mjs && node test/glm53-exl3-recipe.test.mjs && node test/host-memory.test.mjs && node test/runtime-policy.test.mjs && node --test test/runtime-queue.test.mjs test/federated-priority.test.mjs test/runtime-supervisor.test.mjs && node test/interchange.test.mjs && node test/server-resilience.test.mjs && node test/metrics-persistence.test.mjs && node test/model-failover.test.mjs && node test/qwen38-vllm-recipe.test.mjs && node test/reasoning-normalize.test.mjs && node test/reasoning-effort.test.mjs && node test/route-control.test.mjs && node test/spark-route-catalog.test.mjs && node test/chat-lane-canary.test.mjs && node test/runtime-watchdog.test.mjs && node test/tts-catalog.test.mjs && node test/voice-profiles.test.mjs && node test/chatterbox-recipe.test.mjs && npm run test:adoption", + "test:unit": "node test/long-prefill-transport.test.mjs && node test/sparkglm-managed.test.mjs && node test/protocol.test.mjs && node test/protocol-stream.test.mjs && node test/security.test.mjs && node test/community-host.test.mjs && node test/ds4fv-recipe.test.mjs && node test/dspark-hotfixes.test.mjs && node test/glm53-exl3-recipe.test.mjs && node test/host-memory.test.mjs && node test/runtime-policy.test.mjs && node --test test/runtime-queue.test.mjs test/federated-priority.test.mjs test/runtime-supervisor.test.mjs && node test/interchange.test.mjs && node test/server-resilience.test.mjs && node test/metrics-persistence.test.mjs && node test/model-failover.test.mjs && node test/qwen38-vllm-recipe.test.mjs && node test/reasoning-normalize.test.mjs && node test/reasoning-effort.test.mjs && node test/route-control.test.mjs && node test/spark-route-catalog.test.mjs && node test/chat-lane-canary.test.mjs && node test/runtime-watchdog.test.mjs && node test/tts-catalog.test.mjs && node test/voice-profiles.test.mjs && node test/chatterbox-recipe.test.mjs && npm run test:adoption", "test:cluster": "node test/cluster.test.mjs", "test": "npm run test:cluster && npm run test:unit && npm run test:entity-stagger && npm run smoke && npm run community:check", "test:entity-stagger": "node test/entity-stagger-benchmark.test.mjs", diff --git a/src/server.mjs b/src/server.mjs index 41c9a15..21569b9 100644 --- a/src/server.mjs +++ b/src/server.mjs @@ -1284,7 +1284,7 @@ function finalizeMetricBucket(bucket) { }; } -async function fetchUpstream({ backend, path, body, headers = {}, signal, dispatcher }) { +async function fetchUpstream({ backend, path, body, headers = {}, signal, dispatcher = longRunningMediaDispatcher }) { const timeoutMs = backend.timeoutMs ?? 1800000; const fetchSignal = upstreamSignal(signal, timeoutMs); try { diff --git a/test/long-prefill-transport.test.mjs b/test/long-prefill-transport.test.mjs new file mode 100644 index 0000000..85d3d25 --- /dev/null +++ b/test/long-prefill-transport.test.mjs @@ -0,0 +1,75 @@ +import assert from 'node:assert/strict'; +import http from 'node:http'; +import { Agent, getGlobalDispatcher, setGlobalDispatcher } from 'undici'; +import { createLloomServer } from '../src/server.mjs'; +const listen = (server) => + new Promise((resolve) => server.listen(0, '127.0.0.1', () => resolve(server.address().port))); +const previous = getGlobalDispatcher(); +const shortDefault = new Agent({ headersTimeout: 10, bodyTimeout: 10 }); +const client = new Agent({ headersTimeout: 5000, bodyTimeout: 5000 }); +setGlobalDispatcher(shortDefault); +try { + for (const phase of ['headers', 'body', 'buffered']) { + const upstream = http.createServer((req, res) => { + req.resume(); + if (phase === 'body') { + res.writeHead(200, { 'content-type': 'text/event-stream' }); + res.write(': waiting\n\n'); + } + const timer = setTimeout(() => { + if (phase === 'buffered') { + res.writeHead(200, { 'content-type': 'application/json' }); + res.end( + JSON.stringify({ + choices: [{ index: 0, message: { role: 'assistant', content: 'OK' }, finish_reason: 'stop' }] + }) + ); + return; + } + if (!res.headersSent) res.writeHead(200, { 'content-type': 'text/event-stream' }); + res.end('data: {"choices":[{"index":0,"delta":{"content":"OK"},"finish_reason":"stop"}]}\n\ndata: [DONE]\n\n'); + }, 1600); + res.on('close', () => clearTimeout(timer)); + }); + const upPort = await listen(upstream); + const app = createLloomServer( + { + server: { host: '127.0.0.1', port: 0 }, + security: { allowMissingAuth: true, apiKeys: [] }, + defaults: { chatModel: 'test' }, + backends: { local: { type: 'openai', baseUrl: `http://127.0.0.1:${upPort}/v1`, timeoutMs: 5000 } }, + models: [ + { id: 'test', backend: 'local', upstreamModel: 'test', kind: 'chat', contextWindow: 1, maxPromptTokens: 0 } + ], + runtimes: {} + }, + { logger: { error() {}, warn() {} } } + ); + const port = await listen(app.server); + try { + const r = await fetch(`http://127.0.0.1:${port}/v1/chat/completions`, { + method: 'POST', + dispatcher: client, + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ + model: 'test', + messages: [{ role: 'user', content: 'test' }], + stream: phase !== 'buffered' + }) + }); + const body = await r.text(); + assert.equal(r.status, 200); + assert.match(body, /"content":"OK"/); + assert.doesNotMatch(body, /server_error/); + console.log(`${phase}: cold prefill outlives default transport timeout`); + } finally { + await app.close({ stopRuntimes: false }); + upstream.closeAllConnections(); + await new Promise((resolve) => upstream.close(resolve)); + } + } +} finally { + setGlobalDispatcher(previous); + await shortDefault.close(); + await client.close(); +} diff --git a/test/sparkglm-managed.test.mjs b/test/sparkglm-managed.test.mjs new file mode 100644 index 0000000..9406e73 --- /dev/null +++ b/test/sparkglm-managed.test.mjs @@ -0,0 +1,229 @@ +import assert from 'node:assert/strict'; +import { materialize } from '../backends/sparkglm/materialize.mjs'; +import { planRecipe } from '../src/recipes.mjs'; +import { classifyRuntimeWatchdogOutcome } from '../src/runtime-manager.mjs'; +const image = 'sha256:' + 'a'.repeat(64); +const recipe = await materialize({ image, sourceRevision: 'b'.repeat(40) }); +const plan = planRecipe( + recipe, + { models: [], runtimes: {} }, + { + modelRoot: '/models', + platform: 'linux', + arch: 'arm64', + backendIds: new Set(['docker-vllm']), + checkLocalReferences: false + } +); +assert.deepEqual(plan.validationErrors, []); +assert.equal(recipe.models[0].gatewayModel, 'glm-5.3-flash-exl3'); +assert.equal(recipe.models[0].settings.keepWarm, false); +const members = recipe.models[0].settings.placement.members; +assert.deepEqual( + members.map((v) => v.role), + ['worker', 'head'] +); +for (const m of members) { + assert.equal(m.runtimeSettings.management, 'managed'); + assert.equal(m.runtimeSettings.bootstrap.image, image); + assert.equal(m.runtimeSettings.bootstrap.pull, false); + const createArgs = m.runtimeSettings.bootstrap.createArgs; + assert.equal(createArgs[createArgs.indexOf('--restart') + 1], 'no'); + const args = m.runtimeSettings.bootstrap.createArgs.join(' '); + assert.match(args, /backends\/sparkglm\/entrypoint.sh/); + assert.match(args, /EXL3_GROUPED_PREFILL_K4=1/); + assert.match(args, /GLM53_MIXED_PREFILL_CHUNK=0/); + assert.doesNotMatch(args, /backends\/glm53-exl3/); +} +await assert.rejects(materialize({ image: 'sparkglm:latest', sourceRevision: 'b'.repeat(40) })); +await assert.rejects(materialize({ image, sourceRevision: 'main' })); +console.log('SparkGLM managed runtime plan passed'); + +const tiny = await materialize({ image, sourceRevision: 'b'.repeat(40), tiny: true, e3: true }); +assert.equal(tiny.models[0].gatewayModel, 'sparkglm-tiny'); +assert.deepEqual(tiny.models[0].aliases, []); +assert.equal(tiny.setup.steps.length, 1); +const nv = await materialize({ image, sourceRevision: 'b'.repeat(40), nvfp4: true }); +assert.equal(nv.models[0].gatewayModel, 'sparkglm-nvfp4'); +assert.deepEqual(nv.models[0].aliases, []); +assert.equal( + nv.setup.steps.find((v) => v.id === 'download-target').revision, + '240131d6a447c8d89acd428c5ddfc85598651744' +); +await assert.rejects(materialize({ image, sourceRevision: 'b'.repeat(40), tiny: true, nvfp4: true })); +for (const candidate of [tiny, nv]) { + for (const member of candidate.models[0].settings.placement.members) { + if (member.runtimeSettings.warmup?.body) { + assert.equal(member.runtimeSettings.warmup.body.model, candidate.models[0].upstreamModel); + } + } + const p = planRecipe( + candidate, + { models: [], runtimes: {} }, + { + modelRoot: '/models', + platform: 'linux', + arch: 'arm64', + backendIds: new Set(['docker-vllm']), + checkLocalReferences: false + } + ); + assert.deepEqual(p.validationErrors, []); +} + +const split = await materialize({ image, workerImage: 'sha256:' + 'c'.repeat(64), sourceRevision: 'b'.repeat(40) }); +assert.equal(split.models[0].settings.placement.members[0].runtimeSettings.bootstrap.image, 'sha256:' + 'c'.repeat(64)); +assert.equal(split.models[0].settings.placement.members[1].runtimeSettings.bootstrap.image, image); +await assert.rejects(materialize({ image, workerImage: 'latest', sourceRevision: 'b'.repeat(40) })); + +const nt = await materialize({ image, sourceRevision: 'b'.repeat(40), nvfp4Tiny: true }); +assert.equal(nt.models[0].gatewayModel, 'sparkglm-tiny-nvfp4'); +assert.deepEqual(nt.models[0].aliases, []); +assert.deepEqual( + planRecipe( + nt, + { models: [], runtimes: {} }, + { + modelRoot: '/models', + platform: 'linux', + arch: 'arm64', + backendIds: new Set(['docker-vllm']), + checkLocalReferences: false + } + ).validationErrors, + [] +); +await assert.rejects(materialize({ image, sourceRevision: 'b'.repeat(40), nvfp4Tiny: true, e3: true })); + +const matched = await materialize({ image, sourceRevision: 'b'.repeat(40), nvfp4Budget: true }); +assert.equal(matched.models[0].gatewayModel, recipe.models[0].gatewayModel); +assert.equal(matched.models[0].settings.contextWindow, nv.models[0].settings.contextWindow); +for (let rank = 0; rank < 2; rank += 1) { + const exl = matched.models[0].settings.placement.members[rank]; + const fp4 = nv.models[0].settings.placement.members[rank]; + const budget = (member) => + member.runtimeSettings.bootstrap.createArgs.filter((v) => + /^(MAX_MODEL_LEN|KV_CACHE_MEMORY_BYTES|GPU_MEMORY_UTILIZATION|MAX_NUM_BATCHED_TOKENS|MAX_NUM_SEQS)=/.test(v) + ); + assert.deepEqual(budget(exl), budget(fp4)); + assert.equal(exl.resources.memoryGb, fp4.resources.memoryGb); +} +await assert.rejects(materialize({ image, sourceRevision: 'b'.repeat(40), tiny: true, nvfp4Budget: true })); + +const tuned = await materialize({ + image, + sourceRevision: 'b'.repeat(40), + nvfp4: true, + mxfp8Draft: true, + draftTp: 2, + prefillTokens: 1024, + moeBackend: 'humming' +}); +assert.deepEqual( + planRecipe( + tuned, + { models: [], runtimes: {} }, + { + modelRoot: '/models', + platform: 'linux', + arch: 'arm64', + backendIds: new Set(['docker-vllm']), + checkLocalReferences: false + } + ).validationErrors, + [] +); +const draft = tuned.setup.steps.find((step) => step.id === 'download-dflash2'); +assert.equal(draft.revision, '610aa967a92bfeb97e3d848dcb8693553e8b6a55'); +for (const member of tuned.models[0].settings.placement.members) { + const args = member.runtimeSettings.bootstrap.createArgs; + assert(args.includes('DFLASH_MODEL_DIR=/models/' + draft.model.replace('/', '--'))); + assert(args.includes('DFLASH_DRAFT_TP=2')); + assert(args.includes('MAX_NUM_BATCHED_TOKENS=1024')); +} +await assert.rejects(materialize({ image, sourceRevision: 'b'.repeat(40), draftTp: 3 })); +await assert.rejects(materialize({ image, sourceRevision: 'b'.repeat(40), prefillTokens: NaN })); +await assert.rejects(materialize({ image, sourceRevision: 'b'.repeat(40), tiny: true, mxfp8Draft: true })); +const concurrent = await materialize({ + image, + sourceRevision: 'b'.repeat(40), + e3: true, + tiny: true, + e3Policy: 'concurrent', + e3Trace: true +}); +for (const member of concurrent.models[0].settings.placement.members) { + assert(member.runtimeSettings.bootstrap.createArgs.includes('SPARKGLM_EXL3_E3_POLICY=concurrent')); +} +await assert.rejects(materialize({ image, sourceRevision: 'b'.repeat(40), e3Policy: 'concurrent' })); + +const bounded = await materialize({ + image, + sourceRevision: 'b'.repeat(40), + nvfp4: true, + contextTokens: 49152, + kvCacheGiB: 6 +}); +assert.equal(bounded.models[0].settings.contextWindow, 49152); +assert.equal(bounded.models[0].settings.maxPromptTokens, 0); +assert.equal(bounded.models[0].settings.memoryGb, nv.models[0].settings.memoryGb); +const largerCache = await materialize({ image, sourceRevision: 'b'.repeat(40), nvfp4: true, kvCacheGiB: 16 }); +assert.equal(largerCache.models[0].settings.memoryGb, nv.models[0].settings.memoryGb + 8); +assert.ok(largerCache.models[0].settings.placement.members.every((m) => m.resources.memoryGb === 120)); +for (const member of bounded.models[0].settings.placement.members) { + const args = member.runtimeSettings.bootstrap.createArgs; + assert.equal(args.filter((v) => String(v).startsWith('KV_CACHE_MEMORY_BYTES=')).length, 1); + assert.ok(args.includes('KV_CACHE_MEMORY_BYTES=6442450944')); + assert.ok(args.includes('MAX_MODEL_LEN=49152')); +} +for (const opts of [{ contextTokens: 0 }, { contextTokens: 1048577 }, { kvCacheGiB: 0 }, { kvCacheGiB: 2.5 }]) + await assert.rejects(materialize({ image, sourceRevision: 'b'.repeat(40), ...opts })); +await materialize({ image, sourceRevision: 'b'.repeat(40), nvfp4Tiny: true, moeBackend: 'humming' }); + +const lowerCap = await materialize({ image, sourceRevision: 'b'.repeat(40), e3: true, exl3TempRows: 32 }); +assert.ok( + lowerCap.models[0].settings.placement.members.every((m) => + m.runtimeSettings.bootstrap.createArgs.includes('EXL3_TEMP_ROWS_FUSED=32') + ) +); +await assert.rejects(materialize({ image, sourceRevision: 'b'.repeat(40), exl3TempRows: 0 })); +await assert.rejects(materialize({ image, sourceRevision: 'b'.repeat(40), nvfp4: true, exl3TempRows: 32 })); + +await assert.rejects(materialize({ image, sourceRevision: 'b'.repeat(40), draftTp: 1 }), /not implemented/); + +// A different GLM or synthetic fixture on the same appliance port is not healthy +// for this profile. Cover every model identity after its final rewrite. +for (const candidate of [ + recipe, + tiny, + nv, + await materialize({ image, sourceRevision: 'b'.repeat(40), nvfp4Tiny: true }) +]) { + const model = candidate.models[0]; + assert.equal(model.settings.healthPath, '/v1/models'); + assert.equal(model.settings.healthModel, model.upstreamModel); + const head = model.settings.placement.members.find((member) => member.role === 'head'); + assert.equal(head.runtimeSettings.healthUrl, 'http://${leaderAddress}:8890/v1/models'); + assert.equal(head.runtimeSettings.healthModel, model.upstreamModel); + assert.equal(model.settings.placement.members.find((member) => member.role === 'worker').healthStrategy, 'container'); +} + +const fullWindow = await materialize({ + image, + sourceRevision: 'b'.repeat(40), + nvfp4: true, + contextTokens: 1048576, + kvCacheGiB: 11 +}); +assert.equal(fullWindow.models[0].settings.watchdog.minNoProgressMs, 1800000); +assert.equal(fullWindow.models[0].settings.watchdog.enabled, true); + +const fullWindowRuntime = { ...fullWindow.models[0].settings, management: 'managed' }; +assert.equal( + classifyRuntimeWatchdogOutcome(fullWindowRuntime, { status: 504, durationMs: 600012, stream: true }).kind, + 'ignored' +); +assert.equal( + classifyRuntimeWatchdogOutcome(fullWindowRuntime, { status: 504, durationMs: 1800000, stream: true }).kind, + 'no-progress-failure' +);