From b88e36b55a87788b07b072b80292cff9f82fa386 Mon Sep 17 00:00:00 2001 From: ChenYCL Date: Thu, 24 Sep 2026 13:50:48 +0800 Subject: [PATCH 1/9] feat(jev-browser): local GGUF readout backend MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Answers the skill's System One questions from a local llama.cpp server: the question is rendered with labelled options, exactly one token is generated, and the probability mass on each option label at that position is the answer. No API key, no Python, no training, no output parsing — and it speaks the same /v1/systemone contract as hosted Jev, so a run only has to point TYPESAFE_BASE_URL at 127.0.0.1. Measured on 20 construction-graded items (experiments/gguf-provider/RESULTS.md): 0.80 for the shipped Qwen3.5-4B Q4_K_M, against 0.50 for the 0.8B reference entry, 0.55 for "always answer the option listed first" and 0.95 for hosted Jev on the same items — the first local candidate to clear the positional prior. One real browser step (5 questions, 16.1 kB state, 13,729 prompt tokens cold) is 18,298 ms cold / 9,002 ms warm at 3,362 MiB RSS; loopback tokens cost $0. Rotation shows the 4B reads option text where the 0.8B reads position, and its confidence is usable as an abstention signal (>= 0.5 keeps 8/12 answers with 7/8 correct). Models are data, not code: lib/local-models.json carries each entry's file, resolve URL, exact byte size and label, so swapping one is an edit — --list-models prints the registry, --model-name picks per run, --ctx sets the llama.cpp context (16k default: a 100-candidate step renders to ~11.7k tokens). Blobs download into ~/.jev-browser/models and are never vendored. test:local runs the new suites; npm test picks them up through the same glob. --- package.json | 3 +- skills/jev-browser/bin/jev-local.mjs | 330 +++++++++ skills/jev-browser/lib/local-models.json | 18 + skills/jev-browser/lib/local.mjs | 693 ++++++++++++++++++ .../jev-browser/tests/local-models.test.mjs | 134 ++++ skills/jev-browser/tests/local.test.mjs | 127 ++++ 6 files changed, 1304 insertions(+), 1 deletion(-) create mode 100755 skills/jev-browser/bin/jev-local.mjs create mode 100644 skills/jev-browser/lib/local-models.json create mode 100644 skills/jev-browser/lib/local.mjs create mode 100644 skills/jev-browser/tests/local-models.test.mjs create mode 100644 skills/jev-browser/tests/local.test.mjs diff --git a/package.json b/package.json index b3111cc..c51b10c 100644 --- a/package.json +++ b/package.json @@ -12,8 +12,9 @@ "node": ">=22" }, "scripts": { - "test": "node --test --test-timeout=300000 \"tests/unit/*.test.mjs\" \"tests/e2e/*.test.mjs\"", + "test": "node --test --test-timeout=300000 \"tests/unit/*.test.mjs\" \"skills/jev-browser/tests/*.test.mjs\" \"tests/e2e/*.test.mjs\"", "test:unit": "node --test \"tests/unit/*.test.mjs\"", + "test:local": "node --test \"skills/jev-browser/tests/*.test.mjs\"", "test:e2e": "node --test --test-timeout=300000 \"tests/e2e/*.test.mjs\"", "test:e2e:mock": "JEV_BROWSER_TEST_MODE=mock node --test --test-timeout=300000 \"tests/e2e/*.test.mjs\"", "doctor": "node skills/jev-browser/bin/jev-browser.mjs doctor", diff --git a/skills/jev-browser/bin/jev-local.mjs b/skills/jev-browser/bin/jev-local.mjs new file mode 100755 index 0000000..bac3970 --- /dev/null +++ b/skills/jev-browser/bin/jev-local.mjs @@ -0,0 +1,330 @@ +#!/usr/bin/env node +// One command for the fully local (experimental) model backend of jev-browser: download the +// GGUF, start llama.cpp if nothing is serving on the llama port, and serve the TypeSafe +// `/v1/systemone` contract on 127.0.0.1:8092 (see ../lib/local.mjs). +// +// Which model is data, not code — skills/jev-browser/lib/local-models.json. Switch it with +// --model-name for one run, or by pointing the registry's "default" at another entry. +// +// node skills/jev-browser/bin/jev-local.mjs +// node skills/jev-browser/bin/jev-local.mjs --list-models +// TYPESAFE_BASE_URL=http://127.0.0.1:8092 TYPESAFE_API_KEY=local \ +// node skills/jev-browser/bin/jev-browser.mjs judge --state-file s.json --questions-file q.json +// +// Everything except the final `TYPESAFE_BASE_URL=... TYPESAFE_API_KEY=local` line goes to +// stderr, so the line can be piped or eval'd. Read experiments/gguf-provider/RESULTS.md for what +// the local model is good at (short states) and what it is not (goal_done). +import fs from "node:fs"; +import fsp from "node:fs/promises"; +import http from "node:http"; +import path from "node:path"; +import { spawn } from "node:child_process"; +import { Readable } from "node:stream"; +import { pipeline } from "node:stream/promises"; + +// Static on purpose: --help must work even while the registry is being edited. +const HELP = `jev-local — serve the local (experimental) Jev backend for jev-browser + +Usage: + jev-local [--model-name | --model ] [--ctx 16384] [--port 8092] [--llama-port 8090] + jev-local --list-models print the model registry and exit + jev-local --download-only fetch the model file and exit + +What it does, in order: + 1. finds llama-server on PATH, then /opt/homebrew/bin (brew install llama.cpp) + 2. downloads the selected GGUF into ~/.jev-browser/models (streamed to .partial, then renamed) + 3. starts llama-server on 127.0.0.1:8090 with -c 16384 unless something already answers there + 4. serves /health, /v1/models and /v1/systemone on 127.0.0.1:8092 and prints: + TYPESAFE_BASE_URL=http://127.0.0.1:8092 TYPESAFE_API_KEY=local + +Models (skills/jev-browser/lib/local-models.json): + --list-models every entry: id, label, size, default marker + --model-name serve that registry entry (default: the registry's "default") + --model serve a GGUF you already have (no download unless --model-url is given) + --model-url override the download URL for this run + +Tuning: + --ctx llama.cpp context size (default 16384) + --port N port of the Jev-compatible server (default 8092) + --llama-port N port of the llama.cpp server (default 8090) + --download-only fetch the model file and exit + -h, --help this text + +Env: + JEV_LOCAL_MODEL_URL same as --model-url (the flag wins) + +A llama.cpp already serving the selected file on the llama port is reused and left untouched +on exit; one this launcher started is stopped together with it. No API key, no cost, no +network except the one-time model download. +`; + +const argv = process.argv.slice(2); +const flag = (name) => argv.includes(name); +const arg = (name, fallback) => (argv.includes(name) ? argv[argv.indexOf(name) + 1] : fallback); +const log = (message) => process.stderr.write(`${message}\n`); +const humanBytes = (bytes) => (bytes >= 1024 ** 3 ? `${(bytes / 1024 ** 3).toFixed(1)} GiB` : `${Math.round(bytes / 1024 ** 2)} MiB`); +const mib = (bytes) => (bytes / 1024 / 1024).toFixed(0); + +const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); +const removeQuietly = (file) => fsp.rm(file, { force: true }).catch(() => {}); +/** Usage/config problem: the user has to change a flag, a path or the registry — exit 2. */ +const configError = (message) => Object.assign(new Error(message), { config: true }); +const isHealthy = async (getJson, url) => ((await getJson(url, 1500)) ?? {}).status === "ok"; + +// Safety net: if this launcher dies for any reason, it never leaves a llama-server it started +// behind. A llama.cpp it merely reused is not in here and stays untouched. +let spawnedLlama = null; +process.on("exit", () => { + if (spawnedLlama && spawnedLlama.exitCode === null && spawnedLlama.signalCode === null) { + try { + spawnedLlama.kill("SIGTERM"); + } catch { + // already gone + } + } +}); + +/** Print the registry: id, label, size, downloaded state, default marker. */ +function listModels({ loadLocalModels, localPaths }) { + const registry = loadLocalModels(); + const modelsDir = localPaths().models; + const rows = Object.entries(registry.models).map(([id, entry]) => ({ id, ...entry, downloaded: fs.existsSync(path.join(modelsDir, entry.file)) })); + const idWidth = Math.max(...rows.map((row) => row.id.length)); + const labelWidth = Math.max(...rows.map((row) => row.label.length)); + process.stdout.write(`jev-local models — ${registry.path}\n\n`); + for (const row of rows) { + const marker = row.id === registry.default ? "*" : " "; + process.stdout.write(`${marker} ${row.id.padEnd(idWidth)} ${row.label.padEnd(labelWidth)} ${humanBytes(row.bytes).padStart(8)} ${row.file}${row.downloaded ? " [downloaded]" : ""}\n`); + } + process.stdout.write(`\n* = default. Serve one with --model-name ; switch permanently by editing "default".\n`); + return 0; +} + +/** Stream the model to `.partial`, then rename, so a half-download is never used. */ +async function ensureModel({ file, url, expectedBytes }) { + const existing = await fsp.stat(file).catch(() => null); + if (existing) { + if (expectedBytes && existing.size !== expectedBytes) throw configError(`${file} is ${existing.size} bytes but the registry says ${expectedBytes}; delete it and run again to re-download`); + log(`[local] model ready: ${file} (${humanBytes(existing.size)})`); + return { downloaded: false, bytes: existing.size }; + } + await fsp.mkdir(path.dirname(file), { recursive: true }); + const partial = `${file}.partial`; + log(`[local] downloading ${url}`); + const response = await fetch(url); + if (!response.ok) throw new Error(`download failed: HTTP ${response.status} ${response.statusText}`); + const contentLength = Number(response.headers.get("content-length")) || 0; + const total = expectedBytes || contentLength; + let received = 0; + let lastReport = 0; + const body = Readable.fromWeb(response.body); + body.on("data", (chunk) => { + received += chunk.length; + const now = Date.now(); + if (now - lastReport < 500 && received !== total) return; + lastReport = now; + const share = total ? ` / ${mib(total)} MiB (${((received / total) * 100).toFixed(0)}%)` : " MiB"; + process.stderr.write(`\r[local] ${mib(received)}${share}`); + }); + try { + await pipeline(body, fs.createWriteStream(partial)); + } catch (error) { + await removeQuietly(partial); + throw error; + } + process.stderr.write("\n"); + if (total && received !== total) { + await removeQuietly(partial); + throw new Error(`download truncated: got ${received} of ${total} bytes`); + } + await fsp.rename(partial, file); + log(`[local] model ready: ${file} (${humanBytes(received)})`); + return { downloaded: true, bytes: received }; +} + +/** Start llama-server; only a process started here is ever killed here. */ +function spawnLlamaServer(bin, { modelFile, llamaPort, ctx }) { + const args = ["-m", modelFile, "--host", "127.0.0.1", "--port", String(llamaPort), "-c", String(ctx), "-ngl", "99", "--cache-type-k", "q8_0", "--cache-type-v", "q8_0", "-t", "8"]; + log(`[local] starting llama-server: ${bin} ${args.join(" ")}`); + return spawn(bin, args, { stdio: ["ignore", "ignore", "inherit"] }); +} + +async function waitForLlama(getJson, child, url, { timeoutMs = 180_000 } = {}) { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + if (await isHealthy(getJson, url)) return true; + if (child.exitCode !== null || child.signalCode !== null) return false; + await sleep(500); + } + return false; +} + +async function main() { + if (flag("--help") || flag("-h")) { + process.stdout.write(HELP); + return 0; + } + + let local; + try { + local = await import("../lib/local.mjs"); + } catch (error) { + log(`[local] ${error.message}`); + return 2; + } + const { LOCAL_DEFAULTS, LocalProvider, SERVICE, defaultLocalModel, findLlamaServer, getJson, handleLocalRequest, llamaServedModel, localModel, localPaths } = local; + + const port = Number(arg("--port", LOCAL_DEFAULTS.port)); + const llamaPort = Number(arg("--llama-port", LOCAL_DEFAULTS.llamaPort)); + const ctx = Number(arg("--ctx", LOCAL_DEFAULTS.ctx)); + const explicitPath = arg("--model", null); + const requestedId = arg("--model-name", null); + const urlOverride = arg("--model-url", "")?.trim() || process.env.JEV_LOCAL_MODEL_URL?.trim() || null; + const downloadOnly = flag("--download-only"); + const serviceUrl = `http://127.0.0.1:${port}`; + const llamaUrl = `http://127.0.0.1:${llamaPort}`; + const envLine = `TYPESAFE_BASE_URL=${serviceUrl} TYPESAFE_API_KEY=local`; + + if (!Number.isInteger(port) || port < 1 || port > 65535) throw configError(`--port must be 1..65535 (got ${arg("--port")})`); + if (!Number.isInteger(llamaPort) || llamaPort < 1 || llamaPort > 65535) throw configError(`--llama-port must be 1..65535 (got ${arg("--llama-port")})`); + if (!Number.isInteger(ctx) || ctx < 512) throw configError(`--ctx must be an integer >= 512 (got ${arg("--ctx")})`); + if (ctx < 12_000) log(`[local] warning: --ctx ${ctx} is below the ~11,700 tokens a 100-candidate step renders to; long pages will fail`); + if (explicitPath && requestedId) throw configError("use either --model-name or --model , not both"); + + if (flag("--list-models")) return listModels(local); + + // Resolve which GGUF to serve: a registry entry (default or --model-name), or an explicit path. + const entry = requestedId ? localModel(requestedId) : explicitPath ? null : defaultLocalModel(); + const file = explicitPath ? path.resolve(explicitPath) : localPaths(undefined, entry).modelFile; + const url = urlOverride ?? entry?.url ?? null; + const expectedBytes = entry && url === entry.url ? entry.bytes : null; + const modelName = entry?.id ?? path.basename(file).replace(/\.gguf$/i, ""); + const runDir = localPaths().run; + const pidFile = path.join(runDir, `jev-local-${port}.pid`); + + const existing = await fsp.stat(file).catch(() => null); + if (!existing && !url) throw configError(`model file not found: ${file}\npass --model-name to use the registry, or --model-url to download it`); + await ensureModel({ file, url, expectedBytes }); + if (downloadOnly) { + log(`[local] ${modelName} is ready; not starting anything (--download-only)`); + return 0; + } + + // Already serving? Reuse only when it is the same model; otherwise say exactly what is wrong. + const running = await getJson(`${serviceUrl}/health`, 1500); + if (running?.service === SERVICE) { + if (running.model && running.model !== modelName) { + log(`[local] ${serviceUrl} is already serving "${running.model}", not "${modelName}".`); + log(`[local] stop that launcher first (Ctrl-C in its terminal, or kill $(cat ${pidFile})), then run again.`); + return 2; + } + log(`[local] already serving on ${serviceUrl}; reusing it`); + process.stdout.write(`${envLine}\n`); + return 0; + } + + const bin = findLlamaServer(); + if (!bin) { + log("[local] llama-server not found (looked on PATH and in /opt/homebrew/bin).\n\nInstall it with:\n\n brew install llama.cpp\n"); + return 2; + } + log(`[local] llama-server: ${bin}`); + + let llama = null; + if (await isHealthy(getJson, `${llamaUrl}/health`)) { + const served = await llamaServedModel(llamaUrl); + if (served && path.resolve(served) !== file) { + log(`[local] llama.cpp on ${llamaUrl} is serving ${served}, not ${file}.`); + log(`[local] stop that server, or keep both by giving this one its own port: --llama-port ${llamaPort + 1}`); + return 2; + } + log(`[local] llama.cpp already serving on ${llamaUrl}${served ? "" : " (could not read its model path)"} (left untouched on exit)`); + } else { + llama = spawnLlamaServer(bin, { modelFile: file, llamaPort, ctx }); + spawnedLlama = llama; + let spawnError = null; + llama.once("error", (error) => { + spawnError = error; + }); + const stopLlama = () => { + try { + llama.kill("SIGTERM"); + } catch { + // already gone + } + }; + let ready = false; + try { + ready = await waitForLlama(getJson, llama, `${llamaUrl}/health`); + } catch (error) { + stopLlama(); // never leave a half-started server behind + throw error; + } + if (!ready) { + stopLlama(); + throw new Error(spawnError ? `llama-server failed to start: ${spawnError.message}` : `llama-server did not become ready on ${llamaUrl} within 180s (see its output above)`); + } + log(`[local] llama.cpp serving on ${llamaUrl} (pid ${llama.pid}, -c ${ctx})`); + } + + const provider = new LocalProvider({ url: llamaUrl, model: modelName }); + const server = http.createServer((req, res) => { + handleLocalRequest(req, res, provider).catch(() => {}); // the handler answers every path + }); + try { + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(port, "127.0.0.1", resolve); + }); + } catch (error) { + const occupant = await fetchJson(`${serviceUrl}/health`, 1500); + if (occupant?.service === SERVICE) { + process.stdout.write(`${envLine}\n`); + return 0; + } + if (llama) llama.kill("SIGTERM"); + throw new Error(`cannot listen on 127.0.0.1:${port}: ${error.message}`); + } + + await fsp.mkdir(runDir, { recursive: true }); + await fsp.writeFile(pidFile, `${process.pid}\n`); + let llamaPidFile = null; + if (llama) { + llamaPidFile = path.join(runDir, `llama-${llamaPort}.pid`); + await fsp.writeFile(llamaPidFile, `${llama.pid}\n`); + } + + let stopping = false; + const shutdown = async (signal) => { + if (stopping) return; + stopping = true; + log(`[local] ${signal} — stopping${llama ? " (llama-server was started here, stopping it too)" : ""}`); + await new Promise((resolve) => server.close(resolve)); + if (llama) { + llama.kill("SIGTERM"); + await Promise.race([new Promise((resolve) => llama.once("exit", resolve)), sleep(5000)]); + } + await removeQuietly(pidFile); + if (llamaPidFile) await removeQuietly(llamaPidFile); + process.exit(0); + }; + process.on("SIGINT", () => void shutdown("SIGINT")); + process.on("SIGTERM", () => void shutdown("SIGTERM")); + llama?.on("exit", (code) => { + if (!stopping) log(`[local] llama-server exited (code ${code}); the Jev server will report backend-down`); + }); + + log(`[local] serving /v1/systemone on ${serviceUrl} (${modelName}${entry?.label ? ` · ${entry.label}` : ""}); Ctrl-C to stop`); + log("[local] limits: short states only, no images, no few-shot — see experiments/gguf-provider/RESULTS.md"); + process.stdout.write(`${envLine}\n`); + return null; // keep running +} + +main() + .then((code) => { + if (code !== null) process.exit(code); + }) + .catch((error) => { + log(`[local] ${error.message}`); + process.exit(error.config || error.name === "LocalModelRegistryError" ? 2 : 1); + }); diff --git a/skills/jev-browser/lib/local-models.json b/skills/jev-browser/lib/local-models.json new file mode 100644 index 0000000..2cf1818 --- /dev/null +++ b/skills/jev-browser/lib/local-models.json @@ -0,0 +1,18 @@ +{ + "_comment": "Model registry for the fully local backend (skills/jev-browser/bin/jev-local.mjs). Swapping models is a data edit: point \"default\" at another id, or pick one per run with --model-name . Every entry needs file (bare .gguf filename, downloaded into ~/.jev-browser/models), url (https resolve URL), bytes (exact size), label (shown by --list-models and doctor).", + "default": "qwen3.5-4b-q4-k-m", + "models": { + "qwen3.5-0.8b-q8": { + "file": "Qwen3.5-0.8B-Q8_0.gguf", + "url": "https://huggingface.co/unsloth/Qwen3.5-0.8B-GGUF/resolve/main/Qwen3.5-0.8B-Q8_0.gguf", + "bytes": 811843840, + "label": "Qwen3.5-0.8B Q8_0" + }, + "qwen3.5-4b-q4-k-m": { + "file": "Qwen3.5-4B-Q4_K_M.gguf", + "url": "https://huggingface.co/unsloth/Qwen3.5-4B-GGUF/resolve/main/Qwen3.5-4B-Q4_K_M.gguf", + "bytes": 2740937888, + "label": "Qwen3.5-4B Q4_K_M" + } + } +} diff --git a/skills/jev-browser/lib/local.mjs b/skills/jev-browser/lib/local.mjs new file mode 100644 index 0000000..ab3f1ee --- /dev/null +++ b/skills/jev-browser/lib/local.mjs @@ -0,0 +1,693 @@ +// Fully local (experimental) model backend: the TypeSafe `/v1/systemone` contract served +// by a local llama.cpp server — no API key, no Python, no npm packages, no outbound network. +// +// The answer to a question is read straight out of the model's first generated token: the +// question is rendered with labelled options, exactly ONE token is generated, and the +// probability mass on each option label at that position IS the answer. No training, no +// text generation, no output parsing (mirrors ekzhang/openjev-sglang). +// +// The readout algorithm is a copy of experiments/gguf-provider (lib/readout.mjs, +// lib/render.mjs, lib/labels.mjs, lib/provider.mjs); experiments/gguf-provider/RESULTS.md +// holds the measured numbers and the limits. Keep this file and the experiment in sync. +import fs from "node:fs"; +import fsp from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { validateQuestions } from "./typesafe.mjs"; + +export const SERVICE = "jev-local"; + +// Which GGUF to serve is data, not code: lib/local-models.json is the registry, addressed +// relative to this module so the launcher and doctor work from any cwd. Swapping models is a +// JSON edit (point "default" at another id) or --model-name for a single run. +export const REGISTRY_FILE = fileURLToPath(new URL("./local-models.json", import.meta.url)); + +export class LocalModelRegistryError extends Error { + constructor(message) { + super(message); + this.name = "LocalModelRegistryError"; + } +} + +/** + * Read and validate the model registry (`lib/local-models.json`). + * + * Throws LocalModelRegistryError naming the offending path and field when the file is + * missing or malformed — a broken registry never falls back to a hard-coded model. + * @returns {{path: string, default: string, models: Record}} + */ +export function loadLocalModels({ file = REGISTRY_FILE } = {}) { + let text; + try { + text = fs.readFileSync(file, "utf8"); + } catch (error) { + throw new LocalModelRegistryError(`local model registry not readable: ${file} (${error.message})`); + } + let parsed; + try { + parsed = JSON.parse(text); + } catch (error) { + throw new LocalModelRegistryError(`local model registry is not valid JSON: ${file} (${error.message})`); + } + if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) { + throw new LocalModelRegistryError(`local model registry must be a JSON object: ${file}`); + } + const models = parsed.models; + if (!models || typeof models !== "object" || Array.isArray(models) || Object.keys(models).length === 0) { + throw new LocalModelRegistryError(`local model registry needs a non-empty "models" object: ${file}`); + } + for (const [id, entry] of Object.entries(models)) { + if (!entry || typeof entry !== "object" || Array.isArray(entry)) throw new LocalModelRegistryError(`local model registry: model "${id}" must be an object: ${file}`); + for (const key of ["file", "url", "label"]) { + if (typeof entry[key] !== "string" || !entry[key].trim()) throw new LocalModelRegistryError(`local model registry: model "${id}" needs a non-empty "${key}" string: ${file}`); + } + if (path.basename(entry.file) !== entry.file) throw new LocalModelRegistryError(`local model registry: model "${id}" file must be a bare filename (no directories): ${file}`); + if (!/^https?:\/\//.test(entry.url)) throw new LocalModelRegistryError(`local model registry: model "${id}" url must start with http(s): ${file}`); + if (!Number.isInteger(entry.bytes) || entry.bytes <= 0) throw new LocalModelRegistryError(`local model registry: model "${id}" needs a positive integer "bytes": ${file}`); + } + if (typeof parsed.default !== "string" || !models[parsed.default]) { + throw new LocalModelRegistryError(`local model registry: "default" is ${JSON.stringify(parsed.default)}, not one of ${Object.keys(models).join(", ")}: ${file}`); + } + return { path: file, default: parsed.default, models }; +} + +function entryOf(registry, id) { + const entry = registry.models[id]; + if (!entry) throw new LocalModelRegistryError(`unknown local model "${id}"; available: ${Object.keys(registry.models).join(", ")}`); + return { id, ...entry }; +} + +/** One registry entry by id (with its `id`), or a clear error listing what exists. */ +export function localModel(id, { file = REGISTRY_FILE } = {}) { + return entryOf(loadLocalModels({ file }), id); +} + +/** The registry's default entry. */ +export function defaultLocalModel({ file = REGISTRY_FILE } = {}) { + const registry = loadLocalModels({ file }); + return entryOf(registry, registry.default); +} + +// Resolved once at import. LOCAL_MODEL_FILE / LOCAL_MODEL_URL keep working for older callers, +// derived from the registry default; a malformed registry throws here rather than degrading. +export const LOCAL_MODELS = loadLocalModels(); +export const LOCAL_MODEL_ID = LOCAL_MODELS.default; +export const LOCAL_MODEL_FILE = LOCAL_MODELS.models[LOCAL_MODEL_ID].file; +export const LOCAL_MODEL_URL = LOCAL_MODELS.models[LOCAL_MODEL_ID].url; +export const LOCAL_MODEL_BYTES = LOCAL_MODELS.models[LOCAL_MODEL_ID].bytes; +export const LOCAL_MODEL_LABEL = LOCAL_MODELS.models[LOCAL_MODEL_ID].label; + +export const LOCAL_DEFAULTS = Object.freeze({ + port: 8092, + llamaPort: 8090, + // 16384 tokens: a 100-candidate step renders to ~11.7k prompt tokens, so the old 8192 default + // could not serve a real browser step (see experiments/gguf-provider/RESULTS.md). + ctx: 16384, + model: LOCAL_MODEL_ID, + llamaUrl: "http://127.0.0.1:8090", + nProbs: 512, +}); + +/** Below this share of the option-label mass the answer is treated as "not read out" (HTTP 422). */ +export const MIN_LABEL_MASS = 0.5; + +/** Files the one-command launcher uses (~/.jev-browser/{models,run}) for one registry entry. */ +export function localPaths(home = os.homedir(), model = { id: LOCAL_MODEL_ID, file: LOCAL_MODEL_FILE }) { + const root = path.join(home, ".jev-browser"); + const models = path.join(root, "models"); + return { root, models, run: path.join(root, "run"), id: model.id, file: model.file, modelFile: path.join(models, model.file) }; +} + +/** `llama-server` on PATH, then the Homebrew prefix (macOS default). Never throws. */ +export function findLlamaServer({ env = process.env, extra = ["/opt/homebrew/bin"] } = {}) { + const dirs = [...new Set([...(env.PATH ?? "").split(path.delimiter).filter(Boolean), ...extra])]; + for (const dir of dirs) { + const candidate = path.join(dir, "llama-server"); + try { + const stat = fs.statSync(candidate); + if (stat.isFile() && (stat.mode & 0o111) !== 0) return candidate; + } catch { + // not here — keep looking + } + } + return null; +} + +/** GET JSON with a short timeout; null on any failure (unreachable, non-JSON, timeout). */ +export async function getJson(url, timeoutMs = 1500) { + try { + const response = await fetch(url, { signal: AbortSignal.timeout(timeoutMs) }); + return await response.json(); + } catch { + return null; + } +} + +/** + * The GGUF path a llama.cpp server says it is serving — `/props.model_path`, falling back to the + * name llama.cpp puts in `/v1/models`. null when the server does not tell us, so callers can warn + * instead of guessing. + */ +export async function llamaServedModel(url, timeoutMs = 2000) { + const base = url.replace(/\/$/, ""); + const props = await getJson(`${base}/props`, timeoutMs); + if (typeof props?.model_path === "string" && props.model_path) return props.model_path; + const models = await getJson(`${base}/v1/models`, timeoutMs); + const name = models?.models?.[0]?.name ?? models?.data?.[0]?.id; + return typeof name === "string" && name ? name : null; +} + +/** + * What `doctor` reports for the local backend: the binary, which registry entry is active and + * how much of its file is on disk, and whether either server is up (and which model it serves). + */ +export async function localStatus({ home = os.homedir(), port = LOCAL_DEFAULTS.port, llamaPort = LOCAL_DEFAULTS.llamaPort, model = LOCAL_MODEL_ID } = {}) { + const entry = LOCAL_MODELS.models[model] ? { id: model, ...LOCAL_MODELS.models[model] } : null; + const paths = localPaths(home, entry ?? { id: model, file: LOCAL_MODEL_FILE }); + const status = { + ...paths, + port, + llamaPort, + registry: { path: LOCAL_MODELS.path, default: LOCAL_MODELS.default, ids: Object.keys(LOCAL_MODELS.models) }, + id: entry?.id ?? model, + label: entry?.label ?? null, + expectedBytes: entry?.bytes ?? 0, + llamaServer: null, + bytes: 0, + llama: false, + serving: false, + servingModel: null, + endpoint: null, // what answers on `port` when it is not this launcher's wrapper + }; + try { + status.llamaServer = findLlamaServer(); + } catch { + status.llamaServer = null; + } + try { + status.bytes = (await fsp.stat(paths.modelFile)).size; + } catch { + status.bytes = 0; + } + status.llama = ((await getJson(`http://127.0.0.1:${llamaPort}/health`, 1000)) ?? {}).status === "ok"; + const live = await getJson(`http://127.0.0.1:${port}/health`, 1000); + status.serving = live?.service === SERVICE; + status.servingModel = status.serving ? live.model ?? null : null; + // This port may be serving something that is not this launcher: a Kev checkpoint, say, when + // TYPESAFE_BASE_URL points at 8008 and the GGUF wrapper is not running. `port` is the port the + // caller cares about, so report whatever answers there instead of implying the port is dead. + status.endpoint = null; + if (!status.serving) { + const cards = await getJson(`http://127.0.0.1:${port}/v1/models`, 1000); + const card = cards?.models?.[0] ?? cards?.data?.[0]; + if (card) { + status.endpoint = { + port, + name: card.name ?? card.id ?? null, + kind: card.run || card.base ? "kev" : "readout", + run: card.run ?? null, + base: card.base ?? null, + }; + } + } + return status; +} + +export class ReadoutError extends Error { + constructor(message, details) { + super(message); + this.name = "ReadoutError"; + this.details = details; + if (details?.status) this.status = details.status; // llama.cpp's own status, e.g. 400 for an oversized prompt + } +} + +/** Sampler settings that leave the reported distribution untouched (true next-token softmax). */ +export const UNMASKED_SAMPLERS = Object.freeze({ + temperature: 1.0, + top_k: 0, // 0 = disabled (no top-k truncation of the reported candidates) + top_p: 1.0, + min_p: 0.0, + typical_p: 1.0, + repeat_penalty: 1.0, + presence_penalty: 0.0, + frequency_penalty: 0.0, + mirostat: 0, + seed: 1234, +}); + +export class LlamaServer { + constructor({ url = LOCAL_DEFAULTS.llamaUrl, nProbs = LOCAL_DEFAULTS.nProbs, cachePrompt = true, timeoutMs = 120_000 } = {}) { + this.url = url.replace(/\/$/, ""); + this.nProbs = nProbs; + this.cachePrompt = cachePrompt; + this.timeoutMs = timeoutMs; + } + + async #post(route, body) { + const response = await fetch(`${this.url}${route}`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(body), + signal: AbortSignal.timeout(this.timeoutMs), + }); + const text = await response.text(); + if (!response.ok) { + let detail = text; + try { + const parsed = JSON.parse(text); + detail = parsed?.error?.message ?? parsed?.message ?? text; + } catch { + // not JSON — keep the raw body + } + throw new ReadoutError(`llama.cpp ${route} failed (${response.status}): ${String(detail).trim().slice(0, 400)}`, { status: response.status, body: text.slice(0, 2000) }); + } + return JSON.parse(text); + } + + async healthy() { + return ((await getJson(`${this.url}/health`, 2000)) ?? {}).status === "ok"; + } + + /** Token ids for a string, using the served model's real tokenizer. */ + async tokenize(content) { + return (await this.#post("/tokenize", { content })).tokens; + } + + /** Render chat messages with the model's real chat template (llama-server --jinja). */ + async applyTemplate(messages, opts = {}) { + return (await this.#post("/apply-template", { messages, ...opts })).prompt; + } + + /** + * Probabilities of the FIRST token that follows `prompt`. + * @returns {Promise<{candidates: Array<{id:number, token:string, logprob:number}>, sampled: object|null, usage: object, ms: number}>} + */ + async firstTokenDistribution(prompt) { + const started = performance.now(); + const data = await this.#post("/completion", { + prompt, + n_predict: 1, + n_probs: this.nProbs, + cache_prompt: this.cachePrompt, + stream: false, + ...UNMASKED_SAMPLERS, + }); + const ms = performance.now() - started; + const entry = data.completion_probabilities?.[0]; + if (!entry) throw new ReadoutError("/completion returned no completion_probabilities", { data }); + const candidates = (entry.top_logprobs ?? []).map((c) => ({ id: c.id, token: c.token, logprob: c.logprob })); + return { + candidates, + sampled: { id: entry.id, token: entry.token, logprob: entry.logprob }, + usage: { + prompt_tokens: data.timings?.prompt_n ?? null, + cached_tokens: data.timings?.cache_n ?? null, + completion_tokens: data.timings?.predicted_n ?? 1, + prompt_ms: data.timings?.prompt_ms ?? null, + predicted_ms: data.timings?.predicted_ms ?? null, + }, + ms, + }; + } +} + +/** + * Read a distribution over option labels out of a first-token distribution. + * + * Each label is looked up under the space-prefixed surface form the model emits after + * "Answer:" (" A"); the probability of every token id that stands for the label is summed. + * Labels that are not single tokens in the served tokenizer are reported in `multiToken` + * and scored 0 (their first token is shared with a longer candidate, so no exact readout + * exists) — that is what `label_mass` below 1 measures. + * + * @param {Array<{id:number,token:string,logprob:number}>} candidates + * @param {string[]} labels + * @param {Map} tokenIdsByLabel label -> candidate token ids (from the tokenizer) + */ +export function readoutLabels(candidates, labels, tokenIdsByLabel) { + const byId = new Map(); + for (const candidate of candidates) byId.set(candidate.id, candidate); + const raw = {}; + const missing = []; + const multiToken = []; + for (const label of labels) { + const ids = tokenIdsByLabel.get(label) ?? []; + if (ids.length === 0) { + raw[label] = 0; + multiToken.push(label); + continue; + } + let probability = 0; + let seen = false; + for (const id of ids) { + const candidate = byId.get(id); + if (candidate) { + probability += Math.exp(candidate.logprob); + seen = true; + } + } + raw[label] = probability; + if (!seen) missing.push(label); + } + const total = Object.values(raw).reduce((a, b) => a + b, 0); + const probabilities = {}; + for (const label of labels) probabilities[label] = total > 0 ? raw[label] / total : 0; + return { probabilities, raw, total, missing, multiToken }; +} + +/** openjev's confidence: 1 - H(p)/log(n), clamped to [0,1] (0 = uniform, 1 = point mass). */ +export function confidenceOf(probabilities) { + const values = Object.values(probabilities).filter((v) => v > 0); + const n = Object.keys(probabilities).length; + if (n < 2) return values.length ? 1 : 0; + const entropy = -values.reduce((acc, p) => acc + p * Math.log(p), 0); + return Math.min(1, Math.max(0, 1 - entropy / Math.log(n))); +} + +/** 0 -> "A", 25 -> "Z", 26 -> "AA", ... (bijective base-26; integer labels are multi-token). */ +function labelForIndex(index) { + let n = index + 1; + let out = ""; + while (n > 0) { + const rem = (n - 1) % 26; + out = String.fromCharCode(65 + rem) + out; + n = Math.floor((n - 1) / 26); + } + return out; +} + +/** + * Canonical option list for a question, in a stable order. + * - noul -> true/false + * - choice -> criteria keys, in insertion order (matches normalizeAnswers' expected key set) + * - score -> criteria levels, keyed "0".."n-1" + */ +function optionsFor(question) { + if (question.type === "noul") { + const criteria = question.criteria ?? { true: "True", false: "False" }; + return [ + { key: "true", text: String(criteria.true ?? "True") }, + { key: "false", text: String(criteria.false ?? "False") }, + ]; + } + if (question.type === "choice") return Object.entries(question.criteria ?? {}).map(([key, text]) => ({ key, text: String(text) })); + if (question.type === "score") return (question.criteria ?? []).map((text, i) => ({ key: String(i), text: String(text) })); + throw new Error(`unsupported question type: ${question.type}`); +} + +function instructionsText(instructions) { + if (typeof instructions === "string") return instructions; + const parts = []; + if (instructions?.question) parts.push(instructions.question); + for (const rule of instructions?.rules ?? []) parts.push(`- ${rule}`); + return parts.join("\n"); +} + +/** The question block appended to the user turn (the state is a separate, shared part). */ +function renderQuestionBlock(question, labels) { + const options = optionsFor(question); + const lines = options.map((o, i) => `${labels[i]}: ${o.text}`); + const note = question.type === "noul" ? `Option ${labels[0]} means True, option ${labels[1]} means False.` : null; + return { + options, + text: [ + instructionsText(question.instructions), + ...(note ? [note] : []), + "", + "Options:", + ...lines, + "", + `Answer with the letter of the single best option (${labels[0]}-${labels[options.length - 1]}).`, + "Answer:", + ].join("\n"), + }; +} + +const SYSTEM_PROMPT = + "You are an expert classification model. You are given a state and one question about it. " + + "You always answer with the label letter of exactly one option, and nothing else."; + +/** Build one prompt per question; each ends with the "Answer:" cue the readout reads after. */ +async function renderPrompts({ server, state, questions, labels }) { + const stateText = typeof state === "string" ? state : JSON.stringify(state, null, 2); + const prompts = {}; + const optionMeta = {}; + for (const [id, question] of Object.entries(questions)) { + const block = renderQuestionBlock(question, labels); + const messages = [ + { role: "system", content: SYSTEM_PROMPT }, + { role: "user", content: `State:\n${stateText}\n\n${block.text}` }, + ]; + let full; + try { + full = await server.applyTemplate(messages, { chat_template_kwargs: { enable_thinking: false } }); + } catch { + full = await server.applyTemplate(messages); + } + prompts[id] = `${trimAssistantHeader(full)}Answer:`; + optionMeta[id] = { options: block.options, labels: labels.slice(0, block.options.length) }; + } + return { prompts, optionMeta }; +} + +/** Drop the trailing "<|im_start|>assistant\n" the template appends, so we own the cue. */ +function trimAssistantHeader(text) { + const markers = ["<|im_start|>assistant\n", "<|assistant|>\n", "model\n", "assistant\n"]; + for (const marker of markers) if (text.endsWith(marker)) return text.slice(0, -marker.length); + return text; +} + +/** Token ids to look up for each label — only the space-prefixed form the model emits here. */ +async function labelTokenIds(server, labels) { + const map = new Map(); + for (const label of labels) { + const ids = []; + const tokens = await server.tokenize(` ${label}`); + if (tokens.length === 1) ids.push(tokens[0]); + map.set(label, ids); + } + return map; +} + +/** Alphabet of labels that are single tokens in the served tokenizer (skips "AY", "BQ", ...). */ +async function verifiedAlphabet(server, count) { + const out = []; + for (let i = 0; out.length < count && i < count * 4 + 64; i++) { + const label = labelForIndex(i); + const ids = await labelTokenIds(server, [label]); + if ((ids.get(label) ?? []).length > 0) out.push(label); + } + if (out.length < count) throw new Error(`only ${out.length} single-token labels available, needed ${count}`); + return out; +} + +/** + * Local provider: answers a batch of Jev questions about one state by first-token readout. + * `systemOne` and `health` are the interface `handleLocalRequest` needs; any object with + * those two methods works (that is how the unit tests stub the llama.cpp server). + */ +export class LocalProvider { + constructor({ url = LOCAL_DEFAULTS.llamaUrl, model = LOCAL_DEFAULTS.model, nProbs = LOCAL_DEFAULTS.nProbs } = {}) { + this.server = new LlamaServer({ url, nProbs }); + this.model = model; + this.#alphabetCache = new Map(); // option count -> { labels, tokenIds } + } + + #alphabetCache; + + /** Verified single-token alphabet big enough for `count` options (built once per size). */ + async #alphabet(count) { + const cached = this.#alphabetCache.get(count); + if (cached) return cached; + const labels = await verifiedAlphabet(this.server, count); + const tokenIds = await labelTokenIds(this.server, labels); + const entry = { labels, tokenIds }; + this.#alphabetCache.set(count, entry); + return entry; + } + + /** Whether the llama.cpp server behind this provider is up. */ + async health() { + return this.server.healthy(); + } + + /** + * Answer a batch of questions about one state. + * @returns {Promise<{model:string, answers:object, usage:object}>} — the TypeSafe HTTP contract + */ + async systemOne({ state, questions, model = this.model }) { + const started = performance.now(); + validateQuestions(questions); + const entries = Object.entries(questions); + const maxOptions = Math.max(...entries.map(([, question]) => optionsFor(question).length)); + const { labels, tokenIds } = await this.#alphabet(maxOptions); + const alphabetMs = performance.now() - started; + + const { prompts, optionMeta } = await renderPrompts({ server: this.server, state, questions, labels }); + const renderMs = performance.now() - started - alphabetMs; + + const answers = {}; + const perQuestion = {}; + let promptTokens = 0; + let completionTokens = 0; + let readoutMs = 0; + + for (const [id, question] of entries) { + const meta = optionMeta[id]; + const activeTokenIds = new Map(meta.labels.map((label) => [label, tokenIds.get(label) ?? []])); + const readStarted = performance.now(); + const distribution = await this.server.firstTokenDistribution(prompts[id]); + const read = readoutLabels(distribution.candidates, meta.labels, activeTokenIds); + readoutMs += performance.now() - readStarted; + promptTokens += distribution.usage.prompt_tokens ?? 0; + completionTokens += distribution.usage.completion_tokens ?? 0; + + const probabilities = {}; + meta.labels.forEach((label, i) => { + probabilities[meta.options[i].key] = read.probabilities[label] ?? 0; + }); + const ranked = Object.entries(probabilities).sort((a, b) => b[1] - a[1]); + const topKey = ranked[0]?.[0] ?? null; + + if (question.type === "noul") { + answers[id] = { type: "noul", noul: probabilities.true ?? 0 }; + } else if (question.type === "choice") { + answers[id] = { type: "choice", probabilities, choice: topKey, confidence: confidenceOf(probabilities) }; + } else if (question.type === "score") { + answers[id] = { + type: "score", + probabilities, + score: Object.entries(probabilities).reduce((acc, [key, p]) => acc + Number(key) * p, 0), + legend: (question.criteria ?? []).map(String), + confidence: confidenceOf(probabilities), + }; + } else { + throw new Error(`unsupported question type for ${id}: ${question.type}`); + } + + perQuestion[id] = { + ms: Math.round(distribution.ms), + prompt_tokens: distribution.usage.prompt_tokens, + cached_tokens: distribution.usage.cached_tokens, + label_mass: read.total, + missing_labels: read.missing, + multi_token_labels: read.multiToken, + top: ranked.slice(0, 3).map(([key, p]) => [key, Number(p.toFixed(4))]), + }; + } + + return { + model, + answers, + usage: { + input_tokens: promptTokens, + output_tokens: completionTokens, + ms_total: Math.round(performance.now() - started), + ms_alphabet: Math.round(alphabetMs), + ms_render: Math.round(renderMs), + ms_readout: Math.round(readoutMs), + per_question: perQuestion, + }, + }; + } +} + +/** + * Questions whose captured option-label mass fell short of `threshold`: the model put its + * probability elsewhere, so the answer would be an artefact of zero-filled labels. + */ +export function lowMassQuestions(response, threshold = MIN_LABEL_MASS) { + const out = []; + for (const [id, detail] of Object.entries(response?.usage?.per_question ?? {})) { + const mass = Number(detail?.label_mass); + if (Number.isFinite(mass) && mass < threshold) out.push({ id, mass }); + } + return out; +} + +const httpError = (status, message, code, extra = {}) => Object.assign(new Error(message), { status, code, ...extra }); + +function sendJson(res, status, body) { + const text = JSON.stringify(body); + res.writeHead(status, { "content-type": "application/json", "content-length": Buffer.byteLength(text) }); + res.end(text); +} + +function readBody(req, limit = 8 * 1024 * 1024) { + return new Promise((resolve, reject) => { + const chunks = []; + let size = 0; + req.on("data", (chunk) => { + size += chunk.length; + if (size > limit) { + reject(httpError(413, `request body larger than ${limit} bytes`, "BODY_TOO_LARGE")); + req.destroy(); + return; + } + chunks.push(chunk); + }); + req.on("end", () => resolve(Buffer.concat(chunks).toString("utf8"))); + req.on("error", reject); + }); +} + +/** + * Serve the Jev HTTP contract from `provider` (anything with `health()` and + * `systemOne({state, questions, model})`): + * + * GET /health -> {status: "ok"} | 503 {status: "backend-down"} (both tagged jev-local) + * GET /v1/models -> {models: [{name, ...}]} (the shape the skill's client reads) + * POST /v1/systemone -> {model, answers, usage} 422 when an answer was not read out + * + * The 422 (`LOW_LABEL_MASS`) names the question: fewer than half of the probability mass + * landed on its option labels, so answering would be guesswork rather than a readout. + */ +export async function handleLocalRequest(req, res, provider) { + const started = performance.now(); + try { + const route = new URL(req.url, "http://127.0.0.1").pathname; + + if (req.method === "GET" && (route === "/health" || route === "/health/live")) { + const healthy = await provider.health(); + return sendJson(res, healthy ? 200 : 503, { status: healthy ? "ok" : "backend-down", service: SERVICE, model: provider.model ?? LOCAL_DEFAULTS.model }); + } + + if (req.method === "GET" && route.startsWith("/v1/models")) { + return sendJson(res, 200, { + models: [{ name: provider.model ?? LOCAL_DEFAULTS.model, description: "Local llama.cpp first-token readout backend (experimental)", modalities: ["text"] }], + }); + } + + if (req.method !== "POST" || route !== "/v1/systemone") { + return sendJson(res, 404, { error: { message: `not found: ${req.method} ${route}`, code: "NOT_FOUND" } }); + } + + let body; + try { + body = JSON.parse(await readBody(req)); + } catch (error) { + if (error.status) throw error; + throw httpError(400, `request body is not JSON: ${error.message}`, "INVALID_JSON"); + } + + validateQuestions(body.questions); + const response = await provider.systemOne({ state: body.state, questions: body.questions, model: body.model ?? provider.model }); + + const low = lowMassQuestions(response); + if (low.length > 0) { + const message = low + .map(({ id, mass }) => `question ${id}: only ${(mass * 100).toFixed(1)}% of the probability mass landed on its option labels (needs >= ${MIN_LABEL_MASS * 100}%)`) + .join("; "); + throw httpError(422, message, "LOW_LABEL_MASS", { questions: low }); + } + + return sendJson(res, 200, { ...response, usage: { ...response.usage, ms_wall: Math.round(performance.now() - started) } }); + } catch (error) { + const status = error.status ?? (error.code === "INVALID_QUESTIONS" ? 422 : 500); + const code = error.code ?? "PROVIDER_ERROR"; + process.stderr.write(`[local] ${code}: ${error.message}\n`); + return sendJson(res, status, { error: { message: error.message, code, ...(error.questions ? { questions: error.questions } : {}) } }); + } +} diff --git a/skills/jev-browser/tests/local-models.test.mjs b/skills/jev-browser/tests/local-models.test.mjs new file mode 100644 index 0000000..8004420 --- /dev/null +++ b/skills/jev-browser/tests/local-models.test.mjs @@ -0,0 +1,134 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { spawnSync } from "node:child_process"; +import { fileURLToPath } from "node:url"; +import { LOCAL_DEFAULTS, LOCAL_MODEL_BYTES, LOCAL_MODEL_FILE, LOCAL_MODEL_ID, LOCAL_MODEL_LABEL, LOCAL_MODEL_URL, LOCAL_MODELS, defaultLocalModel, loadLocalModels, localModel, localPaths } from "../lib/local.mjs"; + +const BIN = path.join(path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."), "bin", "jev-local.mjs"); + +/** Run the launcher offline: no llama-server, no registry download, no inherited URL override. */ +function jevLocal(args) { + const env = { ...process.env }; + delete env.JEV_LOCAL_MODEL_URL; + return spawnSync(process.execPath, [BIN, ...args], { encoding: "utf8", timeout: 30_000, env }); +} + +/** A throwaway registry file; the callback gets its path. */ +function withRegistryFile(contents, fn) { + const file = path.join(fs.mkdtempSync(path.join(os.tmpdir(), "jev-local-registry-")), "local-models.json"); + if (contents !== null) fs.writeFileSync(file, contents); + try { + return fn(file); + } finally { + fs.rmSync(path.dirname(file), { recursive: true, force: true }); + } +} + +const ENTRY = { file: "Qwen3.5-0.8B-Q8_0.gguf", url: "https://huggingface.co/unsloth/Qwen3.5-0.8B-GGUF/resolve/main/Qwen3.5-0.8B-Q8_0.gguf", bytes: 811843840, label: "Qwen3.5-0.8B Q8_0" }; +const ENTRY_4B = { file: "Qwen3.5-4B-Q4_K_M.gguf", url: "https://huggingface.co/unsloth/Qwen3.5-4B-GGUF/resolve/main/Qwen3.5-4B-Q4_K_M.gguf", bytes: 2740937888, label: "Qwen3.5-4B Q4_K_M" }; + +test("the shipped registry defaults to the 4B, and the derived constants follow it", () => { + const registry = loadLocalModels(); + assert.equal(registry.default, "qwen3.5-4b-q4-k-m"); + assert.deepEqual(LOCAL_MODELS.models[registry.default], ENTRY_4B); + assert.deepEqual(localModel(registry.default), { id: registry.default, ...ENTRY_4B }); + assert.deepEqual(defaultLocalModel(), { id: registry.default, ...ENTRY_4B }); + + // The 0.8B stays in the registry, selectable by id, with its verified fields. + assert.deepEqual(localModel("qwen3.5-0.8b-q8"), { id: "qwen3.5-0.8b-q8", ...ENTRY }); + + assert.equal(LOCAL_MODEL_ID, registry.default); + assert.equal(LOCAL_MODEL_FILE, ENTRY_4B.file); + assert.equal(LOCAL_MODEL_URL, ENTRY_4B.url); + assert.equal(LOCAL_MODEL_BYTES, ENTRY_4B.bytes); + assert.equal(LOCAL_MODEL_LABEL, ENTRY_4B.label); + + // Local paths follow the entry, so swapping the default swaps the file that gets served. + assert.equal(localPaths("/home/x").modelFile, `/home/x/.jev-browser/models/${ENTRY_4B.file}`); + assert.equal(localPaths("/home/x", { id: "old", file: ENTRY.file }).modelFile, `/home/x/.jev-browser/models/${ENTRY.file}`); + assert.equal(LOCAL_DEFAULTS.model, registry.default); + assert.equal(LOCAL_DEFAULTS.ctx, 16384); +}); + +test("an unknown model id names the ids that do exist", () => { + assert.throws(() => localModel("nope"), (error) => { + assert.equal(error.name, "LocalModelRegistryError"); + assert.match(error.message, /unknown local model "nope"/); + assert.match(error.message, /available: qwen3\.5-0\.8b-q8, qwen3\.5-4b-q4-k-m/); + return true; + }); +}); + +test("a missing or malformed registry fails loudly instead of falling back", () => { + withRegistryFile(null, (file) => { + assert.throws(() => loadLocalModels({ file }), /registry not readable: .* \(ENOENT/); + }); + withRegistryFile("{ not json", (file) => { + assert.throws(() => loadLocalModels({ file }), /not valid JSON/); + }); + withRegistryFile(JSON.stringify({ default: "a", models: {} }), (file) => { + assert.throws(() => loadLocalModels({ file }), /needs a non-empty "models" object/); + }); + withRegistryFile(JSON.stringify({ default: "ghost", models: { a: ENTRY } }), (file) => { + assert.throws(() => loadLocalModels({ file }), /"default" is "ghost", not one of a/); + }); + withRegistryFile(JSON.stringify({ default: "a", models: { a: { ...ENTRY, url: "" } } }), (file) => { + assert.throws(() => loadLocalModels({ file }), /model "a" needs a non-empty "url" string/); + }); + withRegistryFile(JSON.stringify({ default: "a", models: { a: { ...ENTRY, file: "sub/model.gguf" } } }), (file) => { + assert.throws(() => loadLocalModels({ file }), /file must be a bare filename/); + }); + withRegistryFile(JSON.stringify({ default: "a", models: { a: { ...ENTRY, bytes: "big" } } }), (file) => { + assert.throws(() => loadLocalModels({ file }), /positive integer "bytes"/); + }); +}); + +test("--list-models prints the registry with the default marked and exits 0", () => { + const result = jevLocal(["--list-models"]); + assert.equal(result.status, 0); + assert.match(result.stdout, /\* qwen3\.5-4b-q4-k-m/); // default marker + assert.match(result.stdout, /Qwen3\.5-4B Q4_K_M/); + assert.match(result.stdout, /2\.6 GiB/); + assert.match(result.stdout, /Qwen3\.5-4B-Q4_K_M\.gguf/); + assert.match(result.stdout, /^ {2}qwen3\.5-0\.8b-q8/m); // still listed, not default + assert.match(result.stdout, /Qwen3\.5-0\.8B Q8_0/); + assert.match(result.stdout, /\* = default/); + assert.equal(result.stderr, ""); +}); + +test("--help documents the registry flags and exits 0", () => { + const result = jevLocal(["--help"]); + assert.equal(result.status, 0); + for (const needle of ["--model-name", "--model-url", "--list-models", "--ctx", "--download-only"]) assert.match(result.stdout, new RegExp(needle)); + // Drift guard: the numbers in the help text are the ones the code uses. + assert.match(result.stdout, new RegExp(`--ctx .*${LOCAL_DEFAULTS.ctx}`)); + assert.match(result.stdout, new RegExp(`--port N.*${LOCAL_DEFAULTS.port}`)); + assert.match(result.stdout, new RegExp(`--llama-port N.*${LOCAL_DEFAULTS.llamaPort}`)); +}); + +test("--model-name nope exits 2 and lists the valid ids", () => { + const result = jevLocal(["--model-name", "nope", "--port", "65397"]); + assert.equal(result.status, 2); + assert.match(result.stderr, /unknown local model "nope"/); + assert.match(result.stderr, /available: qwen3\.5-0\.8b-q8/); + assert.equal(result.stdout, ""); +}); + +test("--model without a url refuses to download, and bad flags exit 2", () => { + const missing = path.join(os.tmpdir(), "jev-local-does-not-exist.gguf"); + const result = jevLocal(["--model", missing, "--port", "65396"]); + assert.equal(result.status, 2); + assert.match(result.stderr, /model file not found/); + assert.match(result.stderr, /--model-name to use the registry/); + + const both = jevLocal(["--model-name", "qwen3.5-0.8b-q8", "--model", missing, "--port", "65396"]); + assert.equal(both.status, 2); + assert.match(both.stderr, /use either --model-name or --model , not both/); + + const ctx = jevLocal(["--ctx", "10", "--port", "65396"]); + assert.equal(ctx.status, 2); + assert.match(ctx.stderr, /--ctx must be an integer >= 512/); +}); diff --git a/skills/jev-browser/tests/local.test.mjs b/skills/jev-browser/tests/local.test.mjs new file mode 100644 index 0000000..d4fa282 --- /dev/null +++ b/skills/jev-browser/tests/local.test.mjs @@ -0,0 +1,127 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import http from "node:http"; +import { MIN_LABEL_MASS, handleLocalRequest, lowMassQuestions, readoutLabels } from "../lib/local.mjs"; +import { normalizeAnswers } from "../lib/typesafe.mjs"; + +// A first-token distribution as llama.cpp reports it, for the five labels A..E: +// A and B are emitted (B through two token ids), C is a single-token label the model +// did not put any mass on, D is not a single token in the tokenizer (empty id list). +const distribution = [ + { id: 10, token: " A", logprob: Math.log(0.6) }, + { id: 11, token: " B", logprob: Math.log(0.25) }, + { id: 14, token: " B", logprob: Math.log(0.05) }, + { id: 99, token: " the", logprob: Math.log(0.1) }, +]; +const tokenIds = new Map([ + ["A", [10]], + ["B", [11, 14]], + ["C", [12]], + ["D", []], +]); +const labels = ["A", "B", "C", "D"]; +const close = (actual, expected, message) => assert.ok(Math.abs(actual - expected) < 1e-9, `${message}: ${actual} != ${expected}`); + +function stubProvider(overrides = {}) { + return { + model: "stub-local", + health: async () => true, + systemOne: async ({ model }) => ({ + model, + answers: { + urgent: { type: "noul", noul: 0.82 }, + team: { type: "choice", probabilities: { billing: 0.7, shipping: 0.25, returns: 0.05 }, choice: "billing", confidence: 0.6 }, + }, + usage: { input_tokens: 123, output_tokens: 2, per_question: { urgent: { label_mass: 0.94 }, team: { label_mass: 0.88 } } }, + ...overrides, + }), + }; +} + +async function withServer(provider, fn) { + const server = http.createServer((req, res) => void handleLocalRequest(req, res, provider)); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + try { + await fn(`http://127.0.0.1:${server.address().port}`); + } finally { + await new Promise((resolve) => server.close(resolve)); + } +} + +test("readoutLabels normalizes the mass on the option labels, reports what it could not read", () => { + const read = readoutLabels(distribution, labels, tokenIds); + + close(read.total, 0.9, "captured label mass"); + assert.deepEqual(read.missing, ["C"], "labels whose token id was not in the distribution"); + assert.deepEqual(read.multiToken, ["D"], "labels that are not single tokens"); + close(read.raw.A, 0.6, "label A"); + close(read.raw.B, 0.3, "label B summed over its two token ids"); + close(Object.values(read.probabilities).reduce((a, b) => a + b, 0), 1, "probabilities sum to 1"); + close(read.probabilities.A, 0.6 / 0.9, "label A normalized"); + close(read.probabilities.B, 0.3 / 0.9, "label B normalized"); + assert.equal(read.probabilities.C, 0, "unread label keeps zero probability"); + assert.equal(read.probabilities.D, 0, "multi-token label keeps zero probability"); +}); + +test("lowMassQuestions flags a question only when the readout missed most of the mass", () => { + const response = { usage: { per_question: { a: { label_mass: 0.9 }, b: { label_mass: 0.31 }, c: { label_mass: 0.5 } } } }; + assert.deepEqual(lowMassQuestions(response), [{ id: "b", mass: 0.31 }]); + assert.equal(MIN_LABEL_MASS, 0.5); + assert.deepEqual(lowMassQuestions({ usage: {} }), []); +}); + +test("POST /v1/systemone answers with the shape the skill's client parses", async () => { + const questions = { + urgent: { type: "noul", instructions: "Does `ticket` need an urgent reply?" }, + team: { type: "choice", instructions: "Which team owns `ticket`?", criteria: { billing: "Billing", shipping: "Shipping", returns: "Returns" } }, + }; + await withServer(stubProvider(), async (base) => { + const health = await (await fetch(`${base}/health`)).json(); + assert.equal(health.status, "ok"); + assert.equal(health.service, "jev-local"); + + const models = await (await fetch(`${base}/v1/models`)).json(); + assert.equal(models.models[0].name, "stub-local"); + + const response = await fetch(`${base}/v1/systemone`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ state: { ticket: "My card was charged twice" }, questions }), + }); + assert.equal(response.status, 200); + const body = await response.json(); + assert.equal(body.model, "stub-local"); + assert.equal(body.answers.urgent.type, "noul"); + assert.equal(body.answers.team.type, "choice"); + assert.equal(body.usage.input_tokens, 123); + assert.ok(Number.isFinite(body.usage.ms_wall)); + + // The client's own normalization must accept these answers untouched. + const normalized = normalizeAnswers(body.answers, questions); + assert.equal(normalized.urgent.top, "true"); + assert.equal(normalized.team.choice, "billing"); + assert.deepEqual(normalized.team.ranked, [["billing", 0.7], ["shipping", 0.25], ["returns", 0.05]]); + }); +}); + +test("POST /v1/systemone returns 422 naming the question when the label mass is too low", async () => { + const provider = stubProvider({ usage: { input_tokens: 123, output_tokens: 2, per_question: { urgent: { label_mass: 0.94 }, team: { label_mass: 0.31 } } } }); + await withServer(provider, async (base) => { + const response = await fetch(`${base}/v1/systemone`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + state: { ticket: "…" }, + questions: { + urgent: { type: "noul", instructions: "Urgent?" }, + team: { type: "choice", instructions: "Which team?", criteria: { billing: "Billing", shipping: "Shipping" } }, + }, + }), + }); + assert.equal(response.status, 422); + const body = await response.json(); + assert.equal(body.error.code, "LOW_LABEL_MASS"); + assert.match(body.error.message, /team/); + assert.deepEqual(body.error.questions, [{ id: "team", mass: 0.31 }]); + }); +}); From de61029b06a060a9107fcf8e6603332f148babbd Mon Sep 17 00:00:00 2001 From: ChenYCL Date: Thu, 24 Sep 2026 13:51:32 +0800 Subject: [PATCH 2/9] fix(jev-browser): typeable ARIA combobox inputs, loopback runs priced at $0, chrome needs_user hint MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three defects the first real local-backend run found (docs/local-backend-run-smoke.md §6), each one silent before: - ARIA 1.2 puts role="combobox" either on the itself (DuckDuckGo) or on a wrapper (Wikipedia). The first form was observed as a plain combobox, so `editable` was 0, the type/type_target/type_value questions were never generated, and runs could only click, scroll, wait or stop: R3 and R8 clicked the search box and never typed. isComboboxField() classifies the field as textbox — a combobox element with no input/textarea/contenteditable inside stays non-editable and is still offered as a click target. - A loopback baseUrl is a local model that bills nothing, yet runner copied config.pricePerMtok (0.042/Mtok) into the client: R8 reported costUsd=0.002705 for tokens that are $0, and budgetUsd counted spend that never happened. isLoopbackBaseUrl()/pricePerMtokFor() now price 127.0.0.1, localhost and ::1 at 0 while still counting tokens. - A closed chrome session was still announced with the ego wording ("the browser was handed to you; resume with --space-id …"). In R6 the run was genuinely blocked but handOff was null and headless had already closed the page, so following the hint found no browser. Without a space id the summary now says the browser was closed rather than handed over, and points at --keep. --- skills/jev-browser/bin/jev-browser.mjs | 15 +++++-- skills/jev-browser/lib/observe.mjs | 14 +++++- skills/jev-browser/lib/typesafe.mjs | 17 ++++++- tests/e2e/combobox.test.mjs | 62 ++++++++++++++++++++++++++ tests/fixtures/server.mjs | 1 + tests/unit/typesafe.test.mjs | 21 +++++++++ 6 files changed, 124 insertions(+), 6 deletions(-) create mode 100644 tests/e2e/combobox.test.mjs diff --git a/skills/jev-browser/bin/jev-browser.mjs b/skills/jev-browser/bin/jev-browser.mjs index 2c12898..87dcfb0 100755 --- a/skills/jev-browser/bin/jev-browser.mjs +++ b/skills/jev-browser/bin/jev-browser.mjs @@ -125,7 +125,14 @@ function summarize(result) { const cost = result.usage ? `$${result.usage.costUsd.toFixed(4)} (${result.usage.requests} requests, ${result.usage.inputTokens} tokens)` : "n/a"; const lines = [`status: ${result.status}${result.reason ? ` — ${result.reason}` : ""}`, `steps: ${result.steps} cost: ${cost} elapsed: ${Math.round((result.elapsedMs ?? 0) / 1000)}s`]; if (result.finalUrl) lines.push(`final: ${result.finalTitle ?? ""} <${result.finalUrl}>`); - if (result.status === "needs_user") lines.push(`blocker: ${result.blocker} — the browser was handed to you; resume with --space-id ${result.resume?.spaceId ?? ""} once done`); + if (result.status === "needs_user") { + const spaceId = result.resume?.spaceId; + lines.push( + spaceId + ? `blocker: ${result.blocker} — the browser was handed to you; resume with --space-id ${spaceId} once done` + : `blocker: ${result.blocker} — the browser was closed, not handed over; re-run when you can act in it (--keep leaves the page open)`, + ); + } if (result.journalDir) lines.push(`journal: ${result.journalDir}`); if (result.screenshot) lines.push(`screenshot: ${result.screenshot}`); return lines.join("\n"); @@ -202,8 +209,10 @@ async function main(argv) { return 0; } case "doctor": { - const { config, sources } = await loadConfig({ flags: flagsFromValues(values) }); - const report = await doctor({ config, sources, skillDir: SKILL_DIR, live: !values.offline }); + // `doctor --home ` has to load that home's config, the same way config show/set do: + // the whole point of the flag is to diagnose a config file that is not the default one. + const { config, sources } = await loadConfig({ flags: flagsFromValues(values), ...(values.home ? { home: values.home } : {}) }); + const report = await doctor({ config, sources, skillDir: SKILL_DIR, home: values.home, live: !values.offline }); if (values.json) print(report, { json: true }); else process.stdout.write(`${formatDoctor(report)}\n`); return report.ok ? 0 : 1; diff --git a/skills/jev-browser/lib/observe.mjs b/skills/jev-browser/lib/observe.mjs index d9fa6ca..a576a04 100644 --- a/skills/jev-browser/lib/observe.mjs +++ b/skills/jev-browser/lib/observe.mjs @@ -34,10 +34,22 @@ export const ENUMERATOR_SOURCE = String.raw` if (rect.bottom < -vh * 2 || rect.top > vh * 4) return false; // far outside; still counted as scrollable content return true; } + // ARIA 1.2 comboboxes are text inputs: role="combobox" may sit on the input itself + // (DuckDuckGo) or on a wrapper element (Wikipedia). Either way the field stays typeable. + function isComboboxField(el, role) { + var tag = el.tagName.toLowerCase(); + if (tag !== "input" && tag !== "textarea" && !el.isContentEditable) return false; + if (role === "combobox") return true; + var wrapper = el.closest("[role=combobox]"); + return !!wrapper && wrapper !== el; + } function roleOf(el) { var tag = el.tagName.toLowerCase(); var role = el.getAttribute("role"); - if (role) return role.toLowerCase(); + if (role) { + var named = role.toLowerCase(); + return isComboboxField(el, named) ? "textbox" : named; + } if (tag === "a") return el.hasAttribute("href") ? "link" : "text"; if (tag === "button") return "button"; if (tag === "select") return "select"; diff --git a/skills/jev-browser/lib/typesafe.mjs b/skills/jev-browser/lib/typesafe.mjs index 99774d5..f2734a4 100644 --- a/skills/jev-browser/lib/typesafe.mjs +++ b/skills/jev-browser/lib/typesafe.mjs @@ -52,6 +52,19 @@ export function validateQuestions(questions) { export const estimateCostUsd = (inputTokens, pricePerMtok = 0.042) => (Number(inputTokens) || 0) * (pricePerMtok / 1e6); +/** True when the contract is served from this machine (a local llama.cpp backend costs nothing). */ +export function isLoopbackBaseUrl(baseUrl) { + try { + const host = new URL(String(baseUrl)).hostname.toLowerCase().replace(/^\[|\]$/g, ""); + return host === "localhost" || host === "127.0.0.1" || host === "::1"; + } catch { + return false; + } +} + +/** USD per million input tokens actually charged for `baseUrl`: loopback tokens are free, so 0. */ +export const pricePerMtokFor = (baseUrl, pricePerMtok = 0.042) => (isLoopbackBaseUrl(baseUrl) ? 0 : pricePerMtok); + /** * Normalize answers so every question exposes `probabilities`, `top`, and `ranked`. * Nouls become {true, false}; choice/score keep the native map. @@ -93,7 +106,7 @@ export class TypeSafeClient { * @param {string} [options.model] * @param {number} [options.timeoutMs] * @param {number} [options.maxRetries] - * @param {number} [options.pricePerMtok] + * @param {number} [options.pricePerMtok] USD per million input tokens; ignored (0) for a loopback baseUrl * @param {Function} [options.fetchImpl] * @param {(row: object) => any} [options.onRequest] journal hook * @param {boolean} [options.cache] reuse answers for identical (model,state,questions) @@ -107,7 +120,7 @@ export class TypeSafeClient { this.model = model; this.timeoutMs = timeoutMs; this.maxRetries = maxRetries; - this.pricePerMtok = pricePerMtok; + this.pricePerMtok = pricePerMtokFor(this.baseUrl, pricePerMtok); this.fetch = fetchImpl ?? globalThis.fetch; this.onRequest = onRequest; this.cacheEnabled = cache; diff --git a/tests/e2e/combobox.test.mjs b/tests/e2e/combobox.test.mjs new file mode 100644 index 0000000..ce87e93 --- /dev/null +++ b/tests/e2e/combobox.test.mjs @@ -0,0 +1,62 @@ +// ARIA 1.2 comboboxes are text fields, but `role="combobox"` may sit on the itself +// (DuckDuckGo) or on a wrapper element (Wikipedia). Both must stay typeable — before this was +// fixed the input was observed as a plain `combobox`, `type` vanished from allowedActions and +// the type_target / type_value / submit_after_type questions were never generated. +import test, { before, after } from "node:test"; +import assert from "node:assert/strict"; +import { createContext, hasChrome } from "../helpers/env.mjs"; +import { executeJob } from "../../skills/jev-browser/lib/runner.mjs"; + +const backend = "chrome"; +const available = await hasChrome(); +const skip = available ? false : "Chrome not installed"; +let ctx; + +before(async () => { + if (available) ctx = await createContext({ backend, headless: true }); +}); +after(async () => { + await ctx?.close(); +}); + +/** Ids of the offered elements whose model-facing description matches. */ +const idsMatching = (state, re) => state.page.elements.filter((e) => re.test(e.description)).map((e) => e.id); + +test(`${backend} (headless): an input carrying role=combobox is typeable`, { skip }, async () => { + const config = { ...ctx.config, backend }; + const dry = await executeJob({ + config, + job: { mode: "dry-run", goal: "Search for the widget documentation", startUrl: ctx.url("/combobox"), inputs: { query: "widgets" } }, + log: () => {}, + }); + const { state, questions, meta } = dry; + + // DuckDuckGo markup: role="combobox" on the input itself. + const ddg = idsMatching(state, /^text field 'Search with DuckDuckGo'/); + assert.equal(ddg.length, 1, state.page.elements.map((e) => e.description).join("\n")); + assert.match(state.page.elements.find((e) => e.id === ddg[0]).description, /placeholder "Search the web without being tracked", empty/); + + // Wikipedia markup: role="combobox" on the wrapper, a plain inside it. + const wiki = idsMatching(state, /^text field 'Search Wikipedia'/); + assert.equal(wiki.length, 1, state.page.elements.map((e) => e.description).join("\n")); + + // The fixture site's own header field and a plain input are typeable as before. + const header = idsMatching(state, /^text field 'Search' \(.*placeholder "Search products"/); + assert.equal(header.length, 1, state.page.elements.map((e) => e.description).join("\n")); + const plain = idsMatching(state, /^text field 'Plain field'/); + assert.equal(plain.length, 1, state.page.elements.map((e) => e.description).join("\n")); + + // A combobox-role element with no field inside is offered, but is not something to type into. + const notAField = idsMatching(state, /^combobox 'Recent searches'/); + assert.equal(notAField.length, 1, state.page.elements.map((e) => e.description).join("\n")); + + assert.ok(meta.allowedActions.includes("type"), `allowedActions=${meta.allowedActions.join(",")}`); + for (const id of [...ddg, ...wiki, ...header, ...plain]) assert.ok(meta.editableIds.includes(id), `${id} is editable`); + assert.equal(meta.editableIds.includes(notAField[0]), false, "the combobox div is not editable"); + + const criteria = Object.keys(questions.type_target.criteria); + for (const id of [...ddg, ...wiki, ...header, ...plain]) assert.ok(criteria.includes(id), `${id} is offered to type_target`); + assert.equal(criteria.includes(notAField[0]), false, "the combobox div is not offered to type_target"); + assert.ok(questions.type_value && questions.submit_after_type, `questions=${Object.keys(questions).join(",")}`); + assert.ok(Object.keys(questions.click_target.criteria).includes(notAField[0]), "the combobox div is still offered as a click target"); +}); diff --git a/tests/fixtures/server.mjs b/tests/fixtures/server.mjs index 91d3c39..c3c7ee8 100644 --- a/tests/fixtures/server.mjs +++ b/tests/fixtures/server.mjs @@ -94,6 +94,7 @@ export function createSite() { return send(layout("Thanks", `

Thanks, ${esc(body.name)}!

Your ${esc(body.topic || "general")} message was received. We will reply to ${esc(body.email)}.

`, { user })); } if (p === "/consent") return send(layout("Consent", `

We use cookies

Accept to continue.

Blog

Latest article: Widgets in 2026.

Products`, { user })); + if (p === "/combobox") return send(layout("Combobox", `

Search the site

  • widgets
  • gadgets

ARIA 1.2 wrapper

Combobox with nothing to type into

Recent searches

Plain input

`, { user })); return send(layout("Not found", `

Page not found

The page ${esc(p)} does not exist.

`), 404); }); return { diff --git a/tests/unit/typesafe.test.mjs b/tests/unit/typesafe.test.mjs index f1b2b28..6b9031e 100644 --- a/tests/unit/typesafe.test.mjs +++ b/tests/unit/typesafe.test.mjs @@ -69,3 +69,24 @@ test("client surfaces 401 without retrying and never leaks the key", async () => test("client requires an API key", () => { assert.throws(() => new TypeSafeClient({}), /Missing TypeSafe API key/); }); + +test("a loopback client accrues no cost; a hosted one does", async () => { + const answerWith = (inputTokens) => async () => ({ + ok: true, + status: 200, + headers: new Headers(), + text: async () => JSON.stringify({ model: "m", answers: { q: { type: "noul", noul: 0.6 } }, usage: { input_tokens: inputTokens, output_tokens: 0 } }), + }); + const questions = { q: { type: "noul", instructions: "Is it urgent?" } }; + for (const baseUrl of ["http://127.0.0.1:8092", "http://localhost:8092", "http://[::1]:8092"]) { + const client = new TypeSafeClient({ apiKey: "k", baseUrl, fetchImpl: answerWith(1_000_000), cache: false }); + const result = await client.systemOne({ state: "x", questions }); + assert.equal(result.costUsd, 0, `${baseUrl} is free`); + assert.equal(client.totals.costUsd, 0, `${baseUrl} totals are free too`); + assert.equal(client.totals.inputTokens, 1_000_000, "tokens are still counted"); + } + const hosted = new TypeSafeClient({ apiKey: "k", baseUrl: "https://api.typesafe.ai", fetchImpl: answerWith(1_000_000), cache: false }); + const result = await hosted.systemOne({ state: "x", questions }); + assert.equal(result.costUsd, 0.042); + assert.equal(hosted.totals.costUsd, 0.042); +}); From 0491e7b6bcf576ddbcd6570ecf10b0b29f411539 Mon Sep 17 00:00:00 2001 From: ChenYCL Date: Thu, 24 Sep 2026 13:52:00 +0800 Subject: [PATCH 3/9] feat(jev-browser): Kev 4B accuracy tier MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The GGUF readout is the default because it is cheap; this adds the tier for when it is not accurate enough. One command fetches jaredpalmer/kev-4b and its Qwen3.5-4B-Base parent into the HuggingFace cache layout the Kev loader expects, verifies every file against a manifest pinned to one commit (sha256 for LFS blobs, git blob sha1 for the small text files), starts the MLX server on 127.0.0.1:8008 and prints the skill's env line last, on stdout, so it can be eval'd. Why the extra tier earns its 9.34 GB base: 19/20 = 0.95 on the same 20 construction-graded items the GGUF candidates were measured on — tying hosted Jev on that set, and above the readout's 0.80 and the 0.8B's 0.50. 18/20 at the released row limit, where the server refuses the two 55-option click_target questions with HTTP 422; --patch-row-limit exists for exactly that, is never applied implicitly, and prints its diff. Bring-up cost measured end to end (docs/local-kev-bringup.md): clone 32 s, uv sync 58 s, first start to first healthy response ≈ 96 s, adapter 159.7 MB in 58 s, base 9.34 GB over two streams from ModelScope in ~22.5 min. Steady state: ~18 GB idle and ~36 GB of GPU memory under load at the raised limit (heavy swap on a 48 GB machine), mean 2.2 s per item, 12.2 s worst — and $0 per token, since the base URL is loopback. The /v1/models card this server returns names the checkpoint it loaded (run, base), which is what lets a run tell Kev from the readout without being told; the per-backend bar that depends on it lands next. Kev is a trained pointer head, not a first-token logprob readout: on the shared fixture page it reads goal_done 0.0607 where the GGUF 0.8B read 0.6952 — the thresholds are not shared. --- skills/jev-browser/bin/jev-kev.mjs | 627 +++++++++++++++++++++++++++++ 1 file changed, 627 insertions(+) create mode 100755 skills/jev-browser/bin/jev-kev.mjs diff --git a/skills/jev-browser/bin/jev-kev.mjs b/skills/jev-browser/bin/jev-kev.mjs new file mode 100755 index 0000000..c0ffd32 --- /dev/null +++ b/skills/jev-browser/bin/jev-kev.mjs @@ -0,0 +1,627 @@ +#!/usr/bin/env node +// One command for the Kev accuracy tier of jev-browser (experimental): fetch the checkpoint and its +// base into the HuggingFace cache layout the Kev loader expects, start the Kev server, and print the +// TypeSafe env line for the skill. +// +// This is the accuracy tier, NOT the default. The default stays the zero-Python llama.cpp + GGUF +// readout (bin/jev-local.mjs); Kev is a trained pointer-head checkpoint served by its own MLX runtime +// and costs a Python venv, a 9.34 GB base plus the adapter, ~18 GB idle / 36 GB GPU footprint under +// load at the raised limit, and ~2.2 s mean / 12.2 s worst per item. +// +// node skills/jev-browser/bin/jev-kev.mjs +// node skills/jev-browser/bin/jev-kev.mjs --verify-only # re-hash the cache, download nothing +// node skills/jev-browser/bin/jev-kev.mjs --patch-row-limit # optional, explicit, prints the diff +// TYPESAFE_BASE_URL=http://127.0.0.1:8008 TYPESAFE_API_KEY=local \ +// node skills/jev-browser/bin/jev-browser.mjs judge --state-file s.json --questions-file q.json +// +// Everything except the final `TYPESAFE_BASE_URL=... TYPESAFE_API_KEY=local` line goes to stderr, so +// that line can be piped or eval'd. See experiments/kev-4b/README.md for what each tier scores. +import fs from "node:fs"; +import fsp from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { createHash } from "node:crypto"; +import { spawn } from "node:child_process"; +import { Readable } from "node:stream"; +import { pipeline } from "node:stream/promises"; + +// ---------------------------------------------------------------------------- assets + +// Pinned to a commit: names, sizes and hashes come from the Hub tree metadata at that commit and are +// what every download is verified against. `hash` is the sha256 of the file for LFS blobs, and the +// git blob sha1 for the small text files — the same rule the Hub uses. Injected by +// experiments/kev-4b/make-manifest.mjs from the verified cache; see that file for provenance. +const KEV_ASSETS = { + run: { + repo: "jaredpalmer/kev-4b", + commit: "485ace8703592fcf405488b262449990824cfed1", + files: [ + { name: ".gitattributes", size: 1570, kind: "git-sha1", hash: "52373fe24473b1aa44333d318f578ae6bf04b49b" }, + { name: "adapter_config.json", size: 1271, kind: "git-sha1", hash: "ea5c33529f3e0e9521f52b42485d44a23d333ba7" }, + { name: "adapter_model.safetensors", size: 129924032, kind: "sha256", hash: "9797de69a42188e411b17b7b4fcb66a23374dcebc21d71a7a66f836b5d34df2b" }, + { name: "added_tokens.json", size: 707, kind: "git-sha1", hash: "b54f9135e44c1e81047e8d05cb027af8bc039eed" }, + { name: "head.pt", size: 5248767, kind: "sha256", hash: "d8f796da36ff7bd7c0fb9496b452139bb7851af4fc82b07b500b682d3f721d6a" }, + { name: "merges.txt", size: 1671853, kind: "git-sha1", hash: "31349551d90c7606f325fe0f11bbb8bd5fa0d7c7" }, + { name: "provenance.json", size: 3140, kind: "git-sha1", hash: "318576caaa383eb7bdd2e10ba320e0b7330de591" }, + { name: "README.md", size: 11699, kind: "git-sha1", hash: "28d99f291e91401d81ef48b8f4ddfc2c9b8563a4" }, + { name: "result.json", size: 73953, kind: "git-sha1", hash: "b75a6b940b022394255f19484ebb63d2bfde33cc" }, + { name: "special_tokens_map.json", size: 616, kind: "git-sha1", hash: "17305b3603dfb19ccc0f658ec2cd2cd3adff4a58" }, + { name: "tokenizer_config.json", size: 1128, kind: "git-sha1", hash: "fa833096a2f03c2ff89094eb1ba7716fbd98b102" }, + { name: "tokenizer.json", size: 19989325, kind: "sha256", hash: "06b9509352d2af50381ab2247e083b80d32d5c0aba91c272ca9ff729b6a0e523" }, + { name: "train.log", size: 3389, kind: "git-sha1", hash: "a62bf77835de9283696656eca66afe4efcdac534" }, + { name: "training_config.json", size: 1586, kind: "git-sha1", hash: "07c40996cb96787387db7a4f89df876cb1361099" }, + { name: "training_metrics.json", size: 320, kind: "git-sha1", hash: "b5bae5408ab4e831f84247cb28a54fd46abb2951" }, + { name: "vocab.json", size: 2776833, kind: "git-sha1", hash: "4783fe10ac3adce15ac8f358ef5462739852c569" }, + ], + }, + base: { + repo: "Qwen/Qwen3.5-4B-Base", + commit: "1001bb4d826a52d1f399e183466143f4da7b741b", + files: [ + { name: ".gitattributes", size: 1570, kind: "git-sha1", hash: "52373fe24473b1aa44333d318f578ae6bf04b49b" }, + { name: "config.json", size: 3161, kind: "git-sha1", hash: "557d961b205319c6a7da5f757f565b69b3967b7d" }, + { name: "LICENSE", size: 11343, kind: "git-sha1", hash: "1d5180a42f1c3383ba7c7bd0a50f0837ef0168df" }, + { name: "merges.txt", size: 3353259, kind: "git-sha1", hash: "a494e019ca1502219fd0128658b979e5f05ae8e8" }, + { name: "model.safetensors-00001-of-00002.safetensors", size: 5329398712, kind: "sha256", hash: "df547074dce70532a0493e5433152bd17a65efb89088cfabc2e7e2371a93d712" }, + { name: "model.safetensors-00002-of-00002.safetensors", size: 3990429344, kind: "sha256", hash: "590fbaac095dd31db886c322d9d2f7df47777966391acf306ddddc3e4e3a15ef" }, + { name: "model.safetensors.index.json", size: 76196, kind: "git-sha1", hash: "7586335c0c85f13864338166a651bc2afbf49849" }, + { name: "preprocessor_config.json", size: 390, kind: "git-sha1", hash: "2ea84a437d448ff71b08df68fdd949d5cc4ebb64" }, + { name: "README.md", size: 3722, kind: "git-sha1", hash: "f66751fce75ca423e0993107b38d4c5408f677fb" }, + { name: "tokenizer_config.json", size: 16713, kind: "git-sha1", hash: "ae8d254e44c51d0cb0907bcb221f18efca829d3e" }, + { name: "tokenizer.json", size: 12807196, kind: "sha256", hash: "fe000e3ed39ed12b8d2481d527d44f93c65d37e87645d2dcc80d1bf9d50d2927" }, + { name: "video_preprocessor_config.json", size: 386, kind: "git-sha1", hash: "37900b3ff9295e1aa7e211378466356b52e64e55" }, + { name: "vocab.json", size: 6722759, kind: "git-sha1", hash: "0aa0ce0658d60ac4a5d609f4eadb0e8e43514176" }, + ], + }, +}; + +// Where the bytes come from, best first. ModelScope is the only independent mirror and measured ~5.5 +// MB/s against HF's 1.6–2.3; hf-mirror redirects to HF but is kept as a third rung. A source that +// yields a file failing verification is demoted for the rest of the run (ModelScope serves a +// different .gitattributes, so its small files are expected to fail and fall through to HF). +const SOURCES = ["modelscope", "hf", "hf-mirror"]; + +const DEFAULT_CLONE = path.join(os.homedir(), ".local", "share", "jev-browser", "kev"); +const CACHE = path.join(os.homedir(), ".cache", "huggingface", "hub"); +const RUN_ID = KEV_ASSETS.run.repo; +const BASE_ID = KEV_ASSETS.base.repo; +const SMALL_FILE = 1024 * 1024; +const STALL_MS = 120_000; // no bytes for this long on one source → try the next +const READY_MS = 420_000; // model assembly (~22 s for the 4B) + server start + +const HELP = `jev-kev — serve the Kev accuracy tier of the local Jev backend for jev-browser + +Usage: + jev-kev [--port 8008] [--run ${RUN_ID}] [--clone ] + jev-kev --verify-only re-hash the whole cache; download nothing + jev-kev --download-only fetch the checkpoint and its base, start nothing + jev-kev --list-files print the pinned manifest and exit + jev-kev --patch-row-limit apply the optional row-limit patch to the local clone + jev-kev --unpatch-row-limit revert it + +What it does, in order: + 1. fetches ${RUN_ID} (16 files, 152 MiB) and its base ${BASE_ID} + (13 files, 9.34 GB) into ~/.cache/huggingface/hub, resuming .incomplete parts and never + re-downloading a file whose size and hash already match + 2. starts the Kev server (${DEFAULT_CLONE.replace(os.homedir(), "~")}/.venv/bin/python -m kev.serve) + 3. prints, and only then prints: + TYPESAFE_BASE_URL=http://127.0.0.1:8008 TYPESAFE_API_KEY=local + +Sources (fastest first, falls through on a stall or a verification failure): + --source pin one: modelscope | hf | hf-mirror + --sources set the whole order + +Optional row-limit patch — off by default, and never applied silently: + The released server caps a request at state + one branch <= 8192 tokens (kev/model.py), so the + 55-option click_target questions on a real page are REFUSED with HTTP 422. Published behaviour is + 18/20 on the 20 graded items with that 422; the patch makes the cap settable and reaches 19/20. + --patch-row-limit one explicit edit to the local clone, prints the diff and the revert command + --row-limit serve with the cap raised (requires --patch-row-limit; without the patch + the server has no such knob and this exits 2) + +Tuning: + --port N port of the Kev server (default 8008) + --run checkpoint to serve (default ${RUN_ID}) + --clone Kev checkout to use (default ${DEFAULT_CLONE.replace(os.homedir(), "~")}) + --timeout wait for the server to become ready (default ${READY_MS / 1000}) + -h, --help this text + +Env (passed through to the server): KEV_TEMPERATURE, KEV_BACKEND, KEV_DTYPE, KEV_PREFIX_CACHE, ... + HF_HUB_OFFLINE=1 is set for the server: the launcher has already verified the cache. + +Exit codes: 0 ok · 1 runtime failure · 2 usage/config (missing venv, port busy, wrong checkpoint) + · 3 a file failed size/hash verification and was removed + +A server already answering on --port is reused when it serves the same checkpoint and reported as a +conflict otherwise; one this launcher started is stopped with it. +`; + +const argv = process.argv.slice(2); +const flag = (name) => argv.includes(name); +const arg = (name, fallback) => (argv.includes(name) ? argv[argv.indexOf(name) + 1] : fallback); +const log = (message) => process.stderr.write(`${message}\n`); +const humanBytes = (bytes) => + bytes >= 1024 ** 3 ? `${(bytes / 1024 ** 3).toFixed(1)} GiB` : bytes >= 1024 ** 2 ? `${(bytes / 1024 ** 2).toFixed(1)} MiB` : `${(bytes / 1024).toFixed(0)} KiB`; +const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); +const configError = (message) => Object.assign(new Error(message), { config: true }); + +// ---------------------------------------------------------------------------- fetching + +const sha256File = (file) => + new Promise((resolve, reject) => { + const hash = createHash("sha256"); + fs.createReadStream(file) + .on("data", (chunk) => hash.update(chunk)) + .on("error", reject) + .on("end", () => resolve(hash.digest("hex"))); + }); + +/** The git blob id of a file: sha1("blob \0" + content) — how the Hub names small files. */ +const gitBlobOid = (file) => + new Promise((resolve, reject) => { + const hash = createHash("sha1"); + hash.update(`blob ${fs.statSync(file).size}\0`); + fs.createReadStream(file) + .on("data", (chunk) => hash.update(chunk)) + .on("error", reject) + .on("end", () => resolve(hash.digest("hex"))); + }); + +const repoDir = (repo) => path.join(CACHE, `models--${repo.replace("/", "--")}`); + +/** Return { ok, how, want, got } for one cached blob against its pinned metadata. */ +async function verifyBlob(target, entry) { + const stat = await fsp.stat(target).catch(() => null); + if (!stat) return { ok: false, how: "missing", want: null, got: null }; + if (stat.size !== entry.size) return { ok: false, how: "size", want: entry.size, got: stat.size }; + const got = entry.kind === "sha256" ? await sha256File(target) : await gitBlobOid(target); + return { ok: got === entry.hash, how: entry.kind, want: entry.hash, got }; +} + +const sourceUrl = (source, repo, commit, name) => { + if (source === "modelscope") return `https://modelscope.cn/api/v1/models/${repo}/repo?Revision=master&FilePath=${encodeURIComponent(name)}`; + const host = source === "hf-mirror" ? "https://hf-mirror.com" : "https://huggingface.co"; + return `${host}/${repo}/resolve/${commit}/${name}`; +}; + +/** + * Stream one URL into `part`, resuming when the server honours Range. Returns the bytes on disk. + * Throws `{ stall: true }` when no bytes arrive for STALL_MS so the caller can try the next source. + */ +async function transfer(url, part, { expectedBytes }) { + const have = (await fsp.stat(part).catch(() => null))?.size ?? 0; + const headers = { "user-agent": "jev-kev/1" }; + if (have > 0) headers.range = `bytes=${have}-`; + if (!/^https:/.test(url)) throw new Error(`refusing a non-https source: ${url}`); + + const controller = new AbortController(); + let timer = null; + const armStall = () => { + clearTimeout(timer); + timer = setTimeout(() => controller.abort(Object.assign(new Error("stalled"), { stall: true })), STALL_MS); + }; + armStall(); + + let received = have; + const started = Date.now(); + let lastReport = 0; + try { + const response = await fetch(url, { headers, redirect: "follow", signal: controller.signal }); + if (response.status === 416) { + // The server thinks we already have everything; keep the part and let verification judge it. + return have; + } + if (!response.ok) throw Object.assign(new Error(`HTTP ${response.status} ${response.statusText}`), { status: response.status }); + const resuming = response.status === 206 && have > 0; + if (!resuming) received = 0; + if (!response.body) throw new Error("empty response body"); + const total = resuming && response.headers.get("content-range")?.includes("/") + ? Number(response.headers.get("content-range").split("/")[1]) || expectedBytes + : Number(response.headers.get("content-length")) || expectedBytes; + + const body = Readable.fromWeb(response.body); + body.on("data", (chunk) => { + received += chunk.length; + armStall(); + const now = Date.now(); + if (now - lastReport < 1000 && received !== total) return; + lastReport = now; + const mbps = received - (resuming ? have : 0) === 0 ? 0 : ((received - (resuming ? have : 0)) / 1e6 / Math.max((now - started) / 1000, 1e-3)); + process.stderr.write(`\r[kev] ${humanBytes(received)} / ${humanBytes(total)} (${Math.floor((received / total) * 100)}%) ${mbps.toFixed(2)} MB/s `); + }); + await pipeline(body, fs.createWriteStream(part, { flags: resuming ? "a" : "w" })); + process.stderr.write("\r\x1b[K"); + if (total && received !== total) throw new Error(`truncated: got ${received} of ${total} bytes`); + return received; + } finally { + clearTimeout(timer); + } +} + +/** Fetch one file: try each source until one lands bytes that match the pinned metadata. */ +async function ensureFile({ repo, commit, name, entry, demoted, report }) { + await fsp.mkdir(path.join(repoDir(repo), "blobs"), { recursive: true }); + const target = path.join(repoDir(repo), "blobs", entry.hash); + const part = `${target}.incomplete`; + const existing = await verifyBlob(target, entry); + if (existing.ok) { + await fsp.rm(part, { force: true }); + report.skipped.push(name); + return { ok: true, source: "cache", how: existing.how }; + } + + const small = entry.size < SMALL_FILE; + const sources = SOURCES.filter((source) => !demoted.has(`${repo}|${source}${small ? "|small" : ""}`)); + if (sources.length === 0) throw Object.assign(new Error(`every source was demoted for ${name}`), { verify: true }); + + const failures = []; + for (const source of sources) { + const url = sourceUrl(source, repo, commit, name); + const started = Date.now(); + try { + const bytes = await transfer(url, part, { expectedBytes: entry.size }); + const result = await verifyBlob(part, entry); + const seconds = Math.max((Date.now() - started) / 1000, 1e-3); + if (!result.ok) { + failures.push(`${source}: ${result.how} want=${result.want ?? "?"} got=${result.got ?? "?"}`); + await fsp.rm(part, { force: true }); // never keep bytes that failed the published metadata + if (small) demoted.add(`${repo}|${source}|small`); + log(`[kev] ${name}: ${source} served a file that does not match the published metadata (${result.how}) — trying the next source`); + continue; + } + await fsp.rename(part, target); + log(`[kev] ${name} ← ${source} (${humanBytes(bytes)} in ${seconds.toFixed(1)}s, ${(bytes / 1e6 / seconds).toFixed(2)} MB/s, ${result.how} verified)`); + report.fetched.push({ name, source, bytes, seconds: Number(seconds.toFixed(1)) }); + await fsp.rm(`${target}.incomplete`, { force: true }); + return { ok: true, source, how: result.how }; + } catch (error) { + await fsp.rm(part, { force: true }); + failures.push(`${source}: ${error.stall ? `no bytes for ${STALL_MS / 1000}s` : error.message}`); + log(`[kev] ${name}: ${source} failed (${error.stall ? `stalled for ${STALL_MS / 1000}s` : error.message})`); + } + } + const error = new Error(`${name} could not be fetched from any source:\n ${failures.join("\n ")}`); + error.verify = true; + throw error; +} + +/** Fetch + verify one repo, writing the cache layout the Kev loader resolves: blobs, refs, snapshots. */ +async function ensureRepo({ key, verifyOnly }) { + const { repo, commit, files } = KEV_ASSETS[key]; + const dir = repoDir(repo); + const report = { repo, commit, skipped: [], fetched: [], verified: 0, bytes: 0, mismatch: [] }; + const demoted = new Set(); + const total = files.reduce((sum, file) => sum + file.size, 0); + const missing = []; + for (const file of files) { + const ok = (await verifyBlob(path.join(dir, "blobs", file.hash), file)).ok; + if (ok) report.verified += 1; + else missing.push(file); + } + log( + `[kev] ${repo}@${commit.slice(0, 8)}: ${files.length} files, ${humanBytes(total)} — ` + + `${report.verified} verified${verifyOnly || missing.length === 0 ? "" : `, ${missing.length} to fetch (${humanBytes(missing.reduce((sum, f) => sum + f.size, 0))})`}`, + ); + + if (!verifyOnly) { + // Largest first: the big shards grab the fastest source while it is still cheap to switch. + for (const file of [...missing].sort((a, b) => b.size - a.size)) { + await ensureFile({ repo, commit, name: file.name, entry: file, demoted, report }); + report.bytes += file.size; + } + } + + // Re-hash everything on disk and rebuild refs/snapshots from exactly what was verified. + await fsp.mkdir(path.join(dir, "refs"), { recursive: true }); + await fsp.mkdir(path.join(dir, "trees"), { recursive: true }); + await fsp.writeFile(path.join(dir, "refs", "main"), commit); + const treeFiles = {}; + for (const file of files) treeFiles[file.name] = file.kind === "sha256" ? { size: file.size, blob_id: file.hash, lfs_sha256: file.hash } : { size: file.size, blob_id: file.hash }; + await fsp.writeFile(path.join(dir, "trees", `${commit}.json`), JSON.stringify({ format_version: 1, files: treeFiles }, null, 1)); + + const snapshot = path.join(dir, "snapshots", commit); + await fsp.mkdir(snapshot, { recursive: true }); + for (const file of files) { + const result = await verifyBlob(path.join(dir, "blobs", file.hash), file); + const link = path.join(snapshot, file.name); + if (result.ok) { + await fsp.rm(link, { force: true }); + await fsp.symlink(path.relative(snapshot, path.join(dir, "blobs", file.hash)), link); + } else { + report.mismatch.push({ file: file.name, ...result }); + } + } + return report; +} + +// ---------------------------------------------------------------------------- clone + server + +/** A guard for the exact bytes this launcher's patch produces. */ +const PATCH_MARK = "KEV_SERVE_MAX_STATE"; +const PATCH_REVERT = `cd ${DEFAULT_CLONE} && git checkout -- kev/model.py`; +const RELEASED_LINE = "SERVE_MAX_STATE, SERVE_MAX_BRANCH = 8192, 8192"; +const PATCHED_LINES = [ + "# LOCAL EVALUATION PATCH (not upstream): the served row limit is state + one branch, and a Jev-shaped browser", + "# state with a long option list exceeds 8192. Defaults are byte-identical to the released constants; raise with", + "# KEV_SERVE_MAX_STATE / KEV_SERVE_MAX_BRANCH. Everything derived below (SERVE_MAX_PACKED, MAX_TRAIN_STATE) follows.", + 'SERVE_MAX_STATE = int(os.environ.get("KEV_SERVE_MAX_STATE", "8192"))', + 'SERVE_MAX_BRANCH = int(os.environ.get("KEV_SERVE_MAX_BRANCH", "8192"))', +]; + +async function patchRowLimit(clone) { + const file = path.join(clone, "kev", "model.py"); + const source = await fsp.readFile(file, "utf8").catch(() => null); + if (source === null) throw configError(`cannot read ${file} — is --clone pointing at a Kev checkout?`); + if (source.includes(PATCH_MARK)) { + log(`[kev] row-limit patch already applied to ${file}; nothing to do`); + log(`[kev] revert with: ${PATCH_REVERT}`); + return; + } + if (!source.includes(RELEASED_LINE)) throw configError(`${file} does not contain the released line\n ${RELEASED_LINE}\nrefusing to patch — this checkout is not the version the launcher was written against`); + await fsp.writeFile(file, source.replace(RELEASED_LINE, PATCHED_LINES.join("\n"))); + log(`[kev] applied the optional row-limit patch to ${file}\n`); + log(`--- a/kev/model.py`); + log(`+++ b/kev/model.py`); + log(`@@ line 15 @@`); + log(`-${RELEASED_LINE}`); + for (const line of PATCHED_LINES) log(`+${line}`); + log(`\n[kev] this is a LOCAL EVALUATION PATCH on a third-party clone, not upstream, and it is not`); + log(`[kev] applied by default: published behaviour is 18/20 on the 20 graded items with a 422 on`); + log(`[kev] the 55-option click_target questions. The defaults are unchanged (8192); raise the cap`); + log(`[kev] for one run with --row-limit .`); + log(`[kev] revert with: ${PATCH_REVERT}`); +} + +async function unpatchRowLimit(clone) { + const file = path.join(clone, "kev", "model.py"); + const source = await fsp.readFile(file, "utf8").catch(() => null); + if (source === null) throw configError(`cannot read ${file} — is --clone pointing at a Kev checkout?`); + if (!source.includes(PATCH_MARK)) { + log(`[kev] the row-limit patch is not applied to ${file}; nothing to revert`); + return; + } + await fsp.writeFile(file, source.replace(PATCHED_LINES.join("\n"), RELEASED_LINE)); + log(`[kev] reverted the row-limit patch in ${file} — the server is back to the published 8192 cap`); +} + +const patchApplied = async (clone) => (await fsp.readFile(path.join(clone, "kev", "model.py"), "utf8").catch(() => "")).includes(PATCH_MARK); + +/** The venv python, with the exact command that creates it when it is missing. */ +async function findPython(clone) { + const python = path.join(clone, ".venv", "bin", "python"); + if (!fs.existsSync(clone)) { + throw configError( + `Kev checkout not found at ${clone}\n\nClone it with:\n\n git clone --depth 1 https://github.com/jaredpalmer/kev.git ${clone}\n uv sync --extra serve --project ${clone}\n`, + ); + } + if (!fs.existsSync(python)) { + throw configError( + `no Python venv at ${path.join(clone, ".venv")}\n\nCreate it with:\n\n uv sync --extra serve --project ${clone}\n\n` + + `(uv follows the repo's .python-version, 3.13 — not 3.14, which has no torch wheel)`, + ); + } + const probe = await run(python, ["-c", "import torch, mlx_lm; print('ok')"], { cwd: clone, quiet: true }).catch((error) => error); + if (probe instanceof Error || probe.code !== 0) { + const detail = (probe instanceof Error ? probe.message : `${probe.stdout}${probe.stderr}`).trim().split("\n").slice(-3).join("\n"); + throw configError( + `the venv at ${python} cannot import torch and mlx_lm, so the MLX backend is unavailable\n ${detail}\n\n` + + `Install the serving extras with:\n\n uv sync --extra serve --project ${clone}\n`, + ); + } + return python; +} + +function run(application, args, { cwd, env, quiet } = {}) { + return new Promise((resolve, reject) => { + const child = spawn(application, args, { cwd, env: env ?? process.env, stdio: ["ignore", "pipe", "pipe"] }); + let stdout = ""; + let stderr = ""; + child.stdout.on("data", (chunk) => { + stdout += chunk; + if (!quiet) process.stderr.write(chunk); + }); + child.stderr.on("data", (chunk) => { + stderr += chunk; + if (!quiet) process.stderr.write(chunk); + }); + child.once("error", reject); + child.once("close", (code) => resolve({ code, stdout, stderr })); + }); +} + +async function getJson(url, timeoutMs = 1500) { + try { + const response = await fetch(url, { signal: AbortSignal.timeout(timeoutMs) }); + return response.ok ? await response.json() : null; + } catch { + return null; + } +} + +let spawned = null; +process.on("exit", () => { + if (spawned && spawned.exitCode === null && spawned.signalCode === null) { + try { + spawned.kill("SIGTERM"); + } catch { + // already gone + } + } +}); + +// ---------------------------------------------------------------------------- main + +function listFiles() { + process.stdout.write(`Pinned manifest · captured from the Hub tree metadata at these commits\n\n`); + for (const [key, repo] of Object.entries(KEV_ASSETS)) { + const bytes = repo.files.reduce((sum, file) => sum + file.size, 0); + process.stdout.write(`${key === "run" ? "checkpoint" : "base "} ${repo.repo}@${repo.commit}\n ${repo.files.length} files, ${bytes.toLocaleString()} bytes\n`); + for (const file of repo.files) process.stdout.write(` ${String(file.size).padStart(13)} ${file.kind.padEnd(9)} ${file.hash.slice(0, 16)}… ${file.name}\n`); + process.stdout.write(`\n`); + } + process.stdout.write(`${SOURCES.length} sources, tried in this order: ${SOURCES.join(" → ")}\n`); + return 0; +} + +async function main() { + if (flag("--help") || flag("-h")) { + process.stdout.write(HELP); + return 0; + } + + const clone = path.resolve(arg("--clone", DEFAULT_CLONE)); + const port = Number(arg("--port", 8008)); + const run = arg("--run", RUN_ID); + const timeoutMs = Number(arg("--timeout", READY_MS / 1000)) * 1000; + if (!Number.isInteger(port) || port < 1 || port > 65535) throw configError(`--port must be 1..65535 (got ${arg("--port")})`); + if (!Number.isFinite(timeoutMs) || timeoutMs < 10_000) throw configError(`--timeout must be at least 10 seconds (got ${arg("--timeout")})`); + if (run !== RUN_ID) { + throw configError( + `--run ${run} is not the pinned checkpoint (${RUN_ID}).\n` + + `This launcher only fetches and verifies ${RUN_ID} and ${BASE_ID}; serving another run needs its\n` + + `own manifest. Fetch it by hand (experiments/kev-4b/fetch.py) and start the server directly.`, + ); + } + const requestedSources = arg("--sources", null)?.split(",").map((s) => s.trim()).filter(Boolean) ?? (arg("--source", null) ? [arg("--source")] : null); + if (requestedSources) { + const unknown = requestedSources.filter((source) => !SOURCES.includes(source)); + if (unknown.length) throw configError(`unknown source(s): ${unknown.join(", ")} (known: ${SOURCES.join(", ")})`); + SOURCES.splice(0, SOURCES.length, ...requestedSources); + } + + if (flag("--list-files")) return listFiles(); + if (flag("--patch-row-limit")) { + await patchRowLimit(clone); + return 0; + } + if (flag("--unpatch-row-limit")) { + await unpatchRowLimit(clone); + return 0; + } + + const rowLimit = arg("--row-limit", null); + if (rowLimit !== null) { + const tokens = Number(rowLimit); + if (!Number.isInteger(tokens) || tokens < 8192) throw configError(`--row-limit must be an integer >= 8192 (got ${rowLimit})`); + if (!(await patchApplied(clone))) { + throw configError( + `--row-limit ${tokens} needs the optional row-limit patch, which is not applied to ${clone}.\n\n` + + `The released server hard-codes the cap (kev/model.py) and accepts no flag or env var for it.\n` + + `Apply it explicitly first — it prints the diff and the revert command:\n\n jev-kev --patch-row-limit\n`, + ); + } + } + + const verifyOnly = flag("--verify-only"); + const downloadOnly = flag("--download-only"); + // Check the runtime before the (expensive) cache verification: a missing venv should fail in a + // second, not after re-hashing 9.5 GB. --verify-only and --download-only never need it. + const python = verifyOnly || downloadOnly ? null : await findPython(clone); + + const reports = []; + for (const key of ["run", "base"]) { + reports.push(await ensureRepo({ key, verifyOnly })); + } + const fetchedBytes = reports.reduce((sum, report) => sum + report.bytes, 0); + const mismatches = reports.flatMap((report) => report.mismatch.map((row) => ({ repo: report.repo, ...row }))); + if (mismatches.length) { + for (const row of mismatches) log(`[kev] BAD ${row.repo}/${row.file}: ${row.how} want=${row.want} got=${row.got}`); + const error = new Error(`${mismatches.length} file(s) do not match the published metadata and were left out of the cache`); + error.verify = true; + throw error; + } + log(`[kev] cache verified: ${reports.map((r) => `${r.repo} ${r.verified}/${r.verified + r.fetched.length}`).join(", ")} (${humanBytes(fetchedBytes)} fetched this run)`); + if (verifyOnly) { + log(`[kev] --verify-only: nothing downloaded, nothing started`); + return 0; + } + if (downloadOnly) { + log(`[kev] --download-only: assets ready, not starting anything`); + return 0; + } + + const serviceUrl = `http://127.0.0.1:${port}`; + const envLine = `TYPESAFE_BASE_URL=${serviceUrl} TYPESAFE_API_KEY=local`; + + const running = await getJson(`${serviceUrl}/v1/models`, 1500); + const served = running?.models?.find((model) => model.name === "kev-latest"); + if (served) { + if (served.run !== run) { + log(`[kev] ${serviceUrl} is already serving "${served.run}", not "${run}".`); + log(`[kev] stop that server first (Ctrl-C in its terminal), then run again.`); + return 2; + } + log(`[kev] already serving ${served.run} on ${serviceUrl}; reusing it`); + process.stdout.write(`${envLine}\n`); + return 0; + } + + const env = { ...process.env, HF_HUB_OFFLINE: "1" }; + if (rowLimit !== null) { + env.KEV_SERVE_MAX_STATE = String(Number(rowLimit)); + env.KEV_SERVE_MAX_BRANCH = String(Number(rowLimit)); + log(`[kev] row limit raised to ${rowLimit} for this run (optional local patch; published default is 8192)`); + } + const args = ["-m", "kev.serve", "--run", run, "--port", String(port)]; + log(`[kev] starting ${python} ${args.join(" ")}`); + spawned = spawn(python, args, { cwd: clone, env, stdio: ["ignore", "inherit", "inherit"] }); + + let spawnError = null; + spawned.once("error", (error) => { + spawnError = error; + }); + const deadline = Date.now() + timeoutMs; + let ready = null; + while (Date.now() < deadline) { + if (spawned.exitCode !== null || spawned.signalCode !== null) break; + ready = await getJson(`${serviceUrl}/v1/models`, 1500); + if (ready?.models?.length) break; + ready = null; + await sleep(500); + } + if (!ready) { + try { + spawned.kill("SIGTERM"); + } catch { + // already gone + } + if (spawnError) throw new Error(`Kev server failed to start: ${spawnError.message}`); + if (spawned.exitCode !== null) throw configError(`Kev server exited with code ${spawned.exitCode} before becoming ready (see its output above)`); + throw new Error(`Kev server did not become ready on ${serviceUrl} within ${timeoutMs / 1000}s`); + } + + const card = ready.models.find((model) => model.name === "kev-latest"); + if (card.run !== run) { + spawned.kill("SIGTERM"); + throw new Error(`the server came up serving "${card.run}", not "${run}" — refusing to hand out a URL for it`); + } + const shutdown = (signal) => { + log(`[kev] ${signal} — stopping the Kev server${spawned ? ` (pid ${spawned.pid})` : ""}`); + try { + spawned?.kill("SIGTERM"); + } catch { + // already gone + } + process.exit(0); + }; + process.on("SIGINT", () => shutdown("SIGINT")); + process.on("SIGTERM", () => shutdown("SIGTERM")); + spawned.on("exit", (code) => { + log(`[kev] server exited (code ${code})`); + }); + + log(`[kev] serving /v1/systemone on ${serviceUrl} (${card.run} · ${card.base} · ${card.device}/${card.backend} · ${card.dtype} · T=${Number(card.temperature).toFixed(4)})`); + log(`[kev] limits: needs a Python venv + MLX, ~18 GB idle and 36 GB GPU footprint under load at the raised limit;`); + log(`[kev] the released cap is 8192 tokens (state + one branch) — see experiments/kev-4b/README.md`); + process.stdout.write(`${envLine}\n`); + return null; // keep running +} + +main() + .then((code) => { + if (code !== null) process.exit(code); + }) + .catch((error) => { + log(`[kev] ${error.message}`); + process.exit(error.config ? 2 : error.verify ? 3 : 1); + }); From afd3306fcc5b6ca195819cc52da833711811704a Mon Sep 17 00:00:00 2001 From: ChenYCL Date: Thu, 24 Sep 2026 13:52:31 +0800 Subject: [PATCH 4/9] feat(jev-browser): per-backend goal_done threshold profiles MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit goal_done is the question that ends a run, and its value is not the same on the two local backends — 0.85 (the hosted default) is wrong for both. Measured, not guessed: - GGUF readout: over the 15 recorded runs in docs/local-backend-run-smoke.md §9, every run that reached the goal crossed 0.25 and no run that never reached it crossed 0.111, so the bar is the maximin geometric midpoint sqrt(0.111 × 0.273) = 0.174 — 1.57× from each side. At 0.85 only 7/7 reached goals were still recognised, with 3 false stucks. - Kev: the same 15 goals replayed against the checkpoint (46 readings, experiments/kev-4b) show the readout's band does not transfer — at 0.174 Kev calls four not-met pages a success (R5 0.341, p3 0.257, g1 0.195, g4 0.188), each stopping the run before the finishing action. Kev's own sides are 0.341/0.683, giving 0.482 (1.41× each); 0.482 scores 7 correct successes / 0 false / 0 false stucks against 0.85's 5 correct and 2 false stucks. thresholds.profile (auto | hosted | local-readout | kev) chooses the pair, and auto resolves it from the endpoint: a non-loopback baseUrl is hosted, a loopback one is classified by ONE GET /v1/models at run start, because only the endpoint knows which backend it is — a card naming a loaded checkpoint (run/base) is Kev, a name-only card is the GGUF readout. An unreachable or unrecognised loopback endpoint takes the HIGHEST bar and says why: a false stuck stops where you can see it, a false success is silent. A value any config layer set still wins per key (thresholds.configured), and a pinned profile is applied without any network call. The resolution is recorded where a run can be audited: controller writes run.json with the profile, the reason and the applied pair before the first step, so a run that dies later still names the bar it ran under; doctor prints the same as a `goal_done bar` line, with where the value was measured. doctor --home now loads that home's config, which is the whole point of the flag. --- skills/jev-browser/lib/config.mjs | 157 +++++++++++++++++++++++- skills/jev-browser/lib/controller.mjs | 7 +- skills/jev-browser/lib/doctor.mjs | 89 +++++++++++++- skills/jev-browser/lib/runner.mjs | 54 ++++++++- tests/e2e/cli.test.mjs | 20 +++- tests/unit/util-config.test.mjs | 165 +++++++++++++++++++++++++- 6 files changed, 482 insertions(+), 10 deletions(-) diff --git a/skills/jev-browser/lib/config.mjs b/skills/jev-browser/lib/config.mjs index 608779a..9a5acf0 100644 --- a/skills/jev-browser/lib/config.mjs +++ b/skills/jev-browser/lib/config.mjs @@ -3,6 +3,7 @@ import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { deepMerge, expandHome, isRecord, readJson, writeJson } from "./util.mjs"; +import { isLoopbackBaseUrl } from "./typesafe.mjs"; export const CONFIG_VERSION = 1; @@ -24,8 +25,10 @@ export const DEFAULTS = Object.freeze({ settleMs: 400, loadTimeoutMs: 15_000, thresholds: { + profile: "auto", // auto | hosted | local-readout | kev — which goal_done bar to use; auto resolves from the endpoint goalDone: 0.85, // noul probability required to declare success goalDoneFinal: 0.7, // looser check used only for the final verification pass + configured: [], // filled by loadConfig: the keys some layer set, so the run never overrides them blocker: 0.6, // choice probability of a non-"none" blocker that pauses for the user noChangeLimit: 3, // consecutive no-effect actions before giving up regressed: 0.6, // probability mass on "moved away" that triggers go_back @@ -58,6 +61,52 @@ export const DEFAULTS = Object.freeze({ }, }); +/** + * The `goal_done` bar for the *GGUF readout* backend. Scored as a termination rule over 15 recorded + * local runs (docs/local-backend-run-smoke.md §9), the question does separate the two populations — + * but the hosted value 0.85 does not: every run that reached the goal crossed 0.25, and no run that + * never reached it crossed 0.111. The value below is the geometric midpoint √(0.111 × 0.273) of + * the two observed sides, which maximises the minimum relative margin on both (1.57× each). + */ +export const LOCAL_GOAL_DONE = 0.174; + +/** + * The same measurement replayed against the Kev checkpoint (experiments/kev-4b/README.md, same 15 + * goals, same scoring, 46 readings): the readout's band does NOT transfer — at 0.174 Kev calls four + * not-met pages a success. Kev's own sides are 0.341 (worst not-met) and 0.683 (lowest met), so the + * bar below is again the geometric midpoint √(0.341 × 0.683), which sits 1.41× from each side. + */ +export const KEV_GOAL_DONE = 0.482; + +/** + * A named bar per backend. `auto` (the default, see thresholds.profile) picks one of these from the + * endpoint; an explicit name pins it. `hosted` is the measured hosted-Jev pair; the two local ones + * are per-backend measurements and are NOT interchangeable — that is the whole reason this is a map. + */ +export const THRESHOLD_PROFILES = Object.freeze({ + hosted: Object.freeze({ goalDone: 0.85, goalDoneFinal: 0.7 }), + "local-readout": Object.freeze({ goalDone: LOCAL_GOAL_DONE, goalDoneFinal: LOCAL_GOAL_DONE }), + kev: Object.freeze({ goalDone: KEV_GOAL_DONE, goalDoneFinal: KEV_GOAL_DONE }), +}); + +/** Where each profile's pair was measured — printed by doctor so the bar never looks arbitrary. */ +export const PROFILE_MEASURED = Object.freeze({ + hosted: "the shipped Jev default", + "local-readout": "docs/local-backend-run-smoke.md §9 (band 0.111 – 0.273)", + kev: "experiments/kev-4b/README.md (band 0.341 – 0.683)", +}); + +export const PROFILE_NAMES = Object.freeze(["auto", ...Object.keys(THRESHOLD_PROFILES)]); + +/** + * What `auto` falls back to when a loopback endpoint cannot be classified (unreachable, or a card + * this build does not recognise): the HIGHEST bar. A false `stuck` is visible and recoverable — the + * run stops and says so — while a false success is silent, so an unknown backend never gets the + * benefit of the doubt. + */ +export const FALLBACK_PROFILE = "kev"; +export const FALLBACK_REASON = "loopback endpoint not classified (unreachable or unrecognised) → the highest bar; a false stuck is visible, a false success is not"; + export const ENV = Object.freeze({ apiKey: "TYPESAFE_API_KEY", baseUrl: "TYPESAFE_BASE_URL", @@ -97,6 +146,16 @@ function fromEnv(env) { return out; } +/** True when a config patch explicitly provides a (possibly nested) key. */ +function hasKeyPath(node, keyPath) { + let current = node; + for (const part of keyPath.split(".")) { + if (!isRecord(current) || !(part in current)) return false; + current = current[part]; + } + return current !== undefined; +} + /** * Load the effective configuration. * @param {object} options @@ -135,15 +194,43 @@ export async function loadConfig({ flags = {}, env = process.env, cwd = process. sources.push({ kind: "flags", keys: Object.keys(flagPatch) }); } + // Which bar is in force depends on the backend, and only the run knows which backend it is + // talking to (one GET /v1/models — see thresholdProfile/classifyModelsCard and runner.mjs). What + // loadConfig can settle without a network call is a *pinned* profile: `thresholds.profile` set to + // a concrete name means its values, for any threshold no layer configured. + const layers = [user, project, envPatch, flagPatch]; + const configuredThresholds = ["goalDone", "goalDoneFinal"].filter((key) => layers.some((layer) => hasKeyPath(layer, `thresholds.${key}`))); + if (!PROFILE_NAMES.includes(merged.thresholds.profile)) { + throw new Error(`Unknown thresholds.profile "${merged.thresholds.profile}" (expected ${PROFILE_NAMES.join(", ")})`); + } + if (merged.thresholds.profile !== "auto") { + const profile = THRESHOLD_PROFILES[merged.thresholds.profile]; + const applied = {}; + for (const key of ["goalDone", "goalDoneFinal"]) { + if (configuredThresholds.includes(key)) continue; + merged.thresholds[key] = profile[key]; + applied[`thresholds.${key}`] = profile[key]; + } + if (Object.keys(applied).length) { + sources.push({ + kind: "thresholds-profile", + keys: Object.keys(applied), + values: applied, + reason: `thresholds.profile=${merged.thresholds.profile} (pinned by configuration)`, + }); + } + } + merged.journalDir = expandHome(merged.journalDir, home); merged.chrome.userDataDir = expandHome(merged.chrome.userDataDir, home); if (!["ego", "chrome", "safari"].includes(merged.backend)) { throw new Error(`Unknown backend "${merged.backend}" (expected ego, chrome or safari)`); } + merged.thresholds.configured = configuredThresholds; for (const key of ["maxSteps", "budgetUsd", "maxMs", "timeoutMs"]) { if (!Number.isFinite(merged[key]) || merged[key] <= 0) throw new Error(`Config ${key} must be a positive number`); } - return { config: merged, sources, paths: { userFile, projectFile } }; + return { config: merged, sources, paths: { userFile, projectFile }, configuredThresholds }; } /** Persist a patch into the user config file (0600, never printed). */ @@ -177,6 +264,74 @@ export function describeConfig(config) { return copy; } +/** + * Which backend a served `/v1/models` card says it is, from the card the endpoint itself returns. + * + * Pure on purpose: the network call belongs to the run (runner.mjs) and to doctor, so this stays + * unit-testable and cannot make `loadConfig` depend on a live endpoint. + * + * - Kev (and hosted Jev) cards carry `run` and `base` beside `name` — a checkpoint the runtime + * loads, which is what makes Kev's scale so different from a first-token readout's. + * - The GGUF launcher's card carries only `name` (the registry id), description and modalities. + * + * @returns {{profile: string|null, kind: string, reason: string, names: string[]}} + */ +export function classifyModelsCard(cards) { + const list = cards?.models ?? cards?.data ?? []; + const names = list.map((card) => card?.name ?? card?.id).filter(Boolean); + if (list.length === 0) return { profile: null, kind: "empty", reason: "the endpoint returned no model cards", names: [] }; + const loaded = list.find((card) => card?.run || card?.base); + if (loaded) { + const bits = [loaded.run && `run=${loaded.run}`, loaded.base && `base=${loaded.base}`].filter(Boolean).join(" "); + return { profile: "kev", kind: "kev", reason: `model card "${loaded.name ?? names[0]}" names a loaded checkpoint (${bits})`, names }; + } + const readout = list.find((card) => card?.name || card?.id); + if (readout) return { profile: "local-readout", kind: "readout", reason: `model card "${readout.name ?? readout.id}" is name-only (the GGUF readout)`, names }; + return { profile: null, kind: "unknown", reason: "model cards carry neither a name nor a checkpoint", names }; +} + +/** + * Which `goal_done` bar the effective configuration uses, and why. + * + * Order: an explicitly pinned `thresholds.profile` first (it never needs the endpoint), then the + * backend scale — a non-loopback baseUrl is hosted, a loopback one is whatever `classification` + * says it is — and finally, for a loopback endpoint nobody could classify, the highest bar. + * + * @param {object} config effective configuration + * @param {{classification?: {profile: string|null, reason: string}|null}} [options] + */ +export function thresholdProfile(config, { classification = null } = {}) { + const pinned = config.thresholds.profile ?? "auto"; + const loopback = isLoopbackBaseUrl(config.baseUrl); + let profile; + let reason; + if (pinned !== "auto") { + profile = pinned; + reason = `thresholds.profile=${pinned} (pinned by configuration)`; + } else if (!loopback) { + profile = "hosted"; + reason = `baseUrl ${config.baseUrl} is not loopback`; + } else if (classification?.profile) { + profile = classification.profile; + reason = classification.reason; + } else { + profile = FALLBACK_PROFILE; + reason = classification?.reason ? `${FALLBACK_REASON} (${classification.reason})` : FALLBACK_REASON; + } + const defaults = THRESHOLD_PROFILES[profile]; + const custom = ["goalDone", "goalDoneFinal"].filter((key) => config.thresholds[key] !== defaults[key]); + return { + profile, + pinned, + reason, + measured: PROFILE_MEASURED[profile], + goalDone: config.thresholds.goalDone, + goalDoneFinal: config.thresholds.goalDoneFinal, + defaults, + custom, + }; +} + /** Parse "a.b.c" = value assignments from the CLI into a nested patch. */ export function patchFromKeyPath(keyPath, rawValue) { let value = rawValue; diff --git a/skills/jev-browser/lib/controller.mjs b/skills/jev-browser/lib/controller.mjs index 8305702..71824ba 100644 --- a/skills/jev-browser/lib/controller.mjs +++ b/skills/jev-browser/lib/controller.mjs @@ -23,7 +23,7 @@ export const STATUS = Object.freeze({ * handOff?() -> object|void finish({success, keep}) -> void * historyLength?() -> number describe() -> {backend, ...ids for resume} */ -export async function runGoal({ driver, client, config, goal, startUrl, inputs = {}, secrets = {}, log = () => {}, onStep, screenshotPath, stepScreenshotsDir, runId = makeRunId() }) { +export async function runGoal({ driver, client, config, goal, startUrl, inputs = {}, secrets = {}, log = () => {}, onStep, screenshotPath, stepScreenshotsDir, runId = makeRunId(), thresholds = null }) { const startedAt = Date.now(); const thr = config.thresholds; const allInputs = { ...inputs, ...secrets }; @@ -41,7 +41,10 @@ export async function runGoal({ driver, client, config, goal, startUrl, inputs = // blocked: (state, action) pairs that produced no change or an error. // tried: (state, action) pairs already executed once; untried edges are preferred (edge memory). const memory = { blocked: new Set(), tried: new Set(), visits: new Map(), hashes: new Map(), history: [], noChangeStreak: 0 }; - const result = { runId, status: STATUS.error, goal, backend: driver.name, steps: 0, startedAt: nowIso(), journalDir }; + const result = { runId, status: STATUS.error, goal, backend: driver.name, steps: 0, startedAt: nowIso(), journalDir, ...(thresholds ? { thresholds } : {}) }; + // The bar this run is using is decided from the endpoint, so write it down before the first step: + // a run that dies later must still say which profile it ran under. + if (journalDir) await writeJson(path.join(journalDir, "run.json"), redact({ ...result, status: "running" }, secretValues)); let obs = null; let previous = null; let lastAction = null; diff --git a/skills/jev-browser/lib/doctor.mjs b/skills/jev-browser/lib/doctor.mjs index a75f65d..9c1bed3 100644 --- a/skills/jev-browser/lib/doctor.mjs +++ b/skills/jev-browser/lib/doctor.mjs @@ -6,8 +6,9 @@ import path from "node:path"; import { promisify } from "node:util"; import { findChromeExecutable } from "./backends/chrome.mjs"; import { SAFARI_ENABLE_HINT } from "./backends/safari.mjs"; -import { describeConfig, userConfigPath } from "./config.mjs"; -import { TypeSafeClient } from "./typesafe.mjs"; +import { classifyModelsCard, describeConfig, thresholdProfile, userConfigPath } from "./config.mjs"; +import { localStatus } from "./local.mjs"; +import { TypeSafeClient, isLoopbackBaseUrl } from "./typesafe.mjs"; import { installTargets } from "./install.mjs"; const run = promisify(execFile); @@ -30,6 +31,35 @@ async function version(cmd, args = ["--version"]) { } } +/** The port a loopback baseUrl points at, or null. */ +function loopbackPort(baseUrl) { + if (!isLoopbackBaseUrl(baseUrl)) return null; + try { + const url = new URL(String(baseUrl)); + return url.port ? Number(url.port) : url.protocol === "https:" ? 443 : 80; + } catch { + return null; + } +} + +/** + * Which local backend answers on `config.baseUrl`. Reuses the models call doctor already made for + * the `typesafe api` line; when there was none (no api key, or offline) and the endpoint is + * loopback it asks /v1/models directly, because the `goal_done` bar depends on the answer. + */ +async function classifyEndpoint({ config, live, models }) { + if (models) return classifyModelsCard(models); + if (!live || config.thresholds.profile !== "auto" || !isLoopbackBaseUrl(config.baseUrl)) return null; + const url = `${String(config.baseUrl).replace(/\/+$/, "")}/v1/models`; + try { + const response = await fetch(url, { signal: AbortSignal.timeout(2500) }); + if (!response.ok) return { profile: null, kind: "http-error", reason: `${url} answered HTTP ${response.status}`, names: [] }; + return classifyModelsCard(await response.json()); + } catch (error) { + return { profile: null, kind: "unreachable", reason: `${url} unreachable (${error.message})`, names: [] }; + } +} + export async function doctor({ config, sources, home = os.homedir(), skillDir, live = true } = {}) { const checks = []; const add = (name, ok, detail, hint) => checks.push({ name, status: ok === null ? "warn" : ok ? "ok" : "fail", detail, ...(hint ? { hint } : {}) }); @@ -38,16 +68,69 @@ export async function doctor({ config, sources, home = os.homedir(), skillDir, l add("node", major >= 22, `node ${process.version}`, major >= 22 ? undefined : "Node 22+ is required (global fetch + WebSocket)"); add("api key", Boolean(config.apiKey), config.apiKey ? `present (${sources.some((s) => s.kind === "env" && s.keys.includes("apiKey")) ? "env TYPESAFE_API_KEY" : "config file"})` : "missing", config.apiKey ? undefined : "export TYPESAFE_API_KEY=... or run: jev-browser config set-key --from-env"); + let models = null; if (config.apiKey && live) { try { const client = new TypeSafeClient({ apiKey: config.apiKey, baseUrl: config.baseUrl, timeoutMs: 10_000, maxRetries: 0 }); - const models = await client.models(); + models = await client.models(); add("typesafe api", true, `${config.baseUrl} → models: ${(models.models ?? []).map((m) => m.name).join(", ")}; configured model: ${config.model}`); } catch (error) { add("typesafe api", false, error.message, "check the key, network, or baseUrl"); } } + // The fully local backend is optional: report it, never fail on it, never throw. The registry + // is imported dynamically so a half-edited lib/local-models.json shows up as a line here + // instead of breaking doctor. + const endpoint = await classifyEndpoint({ config, live, models }); + try { + const { localStatus } = await import("./local.mjs"); + // Probe the port the *run* uses, so a Kev endpoint on 8008 is reported rather than the GGUF + // default of 8092 sitting idle beside it. + const port = loopbackPort(config.baseUrl) ?? undefined; + const local = await localStatus({ home, ...(port ? { port } : {}) }); + const ready = Boolean(local.llamaServer && local.bytes > 0); + const size = local.bytes > 0 ? `${Math.round(local.bytes / 1024 / 1024)} MiB` : `not downloaded (${Math.round(local.expectedBytes / 1024 / 1024)} MiB expected)`; + const livePort = local.serving + ? `127.0.0.1:${local.port} up${local.servingModel ? ` (serving ${local.servingModel})` : ""}` + : local.endpoint + ? `127.0.0.1:${local.port} up (${local.endpoint.kind}${local.endpoint.run ? ` ${local.endpoint.run}` : ` "${local.endpoint.name}"`})` + : `127.0.0.1:${local.port} not running`; + const detail = [ + local.llamaServer ? `llama-server ${local.llamaServer}` : "llama-server not found", + `${local.id}${local.label ? ` "${local.label}"` : ""} ${size}`, + livePort, + ].join(" · "); + const hint = local.endpoint + ? `the port your baseUrl uses is serving a ${local.endpoint.kind}, not this registry entry — see the goal_done bar line` + : !local.llamaServer + ? "brew install llama.cpp" + : local.bytes === 0 + ? `download it: node /bin/jev-local.mjs --download-only (registry: ${local.registry.path})` + : local.serving + ? undefined + : "start it: node /bin/jev-local.mjs (--list-models shows the registry)"; + add("local model", local.endpoint || ready ? true : null, detail, hint); + } catch (error) { + add("local model", null, `registry problem: ${error.message}`, "fix skills/jev-browser/lib/local-models.json, or run --list-models with a working file"); + } + + // The bar depends on which backend answers. Print the pair a RUN would use: a value some layer + // configured, else the resolved profile's — and name which keys were pinned, so "differs from the + // profile" is never confused with "you configured it". + const bar = thresholdProfile(config, { classification: endpoint }); + const pinnedKeys = config.thresholds.configured ?? []; + const effective = Object.fromEntries( + ["goalDone", "goalDoneFinal"].map((key) => [key, pinnedKeys.includes(key) ? config.thresholds[key] : bar.defaults[key]]), + ); + add( + "goal_done bar", + true, + `${effective.goalDone} per step / ${effective.goalDoneFinal} final — ${bar.profile} profile for ${config.baseUrl}` + + `${pinnedKeys.length ? ` (${pinnedKeys.map((key) => `thresholds.${key}`).join(", ")} pinned)` : ""}`, + `${bar.reason} · bar measured in ${bar.measured}${bar.pinned !== "auto" ? `; unset the pin with: config unset thresholds.profile` : ""}`, + ); + const ego = await version("ego-browser"); const egoApp = process.platform === "darwin" ? await exists("/Applications/ego lite.app") : null; add("ego-browser cli", Boolean(ego), ego ? `ego-browser ${ego}` : "not found on PATH", ego ? undefined : "install ego lite (https://ego.dev) and its ego-browser CLI, or use --backend chrome"); diff --git a/skills/jev-browser/lib/runner.mjs b/skills/jev-browser/lib/runner.mjs index 283e05b..d8ff9fa 100644 --- a/skills/jev-browser/lib/runner.mjs +++ b/skills/jev-browser/lib/runner.mjs @@ -1,14 +1,62 @@ // Binds a job (goal + inputs + mode) to a backend driver and the controller. import path from "node:path"; +import { classifyModelsCard, thresholdProfile } from "./config.mjs"; import { runGoal } from "./controller.mjs"; import { buildStepQuestions } from "./questions.mjs"; import { keywordsFor, pageStateForModel } from "./observe.mjs"; -import { TypeSafeClient } from "./typesafe.mjs"; +import { TypeSafeClient, isLoopbackBaseUrl, pricePerMtokFor } from "./typesafe.mjs"; import { JsonlWriter, ensureDir, runId as makeRunId } from "./util.mjs"; /** Estimate tokens for a JSON payload (rough: 1 token ≈ 3.6 chars of JSON). */ export const estimateTokens = (value) => Math.ceil(JSON.stringify(value).length / 3.6); +/** + * Resolve the `goal_done` bar for this run and apply it to `config.thresholds`. + * + * `auto` needs one GET /v1/models to tell a Kev endpoint from the GGUF readout — the two have + * measured bars 0.482 and 0.174 apart (experiments/kev-4b/README.md) and the hosted default (0.85) + * is wrong for both. This is the only place that asks, so it lives at run start rather than in + * loadConfig: no key, no endpoint, no network call for a hosted or pinned configuration. + * + * A value any configuration layer set always wins; otherwise the profile's pair is applied. + * An unreachable or unrecognised loopback endpoint resolves to the HIGHEST bar (FALLBACK_PROFILE). + * + * @returns {Promise<{profile: string, reason: string, goalDone: number, goalDoneFinal: number, source: string}>} + */ +export async function resolveThresholds({ config, client, log = () => {} }) { + const configuredThresholds = config.thresholds.configured ?? []; + const auto = (config.thresholds.profile ?? "auto") === "auto"; + const loopback = isLoopbackBaseUrl(config.baseUrl); + let classification = null; + if (auto && loopback) { + try { + classification = classifyModelsCard(await client.models()); + } catch (error) { + classification = { profile: null, reason: `the endpoint did not answer GET /v1/models (${error?.code ?? error.message})` }; + } + } + const bar = thresholdProfile(config, { classification }); + const applied = {}; + for (const key of ["goalDone", "goalDoneFinal"]) { + if (configuredThresholds.includes(key)) continue; + if (config.thresholds[key] === bar.defaults[key]) continue; + config.thresholds[key] = bar.defaults[key]; + applied[key] = bar.defaults[key]; + } + const record = { + profile: bar.profile, + reason: bar.reason, + goalDone: config.thresholds.goalDone, + goalDoneFinal: config.thresholds.goalDoneFinal, + applied, + custom: bar.custom, + classification: classification ? { kind: classification.kind, profile: classification.profile, reason: classification.reason, names: classification.names } : null, + }; + const configured = configuredThresholds.length ? `, configured: ${configuredThresholds.map((key) => `thresholds.${key}`).join(", ")}` : ""; + log(`thresholds: goal_done >= ${record.goalDone} per step, ${record.goalDoneFinal} final — ${bar.profile} profile${configured ? " (pinned per key)" : ""} — ${bar.reason}`); + return record; +} + export async function executeJob({ config, job, log = () => {} }) { if (config.backend === "ego") { const { runEgoJob } = await import("./backends/ego.mjs"); @@ -42,7 +90,7 @@ export async function runWithDriver({ driver, config, job, log = () => {} }) { const observation = await driver.observe({ keywords: keywordsFor(job.goal, { ...(job.inputs ?? {}), ...(job.secrets ?? {}) }, Object.keys(job.secrets ?? {})) }); const { state, questions, meta } = buildStepQuestions({ obs: observation, goal: job.goal, inputs: { ...(job.inputs ?? {}), ...(job.secrets ?? {}) }, secretKeys: Object.keys(job.secrets ?? {}) }); await driver.finish({ success: true, keep: job.keep ?? false }); - return { mode, backend: driver.name, state, questions, meta, estimatedInputTokens: estimateTokens({ state, questions }), estimatedCostUsd: (estimateTokens({ state, questions }) * config.pricePerMtok) / 1e6 }; + return { mode, backend: driver.name, state, questions, meta, estimatedInputTokens: estimateTokens({ state, questions }), estimatedCostUsd: (estimateTokens({ state, questions }) * pricePerMtokFor(config.baseUrl, config.pricePerMtok)) / 1e6 }; } if (!job.goal) throw new Error("a goal is required"); const runId = job.runId ?? makeRunId(); @@ -61,6 +109,7 @@ export async function runWithDriver({ driver, config, job, log = () => {} }) { pricePerMtok: config.pricePerMtok, onRequest: requestJournal ? (row) => requestJournal.append(row) : undefined, }); + const thresholds = await resolveThresholds({ config, client, log }); const result = await runGoal({ driver, client, @@ -73,6 +122,7 @@ export async function runWithDriver({ driver, config, job, log = () => {} }) { screenshotPath: job.screenshotPath, stepScreenshotsDir: job.stepScreenshotsDir, runId, + thresholds, }); await requestJournal?.flush(); return result; diff --git a/tests/e2e/cli.test.mjs b/tests/e2e/cli.test.mjs index f31f41f..9f3e2a4 100644 --- a/tests/e2e/cli.test.mjs +++ b/tests/e2e/cli.test.mjs @@ -2,7 +2,8 @@ import test from "node:test"; import assert from "node:assert/strict"; import { execFile } from "node:child_process"; import { promisify } from "node:util"; -import { BIN, createContext, hasChrome } from "../helpers/env.mjs"; +import { BIN, createContext, hasChrome, LIVE } from "../helpers/env.mjs"; +import { KEV_GOAL_DONE, LOCAL_GOAL_DONE } from "../../skills/jev-browser/lib/config.mjs"; const run = promisify(execFile); const available = await hasChrome(); @@ -36,6 +37,23 @@ test("CLI: run / observe / judge / pick / doctor / config round-trip (chrome hea const doctor = await run(process.execPath, [BIN, "doctor", "--json", "--offline"], { env }).catch((e) => e); const report = JSON.parse(doctor.stdout); assert.ok(report.checks.some((c) => c.name === "chrome" && c.status === "ok")); + // createContext points baseUrl at the local mock (loopback) unless the suite runs live Jev. + const bar = report.checks.find((c) => c.name === "goal_done bar"); + assert.ok(bar, "doctor reports the effective goal_done bar"); + assert.ok(bar.detail.includes(LIVE ? "hosted profile" : "kev profile"), bar.detail); + // --offline cannot classify the endpoint, so a loopback baseUrl gets the HIGHEST bar and says why. + assert.ok(bar.detail.includes(LIVE ? "0.85" : String(KEV_GOAL_DONE)), bar.detail); + if (!LIVE) assert.match(bar.hint ?? "", /not classified/, "an unclassified endpoint says so instead of guessing low"); + + if (!LIVE) { + // With the endpoint reachable, the same call classifies it: the mock's card is name-only, i.e. + // the GGUF readout, so the bar drops to the measured readout value. + const online = JSON.parse((await run(process.execPath, [BIN, "doctor", "--json"], { env })).stdout); + const onlineBar = online.checks.find((c) => c.name === "goal_done bar"); + assert.ok(onlineBar.detail.includes("local-readout profile"), onlineBar.detail); + assert.ok(onlineBar.detail.includes(String(LOCAL_GOAL_DONE)), onlineBar.detail); + assert.match(onlineBar.hint ?? "", /local-backend-run-smoke\.md §9/, "a lowered bar says where it was measured"); + } await run(process.execPath, [BIN, "config", "set", "thresholds.goalDone", "0.9", "--home", ctx.home], { env }); const show = await run(process.execPath, [BIN, "config", "show", "--home", ctx.home], { env }); diff --git a/tests/unit/util-config.test.mjs b/tests/unit/util-config.test.mjs index 9d93590..056499e 100644 --- a/tests/unit/util-config.test.mjs +++ b/tests/unit/util-config.test.mjs @@ -4,7 +4,8 @@ import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { deepMerge, extractQuoted, parseKeyValue, rankProbabilities, redact, truncate } from "../../skills/jev-browser/lib/util.mjs"; -import { DEFAULTS, loadConfig, saveUserConfig, unsetUserConfig, patchFromKeyPath, describeConfig } from "../../skills/jev-browser/lib/config.mjs"; +import { DEFAULTS, FALLBACK_PROFILE, KEV_GOAL_DONE, LOCAL_GOAL_DONE, PROFILE_MEASURED, THRESHOLD_PROFILES, classifyModelsCard, loadConfig, saveUserConfig, unsetUserConfig, patchFromKeyPath, describeConfig, thresholdProfile } from "../../skills/jev-browser/lib/config.mjs"; +import { resolveThresholds } from "../../skills/jev-browser/lib/runner.mjs"; test("parseKeyValue splits on the first equals sign", () => { assert.deepEqual(parseKeyValue("query=a=b"), ["query", "a=b"]); @@ -67,3 +68,165 @@ test("config rejects unknown backends and non-positive budgets", async () => { await assert.rejects(loadConfig({ home: os.tmpdir(), cwd: os.tmpdir(), env: { JEV_BROWSER_BACKEND: "firefox" } }), /Unknown backend/); await assert.rejects(loadConfig({ home: os.tmpdir(), cwd: os.tmpdir(), env: { JEV_BROWSER_MAX_STEPS: "0" } }), /maxSteps/); }); + +test("a loopback base URL no longer moves the bar at load time; the run resolves it", async () => { + const home = await fs.mkdtemp(path.join(os.tmpdir(), "jev-bar-home-")); + const cwd = await fs.mkdtemp(path.join(os.tmpdir(), "jev-bar-cwd-")); + try { + const hosted = await loadConfig({ home, cwd, env: { TYPESAFE_API_KEY: "k" } }); + assert.equal(hosted.config.thresholds.goalDone, 0.85); + assert.equal(hosted.config.thresholds.goalDoneFinal, 0.7); + assert.equal(hosted.config.thresholds.profile, "auto"); + assert.deepEqual(hosted.sources.map((s) => s.kind), ["env"]); + for (const baseUrl of ["http://127.0.0.1:8092", "http://localhost:8092", "http://[::1]:8092"]) { + const local = await loadConfig({ home, cwd, env: { TYPESAFE_API_KEY: "k", TYPESAFE_BASE_URL: baseUrl } }); + // Which backend answers is not knowable without asking it, so loadConfig must not guess: + // the values stay at the shipped pair and resolveThresholds moves them at run start. + assert.equal(local.config.thresholds.goalDone, 0.85, baseUrl); + assert.equal(local.config.thresholds.goalDoneFinal, 0.7, baseUrl); + assert.deepEqual(local.configuredThresholds, [], "nothing was configured"); + assert.ok(!local.sources.some((s) => s.kind === "local-model"), "no bar was chosen at load time"); + assert.equal(local.config.thresholds.blocker, DEFAULTS.thresholds.blocker, "the other thresholds never move"); + } + } finally { + await fs.rm(home, { recursive: true, force: true }); + await fs.rm(cwd, { recursive: true, force: true }); + } +}); + +test("each profile name maps to its measured pair, and a pinned profile needs no endpoint", async () => { + assert.deepEqual(Object.keys(THRESHOLD_PROFILES).sort(), ["hosted", "kev", "local-readout"]); + assert.deepEqual(THRESHOLD_PROFILES.hosted, { goalDone: 0.85, goalDoneFinal: 0.7 }); + assert.deepEqual(THRESHOLD_PROFILES["local-readout"], { goalDone: LOCAL_GOAL_DONE, goalDoneFinal: LOCAL_GOAL_DONE }); + assert.deepEqual(THRESHOLD_PROFILES.kev, { goalDone: KEV_GOAL_DONE, goalDoneFinal: KEV_GOAL_DONE }); + for (const name of Object.keys(THRESHOLD_PROFILES)) assert.ok(PROFILE_MEASURED[name], `${name} says where it was measured`); + + const home = await fs.mkdtemp(path.join(os.tmpdir(), "jev-bar-home-")); + const cwd = await fs.mkdtemp(path.join(os.tmpdir(), "jev-bar-cwd-")); + try { + const pinned = await loadConfig({ home, cwd, env: { TYPESAFE_API_KEY: "k", TYPESAFE_BASE_URL: "http://127.0.0.1:8008" }, flags: { thresholds: { profile: "kev" } } }); + assert.equal(pinned.config.thresholds.goalDone, KEV_GOAL_DONE, "a pinned profile is applied at load time"); + assert.equal(pinned.config.thresholds.goalDoneFinal, KEV_GOAL_DONE); + assert.ok(pinned.sources.some((s) => s.kind === "thresholds-profile"), "the pin is recorded as a source"); + const bar = thresholdProfile(pinned.config); + assert.equal(bar.profile, "kev"); + assert.equal(bar.pinned, "kev"); + assert.ok(bar.reason.includes("pinned by configuration")); + assert.deepEqual(bar.custom, [], "nothing was overridden per key"); + await assert.rejects(loadConfig({ home, cwd, env: { TYPESAFE_API_KEY: "k" }, flags: { thresholds: { profile: "medium" } } }), /Unknown thresholds\.profile/); + } finally { + await fs.rm(home, { recursive: true, force: true }); + await fs.rm(cwd, { recursive: true, force: true }); + } +}); + +test("classifyModelsCard reads the backend off the served card", () => { + const kev = classifyModelsCard({ models: [{ name: "kev-latest", run: "jaredpalmer/kev-4b", base: "Qwen/Qwen3.5-4B-Base", temperature: 2.14 }] }); + assert.equal(kev.profile, "kev"); + assert.match(kev.reason, /run=jaredpalmer\/kev-4b/); + const readout = classifyModelsCard({ models: [{ name: "qwen3.5-4b-q4-k-m", description: "Local llama.cpp first-token readout backend (experimental)" }] }); + assert.equal(readout.profile, "local-readout"); + assert.match(readout.reason, /name-only/); + assert.equal(classifyModelsCard({ data: [{ id: "x", run: "owner/run" }] }).profile, "kev", "the OpenAI shape works too"); + assert.equal(classifyModelsCard({ models: [] }).profile, null); + assert.equal(classifyModelsCard(null).profile, null); + assert.equal(classifyModelsCard({}).kind, "empty"); +}); + +test("auto resolves loopback by the endpoint and falls back high, never low", () => { + const base = { baseUrl: "http://127.0.0.1:8008", thresholds: { profile: "auto", goalDone: 0.85, goalDoneFinal: 0.7, configured: [] } }; + const kev = thresholdProfile(base, { classification: classifyModelsCard({ models: [{ name: "kev-latest", run: "jaredpalmer/kev-4b", base: "Qwen/Qwen3.5-4B-Base" }] }) }); + assert.equal(kev.profile, "kev"); + assert.deepEqual(kev.defaults, { goalDone: KEV_GOAL_DONE, goalDoneFinal: KEV_GOAL_DONE }); + const readout = thresholdProfile({ ...base, baseUrl: "http://127.0.0.1:8092" }, { classification: classifyModelsCard({ models: [{ name: "qwen3.5-4b-q4-k-m" }] }) }); + assert.equal(readout.profile, "local-readout"); + assert.deepEqual(readout.defaults, { goalDone: LOCAL_GOAL_DONE, goalDoneFinal: LOCAL_GOAL_DONE }); + const hosted = thresholdProfile({ ...base, baseUrl: "https://api.typesafe.ai" }); + assert.equal(hosted.profile, "hosted", "non-loopback needs no endpoint call"); + assert.deepEqual(hosted.defaults, { goalDone: 0.85, goalDoneFinal: 0.7 }); + const unreachable = thresholdProfile(base, { classification: { profile: null, reason: "the endpoint did not answer GET /v1/models" } }); + assert.equal(unreachable.profile, FALLBACK_PROFILE, "an unclassified loopback endpoint gets the highest bar"); + assert.ok(unreachable.defaults.goalDone > readout.defaults.goalDone, "higher than the readout's, never lower"); + assert.match(unreachable.reason, /false stuck is visible/); +}); + +test("each local bar is the maximin midpoint of its own measured band", () => { + const bands = [ + { bar: LOCAL_GOAL_DONE, worstMiss: 0.111, lowestHit: 0.273, model: "the GGUF readout" }, + { bar: KEV_GOAL_DONE, worstMiss: 0.341, lowestHit: 0.683, model: "the Kev 4B checkpoint" }, + ]; + for (const { bar, worstMiss, lowestHit, model } of bands) { + assert.ok(bar > worstMiss && bar < lowestHit, `${model}: sits inside the observed gap`); + const lowerMargin = bar / worstMiss; + const upperMargin = lowestHit / bar; + assert.ok(Math.abs(lowerMargin - upperMargin) < 0.01, `${model}: relative margins differ (${lowerMargin} vs ${upperMargin})`); + } + assert.ok(KEV_GOAL_DONE > LOCAL_GOAL_DONE, "the two bands do not overlap: one bar cannot serve both backends"); +}); + +test("an explicit goal_done bar wins over every profile", async () => { + const home = await fs.mkdtemp(path.join(os.tmpdir(), "jev-bar-home-")); + const cwd = await fs.mkdtemp(path.join(os.tmpdir(), "jev-bar-cwd-")); + const env = { TYPESAFE_API_KEY: "k", TYPESAFE_BASE_URL: "http://127.0.0.1:8008" }; + try { + await saveUserConfig({ thresholds: { goalDone: 0.9 } }, { home }); + const fromFile = await loadConfig({ home, cwd, env }); + assert.equal(fromFile.config.thresholds.goalDone, 0.9, "the user file wins"); + assert.deepEqual(fromFile.configuredThresholds, ["goalDone"], "and is recorded as configured"); + const fromFlag = await loadConfig({ home, cwd, env, flags: { thresholds: { goalDoneFinal: 0.6 } } }); + assert.equal(fromFlag.config.thresholds.goalDoneFinal, 0.6, "flags win over both"); + assert.equal(fromFlag.config.thresholds.goalDone, 0.9); + assert.deepEqual(fromFlag.configuredThresholds.sort(), ["goalDone", "goalDoneFinal"]); + // A configured value also survives a pinned profile. + await saveUserConfig({ thresholds: { profile: "kev" } }, { home }); + const pinned = await loadConfig({ home, cwd, env }); + assert.equal(pinned.config.thresholds.goalDone, 0.9, "the configured value still wins"); + assert.equal(pinned.config.thresholds.goalDoneFinal, KEV_GOAL_DONE, "the key nobody set follows the profile"); + const profile = thresholdProfile(pinned.config); + assert.deepEqual(profile.custom, ["goalDone"], "doctor can name the values that differ from the profile"); + } finally { + await fs.rm(home, { recursive: true, force: true }); + await fs.rm(cwd, { recursive: true, force: true }); + } +}); + +test("resolveThresholds applies the endpoint's bar at run start and never overrides a configured one", async () => { + const config = (baseUrl, thresholds = {}) => ({ + baseUrl, + thresholds: { profile: "auto", goalDone: 0.85, goalDoneFinal: 0.7, configured: [], ...thresholds }, + }); + const clientWith = (card) => ({ models: async () => card }); + const kevCard = { models: [{ name: "kev-latest", run: "jaredpalmer/kev-4b", base: "Qwen/Qwen3.5-4B-Base" }] }; + const readoutCard = { models: [{ name: "qwen3.5-4b-q4-k-m", description: "readout" }] }; + + const kev = config("http://127.0.0.1:8008"); + const kevRecord = await resolveThresholds({ config: kev, client: clientWith(kevCard) }); + assert.equal(kevRecord.profile, "kev"); + assert.equal(kev.thresholds.goalDone, KEV_GOAL_DONE, "the bar is applied to the config the controller reads"); + assert.equal(kev.thresholds.goalDoneFinal, KEV_GOAL_DONE); + assert.deepEqual(kevRecord.applied, { goalDone: KEV_GOAL_DONE, goalDoneFinal: KEV_GOAL_DONE }); + assert.equal(kevRecord.classification.kind, "kev", "the classification is recorded for the journal"); + + const readout = config("http://127.0.0.1:8092"); + await resolveThresholds({ config: readout, client: clientWith(readoutCard) }); + assert.equal(readout.thresholds.goalDone, LOCAL_GOAL_DONE); + assert.equal(readout.thresholds.goalDoneFinal, LOCAL_GOAL_DONE); + + const unreachable = config("http://127.0.0.1:8008"); + const unreachableRecord = await resolveThresholds({ config: unreachable, client: { models: async () => { throw new Error("fetch failed"); } } }); + assert.equal(unreachableRecord.profile, FALLBACK_PROFILE, "an unreachable endpoint gets the highest bar"); + assert.equal(unreachable.thresholds.goalDone, KEV_GOAL_DONE); + assert.match(unreachableRecord.reason, /false stuck is visible/); + + const hosted = config("https://api.typesafe.ai"); + const hostedRecord = await resolveThresholds({ config: hosted, client: clientWith(kevCard) }); + assert.equal(hostedRecord.profile, "hosted"); + assert.deepEqual(hostedRecord.applied, {}, "a hosted run changes nothing"); + assert.equal(hosted.thresholds.goalDone, 0.85); + + const configured = config("http://127.0.0.1:8008", { goalDone: 0.9, configured: ["goalDone"] }); + const configuredRecord = await resolveThresholds({ config: configured, client: clientWith(kevCard) }); + assert.equal(configured.thresholds.goalDone, 0.9, "the configured value survives the profile"); + assert.equal(configured.thresholds.goalDoneFinal, KEV_GOAL_DONE, "the other key still follows the endpoint"); + assert.deepEqual(configuredRecord.applied, { goalDoneFinal: KEV_GOAL_DONE }); +}); From 85394eea94b256fc525eea4ae0cb9e21306d5995 Mon Sep 17 00:00:00 2001 From: ChenYCL Date: Thu, 24 Sep 2026 13:53:13 +0800 Subject: [PATCH 5/9] docs: measured local-backend results and Kev bring-up MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Puts the numbers behind the three local commits on the record, in three layers: the product tier table (README.md, SKILL.md), the operational detail (references/config.md, references/ questions.md), and the raw experiments with their own methodology and datasets (docs/, experiments/). - README/SKILL: what each tier costs and scores. Readout: 2.6 GiB, no Python, no API key, $0, p50 ≈4.3 s per step (one long option list is ~2/3 of it), 0.80 vs hosted Jev's 0.95 on 20 graded items with the 0.8B entry at 0.50. Kev: 19/20 = 0.95 at the raised row limit and 18/20 at the released one, for a venv + MLX + 9.34 GB base + ~18 GB idle / ~36 GB loaded and 2.2 s mean per item. - references/config.md: the profile table with where each bar was measured, the band each was derived from, the per-backend table that shows why 0.174 must not be reused for Kev (4 false successes), and the explicit statement that re-measuring beats trusting either local bar on a model nobody has scored. references/questions.md follows goal_done to its per-backend values. - docs/ and experiments/: the bring-up logs, the first real end-to-end run of a local backend under `run` (15 runs: success 4, stuck 8, needs_user 2, max_steps 1; 39 steps, no retry, no timeout, no 422), the Kev bring-up and threshold replay with its raw per-step readings, and the model-selection research behind the shipped default. --- README.md | 20 + docs/local-backend-run-smoke.md | 468 +++ docs/local-gguf-probe.md | 349 +++ docs/local-kev-bringup.md | 833 ++++++ docs/local-models-research.md | 505 ++++ experiments/gguf-provider/RESULTS.md | 99 + experiments/gguf-provider/cli.mjs | 51 + experiments/gguf-provider/eval/items.mjs | 207 ++ experiments/gguf-provider/eval/run.mjs | 152 + .../fixtures/judge-questions.json | 148 + .../gguf-provider/fixtures/judge-state.json | 309 ++ .../gguf-provider/fixtures/raw_ddg_typed.txt | 562 ++++ ...ckduckgo_com_html__q_apple_stock_price.txt | 541 ++++ .../raw_en_wikipedia_org_wiki_Main_Page.txt | 834 ++++++ .../fixtures/raw_example_com.txt | 128 + .../fixtures/raw_github_login.txt | 251 ++ .../fixtures/raw_wiki_loginpage.txt | 286 ++ .../gguf-provider/fixtures/raw_wiki_typed.txt | 884 ++++++ .../gguf-provider/lab/check-normalize.mjs | 72 + experiments/gguf-provider/lab/fetch.mjs | 170 ++ experiments/gguf-provider/lab/latency.mjs | 44 + .../gguf-provider/lab/probe-ending.mjs | 50 + .../gguf-provider/lab/probe-noul-bias.mjs | 87 + experiments/gguf-provider/lab/ramp.mjs | 83 + experiments/gguf-provider/lab/throughput.sh | 44 + .../gguf-provider/lab/tokenizer-probe.mjs | 52 + experiments/gguf-provider/lib/labels.mjs | 36 + experiments/gguf-provider/lib/provider.mjs | 135 + experiments/gguf-provider/lib/readout.mjs | 169 ++ experiments/gguf-provider/lib/render.mjs | 153 + .../results/analysis-gemma-3-4b-it.json | 343 +++ .../analysis-qwen3-4b-instruct-2507.json | 343 +++ .../results/analysis-qwen35-08b-c16384.json | 343 +++ .../results/analysis-qwen35-4b.json | 343 +++ .../results/baseline-first-option.json | 168 ++ .../gguf-provider/results/candidates.json | 38 + .../results/ceiling-jev-20items.json | 418 +++ .../results/ceiling-jev-fixture-step.json | 121 + .../results/check-normalize-0.6b-recheck.txt | 230 ++ .../results/check-normalize-0.6b.txt | 230 ++ .../results/check-normalize-0.8b.txt | 249 ++ .../results/eval-0.6b-speculative.json | 49 + .../gguf-provider/results/eval-0.6b.json | 433 +++ .../gguf-provider/results/eval-0.8b.json | 484 +++ .../results/eval-gemma-3-4b-it.json | 427 +++ .../results/eval-qwen3-4b-instruct-2507.json | 494 ++++ .../results/eval-qwen35-08b-c16384.json | 484 +++ .../results/eval-qwen35-08b-c8192.json | 0 .../gguf-provider/results/eval-qwen35-4b.json | 460 +++ .../results/fixture-step-gemma-3-4b-it.json | 228 ++ .../fixture-step-qwen3-4b-instruct-2507.json | 230 ++ .../fixture-step-qwen35-08b-c16384.json | 241 ++ .../results/fixture-step-qwen35-4b.json | 228 ++ .../results/fixture-step-request.json | 459 +++ .../gguf-provider/results/latency-0.6b.txt | 29 + .../gguf-provider/results/latency-0.8b.txt | 29 + .../gguf-provider/results/local-models-4b.md | 335 +++ .../results/probe-ending-0.6b.txt | 51 + .../results/probe-noul-bias-0.6b.txt | 78 + .../results/ramp-gemma-2026-09-24.json | 1 + .../ramp-gemma-sustained-2026-09-24.json | 1 + .../results/ramp-qwen-2026-09-24.json | 1 + .../results/rotate-extra-0.6b.json | 53 + .../results/rotate-extra-0.8b.json | 53 + .../results/skill-judge-0.6b.json | 230 ++ .../results/skill-judge-0.8b.json | 243 ++ .../results/throughput-2026-09-24.json | 2 + .../throughput-summary-2026-09-24.json | 30 + .../gguf-provider/results/tokenizer-0.6b.json | 207 ++ experiments/gguf-provider/serve.mjs | 73 + experiments/kev-4b/README.md | 514 ++++ experiments/kev-4b/fetch.py | 303 ++ experiments/kev-4b/head-sweep.txt | 29 + experiments/kev-4b/make-manifest.mjs | 82 + experiments/kev-4b/probe-supplemental.sh | 59 + experiments/kev-4b/probe-throughput.sh | 60 + experiments/kev-4b/published-state.json | 64 + .../results/eval-kev-0.8b-ctx16384.json | 2602 +++++++++++++++++ experiments/kev-4b/results/eval-kev-0.8b.json | 2254 ++++++++++++++ .../kev-4b/results/eval-kev-4b-ctx16384.json | 2579 ++++++++++++++++ .../kev-4b/results/eval-kev-4b-ctx8192.json | 2235 ++++++++++++++ .../fixture-step-kev-0.8b-ctx16384.json | 320 ++ .../kev-4b/results/fixture-step-kev-0.8b.json | 191 ++ .../results/fixture-step-kev-4b-ctx16384.json | 320 ++ .../results/fixture-step-kev-4b-ctx8192.json | 191 ++ .../raw/kev-0.8b-ctx16384/bq-everest.json | 69 + .../raw/kev-0.8b-ctx16384/bq-paris.json | 69 + .../raw/kev-0.8b-ctx16384/bq-penguins.json | 69 + .../raw/kev-0.8b-ctx16384/bq-python.json | 69 + .../raw/kev-0.8b-ctx16384/bq-trap-failed.json | 69 + .../kev-0.8b-ctx16384/ddg-action-aapl.json | 103 + .../kev-0.8b-ctx16384/ddg-blocker-aapl.json | 89 + .../ddg-click-target-aapl.json | 236 ++ .../kev-0.8b-ctx16384/ddg-goal-done-aapl.json | 72 + .../kev-0.8b-ctx16384/ddg-typed-action.json | 109 + .../kev-0.8b-ctx16384/ddg-typed-submit.json | 69 + .../ddg-typed-type-target.json | 77 + .../github-login-action.json | 97 + .../github-login-blocker.json | 89 + .../github-login-submit.json | 69 + .../github-login-type-target.json | 80 + .../github-login-type-value.json | 80 + .../wiki-login-click-target.json | 371 +++ .../wiki-progress-after-login-click.json | 86 + .../wiki-typed-type-target.json | 77 + .../results/raw/kev-0.8b/bq-everest.json | 69 + .../kev-4b/results/raw/kev-0.8b/bq-paris.json | 69 + .../results/raw/kev-0.8b/bq-penguins.json | 69 + .../results/raw/kev-0.8b/bq-python.json | 69 + .../results/raw/kev-0.8b/bq-trap-failed.json | 69 + .../results/raw/kev-0.8b/ddg-action-aapl.json | 103 + .../raw/kev-0.8b/ddg-blocker-aapl.json | 89 + .../raw/kev-0.8b/ddg-click-target-aapl.json | 14 + .../raw/kev-0.8b/ddg-goal-done-aapl.json | 72 + .../raw/kev-0.8b/ddg-typed-action.json | 109 + .../raw/kev-0.8b/ddg-typed-submit.json | 69 + .../raw/kev-0.8b/ddg-typed-type-target.json | 77 + .../raw/kev-0.8b/github-login-action.json | 97 + .../raw/kev-0.8b/github-login-blocker.json | 89 + .../raw/kev-0.8b/github-login-submit.json | 69 + .../kev-0.8b/github-login-type-target.json | 80 + .../raw/kev-0.8b/github-login-type-value.json | 80 + .../raw/kev-0.8b/wiki-login-click-target.json | 371 +++ .../wiki-progress-after-login-click.json | 86 + .../raw/kev-0.8b/wiki-typed-type-target.json | 77 + .../raw/kev-4b-ctx16384/bq-everest.json | 69 + .../results/raw/kev-4b-ctx16384/bq-paris.json | 69 + .../raw/kev-4b-ctx16384/bq-penguins.json | 69 + .../raw/kev-4b-ctx16384/bq-python.json | 69 + .../raw/kev-4b-ctx16384/bq-trap-failed.json | 69 + .../raw/kev-4b-ctx16384/ddg-action-aapl.json | 103 + .../raw/kev-4b-ctx16384/ddg-blocker-aapl.json | 89 + .../ddg-click-target-aapl.json | 236 ++ .../kev-4b-ctx16384/ddg-goal-done-aapl.json | 72 + .../raw/kev-4b-ctx16384/ddg-typed-action.json | 109 + .../raw/kev-4b-ctx16384/ddg-typed-submit.json | 69 + .../ddg-typed-type-target.json | 77 + .../kev-4b-ctx16384/github-login-action.json | 97 + .../kev-4b-ctx16384/github-login-blocker.json | 89 + .../kev-4b-ctx16384/github-login-submit.json | 69 + .../github-login-type-target.json | 80 + .../github-login-type-value.json | 80 + .../wiki-login-click-target.json | 371 +++ .../wiki-progress-after-login-click.json | 86 + .../wiki-typed-type-target.json | 77 + .../raw/kev-4b-ctx8192/bq-everest.json | 69 + .../results/raw/kev-4b-ctx8192/bq-paris.json | 69 + .../raw/kev-4b-ctx8192/bq-penguins.json | 69 + .../results/raw/kev-4b-ctx8192/bq-python.json | 69 + .../raw/kev-4b-ctx8192/bq-trap-failed.json | 69 + .../raw/kev-4b-ctx8192/ddg-action-aapl.json | 103 + .../raw/kev-4b-ctx8192/ddg-blocker-aapl.json | 89 + .../kev-4b-ctx8192/ddg-click-target-aapl.json | 14 + .../kev-4b-ctx8192/ddg-goal-done-aapl.json | 72 + .../raw/kev-4b-ctx8192/ddg-typed-action.json | 109 + .../raw/kev-4b-ctx8192/ddg-typed-submit.json | 69 + .../kev-4b-ctx8192/ddg-typed-type-target.json | 77 + .../kev-4b-ctx8192/github-login-action.json | 97 + .../kev-4b-ctx8192/github-login-blocker.json | 89 + .../kev-4b-ctx8192/github-login-submit.json | 69 + .../github-login-type-target.json | 80 + .../github-login-type-value.json | 80 + .../wiki-login-click-target.json | 371 +++ .../wiki-progress-after-login-click.json | 86 + .../wiki-typed-type-target.json | 77 + .../2026-09-24T02-56-45-072Z-hphny1/run.json | 28 + .../steps.jsonl | 1 + .../2026-09-24T02-56-51-698Z-u46x61/run.json | 28 + .../steps.jsonl | 3 + .../2026-09-24T02-49-11-130Z-cptq13/run.json | 29 + .../steps.jsonl | 5 + .../2026-09-24T02-47-01-646Z-kwow2m/run.json | 28 + .../steps.jsonl | 3 + .../2026-09-24T02-46-50-733Z-4md9mg/run.json | 28 + .../steps.jsonl | 1 + .../2026-09-24T02-51-13-073Z-aygq2f/run.json | 31 + .../steps.jsonl | 1 + .../2026-09-24T02-57-05-363Z-7uoi08/run.json | 29 + .../steps.jsonl | 7 + .../2026-09-24T02-53-13-205Z-bz1i3c/run.json | 29 + .../steps.jsonl | 9 + .../2026-09-24T02-47-36-951Z-yy7ppu/run.json | 28 + .../steps.jsonl | 1 + .../2026-09-24T02-47-21-651Z-ugvzrq/run.json | 28 + .../steps.jsonl | 2 + .../2026-09-24T02-48-29-336Z-h8qpia/run.json | 29 + .../steps.jsonl | 6 + .../2026-09-24T02-46-24-177Z-wewnol/run.json | 28 + .../steps.jsonl | 1 + .../2026-09-24T02-46-38-808Z-chwnfl/run.json | 28 + .../steps.jsonl | 1 + .../2026-09-24T02-47-47-807Z-cqb090/run.json | 28 + .../steps.jsonl | 1 + .../2026-09-24T02-47-58-916Z-uvooil/run.json | 28 + .../steps.jsonl | 4 + .../results/threshold-replay/labels.json | 18 + .../kev-4b/results/threshold-replay/runs.tsv | 17 + .../kev-4b/results/threshold-replay/score.txt | 94 + experiments/kev-4b/run.mjs | 455 +++ experiments/kev-4b/threshold-replay.mjs | 126 + experiments/kev-4b/threshold-replay.sh | 66 + experiments/kev-4b/verify.sh | 69 + skills/jev-browser/SKILL.md | 140 + skills/jev-browser/references/config.md | 62 +- skills/jev-browser/references/questions.md | 2 +- 205 files changed, 37254 insertions(+), 4 deletions(-) create mode 100644 docs/local-backend-run-smoke.md create mode 100644 docs/local-gguf-probe.md create mode 100644 docs/local-kev-bringup.md create mode 100644 docs/local-models-research.md create mode 100644 experiments/gguf-provider/RESULTS.md create mode 100755 experiments/gguf-provider/cli.mjs create mode 100644 experiments/gguf-provider/eval/items.mjs create mode 100755 experiments/gguf-provider/eval/run.mjs create mode 100644 experiments/gguf-provider/fixtures/judge-questions.json create mode 100644 experiments/gguf-provider/fixtures/judge-state.json create mode 100644 experiments/gguf-provider/fixtures/raw_ddg_typed.txt create mode 100644 experiments/gguf-provider/fixtures/raw_duckduckgo_com_html__q_apple_stock_price.txt create mode 100644 experiments/gguf-provider/fixtures/raw_en_wikipedia_org_wiki_Main_Page.txt create mode 100644 experiments/gguf-provider/fixtures/raw_example_com.txt create mode 100644 experiments/gguf-provider/fixtures/raw_github_login.txt create mode 100644 experiments/gguf-provider/fixtures/raw_wiki_loginpage.txt create mode 100644 experiments/gguf-provider/fixtures/raw_wiki_typed.txt create mode 100755 experiments/gguf-provider/lab/check-normalize.mjs create mode 100755 experiments/gguf-provider/lab/fetch.mjs create mode 100755 experiments/gguf-provider/lab/latency.mjs create mode 100644 experiments/gguf-provider/lab/probe-ending.mjs create mode 100755 experiments/gguf-provider/lab/probe-noul-bias.mjs create mode 100755 experiments/gguf-provider/lab/ramp.mjs create mode 100755 experiments/gguf-provider/lab/throughput.sh create mode 100644 experiments/gguf-provider/lab/tokenizer-probe.mjs create mode 100644 experiments/gguf-provider/lib/labels.mjs create mode 100644 experiments/gguf-provider/lib/provider.mjs create mode 100644 experiments/gguf-provider/lib/readout.mjs create mode 100644 experiments/gguf-provider/lib/render.mjs create mode 100644 experiments/gguf-provider/results/analysis-gemma-3-4b-it.json create mode 100644 experiments/gguf-provider/results/analysis-qwen3-4b-instruct-2507.json create mode 100644 experiments/gguf-provider/results/analysis-qwen35-08b-c16384.json create mode 100644 experiments/gguf-provider/results/analysis-qwen35-4b.json create mode 100644 experiments/gguf-provider/results/baseline-first-option.json create mode 100644 experiments/gguf-provider/results/candidates.json create mode 100644 experiments/gguf-provider/results/ceiling-jev-20items.json create mode 100644 experiments/gguf-provider/results/ceiling-jev-fixture-step.json create mode 100644 experiments/gguf-provider/results/check-normalize-0.6b-recheck.txt create mode 100644 experiments/gguf-provider/results/check-normalize-0.6b.txt create mode 100644 experiments/gguf-provider/results/check-normalize-0.8b.txt create mode 100644 experiments/gguf-provider/results/eval-0.6b-speculative.json create mode 100644 experiments/gguf-provider/results/eval-0.6b.json create mode 100644 experiments/gguf-provider/results/eval-0.8b.json create mode 100644 experiments/gguf-provider/results/eval-gemma-3-4b-it.json create mode 100644 experiments/gguf-provider/results/eval-qwen3-4b-instruct-2507.json create mode 100644 experiments/gguf-provider/results/eval-qwen35-08b-c16384.json create mode 100644 experiments/gguf-provider/results/eval-qwen35-08b-c8192.json create mode 100644 experiments/gguf-provider/results/eval-qwen35-4b.json create mode 100644 experiments/gguf-provider/results/fixture-step-gemma-3-4b-it.json create mode 100644 experiments/gguf-provider/results/fixture-step-qwen3-4b-instruct-2507.json create mode 100644 experiments/gguf-provider/results/fixture-step-qwen35-08b-c16384.json create mode 100644 experiments/gguf-provider/results/fixture-step-qwen35-4b.json create mode 100644 experiments/gguf-provider/results/fixture-step-request.json create mode 100644 experiments/gguf-provider/results/latency-0.6b.txt create mode 100644 experiments/gguf-provider/results/latency-0.8b.txt create mode 100644 experiments/gguf-provider/results/local-models-4b.md create mode 100644 experiments/gguf-provider/results/probe-ending-0.6b.txt create mode 100644 experiments/gguf-provider/results/probe-noul-bias-0.6b.txt create mode 100644 experiments/gguf-provider/results/ramp-gemma-2026-09-24.json create mode 100644 experiments/gguf-provider/results/ramp-gemma-sustained-2026-09-24.json create mode 100644 experiments/gguf-provider/results/ramp-qwen-2026-09-24.json create mode 100644 experiments/gguf-provider/results/rotate-extra-0.6b.json create mode 100644 experiments/gguf-provider/results/rotate-extra-0.8b.json create mode 100644 experiments/gguf-provider/results/skill-judge-0.6b.json create mode 100644 experiments/gguf-provider/results/skill-judge-0.8b.json create mode 100644 experiments/gguf-provider/results/throughput-2026-09-24.json create mode 100644 experiments/gguf-provider/results/throughput-summary-2026-09-24.json create mode 100644 experiments/gguf-provider/results/tokenizer-0.6b.json create mode 100755 experiments/gguf-provider/serve.mjs create mode 100644 experiments/kev-4b/README.md create mode 100755 experiments/kev-4b/fetch.py create mode 100644 experiments/kev-4b/head-sweep.txt create mode 100755 experiments/kev-4b/make-manifest.mjs create mode 100755 experiments/kev-4b/probe-supplemental.sh create mode 100755 experiments/kev-4b/probe-throughput.sh create mode 100644 experiments/kev-4b/published-state.json create mode 100644 experiments/kev-4b/results/eval-kev-0.8b-ctx16384.json create mode 100644 experiments/kev-4b/results/eval-kev-0.8b.json create mode 100644 experiments/kev-4b/results/eval-kev-4b-ctx16384.json create mode 100644 experiments/kev-4b/results/eval-kev-4b-ctx8192.json create mode 100644 experiments/kev-4b/results/fixture-step-kev-0.8b-ctx16384.json create mode 100644 experiments/kev-4b/results/fixture-step-kev-0.8b.json create mode 100644 experiments/kev-4b/results/fixture-step-kev-4b-ctx16384.json create mode 100644 experiments/kev-4b/results/fixture-step-kev-4b-ctx8192.json create mode 100644 experiments/kev-4b/results/raw/kev-0.8b-ctx16384/bq-everest.json create mode 100644 experiments/kev-4b/results/raw/kev-0.8b-ctx16384/bq-paris.json create mode 100644 experiments/kev-4b/results/raw/kev-0.8b-ctx16384/bq-penguins.json create mode 100644 experiments/kev-4b/results/raw/kev-0.8b-ctx16384/bq-python.json create mode 100644 experiments/kev-4b/results/raw/kev-0.8b-ctx16384/bq-trap-failed.json create mode 100644 experiments/kev-4b/results/raw/kev-0.8b-ctx16384/ddg-action-aapl.json create mode 100644 experiments/kev-4b/results/raw/kev-0.8b-ctx16384/ddg-blocker-aapl.json create mode 100644 experiments/kev-4b/results/raw/kev-0.8b-ctx16384/ddg-click-target-aapl.json create mode 100644 experiments/kev-4b/results/raw/kev-0.8b-ctx16384/ddg-goal-done-aapl.json create mode 100644 experiments/kev-4b/results/raw/kev-0.8b-ctx16384/ddg-typed-action.json create mode 100644 experiments/kev-4b/results/raw/kev-0.8b-ctx16384/ddg-typed-submit.json create mode 100644 experiments/kev-4b/results/raw/kev-0.8b-ctx16384/ddg-typed-type-target.json create mode 100644 experiments/kev-4b/results/raw/kev-0.8b-ctx16384/github-login-action.json create mode 100644 experiments/kev-4b/results/raw/kev-0.8b-ctx16384/github-login-blocker.json create mode 100644 experiments/kev-4b/results/raw/kev-0.8b-ctx16384/github-login-submit.json create mode 100644 experiments/kev-4b/results/raw/kev-0.8b-ctx16384/github-login-type-target.json create mode 100644 experiments/kev-4b/results/raw/kev-0.8b-ctx16384/github-login-type-value.json create mode 100644 experiments/kev-4b/results/raw/kev-0.8b-ctx16384/wiki-login-click-target.json create mode 100644 experiments/kev-4b/results/raw/kev-0.8b-ctx16384/wiki-progress-after-login-click.json create mode 100644 experiments/kev-4b/results/raw/kev-0.8b-ctx16384/wiki-typed-type-target.json create mode 100644 experiments/kev-4b/results/raw/kev-0.8b/bq-everest.json create mode 100644 experiments/kev-4b/results/raw/kev-0.8b/bq-paris.json create mode 100644 experiments/kev-4b/results/raw/kev-0.8b/bq-penguins.json create mode 100644 experiments/kev-4b/results/raw/kev-0.8b/bq-python.json create mode 100644 experiments/kev-4b/results/raw/kev-0.8b/bq-trap-failed.json create mode 100644 experiments/kev-4b/results/raw/kev-0.8b/ddg-action-aapl.json create mode 100644 experiments/kev-4b/results/raw/kev-0.8b/ddg-blocker-aapl.json create mode 100644 experiments/kev-4b/results/raw/kev-0.8b/ddg-click-target-aapl.json create mode 100644 experiments/kev-4b/results/raw/kev-0.8b/ddg-goal-done-aapl.json create mode 100644 experiments/kev-4b/results/raw/kev-0.8b/ddg-typed-action.json create mode 100644 experiments/kev-4b/results/raw/kev-0.8b/ddg-typed-submit.json create mode 100644 experiments/kev-4b/results/raw/kev-0.8b/ddg-typed-type-target.json create mode 100644 experiments/kev-4b/results/raw/kev-0.8b/github-login-action.json create mode 100644 experiments/kev-4b/results/raw/kev-0.8b/github-login-blocker.json create mode 100644 experiments/kev-4b/results/raw/kev-0.8b/github-login-submit.json create mode 100644 experiments/kev-4b/results/raw/kev-0.8b/github-login-type-target.json create mode 100644 experiments/kev-4b/results/raw/kev-0.8b/github-login-type-value.json create mode 100644 experiments/kev-4b/results/raw/kev-0.8b/wiki-login-click-target.json create mode 100644 experiments/kev-4b/results/raw/kev-0.8b/wiki-progress-after-login-click.json create mode 100644 experiments/kev-4b/results/raw/kev-0.8b/wiki-typed-type-target.json create mode 100644 experiments/kev-4b/results/raw/kev-4b-ctx16384/bq-everest.json create mode 100644 experiments/kev-4b/results/raw/kev-4b-ctx16384/bq-paris.json create mode 100644 experiments/kev-4b/results/raw/kev-4b-ctx16384/bq-penguins.json create mode 100644 experiments/kev-4b/results/raw/kev-4b-ctx16384/bq-python.json create mode 100644 experiments/kev-4b/results/raw/kev-4b-ctx16384/bq-trap-failed.json create mode 100644 experiments/kev-4b/results/raw/kev-4b-ctx16384/ddg-action-aapl.json create mode 100644 experiments/kev-4b/results/raw/kev-4b-ctx16384/ddg-blocker-aapl.json create mode 100644 experiments/kev-4b/results/raw/kev-4b-ctx16384/ddg-click-target-aapl.json create mode 100644 experiments/kev-4b/results/raw/kev-4b-ctx16384/ddg-goal-done-aapl.json create mode 100644 experiments/kev-4b/results/raw/kev-4b-ctx16384/ddg-typed-action.json create mode 100644 experiments/kev-4b/results/raw/kev-4b-ctx16384/ddg-typed-submit.json create mode 100644 experiments/kev-4b/results/raw/kev-4b-ctx16384/ddg-typed-type-target.json create mode 100644 experiments/kev-4b/results/raw/kev-4b-ctx16384/github-login-action.json create mode 100644 experiments/kev-4b/results/raw/kev-4b-ctx16384/github-login-blocker.json create mode 100644 experiments/kev-4b/results/raw/kev-4b-ctx16384/github-login-submit.json create mode 100644 experiments/kev-4b/results/raw/kev-4b-ctx16384/github-login-type-target.json create mode 100644 experiments/kev-4b/results/raw/kev-4b-ctx16384/github-login-type-value.json create mode 100644 experiments/kev-4b/results/raw/kev-4b-ctx16384/wiki-login-click-target.json create mode 100644 experiments/kev-4b/results/raw/kev-4b-ctx16384/wiki-progress-after-login-click.json create mode 100644 experiments/kev-4b/results/raw/kev-4b-ctx16384/wiki-typed-type-target.json create mode 100644 experiments/kev-4b/results/raw/kev-4b-ctx8192/bq-everest.json create mode 100644 experiments/kev-4b/results/raw/kev-4b-ctx8192/bq-paris.json create mode 100644 experiments/kev-4b/results/raw/kev-4b-ctx8192/bq-penguins.json create mode 100644 experiments/kev-4b/results/raw/kev-4b-ctx8192/bq-python.json create mode 100644 experiments/kev-4b/results/raw/kev-4b-ctx8192/bq-trap-failed.json create mode 100644 experiments/kev-4b/results/raw/kev-4b-ctx8192/ddg-action-aapl.json create mode 100644 experiments/kev-4b/results/raw/kev-4b-ctx8192/ddg-blocker-aapl.json create mode 100644 experiments/kev-4b/results/raw/kev-4b-ctx8192/ddg-click-target-aapl.json create mode 100644 experiments/kev-4b/results/raw/kev-4b-ctx8192/ddg-goal-done-aapl.json create mode 100644 experiments/kev-4b/results/raw/kev-4b-ctx8192/ddg-typed-action.json create mode 100644 experiments/kev-4b/results/raw/kev-4b-ctx8192/ddg-typed-submit.json create mode 100644 experiments/kev-4b/results/raw/kev-4b-ctx8192/ddg-typed-type-target.json create mode 100644 experiments/kev-4b/results/raw/kev-4b-ctx8192/github-login-action.json create mode 100644 experiments/kev-4b/results/raw/kev-4b-ctx8192/github-login-blocker.json create mode 100644 experiments/kev-4b/results/raw/kev-4b-ctx8192/github-login-submit.json create mode 100644 experiments/kev-4b/results/raw/kev-4b-ctx8192/github-login-type-target.json create mode 100644 experiments/kev-4b/results/raw/kev-4b-ctx8192/github-login-type-value.json create mode 100644 experiments/kev-4b/results/raw/kev-4b-ctx8192/wiki-login-click-target.json create mode 100644 experiments/kev-4b/results/raw/kev-4b-ctx8192/wiki-progress-after-login-click.json create mode 100644 experiments/kev-4b/results/raw/kev-4b-ctx8192/wiki-typed-type-target.json create mode 100644 experiments/kev-4b/results/threshold-replay/journal/R1/2026-09-24T02-56-45-072Z-hphny1/run.json create mode 100644 experiments/kev-4b/results/threshold-replay/journal/R1/2026-09-24T02-56-45-072Z-hphny1/steps.jsonl create mode 100644 experiments/kev-4b/results/threshold-replay/journal/R2/2026-09-24T02-56-51-698Z-u46x61/run.json create mode 100644 experiments/kev-4b/results/threshold-replay/journal/R2/2026-09-24T02-56-51-698Z-u46x61/steps.jsonl create mode 100644 experiments/kev-4b/results/threshold-replay/journal/R3/2026-09-24T02-49-11-130Z-cptq13/run.json create mode 100644 experiments/kev-4b/results/threshold-replay/journal/R3/2026-09-24T02-49-11-130Z-cptq13/steps.jsonl create mode 100644 experiments/kev-4b/results/threshold-replay/journal/R4/2026-09-24T02-47-01-646Z-kwow2m/run.json create mode 100644 experiments/kev-4b/results/threshold-replay/journal/R4/2026-09-24T02-47-01-646Z-kwow2m/steps.jsonl create mode 100644 experiments/kev-4b/results/threshold-replay/journal/R5/2026-09-24T02-46-50-733Z-4md9mg/run.json create mode 100644 experiments/kev-4b/results/threshold-replay/journal/R5/2026-09-24T02-46-50-733Z-4md9mg/steps.jsonl create mode 100644 experiments/kev-4b/results/threshold-replay/journal/R6/2026-09-24T02-51-13-073Z-aygq2f/run.json create mode 100644 experiments/kev-4b/results/threshold-replay/journal/R6/2026-09-24T02-51-13-073Z-aygq2f/steps.jsonl create mode 100644 experiments/kev-4b/results/threshold-replay/journal/R7/2026-09-24T02-57-05-363Z-7uoi08/run.json create mode 100644 experiments/kev-4b/results/threshold-replay/journal/R7/2026-09-24T02-57-05-363Z-7uoi08/steps.jsonl create mode 100644 experiments/kev-4b/results/threshold-replay/journal/R8/2026-09-24T02-53-13-205Z-bz1i3c/run.json create mode 100644 experiments/kev-4b/results/threshold-replay/journal/R8/2026-09-24T02-53-13-205Z-bz1i3c/steps.jsonl create mode 100644 experiments/kev-4b/results/threshold-replay/journal/g1/2026-09-24T02-47-36-951Z-yy7ppu/run.json create mode 100644 experiments/kev-4b/results/threshold-replay/journal/g1/2026-09-24T02-47-36-951Z-yy7ppu/steps.jsonl create mode 100644 experiments/kev-4b/results/threshold-replay/journal/g3/2026-09-24T02-47-21-651Z-ugvzrq/run.json create mode 100644 experiments/kev-4b/results/threshold-replay/journal/g3/2026-09-24T02-47-21-651Z-ugvzrq/steps.jsonl create mode 100644 experiments/kev-4b/results/threshold-replay/journal/g4/2026-09-24T02-48-29-336Z-h8qpia/run.json create mode 100644 experiments/kev-4b/results/threshold-replay/journal/g4/2026-09-24T02-48-29-336Z-h8qpia/steps.jsonl create mode 100644 experiments/kev-4b/results/threshold-replay/journal/p1/2026-09-24T02-46-24-177Z-wewnol/run.json create mode 100644 experiments/kev-4b/results/threshold-replay/journal/p1/2026-09-24T02-46-24-177Z-wewnol/steps.jsonl create mode 100644 experiments/kev-4b/results/threshold-replay/journal/p2/2026-09-24T02-46-38-808Z-chwnfl/run.json create mode 100644 experiments/kev-4b/results/threshold-replay/journal/p2/2026-09-24T02-46-38-808Z-chwnfl/steps.jsonl create mode 100644 experiments/kev-4b/results/threshold-replay/journal/p3/2026-09-24T02-47-47-807Z-cqb090/run.json create mode 100644 experiments/kev-4b/results/threshold-replay/journal/p3/2026-09-24T02-47-47-807Z-cqb090/steps.jsonl create mode 100644 experiments/kev-4b/results/threshold-replay/journal/p4/2026-09-24T02-47-58-916Z-uvooil/run.json create mode 100644 experiments/kev-4b/results/threshold-replay/journal/p4/2026-09-24T02-47-58-916Z-uvooil/steps.jsonl create mode 100644 experiments/kev-4b/results/threshold-replay/labels.json create mode 100644 experiments/kev-4b/results/threshold-replay/runs.tsv create mode 100644 experiments/kev-4b/results/threshold-replay/score.txt create mode 100755 experiments/kev-4b/run.mjs create mode 100755 experiments/kev-4b/threshold-replay.mjs create mode 100755 experiments/kev-4b/threshold-replay.sh create mode 100755 experiments/kev-4b/verify.sh diff --git a/README.md b/README.md index 83468e8..6cecfa0 100644 --- a/README.md +++ b/README.md @@ -213,6 +213,26 @@ whether the page changed, cost), `requests.jsonl` and `run.json`, with secrets r | `chrome` | unattended runs, CI, no window | own profile dir, `--headless`, or `--cdp-url http://127.0.0.1:9222` to attach | | `safari` | WebKit | enable Develop → Allow Remote Automation once | +**Fully local (experimental).** `node skills/jev-browser/bin/jev-local.mjs` downloads the registry +default (a 2.6 GiB Qwen3.5-4B GGUF), starts `llama.cpp` with a 16k context (Apple Silicon), and +serves the same `/v1/systemone` contract on `127.0.0.1:8092` — first-token readout, no API key, no +Python runtime, $0, p50 ≈4.3 s per step (its one long option list is ~2/3 of it). Models are +data: `skills/jev-browser/lib/local-models.json` holds each entry with its URL, size and label; +pick one with `--model-name ` or edit its `"default"` (`--list-models` prints the registry, +`--ctx` the context size). It is a short-state classifier, not a Jev replacement (no few-shot, no +images; 0.80 vs hosted Jev's 0.95 on 20 graded items, and the smaller 0.8B entry only 0.50): see +[results](experiments/gguf-provider/RESULTS.md). + +**A second local tier trades disk and memory for accuracy.** `node skills/jev-browser/bin/jev-kev.mjs` +fetches the Kev 4B checkpoint and its 9.34 GB base (verifying every file against published hashes), +starts it, and serves the same contract on `127.0.0.1:8008`: **19/20 = 0.95** on the 20 graded items +with its optional row-limit patch applied — 18/20 at the published row limit, where the 55-option +`click_target` questions are refused with HTTP 422 — i.e. it ties hosted Jev on this set. The costs +are real: a Python venv + MLX, the 9.34 GB base, ~18 GB idle and ~36 GB of GPU memory under load at +the raised limit (heavy swap on a 48 GB machine), and mean 2.2 s per item (12.2 s worst). The +`goal_done` bar is resolved per backend, so a Kev run does not inherit the readout's. Tier table and +detail: [SKILL.md](skills/jev-browser/SKILL.md#fully-local-experimental). + ## MCP server `jev-browser mcp` speaks MCP over stdio with zero dependencies. Tools: `jev_browse`, `jev_observe`, diff --git a/docs/local-backend-run-smoke.md b/docs/local-backend-run-smoke.md new file mode 100644 index 0000000..2bc865e --- /dev/null +++ b/docs/local-backend-run-smoke.md @@ -0,0 +1,468 @@ +# 本地后端「真跑一轮」实测：`jev-browser run` 能不能被本地模型驱动完成 + +目标：回答一个此前从未测过的问题 —— 本地 GGUF 后端（`node skills/jev-browser/bin/jev-local.mjs`） +除了能回答单次 judge，能不能被 `jev-browser run` 的 code controller 真正驱动，跑完一整轮浏览器任务。 + +- 机器：M3 Max / 48 GB / macOS 25.6.0 arm64，node v26.9.0，`llama-server` 0.4.0（Homebrew） +- 后端：本机全本地 `Qwen3.5-4B-Q4_K_M`（registry 默认项），llama-server `:8090` + Jev 契约服务 `:8092` +- 浏览器：`--backend chrome --headless`（每次 run 独立 profile，可复现） +- 实测日期：2026-09-24 +- 原始件（15 个 journal、16 项校准探针、39 步请求日志）：`/tmp/jev-local-smoke/` +- **`skills/**` 与 `docs/**` 之外一行未改：§6 列出的观测/代码层问题只报告、不改动，本次也没有提交任何 commit** + +--- + +## 0. 结论（先说结果） + +**能跑完，但不能可靠地「自己判断跑完了」。** + +| 问题 | 答案 | +|---|---| +| 本地后端能被 `run` 驱动到 `success` 吗？ | **能**：R7 在真实站点（Wikipedia 搜索结果页，64 个可点选项）4 步达成 `status=success`、退出码 0；另有 3 次 1 步达成（example.com / 商品页 / 目录页） | +| 传输层/契约层有没有坏？ | **没有**：39 个步骤请求全部首次尝试即成功（`attempts` 全为 1）、0 次超时、0 次 422 `LOW_LABEL_MASS`（最低 label_mass 0.870，阈值 0.5） | +| 15 次 run 的结局 | `success` 4 · `stuck` 8 · `needs_user` 2 · `max_steps` 1（无 `error`、无 `timeout`） | +| 卡点在哪 | 不是选元素（`click_target`），而是 **`action` 选择** 与 **`goal_done` 语义**：9 个非 `success` 的 run 里，8 个的失败点都是 `action` 把 `stop`/`click`/`wait`/`go_back` 排在正确动作前面，剩下 1 个（R3）是 `type` 结构上没被提供 | +| `goal_done >= 0.85` 这条终止规则本地能不能用 | **阈值 0.85 是错的值，不是错的问题**：按终止语义（首次越线即 success，§9）**`goal_done >= 0.25`（可用带 0.12–0.28）7/7 正确 success、0 假 success、0 假 stuck**；现行 0.85 只有 4/7 且假 stuck 3 次。逐「步」分类确实不可分（达成 0.058–0.995 vs 未达成 ≤0.111），代码侧替代（实体/环检测）更差（各 4 次假 success） | + +判定边界（详见 §8）： +- **可用**：`judge` 单问、`click_target` 选元素、「当前页面是不是已经是目标状态」这一问； +- **不可用**：把 `run` 的 `success` 终止交给 `goal_done`（动作型目标会漏判成 `stuck`）、`action` 动作选择（需要填表/输入时几乎必错）、`blocker` 在表单/报价页上的判定（会假 `needs_user`）。 + +--- + +## 1. 环境与启动（逐字） + +```bash +# 本地后端（8090 llama-server + 8092 jev 契约服务）；启动器只复用「同一个 gguf」的已启动实例 +node skills/jev-browser/bin/jev-local.mjs +# → [local] model ready: ~/.jev-browser/models/Qwen3.5-4B-Q4_K_M.gguf (2.6 GiB) +# → [local] llama-server: /opt/homebrew/bin/llama-server +# → [local] starting llama-server: ... -m .../Qwen3.5-4B-Q4_K_M.gguf --host 127.0.0.1 --port 8090 -c 16384 -ngl 99 --cache-type-k q8_0 --cache-type-v q8_0 -t 8 +# → [local] llama.cpp serving on http://127.0.0.1:8090 (pid 38548, -c 16384) +# → [local] serving /v1/systemone on http://127.0.0.1:8092 (qwen3.5-4b-q4-k-m · Qwen3.5-4B Q4_K_M); Ctrl-C to stop +# → TYPESAFE_BASE_URL=http://127.0.0.1:8092 TYPESAFE_API_KEY=local +``` + +模型加载 2.7 s（Metal），服务就绪后 `/health` 返回 `{"status":"ok","service":"jev-local","model":"qwen3.5-4b-q4-k-m"}`。 + +```bash +# fixture 站点（tests/fixtures/server.mjs，固定 3111 端口） +node --input-type=module -e "const {createSite}=await import('./tests/fixtures/server.mjs');const s=createSite();await new Promise(r=>s.server.listen(3111,'127.0.0.1',r));" +``` + +每次 run 的环境（覆盖仓库外的 user config，不写仓库）： + +```bash +export TYPESAFE_BASE_URL=http://127.0.0.1:8092 TYPESAFE_API_KEY=local +export JEV_BROWSER_CONFIG=/tmp/jev-local-smoke/config-.json # journalDir + chrome.userDataDir 指向 /tmp +node skills/jev-browser/bin/jev-browser.mjs run --goal "" --url \ + --backend chrome --headless --max-steps <8|5|6> --json +``` + +`--max-steps` 全部取 4–8；`timeoutMs` 保持**默认 20000**（除 M 组 2 次刻意放大，见 §5 注）。 +退出码：`success=0`、`needs_user=3`、其余 2（`bin/jev-browser.mjs:132-134`）。 + +--- + +## 2. 各次 run：目标、结局、步数、墙钟、停在哪 + +`steps` = controller 真正执行的动作数（`memory.history.length`）；`needs_user`/首步 `stop` 会是 0 步但已花掉 1 次 judge。 + +| # | goal（简写） | 站点 | 状态 | 退出码 | steps | 墙钟 | in-app | 停在哪 | +|---|---|---|---|---|---|---|---|---| +| R1 | 确认主标题是 "Example Domain" | example.com | **success** | 0 | 1 | 9.1 s | 5.9 s | step1 `goal_done=0.992` | +| R2 | 把 Red Gadget 加入购物车 | fixture /products | stuck | 2 | 1 | 10.1 s | 7.8 s | step2 模型选 `stop`（`goal_done=0.057`），实际只差一次点击 | +| R3 | DDG 搜 python → 打开 python.org | duckduckgo.com | stuck | 2 | 4 | 59.1 s | 56.1 s | step5 模型选 `stop`；全程**没机会输入**（§6.1） | +| R4 | 打开 Docs 并点按钮显示密码 | fixture / | stuck | 2 | 4 | 31.4 s | 26.6 s | step5 模型选 `stop`，**目标其实第 2 步就达成了** | +| R5 | 点 Accept 关掉 cookie 弹窗 | fixture /consent | stuck | 2 | 1 | 16.3 s | 12.6 s | step2 模型选 `stop`，**目标第 1 步已达成** | +| R6 | 从搜索结果打开 python.org | duckduckgo.com/?q=… | **needs_user** | 3 | 0 | 29.5 s | 26.1 s | step1 `blocker=verification_challenge(0.973)` —— **真阳性**（DDG 出人机验证，§4） | +| R7 | 打开 Alan Turing 条目 | en.wikipedia.org Special:Search | **success** | 0 | 4 | 46.4 s | 45.2 s | step4 `goal_done=0.995`（state 65 个元素 / 64 个可点选项里点对 `e20`） | +| R8 | 搜 Wikipedia → 打开条目 | en.wikipedia.org Main_Page | max_steps | 2 | 8 | 96.4 s | 94.8 s | 8 步用尽，末次校验 `goal_done=0.005` | +| p1 | 确认商品页已打开并显示价格 | fixture /products/red-gadget | **success** | 0 | 1 | 5.5 s | 4.5 s | step1 `goal_done=0.982` | +| p2 | 确认目录页已打开 | fixture /products | **success** | 0 | 1 | 4.4 s | 3.5 s | step1 `goal_done=0.988` | +| p3 | 给 Team 套餐开免费试用 | fixture /pricing | **needs_user** | 3 | 0 | 5.5 s | 4.7 s | step1 `blocker=missing_information(0.684)` —— **假阳性**，§4 | +| p4 | 用给的邮箱/密码登录 | fixture /login | stuck | 2 | 3 | 17.1 s | 16.2 s | 连续 3 次无变化动作；**全程没输入**（§5） | +| g1 | 把 Red Gadget 加入购物车（商品页起步） | fixture /products/red-gadget | stuck | 2 | 0 | 6.0 s | 6.0 s | step1 模型选 `stop(0.43)`，页面正中间就是 "Add to cart" | +| g3 | 点按钮显示密码并确认可见 | fixture /docs | stuck | 2 | 1 | 9.6 s | 9.6 s | step2 模型选 `stop`，**目标第 1 步已达成**（`goal_done=0.329`） | +| g4 | 填联系表单并发送 | fixture /contact | stuck | 2 | 0 | 8.3 s | 7.2 s | step1 模型选 `stop(0.30)`，四个字段的值都已提供 | + +补充： +- R6 / p3 的 `steps=0` 但 journal 里有一条 step1 —— 判定发生在动作之前，`steps` 只数动作。 +- R6、p3 的 `handOff` 均为 `null`；`resume` 里只有 `{backend,cdpUrl,targetId}`，没有 `spaceId`。CLI 仍会打印 + “the browser was handed to you; resume with --space-id …”（`bin/jev-browser.mjs:128`），而 chrome 后端在 + `success=false` 且 headless 时会关掉浏览器（`lib/backends/chrome.mjs:406`）—— 这条 `needs_user` 提示对 chrome 无意义（§6.4）。 + +### 2.1 逐 run 归因（journal `steps.jsonl` + `requests.jsonl`） + +| run | 归因 | 依据 | +|---|---|---| +| R1 / p1 / p2 / R7 | **无问题**（正常终止） | R7 由 `goal_done=0.995` 收尾；其余 3 次首步 `goal_done≥0.982` | +| R2 / g1 | **模型误判（action）** | 商品页 `action=stop(0.43/0.32)`，而 `click_target` 的第一名就是页面正中的 `button 'Add to cart'`；元素不缺、也没报错 | +| g4 | **模型误判（action）** | 联系表单页，4 个字段的值都在 `inputs` 里，`editable` 4 个、`type` 在 `allowedActions` 内，`action` 仍选 `stop(0.30)` | +| R4 / R5 / g3 | **模型误判（goal_done 语义）** | 动作本身做对了（R4 第 2 步、R5 第 1 步、g3 第 1 步的目标均已达成，见 `finalTextExcerpt`），但 `goal_done` 只有 0.281/0.586/0.329（<`goalDoneFinal` 0.7）→ 记 `stuck` | +| p4 | **模型误判（action 排序）** | `action` 四选一里 `type` 垫底（0.105），虽然 `type_target`=Email(0.851)、`type_value`=email(0.933) 都对；controller 按模型序执行 ⇒ 3 次无变化 | +| R8 | **模型误判 + controller 放大** | `type` 已提供（100 元素/99 可点/1 可编辑，`allowedActions` 含 `type`）却没被选；controller 的 progress 规则又两次选了走不通的 `go_back`（`no previous page in history`） | +| R3 | **证据缺口（missing evidence）** | `` 在观测里不是 `editable`（§6.1，dry-run 实测 `editable=0`），`type`/`type_target`/`type_value` 三问根本没生成 ⇒ 模型只能在 click/scroll/wait/stop 里挑 | +| p3 | **模型误判（blocker 假阳性）** | 定价页给出 `missing_information(0.684)`；页面三个套餐与按钮都列在元素表里，`inputs` 也不缺 | +| R6 | **真阳性，但 run 无事可做** | DDG 返回人机验证（`visible_text` 逐字可查），`verification_challenge(0.973)` 判对了；这次没有可自动化空间 | +| — | **API 失败：0 次** | 39/39 请求 `status=succeeded`、`attempts=1`；无 `TIMEOUT`/`LOW_LABEL_MASS`/`CONNECTION` | +| — | **后端/driver 异常：0 次** | 无 `error` 状态、无 Chrome 启动失败；唯一反复出现的 `no previous page in history` 是 controller 主动 `go_back` 后被 driver 正确拒绝（§6.3） | + +--- + +## 3. 逐步延迟画像（39 个步骤请求，全部取自 `requests.jsonl`） + +单步 = 一次 HTTP 批次调用里本地 provider 顺序跑完该步的所有问题；`prompt_tokens` 是**未命中前缀缓存、真正需要重算**的 token（`timings.prompt_n`）。 + +| 指标 | p50 | mean | max | +|---|---|---|---| +| 单步请求（客户端观测） | **4,312 ms** | 5,984 ms | **18,151 ms** | +| 本地服务 `ms_total` | 4,310 ms | 5,978 ms | 18,147 ms | +| 单步 prompt tokens（去缓存后） | 3,555 | 4,680 | 12,254 | +| 单步问题数 | 5 | 5.6 | 8 | + +按问题拆（同一批内，第一问承担 state 预填充，其余问题命中 llama.cpp 前缀缓存）： + +| 问题 | n | p50 | mean | max | prompt p50 | cold/warm | label_mass p50 / min | +|---|---|---|---|---|---|---|---| +| `goal_done` | 39 | **1,165 ms** | 2,317 ms | 8,887 ms | 933 | 29/10 | 0.966 / 0.939 | +| `blocker` | 38 | 725 ms | 737 ms | 953 ms | 559 | 1/37 | 0.995 / 0.990 | +| `click_target` | 38 | 713 ms | 1,373 ms | 4,499 ms | 490 | 0/38 | 0.992 / 0.926 | +| `progress` | 23 | 606 ms | 587 ms | 771 ms | 442 | 0/23 | 0.995 / 0.984 | +| `action` | 38 | 595 ms | 606 ms | 798 ms | 464 | 0/38 | 0.985 / 0.870 | +| `select_target` | 7 | 568 ms | 547 ms | 721 ms | 362 | 0/7 | 0.962 / 0.932 | +| `submit_after_type` | 12 | 567 ms | 596 ms | 807 ms | 465 | 0/12 | 0.990 / 0.977 | +| `type_target` | 12 | 547 ms | 596 ms | 1,013 ms | 433 | 0/12 | 0.983 / 0.971 | +| `type_value` | 12 | 546 ms | 553 ms | 748 ms | 418 | 0/12 | 0.977 / 0.968 | + +结论与既有文献的两点修正： + +1. **`click_target` 不再是最贵的**。`experiments/gguf-provider/RESULTS.md` 里 “click_target 6.1 s、整步 9 s（热）/18.3 s（冷）” + 是在**不命中前缀缓存**的口径下量的；真实 run 里 provider 顺序发问，state 只算一次（`goal_done` 那一问承担， + p50 1.2 s、max 8.9 s），后面每一问只付「问题块」的增量（400–500 tok，0.5–0.7 s）。整步 p50 落在 **4.3 s**。 +2. **但默认 20 s 客户端超时是真的紧**。R8 step8 的状态是 12,254 tok，单步 **18,151 ms**，距 `timeoutMs=20_000` + 只剩 **1.85 s**；`lib/typesafe.mjs:201-207` 的超时标 `retryable`，一旦触发会把整批 5–8 问重跑（最多 3 次）， + 本次 15 次 run 没有触发，但更大页面/更慢机型会踩。39 步里 0 次超时、0 次 `LOW_LABEL_MASS`。 + +--- + +## 4. `goal_done` 分离分析：有没有一条阈值能分开「已达成 / 未达成」 + +数据两部分：(a) **首步校准探针 16 项**——用 skill 自己的 `run --dry-run` 构造第一步 state（含 goal/inputs/page）， +只把 `goal_done` 这一问发给本地服务，真值由页面构造决定；(b) **15 次 run 的 39 个步骤判定**，真值由 journal + 最终页面人工判定。 + +### 4.1 首步校准（16 项，state 与 controller 实际发送的完全一致） + +| id | 真值 | goal_done | id | 真值 | goal_done | +|---|---|---|---|---|---| +| T1 商品页已打开 | 达成 | **0.981** | F1 加入购物车 | 未达成 | 0.008 | +| T2 目录页已打开 | 达成 | **0.988** | F2 登录成功并看 dashboard | 未达成 | 0.007 | +| T3 联系表单在页面上 | 达成 | **0.954** | F3 Team 试用已开始 | 未达成 | 0.008 | +| T4 登录页在问邮箱密码 | 达成 | **0.969** | F4 已显示密码 | 未达成 | 0.032 | +| T5 三个套餐已列出 | 达成 | **0.987** | F5 弹窗已关闭 | 未达成 | 0.096 | +| T6 页面说需要管理员权限 | 达成 | **0.936** | F6 已打开定价页且 $29 | 未达成 | 0.011 | +| T7 页面在问 cookie 同意 | 达成 | **0.839** | F7 已打开 Docs | 未达成 | 0.043 | +| T8 主标题 Example Domain | 达成 | **0.997** | F8 已从链接打开 IANA | 未达成 | 0.036 | + +**达成 0.839–0.997（n=8）；未达成 0.007–0.096（n=8）。最大负例 0.096 ≪ 最小正例 0.839，中间是一段空档。** +即「这页是不是已经是目标状态」这一问，本地 4B 是可用的；官方阈值 0.85 只有 1/8 真例踩线落空（T7 = 0.839）， +0/8 假阳性。任何落在 **[0.10, 0.83]** 的阈值都能把这 16 项全部分开。 + +### 4.2 run 内的判定（含「做了一步之后」的状态）——分离失效 + +| 类别 | 样本（goal_done） | +|---|---| +| 未达成，首步 | R2 .008 · R3 .004/.011/.012/.006/.006 · R4 .010 · R5 .096 · R6 .005 · p3 .008 · p4 .010 · R7 .023/.015/.014 · R8 .003/.022/.015/.009/.009/.008/.010/.011（+末次 .005）· g1 .050 · g3 .027 · g4 .012 | +| 未达成，但**做了一步**（最危险的一档） | **R4 step2 = 0.111** | +| 已达成，且是首步就能判 | R1 .992 · p1 .982 · p2 .988 | +| 已达成，**靠动作达成** | R7 step4 **.995** · R5 step2 **.586** · g3 step2 **.329** · R4 step3 **.273** · R4 step5 **.281** · **R4 step4 = 0.058** | + +- 未达成侧最大 **0.111**（R4 step2，点了按钮但页面刚变）；已达成侧最小 **0.058**（R4 step4，同一页、密码已可见）。 + **两个区间重叠 ⇒ 不存在能把「已达成 / 未达成」分开的阈值。** +- 这不是「阈值调低一点就行」：把阈值放到 0.2 能救回 g3(0.329)/R4(0.273)，但会救不回 R4 step4(0.058)，而 + 同一页 step2 的 0.111 又高于它 —— 判定顺序本身不稳定（同页同目标：0.111 → 0.273 → 0.058 → 0.281）。 +- 反过来说，`goal_done` 在「页面本身就是答案」的场景（R1/R7/p1/p2、T1–T8）非常干净，0.98 上下； + **它坏掉的正是「做完一个动作之后」这一类**——4B 对「动作产生了结果」的确认能力不足。 + +### 4.3 对控制器的直接后果 + +- `goal_done >= 0.85`（`lib/config.mjs:27`）在动作型目标上**永远不会触发**：本次 6 个「已达成」样本里 5 个 < 0.85。 +- 于是 run 只能靠 `stop` 分支收尾，而 `stop` 只有在 `goal_done >= goalDoneFinal(0.7)`（`lib/config.mjs:28`）时才记 `success`： + R4(0.281)、R5(0.586)、g3(0.329) 三个**实际已经完成**的 run 被判 `stuck`，退出码 2。 +- 末次校验（`maxSteps` 用尽后补问一次）用的正是同一个 0.7：R8 得 0.005，判 `max_steps`（这次判对了）。 +- **所以「本地后端能不能跑完」与「本地后端能不能承认自己跑完」是两件事**：前者可以（R7），后者不可信。 + +--- + +## 5. `action` 选择：与页面明显允许的动作不符 + +按严重度排序（全部有 journal 逐字证据）： + +1. **p4（登录页）——模型知道该填哪儿、填什么，就是不选「type」。** 同一 state 的完整判读（`/tmp/jev-local-smoke/login.dryrun.json` 重放，确定性复现）： + ``` + action click 0.430 · stop 0.246 · wait 0.220 · type 0.105 ← 四选一里 type 垫底 + type_target e2 (Email) 0.851 ← 正确 + type_value email 0.933 ← 正确 + click_target e4 ("Sign in" 按钮) 0.892 + submit_after_type 0.183（= "不按回车"，也对） + ``` + controller 按模型动作序建候选（`lib/controller.mjs:263` 的 `for (const [action] of decision.actions)` → `:320-323` 的取候选），于是先挑 `click e4`（空表单，无变化）， + 再挑 `wait`（无变化），第三个才轮到 `type` —— 3 次无变化后 stuck（`noChangeLimit=3`）。 +2. **R3（DDG 首页）/ R8（Wikipedia 主页）——同样从不输入**：R3 依次点了搜索框 combobox、Search 单选、" + Set As Default Search" 链接；R8 点了 Search 按钮、搜索框 combobox、滚屏、返回，最后点进一篇无关条目 + （Hashim Thaçi）。两者原因不同：R3 的 `type` 是**结构上就没被提供**（§6.1）；R8 的 `type` 提供了 + （Main_Page dry-run：100 个元素 / 99 个可点 / 1 个可编辑，`allowedActions=click,type,scroll_down,wait,stop`， + `type_target`/`type_value`/`submit_after_type` 三问齐全，`inputKeys=[query]`）却没被选。 +3. **提前 `stop`（3 次）**：g1/g4 在「页面正中就是 Add to cart / 四个字段都已给值」时第一问就选 `stop` + （0.43 / 0.30）；R2 在商品页选 `stop`（0.32）。`action` 的 confidence 只有 0.06–0.13（无信心）， + 但 controller 不用 confidence 做弃权（`lib/typesafe.mjs:57` 归一化后只给 `confidence` 字段，无人消费）。 +4. **R7 的 `go_back` 空转**：第 1–2 步 `action` 把 `go_back`(0.28) 排在 `click`(e20 = 正确的 Alan Turing 链接) 前面， + 白走两步（`go_back` 的 `click_target` 一直是 e20 —— 选元素没问题，选动作有问题）。 + 第 2 步 `go_back` 失败（`no previous page in history`）。 +5. **R8 的重复无效 `go_back`**：step5/7/8 由 controller 的「progress 退化」规则（`lib/controller.mjs:257-261`） + 自己选了 `go_back`，其中两次报 `no previous page in history`；因为 blocked 记忆的键是 + `${stateHash}|${actionKey}` 而 state 每步都变（含 `previous_page`/`last_action`，`lib/controller.mjs:193-196`）， + 同一台不可能完成的动作被反复重试。 + +对照：**`click_target` 是本次最稳的一环**。R7 在 65 个元素的真实页面上，每一步都把正确条目排在第一 +（`e20 = link 'Alan Turing' → /wiki/Alan_Turing`，0.784 / 0.742 / 0.713；第二名 `e22` 是同一个 `/wiki/Alan_Turing` 的另一处链接， +0.17 / 0.22 / 0.25，两者合计 0.95 —— 即使第一名失手，结果页也是对的）；R2 step1、R4 step1/2、R5 step1、g3 step1 在 fixture 上也都点对。 +RESULTS.md 里 “`action + click_target` 5 题只对 2 题” 的短板，在真机 run 里重现的是其中的 **action 半边**。 + +--- + +## 6. 观测/代码层问题（只报告，未改任何 `skills/**` 文件） + +> **状态更新 2026-09-24 ~01:25（本次 run 之后，工作区并发改动、未提交）**：6.1（combobox 不是 textbox）、 +> 6.4（`needs_user` 的 ego 专属提示）、6.5（本地 token 按 hosted 价格计费）三条**已在 `skills/**` 里被修**： +> `lib/observe.mjs` 新增 `isComboboxField()` + `tests/e2e/combobox.test.mjs`；`bin/jev-browser.mjs` 的 `needs_user` +> 分支改为有 `spaceId` 才提示 `--space-id`，否则说明浏览器已关闭；`lib/typesafe.mjs` 新增 +> `isLoopbackBaseUrl()`/`pricePerMtokFor()`，loopback 计费为 0。6.2/6.3/6.6 未被改动。下面保留的是 +> **实测时的行为**（run 时间 00:54–01:08），行号按当时的工作区版本。 + +### 6.1 `role="combobox"` 的 `` 被当成非文本字段 → 永远没有 `type` 动作 + +> **状态更新 2026-09-24 ~01:25**：本条已被并发修复（工作区未提交）—— `lib/observe.mjs` 新增 `isComboboxField()`， +> 把 `role=combobox` 的 `/