diff --git a/README.md b/README.md index b5c18a0..71fcc1e 100644 --- a/README.md +++ b/README.md @@ -76,11 +76,35 @@ model should have `"input": ["text", "image"]`. Other `/vision` subcommands: | `/vision max-dim ` | Max image dimension for compression (1–8000) | | `/vision quality <1-100>` | JPEG re-encode quality | | `/vision reasoning-effort ` | Default reasoning effort for delegation | +| `/vision system-prompt [\|clear]` | Set/clear a custom system prompt for the vision model (no arg → multi-line editor) | +| `/vision cache ` | Clear the cache or show stats (memory + disk entries) | +| `/vision fallback \|clear` | Set/clear a fallback vision model | | `/vision clear` | Reset config to defaults | +| `/vision-use [provider/model]` | Switch the DELEGATE vision model inline (no arg → picker). **Hotkey: `alt+shift+v`** (rebindable via `keybindings.json`) | Config is stored at `~/.pi/agent/vision.json` (not `vision-tool.json`, so it doesn't collide with the community package during transition). +## Resilience (v0.2.0) + +DELEGATE mode (text-only primary) is resilient + cheap: + +- **Caching.** Successful delegation results are cached by a content-addressed + key (image hash + compression params + prompt + vision model + reasoning). + A second call on the same image costs **zero** vision-model API calls. + In-memory by default; opt into cross-session persistence with **Persist + cache to disk** (LRU-evicted at the configured max entries). Only successes + are cached — failures never are. `/vision cache clear` wipes both layers. +- **Retry + fallback.** On a retryable failure (HTTP 5xx, 429, network), the + primary vision model is retried with exponential backoff (abort-aware — a + cancelled turn stops retrying immediately). On a non-retryable error or + exhausted retries, a configured **fallback vision model** is tried once. + Configure both via the `/vision` panel or `/vision fallback `. +- **Custom system prompt.** A per-workflow framing prepended to the + vision-model request (`/vision system-prompt `, or the panel row). +- **Inline model switch.** `alt+shift+v` (or `/vision-use`) switches the + DELEGATE vision model mid-session without opening the full panel. + ## How it works Two mechanisms combine to guarantee the behavior: @@ -118,6 +142,10 @@ Parameters: | `compress` | boolean? | Optimize the image before delegation (default `true`) | | `reasoning` | enum? | Reasoning effort for the delegation (`off`…`xhigh`) | +When caching or fallback is active, the tool result `details` include +`cached: true` (cache hit) and `fallback: true` (result from the fallback +model) for traceability. + For multimodal primaries you don't call `describe_image` — just reference the image path in your message and the model sees it natively. diff --git a/extensions/vision.ts b/extensions/vision.ts index cc57732..008cc62 100644 --- a/extensions/vision.ts +++ b/extensions/vision.ts @@ -19,13 +19,15 @@ * subcommands (`/vision on`, `/vision model `, …) remain for power users. */ import type { Api, Model } from "@earendil-works/pi-ai"; -import type { ExtensionAPI, ExtensionCommandContext, Theme } from "@earendil-works/pi-coding-agent"; +import type { ExtensionAPI, ExtensionCommandContext, ExtensionContext, Theme } from "@earendil-works/pi-coding-agent"; import { getAgentDir } from "@earendil-works/pi-coding-agent"; +import { join } from "node:path"; import { Type } from "typebox"; import { StringEnum } from "@earendil-works/pi-ai"; import { Container, type Component, + Input, SettingsList, type SettingItem, SelectItem, @@ -43,10 +45,20 @@ import { type VisionConfig, } from "../lib/config.ts"; import { delegateToVisionModel, type DelegateParams } from "../lib/delegate.ts"; +import { VisionCache } from "../lib/cache.ts"; /** Current config. Loaded on session_start, mutated by /vision, saved to disk. */ let config: VisionConfig = { ...DEFAULT_CONFIG }; +/** Content-addressed delegation cache. Rebuilt on session_start + when + * cachePersist/cacheMaxEntries change. Memory-only when cachePersist is off. */ +let cache: VisionCache = new VisionCache(undefined, DEFAULT_CONFIG.cacheMaxEntries); + +function rebuildCache(): void { + const dir = config.cachePersist ? join(getAgentDir(), "vision-cache") : undefined; + cache = new VisionCache(dir, config.cacheMaxEntries); +} + const SUBCOMMANDS = [ "show", "on", @@ -56,6 +68,9 @@ const SUBCOMMANDS = [ "max-dim", "quality", "reasoning-effort", + "system-prompt", + "cache", + "fallback", "clear", ] as const; @@ -68,9 +83,19 @@ function formatConfigStatus(c: VisionConfig): string { ` maxDimension: ${c.maxDimension}px`, ` jpegQuality: ${c.jpegQuality}`, ` reasoning: ${c.defaultReasoningEffort}`, + ` systemPrompt: ${c.systemPrompt ? truncatePreview(c.systemPrompt, 40) : "(none)"}`, + ` cache: ${c.cacheEnabled ? "on" : "off"}${c.cachePersist ? " (persisted, max " + c.cacheMaxEntries + ")" : ""}`, + ` retry: ${c.retryAttempts} attempts, ${c.retryBackoffMs}ms backoff`, + ` fallback: ${c.fallbackProvider && c.fallbackModel ? c.fallbackProvider + "/" + c.fallbackModel : "(none)"}`, ].join("\n"); } +/** Truncate a string for a settings-row preview, appending an ellipsis if it overflows. */ +function truncatePreview(s: string, max: number): string { + const t = s.replace(/\s+/g, " ").trim(); + return t.length <= max ? t : `${t.slice(0, max - 1)}…`; +} + /** Display string for a setting row, from the current config. */ function renderValue(id: string): string { switch (id) { @@ -84,6 +109,20 @@ function renderValue(id: string): string { return `${config.jpegQuality}`; case "reasoning": return config.defaultReasoningEffort; + case "systemPrompt": + return config.systemPrompt ? truncatePreview(config.systemPrompt, 40) : "(none)"; + case "cacheEnabled": + return config.cacheEnabled ? "on" : "off"; + case "cachePersist": + return config.cachePersist ? "on" : "off"; + case "cacheMaxEntries": + return `${config.cacheMaxEntries}`; + case "retryAttempts": + return `${config.retryAttempts}`; + case "retryBackoffMs": + return `${config.retryBackoffMs}ms`; + case "fallbackModel": + return config.fallbackProvider && config.fallbackModel ? `${config.fallbackProvider}/${config.fallbackModel}` : "(none)"; default: return ""; } @@ -94,21 +133,24 @@ function resync(pi: ExtensionAPI, ctx: ExtensionCommandContext): void { syncToolAvailability(pi, ctx.model, { enabled: config.enabled }); } -/** Apply a setting edit, persist, and re-sync visibility if needed. */ +/** Apply a setting edit, persist, re-sync visibility if needed, and rebuild + * the cache when cache-shape fields change. */ function applyAndSave(id: string, value: string, pi: ExtensionAPI, ctx: ExtensionCommandContext): void { config = applySettingChange(config, id, value); saveConfig(config, getAgentDir()); if (id === "enabled" || id === "model") resync(pi, ctx); + if (id === "cachePersist" || id === "cacheMaxEntries") rebuildCache(); } /** Vision-capable authed models from the registry (input includes "image"). */ -function visionCapableModels(ctx: ExtensionCommandContext): Model[] { +function visionCapableModels(ctx: ExtensionContext): Model[] { return ctx.modelRegistry.getAvailable().filter((m) => m.input.includes("image")); } /** Open pi's native select picker over vision-capable models. Sets provider + - * model together. Used by `/vision model` (no arg) as a quick pick. */ -async function pickVisionModel(ctx: ExtensionCommandContext): Promise { + * model together. Used by `/vision model` (no arg), `/vision-use`, and the + * `alt+shift+v` hotkey as a quick pick. */ +async function pickVisionModel(ctx: ExtensionContext): Promise { const models = visionCapableModels(ctx); if (models.length === 0) { ctx.ui.notify( @@ -176,11 +218,61 @@ async function showVisionSettings(pi: ExtensionAPI, ctx: ExtensionCommandContext values: [...REASONING_LEVELS], description: "Default reasoning effort for delegation calls.", }, + // ── v0.2.0 (SPEC-2) rows ──────────────────────────────────────────── + { + id: "systemPrompt", + label: "System prompt", + currentValue: renderValue("systemPrompt"), + description: "Vision-model framing prepended to the request. Enter to edit inline (single-line). For multi-line, use /vision system-prompt.", + submenu: (cur, subDone) => buildSystemPromptInput(cur, subDone), + }, + { + id: "cacheEnabled", + label: "Caching", + currentValue: renderValue("cacheEnabled"), + values: ["on", "off"], + description: "When on, identical delegation calls return a cached description (0 tokens on hit).", + }, + { + id: "cachePersist", + label: "Persist cache to disk", + currentValue: renderValue("cachePersist"), + values: ["on", "off"], + description: "When on, the cache survives session restarts (LRU-evicted at max entries).", + }, + { + id: "cacheMaxEntries", + label: "Cache max entries", + currentValue: renderValue("cacheMaxEntries"), + values: ["64", "128", "256", "512", "1024"], + description: "Max disk-cache entries before LRU eviction.", + }, + { + id: "retryAttempts", + label: "Retry attempts", + currentValue: renderValue("retryAttempts"), + values: ["0", "1", "2", "3", "5"], + description: "Retries after the first failure (total attempts = this + 1). Only 5xx/429/network retry.", + }, + { + id: "retryBackoffMs", + label: "Retry backoff (ms)", + currentValue: renderValue("retryBackoffMs"), + values: ["250", "500", "1000", "2000"], + description: "Base backoff; delay = min(backoffMs * 2^attempt, 8000ms).", + }, + { + id: "fallbackModel", + label: "Fallback vision model", + currentValue: renderValue("fallbackModel"), + description: "Secondary vision model tried when the primary exhausts retries or fails non-retryable. Enter opens a picker.", + submenu: (_cur, subDone) => buildModelSubmenu(theme, ctx, subDone), + }, ]; const settingsList = new SettingsList( items, - 8, + 12, { label: (text, selected) => (selected ? theme.fg("accent", theme.bold(text)) : text), value: (text, selected) => (selected ? theme.fg("accent", text) : theme.fg("muted", text)), @@ -237,10 +329,25 @@ function buildModelSubmenu( return sl; } +/** Build the single-line system-prompt editor shown when Enter is pressed on + * the system-prompt row. An `Input` (the same component SettingsList uses + * for its own search box). Empty submit clears; Escape cancels. */ +function buildSystemPromptInput( + currentValue: string, + subDone: (selectedValue?: string) => void, +): Component { + const input = new Input(); + input.setValue(currentValue === "(none)" ? "" : currentValue); + input.onSubmit = (value) => subDone(value); // "" commits → applySettingChange clears + input.onEscape = () => subDone(); // undefined → cancel (no change) + return input; +} + export default function visionExtension(pi: ExtensionAPI): void { // ── Session lifecycle ─────────────────────────────────────────────────── pi.on("session_start", (_event, ctx) => { config = loadConfig(getAgentDir()); + rebuildCache(); syncToolAvailability(pi, ctx.model, { enabled: config.enabled }); }); @@ -310,7 +417,7 @@ export default function visionExtension(pi: ExtensionAPI): void { reasoning: (params.reasoning ?? config.defaultReasoningEffort) as ReasoningLevel, }; - const result = await delegateToVisionModel(ctx, config, delegateParams, signal); + const result = await delegateToVisionModel(ctx, config, delegateParams, signal, cache); if (result.ok) { return { content: [{ type: "text" as const, text: result.text }], @@ -328,7 +435,7 @@ export default function visionExtension(pi: ExtensionAPI): void { // ── /vision slash command ────────────────────────────────────────────── pi.registerCommand("vision", { description: - "Open the vision settings panel (like /settings). Subcommands: show, on, off, provider

, model [], max-dim , quality <1-100>, reasoning-effort , clear.", + "Open the vision settings panel (like /settings). Subcommands: show, on, off, provider

, model [], max-dim , quality <1-100>, reasoning-effort , system-prompt [|clear], cache , fallback [|clear>, clear.", handler: async (args, ctx) => { const parts = args.trim().split(/\s+/).filter(Boolean); const sub = parts[0] ?? ""; // empty → open the settings panel @@ -426,10 +533,67 @@ export default function visionExtension(pi: ExtensionAPI): void { case "clear": { config = { ...DEFAULT_CONFIG }; saveConfig(config, agentDir); + rebuildCache(); resync(pi, ctx); ctx.ui.notify("Vision config reset to defaults.", "info"); return; } + case "system-prompt": { + const value = parts.slice(1).join(" ").trim(); + if (!value) { + // No arg → multi-line editor (safe: command handler, not inside ctx.ui.custom). + if (ctx.hasUI) { + const edited = await ctx.ui.editor("Vision system prompt", config.systemPrompt ?? ""); + if (edited === undefined) return; // cancelled + config = { ...config, systemPrompt: edited.trim().length > 0 ? edited.trim() : undefined }; + } else { + ctx.ui.notify("Usage: /vision system-prompt (or /vision system-prompt clear)", "warning"); + return; + } + } else if (value === "clear") { + config = { ...config, systemPrompt: undefined }; + } else { + config = { ...config, systemPrompt: value }; + } + saveConfig(config, agentDir); + ctx.ui.notify(config.systemPrompt ? "Vision system prompt set." : "Vision system prompt cleared.", "info"); + return; + } + case "cache": { + const action = parts[1]; + if (action === "clear") { + cache.clear(); + ctx.ui.notify("Vision cache cleared (memory + disk).", "info"); + } else if (action === "show") { + const s = cache.stats(); + ctx.ui.notify(`Vision cache: ${s.memoryEntries} memory, ${s.diskEntries} disk (max ${s.maxEntries}, persisted ${s.persisted}).`, "info"); + } else { + ctx.ui.notify("Usage: /vision cache ", "warning"); + } + return; + } + case "fallback": { + const value = parts.slice(1).join(" ").trim(); + if (!value) { + ctx.ui.notify("Usage: /vision fallback (or /vision fallback clear)", "warning"); + return; + } + if (value === "clear") { + config = { ...config, fallbackProvider: undefined, fallbackModel: undefined }; + saveConfig(config, agentDir); + ctx.ui.notify("Fallback vision model cleared.", "info"); + return; + } + const slash = value.indexOf("/"); + if (slash > 0 && slash < value.length - 1) { + config = { ...config, fallbackProvider: value.slice(0, slash), fallbackModel: value.slice(slash + 1) }; + } else { + config = { ...config, fallbackModel: value }; + } + saveConfig(config, agentDir); + ctx.ui.notify(`Fallback vision model set to ${config.fallbackProvider}/${config.fallbackModel}.`, "info"); + return; + } default: { ctx.ui.notify( `Unknown /vision subcommand: ${sub}\nAvailable: ${SUBCOMMANDS.join(", ")} (or just /vision for the panel)`, @@ -439,4 +603,37 @@ export default function visionExtension(pi: ExtensionAPI): void { } }, }); + + // ── /vision-use command + alt+shift+v hotkey (SPEC-2 gap #5: inline switch) ─ + // Both switch the DELEGATE vision model mid-session without the full panel. + // Tool visibility is unaffected (it tracks the PRIMARY model's capability, + // not the vision model) so no resync is needed. + pi.registerCommand("vision-use", { + description: + "Switch the DELEGATE vision model inline. No arg → picker; → set directly. (Hotkey: alt+shift+v)", + handler: async (args, ctx) => { + const value = args.trim(); + if (!value) { + const picked = await pickVisionModel(ctx); + if (picked) ctx.ui.notify(`Vision model set to ${config.provider}/${config.model}.`, "info"); + return; + } + const slash = value.indexOf("/"); + if (slash > 0 && slash < value.length - 1) { + config = { ...config, provider: value.slice(0, slash), model: value.slice(slash + 1) }; + } else { + config = { ...config, model: value }; + } + saveConfig(config, getAgentDir()); + ctx.ui.notify(`Vision model set to ${config.provider}/${config.model}.`, "info"); + }, + }); + + pi.registerShortcut("alt+shift+v", { + description: "Switch vision model (inline picker)", + handler: async (ctx) => { + const picked = await pickVisionModel(ctx); + if (picked) ctx.ui.notify(`Vision model set to ${config.provider}/${config.model}.`, "info"); + }, + }); } \ No newline at end of file diff --git a/lib/cache.ts b/lib/cache.ts new file mode 100644 index 0000000..b9cc50b --- /dev/null +++ b/lib/cache.ts @@ -0,0 +1,193 @@ +/** + * Content-addressed cache for vision-model delegation results. + * + * A second `describe_image` call on the same image (same prompt, same vision + * model, same compression params, same reasoning) returns the cached + * description WITHOUT calling the vision model — zero tokens, zero latency + * (SPEC-2 gap #2). + * + * Cache key = sha256(sourceHash + compress + maxDimension + jpegQuality + + * prompt + modelId + reasoning). Keying on the ORIGINAL-byte hash (not the + * compressed bytes) makes hits stable regardless of compression + * nondeterminism (worker vs in-process fallback — see PLAN-2 §1.1). + * + * Two layers: + * 1. In-memory `Map` (session-scoped, always active when `cacheEnabled`). + * 2. Optional persisted disk cache (`

/.json`, LRU-evicted by file + * mtime) — active when a `dir` is provided (`cachePersist: true`). + * + * Only successful results are cached (failures are never cached — a transient + * error must not poison the cache). Writes are atomic (tmp + rename). No + * cross-session lockfile (benign races only — see PLAN-2 §1.2). + */ +import { createHash } from "node:crypto"; +import { existsSync, readdirSync, readFileSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import type { DelegateSuccess } from "./delegate.ts"; + +/** A cached delegation result. `storedAt` is informational (LRU uses file + * mtime, which is robust to clock skew across sessions). */ +export interface CacheEntry { + text: string; + details: DelegateSuccess["details"]; + storedAt: number; +} + +export interface CacheStats { + memoryEntries: number; + diskEntries: number; + maxEntries: number; + persisted: boolean; +} + +/** + * Compute the content-addressed cache key for a delegation call. Deterministic + * + collision-safe (sha256 over the full tuple with `\0` separators so no two + * distinct tuples can collide via concatenation ambiguity). + */ +export function cacheKey( + sourceHash: string, + compress: boolean, + maxDimension: number, + jpegQuality: number, + prompt: string, + modelId: string, + reasoning: string, +): string { + const tuple = [sourceHash, compress, maxDimension, jpegQuality, prompt, modelId, reasoning].join("\0"); + return createHash("sha256").update(tuple).digest("hex"); +} + +/** + * Vision description cache. Memory-first, disk-optional. Pure I/O — no pi + * runtime dependency — so it unit-tests with a tmp dir or memory-only. + */ +export class VisionCache { + private readonly memory = new Map(); + private readonly dir?: string; + private readonly maxEntries: number; + + constructor(dir?: string, maxEntries = 256) { + this.dir = dir; + this.maxEntries = Math.max(1, Math.round(maxEntries)); + } + + get persisted(): boolean { + return this.dir !== undefined; + } + + /** Look up a cached entry. Memory first; on miss, disk (promoting a hit + * into memory). A corrupt disk file is treated as a miss + removed. */ + get(key: string): CacheEntry | undefined { + const mem = this.memory.get(key); + if (mem) return mem; + if (this.dir) { + const file = this.fileFor(key); + if (existsSync(file)) { + try { + const entry = JSON.parse(readFileSync(file, "utf8")) as CacheEntry; + if (typeof entry.text === "string" && entry.details) { + this.memory.set(key, entry); // promote disk hit → memory + return entry; + } + } catch { + // corrupt JSON → remove + miss + } + try { + rmSync(file, { force: true }); + } catch { + // best-effort cleanup + } + } + } + return undefined; + } + + /** Store a successful result. Memory always; disk (atomic tmp+rename) when + * persisted. A disk write failure never fails the call (memory still has + * the entry for the session). */ + set(key: string, entry: CacheEntry): void { + this.memory.set(key, entry); + if (this.dir) { + const file = this.fileFor(key); + const tmp = `${file}.tmp`; + try { + writeFileSync(tmp, JSON.stringify(entry), "utf8"); + renameSync(tmp, file); + } catch { + // disk failure → memory-only degradation; don't throw + } + this.evictIfNeeded(); + } + } + + /** Wipe both layers. */ + clear(): void { + this.memory.clear(); + if (this.dir && existsSync(this.dir)) { + try { + for (const f of readdirSync(this.dir)) { + if (f.endsWith(".json")) rmSync(join(this.dir, f), { force: true }); + } + } catch { + // best-effort + } + } + } + + stats(): CacheStats { + let diskEntries = 0; + if (this.dir && existsSync(this.dir)) { + try { + diskEntries = readdirSync(this.dir).filter((f) => f.endsWith(".json")).length; + } catch { + // best-effort + } + } + return { + memoryEntries: this.memory.size, + diskEntries, + maxEntries: this.maxEntries, + persisted: this.dir !== undefined, + }; + } + + private fileFor(key: string): string { + return join(this.dir!, `${key}.json`); + } + + /** LRU eviction by file mtime: if disk entries exceed `maxEntries`, delete + * the oldest until under cap. Benign across concurrent sessions (worst + * case: a redundant eviction). */ + private evictIfNeeded(): void { + if (!this.dir || !existsSync(this.dir)) return; + let files: { name: string; mtime: number }[] = []; + try { + files = readdirSync(this.dir) + .filter((f) => f.endsWith(".json")) + .map((name) => { + let mtime = 0; + try { + mtime = statSync(join(this.dir!, name)).mtimeMs; + } catch { + // unreadable file → mtime 0 (evicted first) + } + return { name, mtime }; + }); + } catch { + return; + } + if (files.length <= this.maxEntries) return; + files.sort((a, b) => a.mtime - b.mtime); // oldest first + const toEvict = files.length - this.maxEntries; + for (let i = 0; i < toEvict; i++) { + const victim = files[i]; + if (!victim) continue; + try { + rmSync(join(this.dir!, victim.name), { force: true }); + } catch { + // best-effort + } + } + } +} \ No newline at end of file diff --git a/lib/config.ts b/lib/config.ts index 106db2c..a6eafa7 100644 --- a/lib/config.ts +++ b/lib/config.ts @@ -39,6 +39,23 @@ export interface VisionConfig { defaultReasoningEffort: ReasoningLevel; /** Master switch. When false, describe_image is hidden + errors if invoked. */ enabled: boolean; + // ── v0.2.0 (SPEC-2) ────────────────────────────────────────────────────── + /** Custom system prompt prepended to the vision-model request (undefined = none, v0.1.0 shape). */ + systemPrompt: string | undefined; + /** When true, successful delegation results are cached (0 tokens on hit). */ + cacheEnabled: boolean; + /** When true, the cache also persists to disk (cross-session hits, LRU-evicted). */ + cachePersist: boolean; + /** Max entries in the disk cache before LRU eviction. */ + cacheMaxEntries: number; + /** Number of retries after the first failure (total attempts = retryAttempts + 1). */ + retryAttempts: number; + /** Base backoff in ms for retry; delay = min(retryBackoffMs * 2^attempt, 8000). */ + retryBackoffMs: number; + /** Fallback vision model provider (used when the primary exhausts retries / fails non-retryable). */ + fallbackProvider: string | undefined; + /** Fallback vision model id under fallbackProvider. */ + fallbackModel: string | undefined; } export const DEFAULT_CONFIG: VisionConfig = { @@ -48,6 +65,15 @@ export const DEFAULT_CONFIG: VisionConfig = { jpegQuality: 85, defaultReasoningEffort: "off", enabled: true, + // v0.2.0 defaults + systemPrompt: undefined, + cacheEnabled: true, + cachePersist: false, + cacheMaxEntries: 256, + retryAttempts: 2, + retryBackoffMs: 500, + fallbackProvider: undefined, + fallbackModel: undefined, }; export const CONFIG_FILENAME = "vision.json"; @@ -67,6 +93,11 @@ function clampInt(value: unknown, min: number, max: number, fallback: number): n return Math.min(max, Math.max(min, Math.round(n))); } +/** Non-empty trimmed string → string; empty/missing → undefined. */ +function strOrUndef(value: unknown): string | undefined { + return typeof value === "string" && value.trim().length > 0 ? value.trim() : undefined; +} + /** * Merge a parsed partial config over the defaults, validating + clamping * every field so a malformed file can never produce an invalid `VisionConfig`. @@ -74,14 +105,23 @@ function clampInt(value: unknown, min: number, max: number, fallback: number): n export function mergeConfig(partial: unknown): VisionConfig { const p = (partial ?? {}) as Partial>; return { - provider: typeof p.provider === "string" && p.provider.trim().length > 0 ? p.provider.trim() : undefined, - model: typeof p.model === "string" && p.model.trim().length > 0 ? p.model.trim() : undefined, + provider: strOrUndef(p.provider), + model: strOrUndef(p.model), maxDimension: clampInt(p.maxDimension, 1, 8000, DEFAULT_CONFIG.maxDimension), jpegQuality: clampInt(p.jpegQuality, 1, 100, DEFAULT_CONFIG.jpegQuality), defaultReasoningEffort: isReasoningLevel(p.defaultReasoningEffort) ? p.defaultReasoningEffort : DEFAULT_CONFIG.defaultReasoningEffort, enabled: typeof p.enabled === "boolean" ? p.enabled : DEFAULT_CONFIG.enabled, + // v0.2.0 fields + systemPrompt: strOrUndef(p.systemPrompt), + cacheEnabled: typeof p.cacheEnabled === "boolean" ? p.cacheEnabled : DEFAULT_CONFIG.cacheEnabled, + cachePersist: typeof p.cachePersist === "boolean" ? p.cachePersist : DEFAULT_CONFIG.cachePersist, + cacheMaxEntries: clampInt(p.cacheMaxEntries, 1, 10000, DEFAULT_CONFIG.cacheMaxEntries), + retryAttempts: clampInt(p.retryAttempts, 0, 10, DEFAULT_CONFIG.retryAttempts), + retryBackoffMs: clampInt(p.retryBackoffMs, 0, 60000, DEFAULT_CONFIG.retryBackoffMs), + fallbackProvider: strOrUndef(p.fallbackProvider), + fallbackModel: strOrUndef(p.fallbackModel), }; } @@ -150,6 +190,37 @@ export function applySettingChange( case "reasoning": if (isReasoningLevel(value)) return { ...config, defaultReasoningEffort: value }; return config; + // ── v0.2.0 fields ────────────────────────────────────────────────────── + case "systemPrompt": + // Empty string (panel Input cleared) → undefined; otherwise the typed text. + return { ...config, systemPrompt: value.trim().length > 0 ? value.trim() : undefined }; + case "cacheEnabled": + return { ...config, cacheEnabled: value === "on" }; + case "cachePersist": + return { ...config, cachePersist: value === "on" }; + case "cacheMaxEntries": { + const n = parseInt(value, 10); + if (!Number.isFinite(n)) return config; + return { ...config, cacheMaxEntries: Math.min(10000, Math.max(1, n)) }; + } + case "retryAttempts": { + const n = parseInt(value, 10); + if (!Number.isFinite(n)) return config; + return { ...config, retryAttempts: Math.min(10, Math.max(0, n)) }; + } + case "retryBackoffMs": { + const n = parseInt(value, 10); + if (!Number.isFinite(n)) return config; + return { ...config, retryBackoffMs: Math.min(60000, Math.max(0, n)) }; + } + case "fallbackModel": { + // "provider/id" → set both; bare id → set fallbackModel only (keeps fallbackProvider) + const slash = value.indexOf("/"); + if (slash > 0 && slash < value.length - 1) { + return { ...config, fallbackProvider: value.slice(0, slash), fallbackModel: value.slice(slash + 1) }; + } + return { ...config, fallbackModel: value.length > 0 ? value : undefined }; + } default: return config; } diff --git a/lib/delegate.ts b/lib/delegate.ts index 64d6624..a4cc78e 100644 --- a/lib/delegate.ts +++ b/lib/delegate.ts @@ -5,16 +5,24 @@ * `describe_image` from multimodal models, so this path never fires for * them under mechanism A). * + * v0.2.0 (SPEC-2) adds four resilience layers: + * 1. Caching — a content-addressed cache (`lib/cache.ts`) returns a stored + * description on a hit, with ZERO vision-model API calls. + * 2. Custom system prompt — `config.systemPrompt` is prepended to the request. + * 3. Retry + fallback — `lib/resilience.ts` retries retryable errors + * (5xx/429/network) with backoff, then falls back to a configured + * secondary vision model on failure. + * 4. Abort-aware — `ctx.signal` stops retry + skips fallback. + * * Clean-room: the OpenAI-compatible `/chat/completions` request shape with a * base64 data-URL image is standard API usage, not copied from pi-vision-tool. - * v0.1.0 assumes the configured vision model exposes an OpenAI-compat - * chat/completions endpoint (Ollama, OpenRouter, most providers do). API-type - * awareness (anthropic-messages, etc.) is a SPEC-2 resilience concern. */ import type { Model, Api } from "@earendil-works/pi-ai"; import type { ExtensionContext } from "@earendil-works/pi-coding-agent"; import { isConfiguredForDelegation, type ReasoningLevel, type VisionConfig } from "./config.ts"; import { loadImage, type LoadedImage } from "./image.ts"; +import { cacheKey, type VisionCache } from "./cache.ts"; +import { AbortError, classifyError, withRetry } from "./resilience.ts"; export interface DelegateParams { image_path: string; @@ -32,12 +40,19 @@ export interface DelegateSuccess { prompt: string; compressed: boolean; reasoning: ReasoningLevel; + /** true if the result came from the cache (0 vision-model calls). */ + cached: boolean; + /** true if the result came from the fallback vision model. */ + fallback: boolean; }; } export interface DelegateFailure { ok: false; error: { code: string; message: string }; + /** Traceability for fallback failures: the primary error + which fallback + * model was attempted. */ + details?: { primaryError?: string; fallbackModel?: string }; } export type DelegateResult = DelegateSuccess | DelegateFailure; @@ -53,8 +68,9 @@ function buildReasoningParams( /** * Call the vision model's OpenAI-compat chat/completions endpoint with the - * image as a data URL + the user's prompt. Returns the model's text response. - * Exported (and fetch-based) so tests can mock `globalThis.fetch`. + * image as a data URL + the user's prompt (and an optional system prompt). + * Returns the model's text response. Exported + fetch-based so tests can mock + * `globalThis.fetch`. */ export async function callVisionModel( visionModel: Model, @@ -64,22 +80,26 @@ export async function callVisionModel( prompt: string, signal: AbortSignal | undefined, reasoning: ReasoningLevel, + systemPrompt?: string, ): Promise { const baseUrl = visionModel.baseUrl.replace(/\/+$/, ""); - const body: Record = { - model: visionModel.id, - messages: [ + const messages: unknown[] = []; + if (systemPrompt && systemPrompt.length > 0) { + messages.push({ role: "system", content: systemPrompt }); + } + messages.push({ + role: "user", + content: [ { - role: "user", - content: [ - { - type: "image_url", - image_url: { url: `data:${image.mimeType};base64,${image.data}` }, - }, - { type: "text", text: prompt }, - ], + type: "image_url", + image_url: { url: `data:${image.mimeType};base64,${image.data}` }, }, + { type: "text", text: prompt }, ], + }); + const body: Record = { + model: visionModel.id, + messages, max_tokens: 4096, temperature: 0, }; @@ -134,6 +154,14 @@ function errorMessage(err: unknown): string { return err instanceof Error ? err.message : String(err); } +/** Short traceability string for a primary error (used when the fallback + * also fails, so the caller can see why the primary was abandoned). */ +function primaryErrorTag(err: unknown): string { + const cls = classifyError(err); + const msg = errorMessage(err).slice(0, 120); + return `${cls}: ${msg}`; +} + const NOT_CONFIGURED_MSG = [ "Vision tool is not configured.", "", @@ -145,16 +173,31 @@ const NOT_CONFIGURED_MSG = [ 'and the model should have `input: ["text", "image"]`.', ].join("\n"); +const FALLBACK_MODEL_NOT_FOUND_MSG = (provider: string, model: string) => + [ + `Vision tool error: fallback model "${provider}/${model}" not found in the model registry.`, + "", + "Make sure the fallback provider + model are defined in ~/.pi/agent/models.json", + 'and the model has `input: ["text", "image"]`.', + "Use /vision fallback to update or /vision fallback clear to remove.", + ].join("\n"); + /** * Run the full DELEGATE pipeline: preflight config/auth checks → load + - * compress the image → call the vision model → return its text response. - * Every failure returns a structured, actionable error (SPEC-1 T5–T8). + * compress the image → cache check → (retry+fallback) call the vision model + * → return its text response. Every failure returns a structured, actionable + * error (SPEC-1 T5–T8 + SPEC-2 resilience). + * + * `cache` is optional: when omitted, caching is skipped (used by tests + the + * v0.1.0 call shape). When provided + `config.cacheEnabled`, hits return + * zero vision-model API calls. */ export async function delegateToVisionModel( ctx: ExtensionContext, config: VisionConfig, params: DelegateParams, signal: AbortSignal | undefined, + cache?: VisionCache, ): Promise { if (!config.enabled) { return { @@ -208,31 +251,138 @@ export async function delegateToVisionModel( }; } - try { - const text = await callVisionModel( - visionModel, - auth.apiKey, - auth.headers, - loaded.image, + const modelId = `${config.provider}/${config.model}`; + const baseDetails = { + model: modelId, + image_path: params.image_path, + prompt: params.prompt, + compressed: params.compress, + reasoning: params.reasoning, + }; + + // ── Cache check (hit = 0 vision-model calls) ─────────────────────────── + if (cache && config.cacheEnabled) { + const key = cacheKey( + loaded.sourceHash, + params.compress, + config.maxDimension, + config.jpegQuality, params.prompt, - signal, + modelId, params.reasoning, ); + const hit = cache.get(key); + if (hit) { + return { + ok: true, + text: hit.text, + details: { ...hit.details, ...baseDetails, cached: true, fallback: false }, + }; + } + // Miss → fall through to the call; store on success (using the same key). + const missKey = key; + const result = await callWithRetryAndFallback(ctx, config, params, signal, visionModel, auth.apiKey, auth.headers, loaded.image, modelId, baseDetails); + if (result.ok && config.cacheEnabled) { + cache.set(missKey, { text: result.text, details: { ...result.details, cached: false }, storedAt: Date.now() }); + } + return result; + } + + // No cache → straight to the resilient call. + return callWithRetryAndFallback(ctx, config, params, signal, visionModel, auth.apiKey, auth.headers, loaded.image, modelId, baseDetails); +} + +/** + * The resilient call: retry the primary vision model with backoff, then fall + * back to a configured secondary model on failure. Abort-aware (no retry, no + * fallback on `ctx.signal` abort). Extracted so the cache-hit path + the + * no-cache path share one implementation. + */ +async function callWithRetryAndFallback( + ctx: ExtensionContext, + config: VisionConfig, + params: DelegateParams, + signal: AbortSignal | undefined, + primaryModel: Model, + apiKey: string | undefined, + headers: Record | undefined, + image: LoadedImage, + modelId: string, + baseDetails: Omit, +): Promise { + try { + const text = await withRetry( + () => callVisionModel(primaryModel, apiKey, headers, image, params.prompt, signal, params.reasoning, config.systemPrompt), + { attempts: config.retryAttempts, backoffMs: config.retryBackoffMs, signal }, + ); + return { ok: true, text, details: { ...baseDetails, cached: false, fallback: false } }; + } catch (err) { + if (err instanceof AbortError) { + return { ok: false, error: { code: "aborted", message: "Vision tool aborted." } }; + } + // Non-abort failure → try the fallback (if configured). + if (!config.fallbackProvider || !config.fallbackModel) { + return { + ok: false, + error: { code: "vision_call_error", message: `Vision tool error: ${errorMessage(err)}` }, + }; + } + return runFallback(ctx, config, params, signal, image, err); + } +} + +/** Resolve + call the fallback vision model (one attempt, no retry). */ +async function runFallback( + ctx: ExtensionContext, + config: VisionConfig, + params: DelegateParams, + signal: AbortSignal | undefined, + image: LoadedImage, + primaryErr: unknown, +): Promise { + const fallbackId = `${config.fallbackProvider}/${config.fallbackModel}`; + const fbModel = ctx.modelRegistry.find(config.fallbackProvider!, config.fallbackModel!); + if (!fbModel) { + return { + ok: false, + error: { code: "model_not_found", message: FALLBACK_MODEL_NOT_FOUND_MSG(config.fallbackProvider!, config.fallbackModel!) }, + details: { primaryError: primaryErrorTag(primaryErr), fallbackModel: fallbackId }, + }; + } + const fbAuth = await ctx.modelRegistry.getApiKeyAndHeaders(fbModel); + if (!fbAuth.ok) { + return { + ok: false, + error: { + code: "auth_error", + message: `Vision tool error: unable to resolve API key for fallback "${config.fallbackProvider}". ${fbAuth.error}`, + }, + details: { primaryError: primaryErrorTag(primaryErr), fallbackModel: fallbackId }, + }; + } + try { + const text = await callVisionModel(fbModel, fbAuth.apiKey, fbAuth.headers, image, params.prompt, signal, params.reasoning, config.systemPrompt); return { ok: true, text, details: { - model: `${config.provider}/${config.model}`, + model: fallbackId, image_path: params.image_path, prompt: params.prompt, compressed: params.compress, reasoning: params.reasoning, + cached: false, + fallback: true, }, }; - } catch (err) { + } catch (fbErr) { + if (fbErr instanceof AbortError) { + return { ok: false, error: { code: "aborted", message: "Vision tool aborted." } }; + } return { ok: false, - error: { code: "vision_call_error", message: `Vision tool error: ${errorMessage(err)}` }, + error: { code: "vision_call_error", message: `Vision tool error (fallback ${fallbackId}): ${errorMessage(fbErr)}` }, + details: { primaryError: primaryErrorTag(primaryErr), fallbackModel: fallbackId }, }; } } \ No newline at end of file diff --git a/lib/image.ts b/lib/image.ts index 27d4fc4..1990b47 100644 --- a/lib/image.ts +++ b/lib/image.ts @@ -10,6 +10,7 @@ import { readFile, stat } from "node:fs/promises"; import { existsSync, statSync } from "node:fs"; import { isAbsolute, resolve as resolvePath } from "node:path"; +import { createHash } from "node:crypto"; import { resizeImage } from "@earendil-works/pi-coding-agent"; /** Cap on source file size (64 MB) — reject up front so we never base64-encode @@ -40,9 +41,16 @@ export interface ImageLoadError { } export type ImageLoadResult = - | { ok: true; image: LoadedImage } + | { ok: true; image: LoadedImage; sourceHash: string } | { ok: false; error: ImageLoadError }; +/** SHA-256 (hex) of the original image bytes — the content-addressed cache + * key base. Computed from the bytes BEFORE compression so the key is stable + * regardless of compression nondeterminism (worker vs in-process fallback). */ +export function hashBytes(bytes: Uint8Array): string { + return createHash("sha256").update(bytes).digest("hex"); +} + export interface LoadOptions { /** Run compression (resize + re-encode) on the loaded image. */ compress: boolean; @@ -98,7 +106,7 @@ function parseDataUrl(input: string): ImageLoadResult { if (!mime) { return { ok: false, error: { code: "unsupported_format", message: "could not determine image format from data URL" } }; } - return { ok: true, image: { data: payload, mimeType: mime } }; + return { ok: true, image: { data: payload, mimeType: mime }, sourceHash: hashBytes(bytes) }; } /** Is `input` plausibly a file path we should try to read (vs raw base64)? */ @@ -124,7 +132,7 @@ function decodeBase64(input: string): ImageLoadResult { if (!mime) { return { ok: false, error: { code: "unsupported_format", message: "could not determine image format from base64 bytes" } }; } - return { ok: true, image: { data: bytes.toString("base64"), mimeType: mime } }; + return { ok: true, image: { data: bytes.toString("base64"), mimeType: mime }, sourceHash: hashBytes(bytes) }; } /** @@ -136,7 +144,7 @@ export async function loadImage(input: string, options: LoadOptions): Promise { - if (!options.compress) return { ok: true, image }; + if (!options.compress) return { ok: true, image, sourceHash }; try { const inputBytes = Buffer.from(image.data, "base64"); const resized = await resizeImage(inputBytes, image.mimeType, { @@ -192,12 +202,12 @@ async function compressIfRequested( jpegQuality: options.jpegQuality, }); if (resized) { - return { ok: true, image: { data: resized.data, mimeType: resized.mimeType } }; + return { ok: true, image: { data: resized.data, mimeType: resized.mimeType }, sourceHash }; } } catch { // resizeImage threw (e.g. Photon unavailable) → degrade to original. } - return { ok: true, image }; + return { ok: true, image, sourceHash }; } function errorMessage(err: unknown): string { diff --git a/lib/resilience.ts b/lib/resilience.ts new file mode 100644 index 0000000..66bda54 --- /dev/null +++ b/lib/resilience.ts @@ -0,0 +1,131 @@ +/** + * Retry + fallback resilience for vision-model delegation (SPEC-2 gap #4). + * + * `withRetry` wraps a call function with exponential-backoff retry on + * retryable errors (HTTP 5xx / 429 / network). Non-retryable client errors + * (4xx) + empty-content responses throw immediately so the caller can fall + * back to a different model. Aborts (`ctx.signal`) are respected — no retry, + * no fallback — via an abort-aware `sleep`. + * + * Pure + unit-testable: `sleepFn` is injectable for fake-clock tests, and the + * error classifier is a pure function over the thrown value. No pi runtime + * dependency. + */ + +/** Sentinel for an abort (user cancelled the turn). Stops retry + skips + * fallback in the delegate pipeline. */ +export class AbortError extends Error { + constructor(message = "Aborted") { + super(message); + this.name = "AbortError"; + } +} + +export type ErrorClass = "retryable" | "client" | "abort" | "no_content"; + +/** + * Classify a thrown error to decide retry behavior. + * + * - `retryable`: HTTP 5xx, 429 (rate limit), network errors (fetch throws + * `TypeError`, ECONNRESET, ETIMEDOUT, …) → retry with backoff. + * - `client`: HTTP 4xx except 429 (bad request, auth, not found) → no retry + * (a different model may still succeed → caller falls back). + * - `no_content`: 2xx but empty response → no retry (same model likely yields + * the same empty) → caller falls back. + * - `abort`: `ctx.signal` aborted → no retry, no fallback (respect the cancel). + * - unknown → `client` (safe default: don't amplify an unknown failure). + */ +export function classifyError(err: unknown): ErrorClass { + if (err instanceof AbortError) return "abort"; + if (err instanceof Error && err.name === "AbortError") return "abort"; // native/DOMException + const msg = err instanceof Error ? err.message : String(err); + if (msg.includes("returned no content")) return "no_content"; + const m = /returned (\d{3}):/.exec(msg); + if (m) { + const status = Number(m[1]); + if (status === 429 || status >= 500) return "retryable"; + if (status >= 400 && status < 500) return "client"; + } + if (err instanceof TypeError) return "retryable"; // fetch network failure + if (/fetch failed|network|ECONNRESET|ETIMEDOUT|ECONNREFUSED|socket hang up|ENOTFOUND/i.test(msg)) { + return "retryable"; + } + return "client"; +} + +/** + * Abort-aware sleep. Resolves after `ms` unless `signal` aborts first, in + * which case it rejects with `AbortError`. If `signal` is already aborted, + * rejects immediately. + */ +export function sleep(ms: number, signal?: AbortSignal): Promise { + if (signal?.aborted) return Promise.reject(new AbortError()); + return new Promise((resolve, reject) => { + if (!signal) { + setTimeout(resolve, ms); + return; + } + const timer = setTimeout(() => { + signal.removeEventListener("abort", onAbort); + resolve(); + }, ms); + const onAbort = () => { + clearTimeout(timer); + reject(new AbortError()); + }; + signal.addEventListener("abort", onAbort, { once: true }); + }); +} + +export interface RetryOptions { + /** Number of retries after the first failure (total attempts = attempts + 1). */ + attempts: number; + /** Base backoff in ms; delay = min(backoffMs * 2^attempt, 8000). */ + backoffMs: number; + /** Agent abort signal — stops retry mid-backoff. */ + signal?: AbortSignal; + /** Injectable sleep for fake-clock tests (default: real `sleep`). */ + sleepFn?: (ms: number, signal?: AbortSignal) => Promise; +} + +/** + * Call `callFn` with retry + exponential backoff. `callFn(attempt)` receives + * the 0-based attempt index. + * + * - retryable error + attempts remaining → `await sleepFn(delay, signal)` then retry. + * - retryable error + last attempt → throw the error (caller falls back). + * - client / no_content → throw immediately (caller falls back). + * - abort → throw `AbortError` (caller: no fallback). + * + * If `signal` aborts during a backoff sleep, `sleepFn` rejects with + * `AbortError`, which propagates out (no further attempts). + */ +export async function withRetry( + callFn: (attempt: number) => Promise, + opts: RetryOptions, +): Promise { + const sleepFn = opts.sleepFn ?? sleep; + const totalAttempts = Math.max(1, Math.round(opts.attempts) + 1); + let lastErr: unknown; + for (let attempt = 0; attempt < totalAttempts; attempt++) { + if (opts.signal?.aborted) throw new AbortError(); + try { + return await callFn(attempt); + } catch (err) { + lastErr = err; + const cls = classifyError(err); + if (cls === "abort") { + throw err instanceof AbortError ? err : new AbortError(); + } + const isLast = attempt === totalAttempts - 1; + if (cls === "client" || cls === "no_content" || isLast) { + throw err; // non-retryable, or out of retries → caller decides fallback + } + // retryable + attempts remaining → backoff then retry + const delay = Math.min(opts.backoffMs * 2 ** attempt, 8000); + await sleepFn(delay, opts.signal); + } + } + // Unreachable: the loop always returns or throws. Defensive. + throw lastErr instanceof Error ? lastErr : new Error(String(lastErr)); +} \ No newline at end of file diff --git a/package.json b/package.json index 66caf5b..9cd9f99 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@getpipher/vision", - "version": "0.1.0", + "version": "0.2.0", "description": "Capability-aware vision + paste extension for the pi coding agent. Delegates image analysis to a vision model only when the active primary model is text-only; passes images through natively for multimodal models (zero delegation).", "keywords": [ "pi-package", diff --git a/tests/cache.test.ts b/tests/cache.test.ts new file mode 100644 index 0000000..9be95f6 --- /dev/null +++ b/tests/cache.test.ts @@ -0,0 +1,159 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync, rmSync, utimesSync, writeFileSync, existsSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { cacheKey, type CacheEntry, VisionCache } from "../lib/cache.ts"; + +function tmpCacheDir(): string { + return mkdtempSync(join(tmpdir(), "vision-cache-")); +} + +function makeEntry(text: string, model = "ollama/minimax-m3:cloud"): CacheEntry { + return { + text, + details: { + model, + image_path: "/tmp/x.png", + prompt: "describe", + compressed: true, + reasoning: "off", + cached: false, + fallback: false, + }, + storedAt: Date.now(), + }; +} + +test("cacheKey: deterministic for identical inputs", () => { + const a = cacheKey("hash1", true, 1568, 85, "describe", "ollama/m", "off"); + const b = cacheKey("hash1", true, 1568, 85, "describe", "ollama/m", "off"); + assert.equal(a, b); + assert.match(a, /^[0-9a-f]{64}$/); +}); + +test("cacheKey: differs when any component changes", () => { + const base = cacheKey("h", true, 1568, 85, "p", "m", "off"); + assert.notEqual(cacheKey("h2", true, 1568, 85, "p", "m", "off"), base); + assert.notEqual(cacheKey("h", false, 1568, 85, "p", "m", "off"), base); + assert.notEqual(cacheKey("h", true, 1024, 85, "p", "m", "off"), base); + assert.notEqual(cacheKey("h", true, 1568, 90, "p", "m", "off"), base); + assert.notEqual(cacheKey("h", true, 1568, 85, "p2", "m", "off"), base); + assert.notEqual(cacheKey("h", true, 1568, 85, "p", "m2", "off"), base); + assert.notEqual(cacheKey("h", true, 1568, 85, "p", "m", "high"), base); +}); + +test("cacheKey: \\0 separator prevents concatenation ambiguity", () => { + // ("a","bc") vs ("ab","c") must NOT collide when joined with \0 + const a = cacheKey("a", true, 1, 1, "bc", "m", "off"); + const b = cacheKey("ab", true, 1, 1, "c", "m", "off"); + assert.notEqual(a, b, "field-boundary separator must prevent collisions"); +}); + +test("VisionCache memory-only: set + get hit", () => { + const cache = new VisionCache(undefined, 256); + const k = cacheKey("h", true, 1568, 85, "p", "m", "off"); + assert.equal(cache.get(k), undefined); + cache.set(k, makeEntry("desc")); + const hit = cache.get(k); + assert.equal(hit?.text, "desc"); + assert.equal(cache.persisted, false); +}); + +test("VisionCache memory-only: unknown key → miss", () => { + const cache = new VisionCache(undefined, 256); + assert.equal(cache.get("nope"), undefined); +}); + +test("VisionCache disk: hit restores after memory wipe (promotion)", () => { + const dir = tmpCacheDir(); + try { + const cache = new VisionCache(dir, 256); + const k = cacheKey("h", true, 1568, 85, "p", "m", "off"); + cache.set(k, makeEntry("disk-desc")); + assert.ok(existsSync(join(dir, `${k}.json`)), "entry persisted to disk"); + + // Simulate a session restart: new cache instance (memory empty), same dir. + const restarted = new VisionCache(dir, 256); + const hit = restarted.get(k); + assert.equal(hit?.text, "disk-desc", "disk hit after memory wipe"); + assert.equal(restarted.get(k)?.text, "disk-desc", "promoted to memory (2nd get is memory hit)"); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("VisionCache disk: LRU evicts oldest at maxEntries+1", () => { + const dir = tmpCacheDir(); + try { + const cache = new VisionCache(dir, 3); + const keys = ["k1", "k2", "k3", "k4"]; + const now = Date.now() / 1000; + keys.forEach((k, i) => { + cache.set(k, makeEntry(`desc-${i}`)); + // Set distinct mtimes so LRU ordering is deterministic. + utimesSync(join(dir, `${k}.json`), now + i, now + i); + }); + // 4 entries inserted, max 3 → oldest (k1) evicted. + assert.equal(existsSync(join(dir, "k1.json")), false, "oldest evicted"); + assert.equal(existsSync(join(dir, "k4.json")), true, "newest retained"); + const stats = cache.stats(); + assert.equal(stats.diskEntries, 3); + assert.equal(stats.maxEntries, 3); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("VisionCache clear(): wipes memory + disk", () => { + const dir = tmpCacheDir(); + try { + const cache = new VisionCache(dir, 256); + const k = cacheKey("h", true, 1568, 85, "p", "m", "off"); + cache.set(k, makeEntry("desc")); + assert.ok(existsSync(join(dir, `${k}.json`))); + cache.clear(); + assert.equal(cache.get(k), undefined, "memory cleared"); + assert.equal(existsSync(join(dir, `${k}.json`)), false, "disk cleared"); + assert.equal(cache.stats().diskEntries, 0); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("VisionCache stats(): counts memory + disk", () => { + const dir = tmpCacheDir(); + try { + const cache = new VisionCache(dir, 256); + cache.set("a", makeEntry("1")); + cache.set("b", makeEntry("2")); + const s = cache.stats(); + assert.equal(s.memoryEntries, 2); + assert.equal(s.diskEntries, 2); + assert.equal(s.persisted, true); + assert.equal(s.maxEntries, 256); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("VisionCache disk: corrupt file → miss + removed", () => { + const dir = tmpCacheDir(); + try { + const cache = new VisionCache(dir, 256); + const k = "badkey"; + writeFileSync(join(dir, `${k}.json`), "{ not valid json"); + assert.equal(cache.get(k), undefined, "corrupt file → miss"); + assert.equal(existsSync(join(dir, `${k}.json`)), false, "corrupt file removed"); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("VisionCache disk: write failure degrades to memory-only (no throw)", () => { + // A non-existent dir path → write throws internally → swallowed; memory still hits. + const cache = new VisionCache(join(tmpdir(), "vision-cache-nonexistent-xyz"), 256); + const k = "memkey"; + assert.doesNotThrow(() => cache.set(k, makeEntry("mem-only"))); + assert.equal(cache.get(k)?.text, "mem-only", "memory hit even when disk write failed"); +}); \ No newline at end of file diff --git a/tests/config.test.ts b/tests/config.test.ts index 83d9b29..cc55a6a 100644 --- a/tests/config.test.ts +++ b/tests/config.test.ts @@ -26,6 +26,14 @@ test("DEFAULT_CONFIG has the expected shape", () => { jpegQuality: 85, defaultReasoningEffort: "off", enabled: true, + systemPrompt: undefined, + cacheEnabled: true, + cachePersist: false, + cacheMaxEntries: 256, + retryAttempts: 2, + retryBackoffMs: 500, + fallbackProvider: undefined, + fallbackModel: undefined, }); }); @@ -204,3 +212,107 @@ test("applySettingChange: reasoning sets valid level, rejects invalid", () => { test("applySettingChange: unknown id → unchanged", () => { assert.deepEqual(applySettingChange(DEFAULT_CONFIG, "nope", "x"), DEFAULT_CONFIG); }); + +// ── v0.2.0 fields (SPEC-2) ────────────────────────────────────────────────── + +test("mergeConfig: v0.1.0-shape config (6 fields) loads with v0.2.0 defaults (forward-compat)", () => { + const c = mergeConfig({ + provider: "ollama", + model: "minimax-m3:cloud", + maxDimension: 1568, + jpegQuality: 85, + defaultReasoningEffort: "off", + enabled: true, + }); + assert.equal(c.systemPrompt, undefined); + assert.equal(c.cacheEnabled, true); + assert.equal(c.cachePersist, false); + assert.equal(c.cacheMaxEntries, 256); + assert.equal(c.retryAttempts, 2); + assert.equal(c.retryBackoffMs, 500); + assert.equal(c.fallbackProvider, undefined); + assert.equal(c.fallbackModel, undefined); +}); + +test("mergeConfig: v0.2.0 fields pass through + validate", () => { + const c = mergeConfig({ + systemPrompt: " You are an analyst. ", + cacheEnabled: false, + cachePersist: true, + cacheMaxEntries: 512, + retryAttempts: 5, + retryBackoffMs: 1000, + fallbackProvider: "openrouter", + fallbackModel: "qwen3.5:cloud", + }); + assert.equal(c.systemPrompt, "You are an analyst.", "trimmed"); + assert.equal(c.cacheEnabled, false); + assert.equal(c.cachePersist, true); + assert.equal(c.cacheMaxEntries, 512); + assert.equal(c.retryAttempts, 5); + assert.equal(c.retryBackoffMs, 1000); + assert.equal(c.fallbackProvider, "openrouter"); + assert.equal(c.fallbackModel, "qwen3.5:cloud"); +}); + +test("mergeConfig: empty systemPrompt → undefined", () => { + assert.equal(mergeConfig({ systemPrompt: " " }).systemPrompt, undefined); + assert.equal(mergeConfig({ systemPrompt: "" }).systemPrompt, undefined); +}); + +test("mergeConfig: clamps cacheMaxEntries to [1, 10000]", () => { + assert.equal(mergeConfig({ cacheMaxEntries: 0 }).cacheMaxEntries, 1); + assert.equal(mergeConfig({ cacheMaxEntries: 999999 }).cacheMaxEntries, 10000); + assert.equal(mergeConfig({ cacheMaxEntries: "128" }).cacheMaxEntries, 128); + assert.equal(mergeConfig({ cacheMaxEntries: "x" }).cacheMaxEntries, 256); +}); + +test("mergeConfig: clamps retryAttempts to [0, 10] + retryBackoffMs to [0, 60000]", () => { + assert.equal(mergeConfig({ retryAttempts: -1 }).retryAttempts, 0); + assert.equal(mergeConfig({ retryAttempts: 99 }).retryAttempts, 10); + assert.equal(mergeConfig({ retryBackoffMs: -5 }).retryBackoffMs, 0); + assert.equal(mergeConfig({ retryBackoffMs: 999999 }).retryBackoffMs, 60000); +}); + +test("mergeConfig: non-boolean cache flags → defaults", () => { + assert.equal(mergeConfig({ cacheEnabled: "yes" }).cacheEnabled, true); + assert.equal(mergeConfig({ cachePersist: 1 }).cachePersist, false); + assert.equal(mergeConfig({ cacheEnabled: false }).cacheEnabled, false); +}); + +test("applySettingChange: systemPrompt set / clear", () => { + const set = applySettingChange(DEFAULT_CONFIG, "systemPrompt", "You are a forensic analyst."); + assert.equal(set.systemPrompt, "You are a forensic analyst."); + const cleared = applySettingChange(set, "systemPrompt", ""); + assert.equal(cleared.systemPrompt, undefined, "empty string clears"); + const trimmed = applySettingChange(DEFAULT_CONFIG, "systemPrompt", " hi "); + assert.equal(trimmed.systemPrompt, "hi"); +}); + +test("applySettingChange: cacheEnabled / cachePersist on/off", () => { + assert.equal(applySettingChange(DEFAULT_CONFIG, "cacheEnabled", "off").cacheEnabled, false); + assert.equal(applySettingChange(DEFAULT_CONFIG, "cacheEnabled", "on").cacheEnabled, true); + assert.equal(applySettingChange(DEFAULT_CONFIG, "cachePersist", "on").cachePersist, true); + assert.equal(applySettingChange(DEFAULT_CONFIG, "cachePersist", "off").cachePersist, false); +}); + +test("applySettingChange: cacheMaxEntries / retryAttempts / retryBackoffMs parse + clamp", () => { + assert.equal(applySettingChange(DEFAULT_CONFIG, "cacheMaxEntries", "512").cacheMaxEntries, 512); + assert.equal(applySettingChange(DEFAULT_CONFIG, "cacheMaxEntries", "99999").cacheMaxEntries, 10000); + assert.equal(applySettingChange(DEFAULT_CONFIG, "retryAttempts", "5").retryAttempts, 5); + assert.equal(applySettingChange(DEFAULT_CONFIG, "retryAttempts", "99").retryAttempts, 10); + assert.equal(applySettingChange(DEFAULT_CONFIG, "retryBackoffMs", "1000").retryBackoffMs, 1000); + assert.equal(applySettingChange(DEFAULT_CONFIG, "retryBackoffMs", "notanum").retryBackoffMs, DEFAULT_CONFIG.retryBackoffMs); +}); + +test("applySettingChange: fallbackModel provider/id splits both; bare id keeps fallbackProvider", () => { + const r = applySettingChange(DEFAULT_CONFIG, "fallbackModel", "openrouter/qwen3.5:cloud"); + assert.equal(r.fallbackProvider, "openrouter"); + assert.equal(r.fallbackModel, "qwen3.5:cloud"); + const base = { ...DEFAULT_CONFIG, fallbackProvider: "openrouter", fallbackModel: "old" }; + const bare = applySettingChange(base, "fallbackModel", "new-fb"); + assert.equal(bare.fallbackProvider, "openrouter"); + assert.equal(bare.fallbackModel, "new-fb"); + const cleared = applySettingChange(base, "fallbackModel", ""); + assert.equal(cleared.fallbackModel, undefined); +}); diff --git a/tests/delegate.test.ts b/tests/delegate.test.ts index b9f88f6..15bda71 100644 --- a/tests/delegate.test.ts +++ b/tests/delegate.test.ts @@ -10,6 +10,7 @@ import { delegateToVisionModel, } from "../lib/delegate.ts"; import { DEFAULT_CONFIG } from "../lib/config.ts"; +import { VisionCache } from "../lib/cache.ts"; const PNG_1x1_B64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAAC0lEQVR42mNk+M8AAAMBEg1+mP0AAAAASUVORK5CYII="; @@ -65,6 +66,26 @@ function mockFetchError(status: number, body: string): { return { calls, restore: () => { globalThis.fetch = original; } }; } +/** Mock fetch that returns a queued sequence of responses (one per call). */ +function mockFetchSeq(responses: { status: number; body: unknown }[]): { + calls: FetchCall[]; + restore: () => void; +} { + const calls: FetchCall[] = []; + const original = globalThis.fetch; + let i = 0; + globalThis.fetch = (async (url: string | URL | Request, init?: RequestInit) => { + calls.push({ url: String(url), init: init ?? {} }); + const r = responses[Math.min(i, responses.length - 1)]!; + i++; + return new Response(JSON.stringify(r.body), { + status: r.status, + headers: { "Content-Type": "application/json" }, + }); + }) as typeof globalThis.fetch; + return { calls, restore: () => { globalThis.fetch = original; } }; +} + function makeCtx(opts: { model?: Model | undefined; authOk?: boolean; @@ -325,4 +346,275 @@ test("delegateToVisionModel: success → returns text + details", async () => { m.restore(); rmSync(dir, { recursive: true, force: true }); } +}); + +// ── v0.2.0 (SPEC-2) tests ─────────────────────────────────────────────── + +test("delegateToVisionModel: success details include cached=false + fallback=false (v0.1.0 path)", async () => { + const dir = tmpDir(); + const m = mockFetch({ status: 200, body: { choices: [{ message: { content: "ok" } }] } }); + try { + const file = join(dir, "pixel.png"); + writeFileSync(file, PNG_BYTES); + const ctx = makeCtx({ cwd: dir }); + const cfg = { ...DEFAULT_CONFIG, provider: "ollama", model: "minimax-m3:cloud" }; + const r = await delegateToVisionModel(ctx, cfg, { + image_path: file, prompt: "describe", compress: false, reasoning: "off", + }, undefined); + assert.equal(r.ok, true); + if (r.ok) { + assert.equal(r.details.cached, false); + assert.equal(r.details.fallback, false); + } + } finally { + m.restore(); + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("delegateToVisionModel: cache hit = 0 vision-model calls (2nd call)", async () => { + const dir = tmpDir(); + const m = mockFetch({ status: 200, body: { choices: [{ message: { content: "a desc" } }] } }); + try { + const file = join(dir, "pixel.png"); + writeFileSync(file, PNG_BYTES); + const ctx = makeCtx({ cwd: dir }); + const cfg = { ...DEFAULT_CONFIG, provider: "ollama", model: "minimax-m3:cloud", cacheEnabled: true }; + const cache = new VisionCache(undefined, 256); + const params = { image_path: file, prompt: "describe", compress: false, reasoning: "off" as const }; + + const r1 = await delegateToVisionModel(ctx, cfg, params, undefined, cache); + assert.equal(r1.ok, true); + if (r1.ok) assert.equal(r1.details.cached, false); + assert.equal(m.calls.length, 1, "first call fetches"); + + const r2 = await delegateToVisionModel(ctx, cfg, params, undefined, cache); + assert.equal(r2.ok, true); + if (r2.ok) { + assert.equal(r2.details.cached, true, "second call is a cache hit"); + assert.equal(r2.text, "a desc"); + } + assert.equal(m.calls.length, 1, "second call = 0 vision-model calls"); + } finally { + m.restore(); + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("delegateToVisionModel: cache miss when prompt changes", async () => { + const dir = tmpDir(); + const m = mockFetch({ status: 200, body: { choices: [{ message: { content: "d" } }] } }); + try { + const file = join(dir, "pixel.png"); + writeFileSync(file, PNG_BYTES); + const ctx = makeCtx({ cwd: dir }); + const cfg = { ...DEFAULT_CONFIG, provider: "ollama", model: "minimax-m3:cloud", cacheEnabled: true }; + const cache = new VisionCache(undefined, 256); + await delegateToVisionModel(ctx, cfg, { image_path: file, prompt: "describe", compress: false, reasoning: "off" }, undefined, cache); + await delegateToVisionModel(ctx, cfg, { image_path: file, prompt: "different", compress: false, reasoning: "off" }, undefined, cache); + assert.equal(m.calls.length, 2, "different prompt → miss → fetches again"); + } finally { + m.restore(); + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("delegateToVisionModel: systemPrompt set → request body has system message first", async () => { + const dir = tmpDir(); + const m = mockFetch({ status: 200, body: { choices: [{ message: { content: "ok" } }] } }); + try { + const file = join(dir, "pixel.png"); + writeFileSync(file, PNG_BYTES); + const ctx = makeCtx({ cwd: dir }); + const cfg = { ...DEFAULT_CONFIG, provider: "ollama", model: "minimax-m3:cloud", systemPrompt: "You are a forensic analyst." }; + await delegateToVisionModel(ctx, cfg, { image_path: file, prompt: "p", compress: false, reasoning: "off" }, undefined); + const body = JSON.parse(m.calls[0]!.init.body as string); + assert.equal(body.messages[0].role, "system"); + assert.equal(body.messages[0].content, "You are a forensic analyst."); + assert.equal(body.messages[1].role, "user"); + } finally { + m.restore(); + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("delegateToVisionModel: no systemPrompt → v0.1.0 shape (user first)", async () => { + const dir = tmpDir(); + const m = mockFetch({ status: 200, body: { choices: [{ message: { content: "ok" } }] } }); + try { + const file = join(dir, "pixel.png"); + writeFileSync(file, PNG_BYTES); + const ctx = makeCtx({ cwd: dir }); + const cfg = { ...DEFAULT_CONFIG, provider: "ollama", model: "minimax-m3:cloud" }; + await delegateToVisionModel(ctx, cfg, { image_path: file, prompt: "p", compress: false, reasoning: "off" }, undefined); + const body = JSON.parse(m.calls[0]!.init.body as string); + assert.equal(body.messages[0].role, "user", "no system message when systemPrompt unset"); + } finally { + m.restore(); + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("delegateToVisionModel: retry on 500 then success (retryAttempts=2 → 3 calls)", async () => { + const dir = tmpDir(); + const m = mockFetchSeq([ + { status: 500, body: { error: "boom" } }, + { status: 500, body: { error: "boom" } }, + { status: 200, body: { choices: [{ message: { content: "recovered" } }] } }, + ]); + try { + const file = join(dir, "pixel.png"); + writeFileSync(file, PNG_BYTES); + const ctx = makeCtx({ cwd: dir }); + const cfg = { ...DEFAULT_CONFIG, provider: "ollama", model: "minimax-m3:cloud", retryAttempts: 2, retryBackoffMs: 1 }; + const r = await delegateToVisionModel(ctx, cfg, { image_path: file, prompt: "p", compress: false, reasoning: "off" }, undefined); + assert.equal(r.ok, true); + if (r.ok) { + assert.equal(r.text, "recovered"); + assert.equal(r.details.fallback, false); + assert.equal(r.details.cached, false); + } + assert.equal(m.calls.length, 3, "3 total attempts on primary"); + } finally { + m.restore(); + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("delegateToVisionModel: 4xx (client) → no retry → fallback fires", async () => { + const dir = tmpDir(); + const m = mockFetchSeq([ + { status: 400, body: "bad request" }, // primary (no retry) + { status: 200, body: { choices: [{ message: { content: "fb-desc" } }] } }, // fallback + ]); + try { + const file = join(dir, "pixel.png"); + writeFileSync(file, PNG_BYTES); + const ctx = makeCtx({ cwd: dir }); + const cfg = { + ...DEFAULT_CONFIG, provider: "ollama", model: "minimax-m3:cloud", + retryAttempts: 3, retryBackoffMs: 1, + fallbackProvider: "openrouter", fallbackModel: "qwen3.5:cloud", + }; + const r = await delegateToVisionModel(ctx, cfg, { image_path: file, prompt: "p", compress: false, reasoning: "off" }, undefined); + assert.equal(r.ok, true); + if (r.ok) { + assert.equal(r.details.fallback, true); + assert.equal(r.details.model, "openrouter/qwen3.5:cloud"); + assert.equal(r.text, "fb-desc"); + } + assert.equal(m.calls.length, 2, "1 primary (no retry) + 1 fallback"); + } finally { + m.restore(); + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("delegateToVisionModel: 5xx exhausts retries → fallback fires", async () => { + const dir = tmpDir(); + const m = mockFetchSeq([ + { status: 500, body: "boom" }, + { status: 500, body: "boom" }, + { status: 500, body: "boom" }, // primary exhausts (attempts=2 → 3 calls) + { status: 200, body: { choices: [{ message: { content: "fb" } }] } }, // fallback + ]); + try { + const file = join(dir, "pixel.png"); + writeFileSync(file, PNG_BYTES); + const ctx = makeCtx({ cwd: dir }); + const cfg = { + ...DEFAULT_CONFIG, provider: "ollama", model: "minimax-m3:cloud", + retryAttempts: 2, retryBackoffMs: 1, + fallbackProvider: "openrouter", fallbackModel: "qwen3.5:cloud", + }; + const r = await delegateToVisionModel(ctx, cfg, { image_path: file, prompt: "p", compress: false, reasoning: "off" }, undefined); + assert.equal(r.ok, true); + if (r.ok) assert.equal(r.details.fallback, true); + assert.equal(m.calls.length, 4, "3 primary + 1 fallback"); + } finally { + m.restore(); + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("delegateToVisionModel: no fallback configured + primary exhausts → primary error", async () => { + const dir = tmpDir(); + const m = mockFetchSeq([ + { status: 500, body: "boom" }, + { status: 500, body: "boom" }, + { status: 500, body: "boom" }, + ]); + try { + const file = join(dir, "pixel.png"); + writeFileSync(file, PNG_BYTES); + const ctx = makeCtx({ cwd: dir }); + const cfg = { ...DEFAULT_CONFIG, provider: "ollama", model: "minimax-m3:cloud", retryAttempts: 2, retryBackoffMs: 1 }; + const r = await delegateToVisionModel(ctx, cfg, { image_path: file, prompt: "p", compress: false, reasoning: "off" }, undefined); + assert.equal(r.ok, false); + if (!r.ok) { + assert.equal(r.error.code, "vision_call_error"); + assert.match(r.error.message, /500/); + assert.equal(r.details, undefined, "no fallback attempted → no primaryError details"); + } + assert.equal(m.calls.length, 3); + } finally { + m.restore(); + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("delegateToVisionModel: fallback also fails → error + primaryError + fallbackModel", async () => { + const dir = tmpDir(); + const m = mockFetchSeq([ + { status: 500, body: "primary boom" }, + { status: 500, body: "primary boom" }, + { status: 500, body: "primary boom" }, + { status: 500, body: "fallback boom" }, // fallback also fails + ]); + try { + const file = join(dir, "pixel.png"); + writeFileSync(file, PNG_BYTES); + const ctx = makeCtx({ cwd: dir }); + const cfg = { + ...DEFAULT_CONFIG, provider: "ollama", model: "minimax-m3:cloud", + retryAttempts: 2, retryBackoffMs: 1, + fallbackProvider: "openrouter", fallbackModel: "qwen3.5:cloud", + }; + const r = await delegateToVisionModel(ctx, cfg, { image_path: file, prompt: "p", compress: false, reasoning: "off" }, undefined); + assert.equal(r.ok, false); + if (!r.ok) { + assert.equal(r.error.code, "vision_call_error"); + assert.match(r.error.message, /fallback/); + assert.equal(r.details?.fallbackModel, "openrouter/qwen3.5:cloud"); + assert.ok(r.details?.primaryError, "primaryError set for traceability"); + } + assert.equal(m.calls.length, 4); + } finally { + m.restore(); + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("delegateToVisionModel: abort → code 'aborted', 0 calls, no fallback", async () => { + const dir = tmpDir(); + const m = mockFetch({ status: 200, body: { choices: [{ message: { content: "ok" } }] } }); + try { + const file = join(dir, "pixel.png"); + writeFileSync(file, PNG_BYTES); + const ctx = makeCtx({ cwd: dir }); + const ac = new AbortController(); + ac.abort(); + const cfg = { + ...DEFAULT_CONFIG, provider: "ollama", model: "minimax-m3:cloud", + retryAttempts: 3, retryBackoffMs: 1, + fallbackProvider: "openrouter", fallbackModel: "qwen3.5:cloud", + }; + const r = await delegateToVisionModel(ctx, cfg, { image_path: file, prompt: "p", compress: false, reasoning: "off" }, ac.signal); + assert.equal(r.ok, false); + if (!r.ok) assert.equal(r.error.code, "aborted"); + assert.equal(m.calls.length, 0, "abort before first attempt → 0 calls + no fallback"); + } finally { + m.restore(); + rmSync(dir, { recursive: true, force: true }); + } }); \ No newline at end of file diff --git a/tests/image.test.ts b/tests/image.test.ts index 7e0d768..bfad727 100644 --- a/tests/image.test.ts +++ b/tests/image.test.ts @@ -3,7 +3,7 @@ import assert from "node:assert/strict"; import { mkdtempSync, rmSync, writeFileSync, mkdirSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { detectMimeType, loadImage, MAX_IMAGE_BYTES } from "../lib/image.ts"; +import { detectMimeType, hashBytes, loadImage, MAX_IMAGE_BYTES } from "../lib/image.ts"; // 1×1 transparent PNG — decodes to bytes starting with the PNG signature // (89 50 4E 47 0D 0A 1A 0A). @@ -153,8 +153,61 @@ test("loadImage: compress=true returns ok (compressed or gracefully degraded)", if (r.ok) { assert.ok(r.image.data.length > 0); assert.ok(r.image.mimeType.startsWith("image/")); + assert.equal(r.sourceHash.length, 64, "sourceHash is sha256 hex (64 chars)"); + assert.match(r.sourceHash, /^[0-9a-f]{64}$/, "sourceHash is lowercase hex"); } } finally { rmSync(dir, { recursive: true, force: true }); } +}); + +test("loadImage: sourceHash is stable across two loads of the same file", async () => { + const dir = tmpDir(); + try { + const file = join(dir, "pixel.png"); + writeFileSync(file, PNG_BYTES); + const a = await loadImage(file, { ...LOAD_OPTS, cwd: dir }); + const b = await loadImage(file, { ...LOAD_OPTS, cwd: dir }); + assert.equal(a.ok, true); + assert.equal(b.ok, true); + if (a.ok && b.ok) assert.equal(a.sourceHash, b.sourceHash, "same bytes → same hash"); + if (a.ok) assert.equal(a.sourceHash, hashBytes(PNG_BYTES), "matches direct hashBytes"); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("loadImage: different bytes → different sourceHash", async () => { + const dir = tmpDir(); + try { + writeFileSync(join(dir, "a.png"), PNG_BYTES); + const otherBytes = Buffer.from( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+M8AAAMBApDk1+wAAAAASUVORK5CYII=", + "base64", + ); + writeFileSync(join(dir, "b.png"), otherBytes); + const a = await loadImage(join(dir, "a.png"), { ...LOAD_OPTS, cwd: dir }); + const b = await loadImage(join(dir, "b.png"), { ...LOAD_OPTS, cwd: dir }); + assert.equal(a.ok, true); + assert.equal(b.ok, true); + if (a.ok && b.ok) assert.notEqual(a.sourceHash, b.sourceHash, "different bytes → different hash"); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("loadImage: data URL + raw base64 both return sourceHash", async () => { + const viaUrl = await loadImage(`data:image/png;base64,${PNG_1x1_B64}`, LOAD_OPTS); + const viaRaw = await loadImage(PNG_1x1_B64, LOAD_OPTS); + assert.equal(viaUrl.ok, true); + assert.equal(viaRaw.ok, true); + if (viaUrl.ok) assert.equal(viaUrl.sourceHash, hashBytes(PNG_BYTES), "data URL hash matches bytes"); + if (viaRaw.ok) assert.equal(viaRaw.sourceHash, hashBytes(PNG_BYTES), "raw base64 hash matches bytes"); + if (viaUrl.ok && viaRaw.ok) + assert.equal(viaUrl.sourceHash, viaRaw.sourceHash, "same bytes via different input shapes → same hash"); +}); + +test("hashBytes: empty + known vector", () => { + assert.equal(hashBytes(Buffer.alloc(0)), "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"); + assert.equal(hashBytes(PNG_BYTES).length, 64); }); \ No newline at end of file diff --git a/tests/integration.test.ts b/tests/integration.test.ts index bbd66cd..e0eec25 100644 --- a/tests/integration.test.ts +++ b/tests/integration.test.ts @@ -64,10 +64,12 @@ interface MockPi { handlers: Map any>>; tools: Map; commands: Map Promise }>; + shortcuts: Map Promise | void }>; active: string[]; on(event: string, handler: any): void; registerTool(def: ToolDefinition): void; registerCommand(name: string, opts: any): void; + registerShortcut(shortcut: string, opts: { description?: string; handler: (ctx: any) => Promise | void }): void; getActiveTools(): string[]; setActiveTools(names: string[]): void; emit(event: string, eventObj: any, ctx: any): Promise; @@ -77,11 +79,13 @@ function createMockPi(initialActive = ["read", "bash", "edit", "write"]): MockPi const handlers = new Map any>>(); const tools = new Map(); const commands = new Map Promise }>(); + const shortcuts = new Map Promise | void }>(); let active = initialActive.slice(); const pi: MockPi = { handlers, tools, commands, + shortcuts, active, on(event, handler) { (handlers.get(event) ?? handlers.set(event, []).get(event)!).push(handler); @@ -92,6 +96,9 @@ function createMockPi(initialActive = ["read", "bash", "edit", "write"]): MockPi registerCommand(name, opts) { commands.set(name, opts); }, + registerShortcut(shortcut, opts) { + shortcuts.set(shortcut, opts); + }, getActiveTools: () => active.slice(), setActiveTools(names) { active = names.slice(); @@ -161,6 +168,23 @@ function mockFetch(body: unknown, status = 200): FetchMock { return { calls, restore: () => { globalThis.fetch = original; } }; } +/** Mock fetch that returns a queued sequence of responses (one per call). */ +function mockFetchSeq(responses: Array<{ status: number; body: unknown }>): FetchMock { + const calls: FetchMock["calls"] = []; + const original = globalThis.fetch; + let i = 0; + globalThis.fetch = (async (url: string | URL | Request, init?: RequestInit) => { + calls.push({ url: String(url), body: init?.body ? JSON.parse(init.body as string) : null }); + const r = responses[Math.min(i, responses.length - 1)]!; + i++; + return new Response(typeof r.body === "string" ? r.body : JSON.stringify(r.body), { + status: r.status, + headers: { "Content-Type": "application/json" }, + }); + }) as typeof globalThis.fetch; + return { calls, restore: () => { globalThis.fetch = original; } }; +} + async function runVisionCommand(pi: MockPi, args: string, model?: Model): Promise { const cmd = pi.commands.get("vision"); assert.ok(cmd, "/vision command registered"); @@ -558,7 +582,123 @@ test("panel: live edit via SettingsList onChange applies + persists", async () = await pi.commands.get("vision")!.handler("", panelCtx); // The live-edit logic itself (applySettingChange + applyAndSave) is covered by // the lib/config tests; this test confirms the panel constructs in TUI mode. -});// Cleanup the temp agent dir after all tests. +});// ── v0.2.0 (SPEC-2) integration tests ──────────────────────────────────── + +test("T11: cache hit via tool execute → 2nd call = 0 vision-model calls + details.cached", async () => { + const pi = createMockPi(); + visionFactory(pi as unknown as ExtensionAPI); + pasteFactory(pi as unknown as ExtensionAPI); + const { dir, file } = tmpImgDir(); + const fm = mockFetch({ choices: [{ message: { content: "a pixel" } }] }); + try { + await pi.emit("session_start", { type: "session_start", reason: "startup" }, makeCtx({ model: TEXT_ONLY, cwd: dir })); + await runVisionCommand(pi, "provider ollama", TEXT_ONLY); + await runVisionCommand(pi, "model minimax-m3:cloud", TEXT_ONLY); + const ctx = makeCtx({ model: TEXT_ONLY, cwd: dir, registry: makeRegistry({ model: VISION_MODEL }) }); + const r1 = await executeTool(pi, { image_path: file, prompt: "describe", compress: false }, ctx); + assert.equal(r1.details.cached, false, "first call is a cache miss"); + assert.equal(fm.calls.length, 1, "first call fetches"); + const r2 = await executeTool(pi, { image_path: file, prompt: "describe", compress: false }, ctx); + assert.equal(r2.details.cached, true, "second call is a cache hit"); + assert.equal(fm.calls.length, 1, "T11 GATE: 2nd call = 0 vision-model API calls"); + assert.match(r2.content[0].text, /a pixel/); + } finally { + fm.restore(); + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("T16: systemPrompt set → request body has system message first (via tool execute)", async () => { + const pi = createMockPi(); + visionFactory(pi as unknown as ExtensionAPI); + pasteFactory(pi as unknown as ExtensionAPI); + const { dir, file } = tmpImgDir(); + const fm = mockFetch({ choices: [{ message: { content: "ok" } }] }); + try { + await pi.emit("session_start", { type: "session_start", reason: "startup" }, makeCtx({ model: TEXT_ONLY, cwd: dir })); + await runVisionCommand(pi, "provider ollama", TEXT_ONLY); + await runVisionCommand(pi, "model minimax-m3:cloud", TEXT_ONLY); + await runVisionCommand(pi, "system-prompt You are a forensic analyst.", TEXT_ONLY); + const ctx = makeCtx({ model: TEXT_ONLY, cwd: dir, registry: makeRegistry({ model: VISION_MODEL }) }); + await executeTool(pi, { image_path: file, prompt: "p", compress: false }, ctx); + assert.equal(fm.calls.length, 1); + assert.equal(fm.calls[0]!.body.messages[0].role, "system"); + assert.equal(fm.calls[0]!.body.messages[0].content, "You are a forensic analyst."); + } finally { + fm.restore(); + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("T20: primary 4xx (no retry) → fallback fires via tool execute + details.fallback", async () => { + const pi = createMockPi(); + visionFactory(pi as unknown as ExtensionAPI); + pasteFactory(pi as unknown as ExtensionAPI); + const { dir, file } = tmpImgDir(); + const fm = mockFetchSeq([ + { status: 400, body: "bad request" }, // primary (no retry) + { status: 200, body: { choices: [{ message: { content: "fb desc" } }] } }, // fallback + ]); + try { + await pi.emit("session_start", { type: "session_start", reason: "startup" }, makeCtx({ model: TEXT_ONLY, cwd: dir })); + await runVisionCommand(pi, "provider ollama", TEXT_ONLY); + await runVisionCommand(pi, "model minimax-m3:cloud", TEXT_ONLY); + await runVisionCommand(pi, "fallback openrouter/qwen3.5:cloud", TEXT_ONLY); + const ctx = makeCtx({ model: TEXT_ONLY, cwd: dir, registry: makeRegistry({ model: VISION_MODEL }) }); + const res = await executeTool(pi, { image_path: file, prompt: "p", compress: false }, ctx); + assert.equal(res.details.fallback, true, "fallback fired"); + assert.equal(res.details.model, "openrouter/qwen3.5:cloud"); + assert.match(res.content[0].text, /fb desc/); + assert.equal(fm.calls.length, 2, "1 primary (no retry) + 1 fallback"); + } finally { + fm.restore(); + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("T24: alt+shift+v shortcut registered + invokes pickVisionModel → config updated", async () => { + const pi = createMockPi(); + visionFactory(pi as unknown as ExtensionAPI); + pasteFactory(pi as unknown as ExtensionAPI); + await pi.emit("session_start", { type: "session_start", reason: "startup" }, makeCtx({ model: TEXT_ONLY })); + const shortcut = pi.shortcuts.get("alt+shift+v"); + assert.ok(shortcut, "alt+shift+v shortcut registered"); + let notified = ""; + const sc = makeCtx({ + model: TEXT_ONLY, + registry: { + getAvailable: () => [MULTIMODAL, VISION_MODEL], + find: () => VISION_MODEL, + getApiKeyAndHeaders: async () => ({ ok: true, apiKey: "k", headers: undefined }), + } as any, + }) as unknown as ExtensionContext; + (sc.ui as any).select = async (_t: string, _o: string[]) => "ollama/minimax-m3:cloud"; + (sc.ui as any).notify = (msg: string) => { notified = msg; }; + await shortcut!.handler(sc); + assert.match(notified, /Vision model set to ollama\//); +}); + +test("T25: /vision-use sets directly (no picker)", async () => { + const pi = createMockPi(); + visionFactory(pi as unknown as ExtensionAPI); + pasteFactory(pi as unknown as ExtensionAPI); + await pi.emit("session_start", { type: "session_start", reason: "startup" }, makeCtx({ model: TEXT_ONLY })); + let notified = ""; + const cmd = pi.commands.get("vision-use"); + assert.ok(cmd, "/vision-use command registered"); + const c = makeCtx({ model: TEXT_ONLY }) as unknown as ExtensionCommandContext; + (c.ui as any).notify = (msg: string) => { notified = msg; }; + await cmd!.handler("openrouter/qwen3.5:cloud", c); + assert.match(notified, /openrouter\/qwen3\.5:cloud/); + // Verify via /vision show + let shown = ""; + const sc = makeCtx({ model: TEXT_ONLY }) as unknown as ExtensionCommandContext; + (sc.ui as any).notify = (msg: string) => { shown = msg; }; + await pi.commands.get("vision")!.handler("show", sc); + assert.match(shown, /qwen3\.5:cloud/); +}); + +// Cleanup the temp agent dir after all tests. test("cleanup", () => { rmSync(TMP_AGENT, { recursive: true, force: true }); }); \ No newline at end of file diff --git a/tests/resilience.test.ts b/tests/resilience.test.ts new file mode 100644 index 0000000..8239a41 --- /dev/null +++ b/tests/resilience.test.ts @@ -0,0 +1,180 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { AbortError, classifyError, sleep, withRetry, type ErrorClass } from "../lib/resilience.ts"; + +function httpError(status: number, body = "err"): Error { + return new Error(`Vision model returned ${status}: ${body}`); +} + +test("classifyError: 500/502/503 → retryable", () => { + assert.equal(classifyError(httpError(500)), "retryable"); + assert.equal(classifyError(httpError(502)), "retryable"); + assert.equal(classifyError(httpError(503)), "retryable"); +}); + +test("classifyError: 429 → retryable (rate limit)", () => { + assert.equal(classifyError(httpError(429)), "retryable"); +}); + +test("classifyError: 400/401/403/404 → client (no retry)", () => { + assert.equal(classifyError(httpError(400)), "client"); + assert.equal(classifyError(httpError(401)), "client"); + assert.equal(classifyError(httpError(403)), "client"); + assert.equal(classifyError(httpError(404)), "client"); +}); + +test("classifyError: 'returned no content' → no_content", () => { + assert.equal(classifyError(new Error("Vision model returned no content in the response")), "no_content"); +}); + +test("classifyError: AbortError → abort (custom + native name)", () => { + assert.equal(classifyError(new AbortError()), "abort"); + const native = new Error("aborted"); + native.name = "AbortError"; + assert.equal(classifyError(native), "abort"); +}); + +test("classifyError: TypeError (fetch network failure) → retryable", () => { + assert.equal(classifyError(new TypeError("fetch failed")), "retryable"); +}); + +test("classifyError: network-like messages → retryable", () => { + assert.equal(classifyError(new Error("ECONNRESET")), "retryable"); + assert.equal(classifyError(new Error("socket hang up")), "retryable"); + assert.equal(classifyError(new Error("ETIMEDOUT")), "retryable"); +}); + +test("classifyError: unknown Error → client (safe default, no retry)", () => { + assert.equal(classifyError(new Error("something weird")), "client"); + assert.equal(classifyError("a string"), "client"); +}); + +test("withRetry: succeeds on first attempt (no retry)", async () => { + let calls = 0; + const r = await withRetry(async () => { calls++; return "ok"; }, { attempts: 3, backoffMs: 10 }); + assert.equal(r, "ok"); + assert.equal(calls, 1); +}); + +test("withRetry: retries on 500 then succeeds on 3rd attempt (attempts=2)", async () => { + const delays: number[] = []; + let calls = 0; + const r = await withRetry( + async () => { + calls++; + if (calls < 3) throw httpError(500); + return "ok"; + }, + { attempts: 2, backoffMs: 100, sleepFn: (ms) => { delays.push(ms); return Promise.resolve(); } }, + ); + assert.equal(r, "ok"); + assert.equal(calls, 3, "total attempts = attempts+1"); + assert.equal(delays.length, 2, "slept twice (between 3 attempts)"); + assert.equal(delays[0], 100, "backoff attempt 0 = backoffMs"); + assert.equal(delays[1], 200, "backoff attempt 1 = 2*backoffMs"); +}); + +test("withRetry: no retry on 400 (client) — throws immediately, one call", async () => { + let calls = 0; + await assert.rejects( + withRetry(async () => { calls++; throw httpError(400); }, { attempts: 3, backoffMs: 10, sleepFn: () => Promise.resolve() }), + /returned 400/, + ); + assert.equal(calls, 1, "client errors are not retried"); +}); + +test("withRetry: no retry on no_content — throws immediately", async () => { + let calls = 0; + await assert.rejects( + withRetry(async () => { calls++; throw new Error("Vision model returned no content in the response"); }, { attempts: 3, backoffMs: 10, sleepFn: () => Promise.resolve() }), + /no content/, + ); + assert.equal(calls, 1); +}); + +test("withRetry: retryable exhausts retries → throws last error", async () => { + let calls = 0; + await assert.rejects( + withRetry(async () => { calls++; throw httpError(503); }, { attempts: 2, backoffMs: 10, sleepFn: () => Promise.resolve() }), + /returned 503/, + ); + assert.equal(calls, 3, "3 total attempts then throw"); +}); + +test("withRetry: abort → AbortError, no further attempts", async () => { + let calls = 0; + await assert.rejects( + withRetry(async () => { calls++; throw new AbortError(); }, { attempts: 3, backoffMs: 10, sleepFn: () => Promise.resolve() }), + (err: unknown) => err instanceof AbortError, + ); + assert.equal(calls, 1, "abort stops immediately"); +}); + +test("withRetry: signal already aborted before first attempt → AbortError, zero calls", async () => { + const ac = new AbortController(); + ac.abort(); + let calls = 0; + await assert.rejects( + withRetry(async () => { calls++; return "ok"; }, { attempts: 3, backoffMs: 10, signal: ac.signal, sleepFn: () => Promise.resolve() }), + (err: unknown) => err instanceof AbortError, + ); + assert.equal(calls, 0); +}); + +test("withRetry: signal aborts during backoff sleep → AbortError, no further attempts", async () => { + const ac = new AbortController(); + let calls = 0; + let sleepCalls = 0; + await assert.rejects( + withRetry( + async () => { calls++; throw httpError(500); }, + { + attempts: 5, backoffMs: 10, signal: ac.signal, + sleepFn: () => { sleepCalls++; ac.abort(); return Promise.reject(new AbortError()); }, + }, + ), + (err: unknown) => err instanceof AbortError, + ); + assert.equal(calls, 1, "only the first attempt ran"); + assert.equal(sleepCalls, 1, "sleep aborted + propagated"); +}); + +test("withRetry: backoff caps at 8000ms", async () => { + const delays: number[] = []; + let calls = 0; + await assert.rejects( + withRetry( + async () => { calls++; throw httpError(500); }, + { attempts: 10, backoffMs: 5000, sleepFn: (ms) => { delays.push(ms); return Promise.resolve(); } }, + ), + /returned 500/, + ); + // delays: 5000, 10000→capped 8000, 8000, ... + assert.equal(delays[0], 5000); + assert.equal(delays[1], 8000, "capped at 8000"); + assert.ok(delays.every((d) => d <= 8000), "no delay exceeds cap"); +}); + +test("sleep: resolves after ms (no signal)", async () => { + const start = Date.now(); + await sleep(30); + assert.ok(Date.now() - start >= 25, "slept ~30ms"); +}); + +test("sleep: rejects immediately if signal already aborted", async () => { + const ac = new AbortController(); + ac.abort(); + await assert.rejects(sleep(100, ac.signal), (err: unknown) => err instanceof AbortError); +}); + +test("sleep: rejects with AbortError if signal aborts during sleep", async () => { + const ac = new AbortController(); + const p = sleep(1000, ac.signal); + ac.abort(); + await assert.rejects(p, (err: unknown) => err instanceof AbortError); +}); + +test("ErrorClass type is exported (compile-time)", () => { + const c: ErrorClass = "retryable"; + assert.equal(c, "retryable"); +}); \ No newline at end of file