diff --git a/extension/manifest.firefox.json b/extension/manifest.firefox.json index e2f7465..6b84e32 100644 --- a/extension/manifest.firefox.json +++ b/extension/manifest.firefox.json @@ -2,7 +2,7 @@ "manifest_version": 3, "name": "Combo-X", "description": "Local-first browser agent — BYOK, vault, sessions, scrape/export, approvals.", - "version": "1.8.1", + "version": "1.8.2", "icons": { "16": "public/icon-16.png", "48": "public/icon-48.png", diff --git a/extension/manifest.json b/extension/manifest.json index 34df510..5e0868e 100644 --- a/extension/manifest.json +++ b/extension/manifest.json @@ -2,7 +2,7 @@ "manifest_version": 3, "name": "Combo-X", "description": "Local-first browser agent — BYOK, vault, sessions, scrape/export, approvals.", - "version": "1.8.1", + "version": "1.8.2", "key": "MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA1VhJsK7cNcf/9iGadvftWC4KBW8O5VrjJB4DCak24azuapM7NXoQVmMMXtNxM2qS4m+YytEsAqXEbWWffmwXCqAfxJ/acHgdxlbaF1eDX/HICqGVR1RIiyFhY4DmuEB0G68TjvfD6b6hs2O5KycxeWEcjOlgENJSP5j534Zv9swlKXrpPMJlbhxhhGhkjquNo/6WoEnAv7ebpmZDj+t1aTEXP6NV/uRrq9ja4tfkXGogKJbZcc7HS8oO3tHBE0WZOZWDcKmGcPXIBCOt1FzfVNuVcMjPuCK8SBw2RknLTMobJrefZNWfdE7215iFaw0hWgrNxyO4gTV8uFyy/SIJ0wIDAQAB", "icons": { "16": "public/icon-16.png", diff --git a/extension/package.json b/extension/package.json index a0aa8af..ff7e7ed 100644 --- a/extension/package.json +++ b/extension/package.json @@ -1,6 +1,6 @@ { "name": "@combo-x/extension", - "version": "1.8.1", + "version": "1.8.2", "private": true, "type": "module", "scripts": { diff --git a/extension/src/sidepanel/App.tsx b/extension/src/sidepanel/App.tsx index 2fbfe6b..57a5fef 100644 --- a/extension/src/sidepanel/App.tsx +++ b/extension/src/sidepanel/App.tsx @@ -341,6 +341,7 @@ const TOOLS_MIGRATE_FLAG = "combo_x_tools_migrate_v169"; const TOOLS_MIGRATE_ADD = [ "parse_data", "get_interactive", + "list_form_fields", "rag_search", "list_attachments", "save_view", diff --git a/extension/src/sidepanel/toolGroups.ts b/extension/src/sidepanel/toolGroups.ts index 07e18e0..c06223a 100644 --- a/extension/src/sidepanel/toolGroups.ts +++ b/extension/src/sidepanel/toolGroups.ts @@ -26,6 +26,7 @@ export const TOOL_GROUPS = { "get_page", "get_links", "get_interactive", + "list_form_fields", "press_key", "click_index", "type_index", diff --git a/packages/core/src/agent/loop.ts b/packages/core/src/agent/loop.ts index d6becc6..454fed3 100644 --- a/packages/core/src/agent/loop.ts +++ b/packages/core/src/agent/loop.ts @@ -100,7 +100,8 @@ import { type AgentBudgetMode, } from "./budget.js"; import { PageTemplateCache } from "./pageTemplateCache.js"; -import { annotateRedirect, RepeatGuard } from "./repeatGuard.js"; +import { annotateRedirect, MUTATION_TOOLS, RepeatGuard } from "./repeatGuard.js"; +import { taskProgress } from "../tasks/store.js"; import { leanHistory, truncateToolResultForLlm, @@ -443,6 +444,9 @@ For interaction prefer get_interactive → click_index / type_index (Nanobrowser Each user turn may include ## Active browser tab (url/title/tabId/time) and ## Picked element(s) — treat those as ground truth for where the user is and what they pointed at; act on the picked element before exploring elsewhere. SEARCH, DON'T DUMP (this is how you stay fast on big apps): - Looking for ONE labelled thing? Go straight to it: get_interactive({filter:"Save"}) or find_text({text:"App content"}). find_text returns interactiveIndex on hits inside a control, so you can click_index immediately. Never list 100 controls to find one. +- Unlabeled icon in a table row? get_interactive({within:{text:"FaqPage"}}) — item.i stays absolute for click_index. +- Chat column / cookie wall flooding reads? excludeSelector:"aside,[data-testid='chat-panel']" on get_page / find_text / get_interactive. +- "What can I fill in?" → list_form_fields (labels + type_index handles; passwords never leak values). - get_page defaults to mode:"main", which drops nav/header/footer. On consoles (Play Console, GSC, admin panels) the chrome is most of the page — mode:"full" is almost always the wrong call. - Reading a long document? get_page({filter:"keyword"}) greps it by line, and exclude:"…" strips boilerplate that repeats on every read. That beats paging through it. - Describe the control you want and let the filters do the work. get_interactive takes exclude (drop icon-ligature/nav noise), state:"enabled" (skip controls you cannot click yet — item.disabled marks them), requireLabel:true (skip unnamed icon buttons), and fields:["text"] (return only what you will read). get_links takes unique:true (nav is usually rendered two or three times) and origin:"internal". find_text takes clickableOnly:true so every hit is click-ready. @@ -453,6 +457,7 @@ NOTHING IS EVER SILENTLY CUT — always read the envelope: - item.i from get_interactive is an absolute handle into the full scan — filtering and paging never invalidate it, so click_index({index:i}) always hits the control you saw. DON'T REPEAT YOURSELF: - Identical arguments return an identical result. If a call yields nothing useful, change the arguments or the tool — a third identical call is refused outright. +- A streak of read-only observations with no click/type/navigate also trips a stuck guard — mutate, switch tool (list_form_fields / scrape_tables / within / excludeSelector), or report BLOCKED. - navigate reports redirected:true + requestedUrl when you did not land where you asked. That path is gated or renamed: reach the page through its on-page link instead of retrying the URL. - If two different approaches both fail, stop and tell the user what blocked you. That is a better answer than burning the step budget. For current facts / news use web_search (or OpenRouter built-in web search when enabled); use web_fetch or navigate for full pages. Prefer web_search over inventing URLs. @@ -636,6 +641,8 @@ interface RunContext { boundTabId?: number; /** Detects identical call+result pairs and refuses the third one. */ repeatGuard?: RepeatGuard; + /** Short description of the last mutating tool call (stop-checkpoint). */ + lastMutation?: string; } export class AgentLoop { @@ -931,14 +938,30 @@ export class AgentLoop { } } runCtx.ephemeralTabIds = []; + + let finalText = outcome.finalText; + let doneMessage = outcome.doneMessage ?? outcome.finalText; + if (outcome.aborted) { + const checkpoint = await this.buildStopCheckpoint(runCtx); + if (checkpoint) { + finalText = checkpoint; + doneMessage = checkpoint; + messages.push({ role: "assistant", content: checkpoint }); + emit({ type: "assistant_delta", message: checkpoint }); + } else if (!finalText.trim()) { + finalText = "Stopped."; + doneMessage = "Stopped."; + } + } + emit({ type: "done", - message: outcome.doneMessage ?? outcome.finalText, + message: doneMessage, usage, }); return { messages, - finalText: outcome.finalText, + finalText, steps, usage, aborted: outcome.aborted, @@ -1404,6 +1427,26 @@ export class AgentLoop { } } + /** + * On abort/stop: one visible transcript line summarizing open tasks + last mutation. + * Returns null when there is nothing useful to say (no todo/doing tasks). + */ + private async buildStopCheckpoint(runCtx: RunContext): Promise { + if (!runCtx.tasks) return null; + try { + const rows = await runCtx.tasks.list( + runCtx.sessionId ? { sessionId: runCtx.sessionId } : {}, + ); + const open = rows.filter((t) => t.status === "todo" || t.status === "doing"); + if (!open.length) return null; + const { done, total } = taskProgress(rows); + const last = runCtx.lastMutation?.trim() || "none"; + return `Stopped. Tasks: ${done}/${total} done; last action: ${last}.`; + } catch { + return null; + } + } + /** * Apply soft context limit: when lean history still exceeds `contextLimit`, * compress older turns into a single summary message and emit a @@ -3083,6 +3126,17 @@ export class AgentLoop { } } + if (runCtx && MUTATION_TOOLS.has(name)) { + const ok = + !result || + typeof result !== "object" || + Array.isArray(result) || + (result as { ok?: boolean }).ok !== false; + if (ok) { + runCtx.lastMutation = describeMutation(name, args); + } + } + const repeat = runCtx?.repeatGuard?.record(name, args, result); if (repeat?.kind === "warn") { result = RepeatGuard.annotate(result, repeat.note); @@ -4523,3 +4577,31 @@ export class AgentLoop { return mcpCall(conn, String(args.tool ?? ""), toolArgs, connectors.getSecret); } } + +/** Short human label for the last mutating tool (stop-checkpoint). */ +function describeMutation(name: string, args: Record): string { + switch (name) { + case "click_index": + return `clicked index ${String(args.index ?? "?")}`; + case "type_index": { + const text = typeof args.text === "string" ? args.text.trim() : ""; + const preview = text.length > 40 ? `${text.slice(0, 40)}…` : text; + return preview + ? `typed into index ${String(args.index ?? "?")}: ${preview}` + : `typed into index ${String(args.index ?? "?")}`; + } + case "click": + return `clicked ${String(args.selector ?? "element")}`; + case "type_text": + return `typed into ${String(args.selector ?? "field")}`; + case "press_key": + return `pressed ${String(args.key ?? "key")}`; + case "navigate": + case "open_tab": + return `navigated to ${String(args.url ?? "url")}`; + case "go_back": + return "went back"; + default: + return name; + } +} diff --git a/packages/core/src/agent/loopGuards.test.ts b/packages/core/src/agent/loopGuards.test.ts index 9a33124..860bbd0 100644 --- a/packages/core/src/agent/loopGuards.test.ts +++ b/packages/core/src/agent/loopGuards.test.ts @@ -6,6 +6,7 @@ import { describe, expect, it, vi } from "vitest"; import type { ChatMessage, OpenRouterClient } from "../llm/openrouter.js"; import { MemoryStore } from "../memory/store.js"; +import { TaskStore } from "../tasks/store.js"; import type { BrowserBridge } from "./loop.js"; import { AgentLoop } from "./loop.js"; @@ -149,6 +150,118 @@ describe("repeat-call guard inside the agent loop", () => { }); }); +describe("stop checkpoint on abort", () => { + it("appends a task summary when stopping with open todo/doing tasks", async () => { + const tasks = new TaskStore(`tasks_${crypto.randomUUID()}`); + const sessionId = "sess-stop"; + await tasks.put({ + id: "t1", + title: "Edit FaqPage meta", + status: "doing", + sessionId, + note: "saved FaqPage", + }); + await tasks.put({ + id: "t2", + title: "Edit Row3 meta", + status: "todo", + sessionId, + }); + + const llm = mockLlm([ + { + content: null, + toolCalls: [ + { id: "1", name: "click_index", args: JSON.stringify({ index: 2 }) }, + ], + }, + { + content: null, + toolCalls: [{ id: "2", name: "get_page", args: "{}" }], + }, + { content: "should not reach" }, + ]); + const runContent = vi.fn(async (req: { op?: string }) => { + if (req.op === "click_index") return { ok: true, data: { clickedIndex: 2 } }; + return { ok: true, data: { title: "t", url: "https://app.test/x", text: "x" } }; + }); + const browser = stubBrowser({ runContent }); + const agent = new AgentLoop(llm, browser, new MemoryStore({ dbName: `g_${crypto.randomUUID()}` })); + const controller = new AbortController(); + + // Abort after the first tool turn completes (mutation recorded), before the next model call. + let toolTurns = 0; + const result = await agent.run({ + model: "mock", + approvalMode: "auto_all", + userMessage: "edit meta tags", + sessionId, + tasks, + signal: controller.signal, + maxSteps: 8, + onEvent: (e) => { + if (e.type === "tool_result") { + toolTurns += 1; + if (toolTurns >= 1) controller.abort(); + } + }, + }); + + expect(result.aborted).toBe(true); + expect(result.finalText).toMatch(/^Stopped\. Tasks: 0\/2 done; last action:/); + expect(result.finalText).toMatch(/clicked index 2/); + }); +}); + +describe("semantic stuck guard inside the agent loop", () => { + it("warns after 6 read-only calls and resets on a click", async () => { + const runContent = vi.fn(async (req: { op?: string }) => { + if (req.op === "click_index") return { ok: true, data: { clickedIndex: 0 } }; + return { ok: true, data: { title: "t", url: "https://app.test/meta", text: "x", items: [] } }; + }); + const browser = stubBrowser({ runContent }); + + const reads = Array.from({ length: 6 }, (_, i) => ({ + id: `r${i}`, + name: "get_page", + args: JSON.stringify({ filter: `noise-${i}` }), + })); + const llm = mockLlm([ + { content: null, toolCalls: reads.slice(0, 3) }, + { content: null, toolCalls: reads.slice(3, 6) }, + { + content: null, + toolCalls: [{ id: "click", name: "click_index", args: JSON.stringify({ index: 0 }) }], + }, + { + content: null, + toolCalls: [{ id: "after", name: "get_page", args: JSON.stringify({ filter: "after" }) }], + }, + { content: "done" }, + ]); + const agent = new AgentLoop(llm, browser, new MemoryStore({ dbName: `g_${crypto.randomUUID()}` })); + + const results: Array<{ tool?: string; result: unknown }> = []; + await agent.run({ + model: "mock", + approvalMode: "auto_all", + userMessage: "edit rows", + maxSteps: 12, + onEvent: (e) => { + if (e.type === "tool_result") results.push({ tool: e.tool, result: e.result }); + }, + }); + + const readResults = results.filter((r) => r.tool === "get_page"); + const sixth = readResults[5]?.result as { _repeat?: string }; + expect(sixth?._repeat).toMatch(/observations without changing anything/); + + const afterClick = results.filter((r) => r.tool === "get_page").slice(6); + expect(afterClick.length).toBeGreaterThanOrEqual(1); + expect((afterClick[0]?.result as { _repeat?: string })?._repeat).toBeUndefined(); + }); +}); + describe("prompt-cache layout", () => { it("keeps the volatile task list out of the cached system prefix", async () => { const seen: ChatMessage[][] = []; diff --git a/packages/core/src/agent/repeatGuard.test.ts b/packages/core/src/agent/repeatGuard.test.ts index 5d892a1..7a4ec25 100644 --- a/packages/core/src/agent/repeatGuard.test.ts +++ b/packages/core/src/agent/repeatGuard.test.ts @@ -92,6 +92,60 @@ describe("RepeatGuard", () => { }); }); +describe("semantic observation stuck guard", () => { + it("warns at 6 consecutive observations with varied args", () => { + const g = new RepeatGuard(); + let last: ReturnType = { kind: "ok" }; + for (let i = 0; i < 6; i++) { + last = g.record("get_page", { filter: `x${i}` }, { ok: true, data: { text: `t${i}` } }); + } + expect(last.kind).toBe("warn"); + if (last.kind === "warn") { + expect(last.note).toMatch(/6 observations without changing anything/); + expect(last.note).toMatch(/list_form_fields|within|excludeSelector/); + } + expect(g.getObservationStreak()).toBe(6); + }); + + it("blocks the 9th observation before it runs", () => { + const g = new RepeatGuard(); + for (let i = 0; i < 9; i++) { + g.record("find_text", { text: `q${i}` }, { ok: true, matches: [] }); + } + expect(g.getObservationStreak()).toBe(9); + const verdict = g.check("get_interactive", { filter: "Save" }); + expect(verdict.kind).toBe("block"); + if (verdict.kind === "block") { + expect(verdict.result.error).toBe("observation_stuck_blocked"); + } + }); + + it("resets the streak on a mutation", () => { + const g = new RepeatGuard(); + for (let i = 0; i < 6; i++) { + g.record("get_interactive", { filter: `f${i}` }, { ok: true, items: [] }); + } + expect(g.getObservationStreak()).toBe(6); + g.record("click_index", { index: 3 }, { ok: true, clickedIndex: 3 }); + expect(g.getObservationStreak()).toBe(0); + expect(g.record("get_page", {}, { ok: true, data: { text: "x" } }).kind).toBe("ok"); + }); + + it("does not trip when the observed URL is changing", () => { + const g = new RepeatGuard(); + for (let i = 0; i < 8; i++) { + const verdict = g.record( + "get_page", + { offset: i }, + { ok: true, data: { text: "loading", url: `https://app.test/step/${i}` } }, + ); + expect(verdict.kind).toBe("ok"); + } + expect(g.check("get_page", {}).kind).toBe("ok"); + expect(g.getObservationStreak()).toBe(1); + }); +}); + describe("annotateRedirect", () => { it("flags a silent redirect and names both URLs", () => { const out = annotateRedirect( diff --git a/packages/core/src/agent/repeatGuard.ts b/packages/core/src/agent/repeatGuard.ts index 70ae694..39a4e4c 100644 --- a/packages/core/src/agent/repeatGuard.ts +++ b/packages/core/src/agent/repeatGuard.ts @@ -7,10 +7,17 @@ * Nothing in the loop noticed that the call was identical AND the outcome was * identical, so there was no pressure to change approach. * - * Policy: the first repeat is annotated (the model usually self-corrects), the - * second is refused with concrete alternatives. Only *identical args producing - * an identical result* count — legitimately re-polling a changing page (a - * loading spinner, a queue) never trips the guard. + * Follow-up (Base44 meta-tags table, 2026-08-03): ~12 consecutive *different* + * read-only calls with zero mutations also burned the budget. The identical- + * call guard stayed silent because args varied. Semantic stuck tracking below + * catches that class: observation streak with no mutation and no URL change. + * + * Policy: the first identical repeat is annotated (the model usually + * self-corrects), the second is refused with concrete alternatives. Only + * *identical args producing an identical result* count for that path — + * legitimately re-polling a changing page (a loading spinner, a queue) never + * trips it. Separately, ≥6 observations without a mutation/URL change warn; + * ≥9 block. */ /** Tools whose whole purpose is to re-observe changing state. */ @@ -19,12 +26,47 @@ const POLLING_TOOLS = new Set([ "get_page", "page_digest", "get_interactive", + "list_form_fields", "screenshot", "list_tabs", "page_metrics", "list_tasks", ]); +/** Read-only tools that do not change the page (semantic stuck streak). */ +export const OBSERVATION_TOOLS = new Set([ + "get_page", + "page_digest", + "get_interactive", + "list_form_fields", + "find_text", + "query_all", + "extract", + "scrape_tables", + "get_links", + "scroll", + "wait", + "screenshot", + "screenshot_viewport", + "screenshot_element", + "screenshot_full", +]); + +/** Tools that mutate the page or navigate — reset the observation streak. */ +export const MUTATION_TOOLS = new Set([ + "click", + "click_index", + "type_text", + "type_index", + "press_key", + "navigate", + "open_tab", + "go_back", +]); + +const STUCK_WARN_AT = 6; +const STUCK_BLOCK_AT = 9; + export type RepeatVerdict = | { kind: "ok" } | { kind: "warn"; repeats: number; note: string } @@ -67,13 +109,13 @@ function alternativesFor(name: string, args: Record): string { return ( `Reading the same page again returns the same bytes. Change the query instead: ` + `get_page({filter:""}) to grep it, get_page({offset:}) to continue, ` + - `or page_digest / find_text to locate the part you need.` + `excludeSelector to drop chat/cookie chrome, or page_digest / find_text to locate the part you need.` ); case "get_interactive": return ( `The control list has not changed. Narrow it instead: ` + - `get_interactive({filter:"