From 70a984424a790d741aaf01cb6b6ad891656fbdca Mon Sep 17 00:00:00 2001 From: Anton Date: Wed, 23 Sep 2026 20:37:45 +0200 Subject: [PATCH 1/2] feat(browser): return AX state and screenshot after Jev runs --- .codex/skills/cate-cli/SKILL.md | 6 ++++-- skills/cate-cli/SKILL.md | 6 ++++-- src/cli/browserJev.test.ts | 5 ++++- src/cli/browserJev.ts | 15 +++++++++++++-- src/cli/cate.test.ts | 21 ++++++++++++++++++++- src/cli/cate.ts | 11 ++++++++++- 6 files changed, 55 insertions(+), 9 deletions(-) diff --git a/.codex/skills/cate-cli/SKILL.md b/.codex/skills/cate-cli/SKILL.md index 020a7d40..6b33a2d1 100644 --- a/.codex/skills/cate-cli/SKILL.md +++ b/.codex/skills/cate-cli/SKILL.md @@ -61,8 +61,10 @@ value as a single word and supply absolute HTTP/HTTPS destination URLs explicitl Supported operations are HTTP/HTTPS navigation, clicks, field replacement, keys, page scrolling, and waits. Runs pin the panel/tab and stop on user takeover, uncertainty, errors, -180 seconds, or the step limit (20 by default, up to 100). `--json` returns -the status, action trace, model-call count, and final URL; only `done` exits zero. +180 seconds, or the step limit (20 by default, up to 100). Each run returns +the final AX state and screenshot. Human output saves the screenshot to a temporary +PNG file; `--json` includes the observation with base64 image data, status, +action trace, model-call count, and final URL. Only `done` exits zero. Completion is a model judgment based on the page; verify important outcomes. ### JavaScript mode diff --git a/skills/cate-cli/SKILL.md b/skills/cate-cli/SKILL.md index ab2952ad..16a51d6f 100644 --- a/skills/cate-cli/SKILL.md +++ b/skills/cate-cli/SKILL.md @@ -61,8 +61,10 @@ value as a single word and supply absolute HTTP/HTTPS destination URLs explicitl Supported operations are HTTP/HTTPS navigation, clicks, field replacement, keys, page scrolling, and waits. Runs pin the panel/tab and stop on user takeover, uncertainty, errors, -180 seconds, or the step limit (20 by default, up to 100). `--json` returns -the status, action trace, model-call count, and final URL; only `done` exits zero. +180 seconds, or the step limit (20 by default, up to 100). Each run returns +the final AX state and screenshot. Human output saves the screenshot to a temporary +PNG file; `--json` includes the observation with base64 image data, status, +action trace, model-call count, and final URL. Only `done` exits zero. Completion is a model judgment based on the page; verify important outcomes. ### JavaScript mode diff --git a/src/cli/browserJev.test.ts b/src/cli/browserJev.test.ts index 790775f8..c00e00c9 100644 --- a/src/cli/browserJev.test.ts +++ b/src/cli/browserJev.test.ts @@ -14,6 +14,7 @@ function fixture(decisions: Array, overrides: Partial if (method.endsWith('getTab')) return { panelId: 'browser', tabId: 'tab' } if (method.endsWith('setValue')) observation = { ...observation, state: `textbox Name value ${args.value}, button Save` } if (method.endsWith('click')) observation = { ...observation, state: 'Saved successfully' } + if (method.endsWith('getAXStateAndScreenshot')) return { ...observation, screenshot: { mimeType: 'image/png', data: 'cG5n', width: 800, height: 600 } } return observation }) const decide = vi.fn(async (request: { state: any; instructions: string; criteria: Record }) => { @@ -31,6 +32,8 @@ describe('Jev browser control', () => { const { options, invoke } = fixture(['setValue', 'c0', { word: 'Hi' }, 'click', 'c0', 'done']) const result = await runBrowserJev(options) expect(result).toMatchObject({ status: 'done', isError: false, modelCalls: 6, actions: [{ method: 'setValue', target: 1 }, { method: 'click', target: 2 }] }) + expect(result.observation).toMatchObject({ state: 'Saved successfully', screenshot: { data: 'cG5n' } }) + expect(invoke.mock.calls.at(-1)?.[0]).toBe('cate.browser.getAXStateAndScreenshot') expect(invoke).toHaveBeenCalledWith('cate.browser.setValue', expect.objectContaining({ value: 'Hi', target: 1, panelId: 'browser', tabId: 'tab', _userInputEpoch: 0 })) expect(invoke.mock.calls.filter(([method]) => method.endsWith('setValue'))).toHaveLength(1) }) @@ -101,7 +104,7 @@ describe('Jev browser control', () => { it('rejects a choice outside the code-owned action set', async () => { const { options, invoke } = fixture(['arbitrary-code']) expect(await runBrowserJev(options)).toMatchObject({ status: 'error', message: 'Invalid OpenRouter Jev Choice response' }) - expect(invoke.mock.calls.every(([method]) => method.endsWith('getTab') || method.endsWith('getAXState'))).toBe(true) + expect(invoke.mock.calls.every(([method]) => method.endsWith('getTab') || method.endsWith('getAXState') || method.endsWith('getAXStateAndScreenshot'))).toBe(true) }) it('stops on low confidence and does not repeat provider failures', async () => { diff --git a/src/cli/browserJev.ts b/src/cli/browserJev.ts index cb45482f..ff0616db 100644 --- a/src/cli/browserJev.ts +++ b/src/cli/browserJev.ts @@ -13,6 +13,7 @@ export interface JevResult { actions: Array<{ method: string; target?: number }> modelCalls: number url?: string + observation?: BrowserObservation isError: boolean } interface JevOptions { @@ -106,7 +107,17 @@ export async function runBrowserJev(options: JevOptions): Promise { const client = new JevClient(options, signal) const actions: JevResult['actions'] = [] let observation: BrowserObservation | undefined - const finish = (status: Status, message: string): JevResult => ({ status, message, actions, modelCalls: client.calls, url: observation?.url, isError: status !== 'done' }) + let binding: { panelId: string; tabId: string } | undefined + const finish = async (status: Status, message: string): Promise => { + if (binding && !signal.aborted) { + try { + const current = object(await options.invoke('cate.browser.getAXStateAndScreenshot', binding)) + if (current.kind === 'ax' && current.panelId === binding.panelId && current.tabId === binding.tabId + && typeof current.state === 'string') observation = current as unknown as BrowserObservation + } catch { /* Keep the original Jev outcome when a final read fails. */ } + } + return { status, message, actions, modelCalls: client.calls, url: observation?.url, observation, isError: status !== 'done' } + } try { if (!options.prompt.trim() || options.prompt.length > 8_000) throw new Error('Jev prompt must contain 1–8000 characters.') if (!Number.isInteger(options.maxSteps) || options.maxSteps < 1 || options.maxSteps > 100) throw new Error('Jev max steps must be between 1 and 100.') @@ -119,7 +130,7 @@ export async function runBrowserJev(options: JevOptions): Promise { } const bound = object(await invoke('getTab', options.panelId ? { panelId: options.panelId } : {})) if (typeof bound.panelId !== 'string' || typeof bound.tabId !== 'string') throw new Error('Browser binding did not resolve a panel and tab.') - const binding = { panelId: bound.panelId, tabId: bound.tabId } + binding = { panelId: bound.panelId, tabId: bound.tabId } let userInputEpoch: number | undefined const observe = async (): Promise => { const next = object(await invoke('getAXState', { ...binding, _userInputEpoch: userInputEpoch, disableDiffing: true })) diff --git a/src/cli/cate.test.ts b/src/cli/cate.test.ts index 76608178..cde2c04a 100644 --- a/src/cli/cate.test.ts +++ b/src/cli/cate.test.ts @@ -337,10 +337,29 @@ describe('output and run loop', () => { }) as typeof fetch expect(await run(['browser', 'jev', 'Verify Saved is visible', '--json'], deps)).toBe(0) expect(JSON.parse(deps.out[0])).toMatchObject({ status: 'done', modelCalls: 1, isError: false }) - expect(rpc.map(item => item.method)).toEqual(['cate.browser.getTab', 'cate.browser.getAXState', 'cate.browser.jevDecision', 'cate.browser.getAXState']) + expect(rpc.map(item => item.method)).toEqual(['cate.browser.getTab', 'cate.browser.getAXState', 'cate.browser.jevDecision', 'cate.browser.getAXState', 'cate.browser.getAXStateAndScreenshot']) + expect(JSON.parse(deps.out[0]).observation.state).toBe('Saved') expect(deps.err).toEqual([]) }) + it('prints Jev AX state and saves its final screenshot', async () => { + const deps = runDeps() + deps.fetch = vi.fn(async (_url, init) => { + const { method } = JSON.parse(String(init?.body)) + const result = method.endsWith('getTab') ? { panelId: 'browser', tabId: 'tab' } + : method.endsWith('jevDecision') ? { answers: { decision: { type: 'choice', choice: 'done', confidence: 1, probabilities: { done: 1 } } } } + : { kind: 'ax', panelId: 'browser', tabId: 'tab', userInputEpoch: 0, state: 'button Saved [2]', elements: [], url: 'https://example.test', + ...(method.endsWith('getAXStateAndScreenshot') ? { screenshot: { mimeType: 'image/png', data: 'cG5n', width: 800, height: 600 } } : {}) } + return new Response(JSON.stringify({ result })) + }) as typeof fetch + const writeImage = vi.fn(async () => '/tmp/jev.png') + expect(await run(['browser', 'jev', 'Verify Saved'], { ...deps, writeImage })).toBe(0) + expect(writeImage).toHaveBeenCalledWith('cG5n') + expect(deps.out[0]).toContain('button Saved [2]') + expect(deps.out[0]).toContain('Screenshot: /tmp/jev.png') + expect(deps.out[0]).not.toContain('cG5n') + }) + it('returns a nonzero Jev status when Cate has no saved key', async () => { const deps = runDeps() deps.fetch = vi.fn(async (_url, init) => { diff --git a/src/cli/cate.ts b/src/cli/cate.ts index cf6fa3c4..18f3a83d 100644 --- a/src/cli/cate.ts +++ b/src/cli/cate.ts @@ -448,7 +448,9 @@ function renderGeneric(value: unknown): string { export function formatHuman(method: string, value: unknown): string { if (method === 'cate.browser.jev') { const result = asObject(value) - return `Jev: ${result?.status} — ${result?.message}\n${Array.isArray(result?.actions) ? result.actions.length : 0} steps, ${result?.modelCalls} model calls${result?.url ? `\n${result.url}` : ''}` + const observation = asObject(result?.observation) + const screenshot = asObject(observation?.screenshot) + return `Jev: ${result?.status} — ${result?.message}\n${Array.isArray(result?.actions) ? result.actions.length : 0} steps, ${result?.modelCalls} model calls${result?.url ? `\n${result.url}` : ''}${typeof observation?.state === 'string' ? `\n${observation.state}` : ''}${typeof screenshot?.path === 'string' ? `\nScreenshot: ${screenshot.path}\nOpen this file with your image-viewing tool to inspect the page visually.` : ''}` } const content = asObject(value)?.content if ((method === 'cate.browser.run' || method === 'cate.browser.reset') && Array.isArray(content)) return content.map((item) => { @@ -631,6 +633,13 @@ export async function run(argv: string[], deps: RunDeps): Promise { progress: parsed.flags.json ? undefined : deps.stderr, }) : await send(request.method, request.args, sendDeps) + if (!parsed.flags.json && request.method === 'cate.browser.jev' && deps.writeImage) { + const screenshot = asObject(asObject(asObject(value)?.observation)?.screenshot) + if (screenshot && typeof screenshot.data === 'string') { + screenshot.path = await deps.writeImage(screenshot.data) + delete screenshot.data + } + } if (!parsed.flags.json && request.method === 'cate.browser.run' && deps.writeImage) { const content = asObject(value)?.content if (Array.isArray(content)) for (const item of content) { From f563415a8ca5a5fcad10b8231b6561404b51aa18 Mon Sep 17 00:00:00 2001 From: Anton Date: Wed, 23 Sep 2026 20:53:11 +0200 Subject: [PATCH 2/2] fix(browser): report failed Jev final capture --- .codex/skills/cate-cli/SKILL.md | 4 +++- skills/cate-cli/SKILL.md | 4 +++- src/cli/browserJev.test.ts | 15 +++++++++++++++ src/cli/browserJev.ts | 29 +++++++++++++++++++++-------- src/cli/cate.test.ts | 20 +++++++++++++++++++- src/cli/cate.ts | 2 +- 6 files changed, 62 insertions(+), 12 deletions(-) diff --git a/.codex/skills/cate-cli/SKILL.md b/.codex/skills/cate-cli/SKILL.md index 6b33a2d1..9b489211 100644 --- a/.codex/skills/cate-cli/SKILL.md +++ b/.codex/skills/cate-cli/SKILL.md @@ -64,7 +64,9 @@ waits. Runs pin the panel/tab and stop on user takeover, uncertainty, errors, 180 seconds, or the step limit (20 by default, up to 100). Each run returns the final AX state and screenshot. Human output saves the screenshot to a temporary PNG file; `--json` includes the observation with base64 image data, status, -action trace, model-call count, and final URL. Only `done` exits zero. +action trace, model-call count, and final URL. If capture fails, the result +reports `observationError` and exits nonzero instead of returning stale state. +Only `done` with a successful capture exits zero. Completion is a model judgment based on the page; verify important outcomes. ### JavaScript mode diff --git a/skills/cate-cli/SKILL.md b/skills/cate-cli/SKILL.md index 16a51d6f..b2375c4c 100644 --- a/skills/cate-cli/SKILL.md +++ b/skills/cate-cli/SKILL.md @@ -64,7 +64,9 @@ waits. Runs pin the panel/tab and stop on user takeover, uncertainty, errors, 180 seconds, or the step limit (20 by default, up to 100). Each run returns the final AX state and screenshot. Human output saves the screenshot to a temporary PNG file; `--json` includes the observation with base64 image data, status, -action trace, model-call count, and final URL. Only `done` exits zero. +action trace, model-call count, and final URL. If capture fails, the result +reports `observationError` and exits nonzero instead of returning stale state. +Only `done` with a successful capture exits zero. Completion is a model judgment based on the page; verify important outcomes. ### JavaScript mode diff --git a/src/cli/browserJev.test.ts b/src/cli/browserJev.test.ts index c00e00c9..5dfc8bb1 100644 --- a/src/cli/browserJev.test.ts +++ b/src/cli/browserJev.test.ts @@ -38,6 +38,21 @@ describe('Jev browser control', () => { expect(invoke.mock.calls.filter(([method]) => method.endsWith('setValue'))).toHaveLength(1) }) + it.each([ + { error: 'browser-screenshot-failed' }, + { kind: 'ax', panelId: 'browser', tabId: 'tab', state: 'Saved successfully' }, + ])('reports a failed final capture without returning stale AX state', async finalRead => { + const { options, invoke } = fixture(['done']) + const original = invoke.getMockImplementation()! + invoke.mockImplementation((method, args) => method.endsWith('getAXStateAndScreenshot') + ? Promise.resolve(finalRead as never) : original(method, args)) + const result = await runBrowserJev(options) + expect(result).toMatchObject({ status: 'done', isError: true, observationError: expect.any(String) }) + expect(result.observation).toBeUndefined() + expect(result.url).toBeUndefined() + expect(invoke.mock.calls.at(-1)?.[0]).toBe('cate.browser.getAXStateAndScreenshot') + }) + it('offers distinct whitespace-separated words verbatim, preserving punctuation and Unicode', async () => { const { options, decide, invoke } = fixture(['setValue', 'c0', { word: 'café!' }, 'done']) const result = await runBrowserJev({ ...options, prompt: ' Enter\tcafé!\ninto Name café! ' }) diff --git a/src/cli/browserJev.ts b/src/cli/browserJev.ts index ff0616db..c3bf81a9 100644 --- a/src/cli/browserJev.ts +++ b/src/cli/browserJev.ts @@ -14,6 +14,7 @@ export interface JevResult { modelCalls: number url?: string observation?: BrowserObservation + observationError?: string isError: boolean } interface JevOptions { @@ -107,16 +108,27 @@ export async function runBrowserJev(options: JevOptions): Promise { const client = new JevClient(options, signal) const actions: JevResult['actions'] = [] let observation: BrowserObservation | undefined - let binding: { panelId: string; tabId: string } | undefined + let finalBinding: { panelId: string; tabId: string } | undefined const finish = async (status: Status, message: string): Promise => { - if (binding && !signal.aborted) { + let finalObservation: BrowserObservation | undefined + let observationError: string | undefined + if (finalBinding) { try { - const current = object(await options.invoke('cate.browser.getAXStateAndScreenshot', binding)) - if (current.kind === 'ax' && current.panelId === binding.panelId && current.tabId === binding.tabId - && typeof current.state === 'string') observation = current as unknown as BrowserObservation - } catch { /* Keep the original Jev outcome when a final read fails. */ } + const current = object(await options.invoke('cate.browser.getAXStateAndScreenshot', finalBinding)) + if (current.error) throw new Error(String(current.error)) + const screenshot = object(current.screenshot) + if (current.kind !== 'ax' || current.panelId !== finalBinding.panelId || current.tabId !== finalBinding.tabId + || typeof current.state !== 'string' || screenshot.mimeType !== 'image/png' || typeof screenshot.data !== 'string') { + throw new Error('Invalid final browser observation') + } + finalObservation = current as unknown as BrowserObservation + } catch (error) { + observationError = error instanceof Error ? error.message : 'Final browser observation failed' + } } - return { status, message, actions, modelCalls: client.calls, url: observation?.url, observation, isError: status !== 'done' } + return { status, message, actions, modelCalls: client.calls, url: finalObservation?.url, + ...(finalObservation ? { observation: finalObservation } : {}), ...(observationError ? { observationError } : {}), + isError: status !== 'done' || observationError !== undefined } } try { if (!options.prompt.trim() || options.prompt.length > 8_000) throw new Error('Jev prompt must contain 1–8000 characters.') @@ -130,7 +142,8 @@ export async function runBrowserJev(options: JevOptions): Promise { } const bound = object(await invoke('getTab', options.panelId ? { panelId: options.panelId } : {})) if (typeof bound.panelId !== 'string' || typeof bound.tabId !== 'string') throw new Error('Browser binding did not resolve a panel and tab.') - binding = { panelId: bound.panelId, tabId: bound.tabId } + const binding = { panelId: bound.panelId, tabId: bound.tabId } + finalBinding = binding let userInputEpoch: number | undefined const observe = async (): Promise => { const next = object(await invoke('getAXState', { ...binding, _userInputEpoch: userInputEpoch, disableDiffing: true })) diff --git a/src/cli/cate.test.ts b/src/cli/cate.test.ts index cde2c04a..ac03eb5d 100644 --- a/src/cli/cate.test.ts +++ b/src/cli/cate.test.ts @@ -332,7 +332,8 @@ describe('output and run loop', () => { rpc.push(body) const result = body.method.endsWith('getTab') ? { panelId: 'browser', tabId: 'tab' } : body.method.endsWith('jevDecision') ? { answers: { decision: { type: 'choice', choice: 'done', confidence: 1, probabilities: { done: 1 } } } } - : { kind: 'ax', panelId: 'browser', tabId: 'tab', userInputEpoch: 0, state: 'Saved', elements: [], url: 'https://example.test' } + : { kind: 'ax', panelId: 'browser', tabId: 'tab', userInputEpoch: 0, state: 'Saved', elements: [], url: 'https://example.test', + ...(body.method.endsWith('getAXStateAndScreenshot') ? { screenshot: { mimeType: 'image/png', data: 'cG5n', width: 800, height: 600 } } : {}) } return new Response(JSON.stringify({ result })) }) as typeof fetch expect(await run(['browser', 'jev', 'Verify Saved is visible', '--json'], deps)).toBe(0) @@ -360,6 +361,23 @@ describe('output and run loop', () => { expect(deps.out[0]).not.toContain('cG5n') }) + it('returns a nonzero Jev result and recovery instruction when final capture fails', async () => { + const deps = runDeps() + deps.fetch = vi.fn(async (_url, init) => { + const { method } = JSON.parse(String(init?.body)) + const result = method.endsWith('getTab') ? { panelId: 'browser', tabId: 'tab' } + : method.endsWith('jevDecision') ? { answers: { decision: { type: 'choice', choice: 'done', confidence: 1, probabilities: { done: 1 } } } } + : method.endsWith('getAXStateAndScreenshot') ? { error: 'browser-screenshot-failed' } + : { kind: 'ax', panelId: 'browser', tabId: 'tab', userInputEpoch: 0, state: 'Prior page', elements: [], url: 'https://example.test' } + return new Response(JSON.stringify({ result })) + }) as typeof fetch + expect(await run(['browser', 'jev', 'Verify Saved'], deps)).toBe(1) + expect(deps.out[0]).toContain('Final browser observation unavailable:') + expect(deps.out[0]).toContain('browser-screenshot-failed') + expect(deps.out[0]).toContain('Run a browser read to inspect the current page.') + expect(deps.out[0]).not.toContain('Prior page') + }) + it('returns a nonzero Jev status when Cate has no saved key', async () => { const deps = runDeps() deps.fetch = vi.fn(async (_url, init) => { diff --git a/src/cli/cate.ts b/src/cli/cate.ts index 18f3a83d..90eab8fb 100644 --- a/src/cli/cate.ts +++ b/src/cli/cate.ts @@ -450,7 +450,7 @@ export function formatHuman(method: string, value: unknown): string { const result = asObject(value) const observation = asObject(result?.observation) const screenshot = asObject(observation?.screenshot) - return `Jev: ${result?.status} — ${result?.message}\n${Array.isArray(result?.actions) ? result.actions.length : 0} steps, ${result?.modelCalls} model calls${result?.url ? `\n${result.url}` : ''}${typeof observation?.state === 'string' ? `\n${observation.state}` : ''}${typeof screenshot?.path === 'string' ? `\nScreenshot: ${screenshot.path}\nOpen this file with your image-viewing tool to inspect the page visually.` : ''}` + return `Jev: ${result?.status} — ${result?.message}\n${Array.isArray(result?.actions) ? result.actions.length : 0} steps, ${result?.modelCalls} model calls${result?.url ? `\n${result.url}` : ''}${typeof observation?.state === 'string' ? `\n${observation.state}` : ''}${typeof screenshot?.path === 'string' ? `\nScreenshot: ${screenshot.path}\nOpen this file with your image-viewing tool to inspect the page visually.` : ''}${typeof result?.observationError === 'string' ? `\nFinal browser observation unavailable: ${result.observationError}\nRun a browser read to inspect the current page.` : ''}` } const content = asObject(value)?.content if ((method === 'cate.browser.run' || method === 'cate.browser.reset') && Array.isArray(content)) return content.map((item) => {