Return AX state and screenshot after Jev runs

.codex/skills/cate-cli/SKILL.md

@@ -61,8 +61,12 @@ value as a single word and supply absolute HTTP/HTTPS destination URLs explicitl
 
 Supported operations are HTTP/HTTPS navigation, clicks, field replacement, keys, page scrolling, and
 waits. Runs pin the panel/tab and stop on user takeover, uncertainty, errors,
-180 seconds, or the step limit (20 by default, up to 100). `--json` returns
-the status, action trace, model-call count, and final URL; only `done` exits zero.
+180 seconds, or the step limit (20 by default, up to 100). Each run returns
+the final AX state and screenshot. Human output saves the screenshot to a temporary
+PNG file; `--json` includes the observation with base64 image data, status,
+action trace, model-call count, and final URL. If capture fails, the result
+reports `observationError` and exits nonzero instead of returning stale state.
+Only `done` with a successful capture exits zero.
 Completion is a model judgment based on the page; verify important outcomes.
 
 ### JavaScript mode

skills/cate-cli/SKILL.md

@@ -61,8 +61,12 @@ value as a single word and supply absolute HTTP/HTTPS destination URLs explicitl
 
 Supported operations are HTTP/HTTPS navigation, clicks, field replacement, keys, page scrolling, and
 waits. Runs pin the panel/tab and stop on user takeover, uncertainty, errors,
-180 seconds, or the step limit (20 by default, up to 100). `--json` returns
-the status, action trace, model-call count, and final URL; only `done` exits zero.
+180 seconds, or the step limit (20 by default, up to 100). Each run returns
+the final AX state and screenshot. Human output saves the screenshot to a temporary
+PNG file; `--json` includes the observation with base64 image data, status,
+action trace, model-call count, and final URL. If capture fails, the result
+reports `observationError` and exits nonzero instead of returning stale state.
+Only `done` with a successful capture exits zero.
 Completion is a model judgment based on the page; verify important outcomes.
 
 ### JavaScript mode

src/cli/browserJev.test.ts

@@ -14,6 +14,7 @@ function fixture(decisions: Array<string | { word: string }>, overrides: Partial
     if (method.endsWith('getTab')) return { panelId: 'browser', tabId: 'tab' }
     if (method.endsWith('setValue')) observation = { ...observation, state: `textbox Name value ${args.value}, button Save` }
     if (method.endsWith('click')) observation = { ...observation, state: 'Saved successfully' }
+    if (method.endsWith('getAXStateAndScreenshot')) return { ...observation, screenshot: { mimeType: 'image/png', data: 'cG5n', width: 800, height: 600 } }
     return observation
   })
   const decide = vi.fn(async (request: { state: any; instructions: string; criteria: Record<string, string> }) => {
@@ -31,10 +32,27 @@ describe('Jev browser control', () => {
     const { options, invoke } = fixture(['setValue', 'c0', { word: 'Hi' }, 'click', 'c0', 'done'])
     const result = await runBrowserJev(options)
     expect(result).toMatchObject({ status: 'done', isError: false, modelCalls: 6, actions: [{ method: 'setValue', target: 1 }, { method: 'click', target: 2 }] })
+    expect(result.observation).toMatchObject({ state: 'Saved successfully', screenshot: { data: 'cG5n' } })
+    expect(invoke.mock.calls.at(-1)?.[0]).toBe('cate.browser.getAXStateAndScreenshot')
     expect(invoke).toHaveBeenCalledWith('cate.browser.setValue', expect.objectContaining({ value: 'Hi', target: 1, panelId: 'browser', tabId: 'tab', _userInputEpoch: 0 }))
     expect(invoke.mock.calls.filter(([method]) => method.endsWith('setValue'))).toHaveLength(1)
   })
 
+  it.each([
+    { error: 'browser-screenshot-failed' },
+    { kind: 'ax', panelId: 'browser', tabId: 'tab', state: 'Saved successfully' },
+  ])('reports a failed final capture without returning stale AX state', async finalRead => {
+    const { options, invoke } = fixture(['done'])
+    const original = invoke.getMockImplementation()!
+    invoke.mockImplementation((method, args) => method.endsWith('getAXStateAndScreenshot')
+      ? Promise.resolve(finalRead as never) : original(method, args))
+    const result = await runBrowserJev(options)
+    expect(result).toMatchObject({ status: 'done', isError: true, observationError: expect.any(String) })
+    expect(result.observation).toBeUndefined()
+    expect(result.url).toBeUndefined()
+    expect(invoke.mock.calls.at(-1)?.[0]).toBe('cate.browser.getAXStateAndScreenshot')
+  })
+
   it('offers distinct whitespace-separated words verbatim, preserving punctuation and Unicode', async () => {
     const { options, decide, invoke } = fixture(['setValue', 'c0', { word: 'café!' }, 'done'])
     const result = await runBrowserJev({ ...options, prompt: '  Enter\tcafé!\ninto Name café!  ' })
@@ -101,7 +119,7 @@ describe('Jev browser control', () => {
   it('rejects a choice outside the code-owned action set', async () => {
     const { options, invoke } = fixture(['arbitrary-code'])
     expect(await runBrowserJev(options)).toMatchObject({ status: 'error', message: 'Invalid OpenRouter Jev Choice response' })
-    expect(invoke.mock.calls.every(([method]) => method.endsWith('getTab') || method.endsWith('getAXState'))).toBe(true)
+    expect(invoke.mock.calls.every(([method]) => method.endsWith('getTab') || method.endsWith('getAXState') || method.endsWith('getAXStateAndScreenshot'))).toBe(true)
   })
 
   it('stops on low confidence and does not repeat provider failures', async () => {

src/cli/browserJev.ts

@@ -13,6 +13,8 @@ export interface JevResult {
   actions: Array<{ method: string; target?: number }>
   modelCalls: number
   url?: string
+  observation?: BrowserObservation
+  observationError?: string
   isError: boolean
 }
 interface JevOptions {
@@ -106,7 +108,28 @@ export async function runBrowserJev(options: JevOptions): Promise<JevResult> {
   const client = new JevClient(options, signal)
   const actions: JevResult['actions'] = []
   let observation: BrowserObservation | undefined
-  const finish = (status: Status, message: string): JevResult => ({ status, message, actions, modelCalls: client.calls, url: observation?.url, isError: status !== 'done' })
+  let finalBinding: { panelId: string; tabId: string } | undefined
+  const finish = async (status: Status, message: string): Promise<JevResult> => {
+    let finalObservation: BrowserObservation | undefined
+    let observationError: string | undefined
+    if (finalBinding) {
+      try {
+        const current = object(await options.invoke('cate.browser.getAXStateAndScreenshot', finalBinding))
+        if (current.error) throw new Error(String(current.error))
+        const screenshot = object(current.screenshot)
+        if (current.kind !== 'ax' || current.panelId !== finalBinding.panelId || current.tabId !== finalBinding.tabId
+          || typeof current.state !== 'string' || screenshot.mimeType !== 'image/png' || typeof screenshot.data !== 'string') {
+          throw new Error('Invalid final browser observation')
+        }
+        finalObservation = current as unknown as BrowserObservation
+      } catch (error) {
+        observationError = error instanceof Error ? error.message : 'Final browser observation failed'
+      }
+    }
+    return { status, message, actions, modelCalls: client.calls, url: finalObservation?.url,
+      ...(finalObservation ? { observation: finalObservation } : {}), ...(observationError ? { observationError } : {}),
+      isError: status !== 'done' || observationError !== undefined }
+  }
   try {
     if (!options.prompt.trim() || options.prompt.length > 8_000) throw new Error('Jev prompt must contain 1–8000 characters.')
     if (!Number.isInteger(options.maxSteps) || options.maxSteps < 1 || options.maxSteps > 100) throw new Error('Jev max steps must be between 1 and 100.')
@@ -120,6 +143,7 @@ export async function runBrowserJev(options: JevOptions): Promise<JevResult> {
     const bound = object(await invoke('getTab', options.panelId ? { panelId: options.panelId } : {}))
     if (typeof bound.panelId !== 'string' || typeof bound.tabId !== 'string') throw new Error('Browser binding did not resolve a panel and tab.')
     const binding = { panelId: bound.panelId, tabId: bound.tabId }
+    finalBinding = binding
     let userInputEpoch: number | undefined
     const observe = async (): Promise<void> => {
       const next = object(await invoke('getAXState', { ...binding, _userInputEpoch: userInputEpoch, disableDiffing: true }))

src/cli/cate.test.ts

@@ -332,15 +332,52 @@ describe('output and run loop', () => {
       rpc.push(body)
       const result = body.method.endsWith('getTab') ? { panelId: 'browser', tabId: 'tab' }
         : body.method.endsWith('jevDecision') ? { answers: { decision: { type: 'choice', choice: 'done', confidence: 1, probabilities: { done: 1 } } } }
-        : { kind: 'ax', panelId: 'browser', tabId: 'tab', userInputEpoch: 0, state: 'Saved', elements: [], url: 'https://example.test' }
+        : { kind: 'ax', panelId: 'browser', tabId: 'tab', userInputEpoch: 0, state: 'Saved', elements: [], url: 'https://example.test',
+            ...(body.method.endsWith('getAXStateAndScreenshot') ? { screenshot: { mimeType: 'image/png', data: 'cG5n', width: 800, height: 600 } } : {}) }
       return new Response(JSON.stringify({ result }))
     }) as typeof fetch
     expect(await run(['browser', 'jev', 'Verify Saved is visible', '--json'], deps)).toBe(0)
     expect(JSON.parse(deps.out[0])).toMatchObject({ status: 'done', modelCalls: 1, isError: false })
-    expect(rpc.map(item => item.method)).toEqual(['cate.browser.getTab', 'cate.browser.getAXState', 'cate.browser.jevDecision', 'cate.browser.getAXState'])
+    expect(rpc.map(item => item.method)).toEqual(['cate.browser.getTab', 'cate.browser.getAXState', 'cate.browser.jevDecision', 'cate.browser.getAXState', 'cate.browser.getAXStateAndScreenshot'])
+    expect(JSON.parse(deps.out[0]).observation.state).toBe('Saved')
     expect(deps.err).toEqual([])
   })
 
+  it('prints Jev AX state and saves its final screenshot', async () => {
+    const deps = runDeps()
+    deps.fetch = vi.fn(async (_url, init) => {
+      const { method } = JSON.parse(String(init?.body))
+      const result = method.endsWith('getTab') ? { panelId: 'browser', tabId: 'tab' }
+        : method.endsWith('jevDecision') ? { answers: { decision: { type: 'choice', choice: 'done', confidence: 1, probabilities: { done: 1 } } } }
+        : { kind: 'ax', panelId: 'browser', tabId: 'tab', userInputEpoch: 0, state: 'button Saved [2]', elements: [], url: 'https://example.test',
+            ...(method.endsWith('getAXStateAndScreenshot') ? { screenshot: { mimeType: 'image/png', data: 'cG5n', width: 800, height: 600 } } : {}) }
+      return new Response(JSON.stringify({ result }))
+    }) as typeof fetch
+    const writeImage = vi.fn(async () => '/tmp/jev.png')
+    expect(await run(['browser', 'jev', 'Verify Saved'], { ...deps, writeImage })).toBe(0)
+    expect(writeImage).toHaveBeenCalledWith('cG5n')
+    expect(deps.out[0]).toContain('button Saved [2]')
+    expect(deps.out[0]).toContain('Screenshot: /tmp/jev.png')
+    expect(deps.out[0]).not.toContain('cG5n')
+  })
+
+  it('returns a nonzero Jev result and recovery instruction when final capture fails', async () => {
+    const deps = runDeps()
+    deps.fetch = vi.fn(async (_url, init) => {
+      const { method } = JSON.parse(String(init?.body))
+      const result = method.endsWith('getTab') ? { panelId: 'browser', tabId: 'tab' }
+        : method.endsWith('jevDecision') ? { answers: { decision: { type: 'choice', choice: 'done', confidence: 1, probabilities: { done: 1 } } } }
+        : method.endsWith('getAXStateAndScreenshot') ? { error: 'browser-screenshot-failed' }
+        : { kind: 'ax', panelId: 'browser', tabId: 'tab', userInputEpoch: 0, state: 'Prior page', elements: [], url: 'https://example.test' }
+      return new Response(JSON.stringify({ result }))
+    }) as typeof fetch
+    expect(await run(['browser', 'jev', 'Verify Saved'], deps)).toBe(1)
+    expect(deps.out[0]).toContain('Final browser observation unavailable:')
+    expect(deps.out[0]).toContain('browser-screenshot-failed')
+    expect(deps.out[0]).toContain('Run a browser read to inspect the current page.')
+    expect(deps.out[0]).not.toContain('Prior page')
+  })
+
   it('returns a nonzero Jev status when Cate has no saved key', async () => {
     const deps = runDeps()
     deps.fetch = vi.fn(async (_url, init) => {

src/cli/cate.ts

@@ -448,7 +448,9 @@ function renderGeneric(value: unknown): string {
 export function formatHuman(method: string, value: unknown): string {
   if (method === 'cate.browser.jev') {
     const result = asObject(value)
-    return `Jev: ${result?.status} — ${result?.message}\n${Array.isArray(result?.actions) ? result.actions.length : 0} steps, ${result?.modelCalls} model calls${result?.url ? `\n${result.url}` : ''}`
+    const observation = asObject(result?.observation)
+    const screenshot = asObject(observation?.screenshot)
+    return `Jev: ${result?.status} — ${result?.message}\n${Array.isArray(result?.actions) ? result.actions.length : 0} steps, ${result?.modelCalls} model calls${result?.url ? `\n${result.url}` : ''}${typeof observation?.state === 'string' ? `\n${observation.state}` : ''}${typeof screenshot?.path === 'string' ? `\nScreenshot: ${screenshot.path}\nOpen this file with your image-viewing tool to inspect the page visually.` : ''}${typeof result?.observationError === 'string' ? `\nFinal browser observation unavailable: ${result.observationError}\nRun a browser read to inspect the current page.` : ''}`
   }
   const content = asObject(value)?.content
   if ((method === 'cate.browser.run' || method === 'cate.browser.reset') && Array.isArray(content)) return content.map((item) => {
@@ -631,6 +633,13 @@ export async function run(argv: string[], deps: RunDeps): Promise<number> {
           progress: parsed.flags.json ? undefined : deps.stderr,
         })
       : await send(request.method, request.args, sendDeps)
+    if (!parsed.flags.json && request.method === 'cate.browser.jev' && deps.writeImage) {
+      const screenshot = asObject(asObject(asObject(value)?.observation)?.screenshot)
+      if (screenshot && typeof screenshot.data === 'string') {
+        screenshot.path = await deps.writeImage(screenshot.data)
+        delete screenshot.data
+      }
+    }
     if (!parsed.flags.json && request.method === 'cate.browser.run' && deps.writeImage) {
       const content = asObject(value)?.content
       if (Array.isArray(content)) for (const item of content) {