diff --git a/evals/games/night-collection.ts b/evals/games/night-collection.ts index 96bb4f967..c2255fc1c 100644 --- a/evals/games/night-collection.ts +++ b/evals/games/night-collection.ts @@ -177,7 +177,7 @@ function parseToolResult(result: unknown): Record | undefined { // ── scoring ──────────────────────────────────────────────────────────────── -export type ReplyMode = 'own-turn' | 'coalesced' | 'lost' +export type ReplyMode = 'own-turn' | 'coalesced' | 'delivered-inferred' | 'lost' /** * Classification is grounded in the DAEMON'S OWN wake evidence @@ -203,8 +203,10 @@ export interface ReplyOutcome { marker: NightMarker /** 'own-turn': a referee turn started on the delivered reply; 'coalesced': * no turn started on it, but a referee turn's input carried it (context - * row of a coalesced wake); 'lost': the referee never saw it at all — the - * headless prose-reply loss. */ + * row of a coalesced wake); 'delivered-inferred': the child never called + * sendMessage — the daemon's #800 inferred reply delivered its final + * output to the referee, explicitly marked; 'lost': the referee never saw + * it at all — the pre-#800-fix headless prose-reply loss. */ mode: ReplyMode /** Turns STARTED on an admitted reply wake whose input carries the * DELIVERED form. Must be ≤ 1. */ @@ -266,13 +268,16 @@ export function scoreNightCollection(inputs: ScoreInputs): NightCollectionScore const token = MARKERS[marker] const delivered = deliveredFormPattern(marker) const contextRow = contextRowPattern(marker) - const ownTurnStarts = replyWakeTurnInputs.filter((input) => delivered.test(input)).length + const ownTurnInputs = replyWakeTurnInputs.filter((input) => delivered.test(input)) + const ownTurnStarts = ownTurnInputs.length const deliveredPromptSightings = inputs.refereePrompts.filter((text) => delivered.test(text)).length const contextRowSightings = inputs.refereePrompts.filter((text) => contextRow.test(text)).length const contentVisible = deliveredPromptSightings + contextRowSightings > 0 let mode: ReplyMode = 'lost' if (ownTurnStarts > 0) { - mode = 'own-turn' + // The #800 inferred reply arrives as an ordinary reply wake whose body + // carries the explicit marker — distinguishable by construction. + mode = ownTurnInputs.some((input) => input.includes('[inferred reply]')) ? 'delivered-inferred' : 'own-turn' } else if (contentVisible && coalescedBudget > 0) { coalescedBudget -= 1 mode = 'coalesced' diff --git a/evals/test/webchat-night-collection.test.ts b/evals/test/webchat-night-collection.test.ts index 102efbf6b..1ac2eaa34 100644 --- a/evals/test/webchat-night-collection.test.ts +++ b/evals/test/webchat-night-collection.test.ts @@ -150,15 +150,17 @@ describe('webchat night collection (scripted)', () => { expect(outcome.postedPublicly).toBe(true) } - // The #905 validation cell — current-main truth: a headless child's - // PROSE answer is lost. The referee never sees it, in any turn input. - expect(byChild.get('seer')!.mode).toBe('lost') - expect(score.lost).toEqual(['seer']) + // The #800 mechanism-fix cell (formerly the #905 validation cell, whose + // current-main truth was 'lost'): a headless child's PROSE answer is no + // longer dropped — the daemon delivers its final output to the referee + // as an INFERRED reply, explicitly marked, and nothing is lost. + expect(byChild.get('seer')!.mode).toBe('delivered-inferred') + expect(score.lost).toEqual([]) - // Daemon-side ground truth: exactly the three correct replies were - // admitted as reply wakes — the prose answer produced none, and no - // verdict above rests on content visibility alone. - expect(score.acceptedReplyWakes).toBe(3) + // Daemon-side ground truth: three direct reports plus the seer's + // inferred delivery — four admitted reply wakes, and no verdict above + // rests on content visibility alone. + expect(score.acceptedReplyWakes).toBe(4) // The referee-mediated relay leg, end to end: wolf-B was woken with // wolf-A's proposal, and its verdict came back. diff --git a/packages/daemon/src/daemon.ts b/packages/daemon/src/daemon.ts index 9678128cb..3900af962 100644 --- a/packages/daemon/src/daemon.ts +++ b/packages/daemon/src/daemon.ts @@ -9338,6 +9338,87 @@ export class Daemon { * `originSessionId`. A root/human turn (no active call metadata) or any other sessionId is * refused — an agent can never inject into an arbitrary session. */ + /** + * The #800 inferred reply — the mechanism half the parked directive fix (#905) could not + * substitute for, measured on the webchat night-collection cell: a COLD needsReply child + * mostly answers its delegation as its ordinary assistant response (a correct answer, + * delivered to nobody) and never reaches for any messaging tool. The pi-intercom pattern: + * when a delegation turn ends cleanly without the child having sent its + * `sendMessage {sessionId}` report, deliver the child's final ordinary output TO the parent + * as the report, explicitly marked inferred — a headless child's answer is never silently + * dropped. + * + * Exactly-one-obligation scoping (the niche boundary): + * - only turns whose OWN trusted CallMeta carries `needsReply` + an origin — i.e. the + * delegation wake itself (and a re-delegation into the same child). Human follow-ups, + * plain calls, continuations, and unrelated turns of the child session never infer; + * - only when the obligation is still open (`replyState === 'awaiting'`) — a report the + * child actually sent this turn, or one that terminally failed, is respected; + * - only clean completions: failed/suppressed turns keep their own semantics + * (`viewSessionStatus` reports those); + * - deferred when the session still has live background tasks — the bg-task wake exists + * precisely to let the child report AFTER its task settles, and that wake turn (which + * carries no CallMeta) will not re-infer; the obligation then resolves through the + * child's own report or stays visibly `awaiting`. + * + * A child whose final output is empty or the no-response sentinel produced NOTHING to + * infer — the parent gets an explicit "finished without reporting" wake instead of + * silence. Delivery reuses `replyToSession` verbatim (origin authorization, hop charge, + * queue/coalesce semantics, `markChildParentReply`), so an inferred report is + * indistinguishable from a real one on every axis EXCEPT the marker the parent (and the + * artifacts) see. Runs while the turn's activeTurnCallMeta is still installed. + */ + private maybeInferParentReply( + childKey: string, + agentId: string, + msg: NormalizedMessage, + callMeta: CallMeta | undefined, + p: { replyText: string; outputSuppressed?: string | undefined } + ): void { + if (this.draining) return + if (!callMeta?.needsReply || callMeta.originSessionId === undefined) return + if (p.outputSuppressed) return + const link = this.childSessionLinks.get(childKey) + if (link && (link.parentSessionId !== callMeta.originSessionId || link.replyState !== 'awaiting')) return + // Live background tasks: the child may legitimately be waiting to report until its + // task settles (see wakeOnBackgroundTaskDone). Do not preempt that with a premature + // inference of "I started the task…" narration. `armedWakes` closes the settle race + // (review): a task that just SETTLED leaves `tasks` before its wake timer fires — + // and that wake is deferred while this very dispatch finalizes — so a tasks-only + // check would see zero and infer the narration while the bg wake is still owed. + const sessionId = this.store.getSession(childKey)?.acpSessionId ?? undefined + const lease = sessionId !== undefined ? this.sdkLease.get(sdkLeaseKey(agentId, sessionId)) : undefined + if (lease !== undefined && (lease.tasks.size > 0 || lease.armedWakes > 0)) return + const finalOutput = p.replyText.trim() + const text = + finalOutput && !isNoResponseBody(finalOutput) + ? `[inferred reply] The delegated session finished its turn without sending its report ` + + `(no sendMessage {"sessionId"} call). This is its final output, delivered on its behalf:\n\n${finalOutput}` + : `[inferred reply] The delegated session finished its turn without sending its report and ` + + `produced no final output. Treat the delegation as ended without a result.` + this.log.info( + `inferred parent reply: ${agentId} (${childKey}) → session ${callMeta.originSessionId} ` + + `(turn ended with obligation open; output ${finalOutput ? `${finalOutput.length} chars` : 'empty'})` + ) + void this.replyToSession({ + callerAgentId: agentId, + platform: msg.platform, + ...(msg.transportScope !== undefined ? { callerTransportScope: msg.transportScope } : {}), + callerChannel: msg.channel, + callerThread: msg.thread ?? msg.msgId, + sessionId: callMeta.originSessionId, + text + }) + .then((result) => { + if (!result.delivered) { + this.log.warn( + `inferred parent reply not delivered for ${childKey}: ${result.reason ?? 'unknown'} — obligation stays visible via viewSessionStatus` + ) + } + }) + .catch((err) => this.log.error(`inferred parent reply dispatch failed for ${childKey}: ${formatErr(err)}`)) + } + private markChildParentReply( childSessionKey: string, parentSessionId: string, @@ -15342,6 +15423,18 @@ export class Daemon { } : {}) }) + // #800 mechanism fix, the inferred reply: a needsReply delegation turn that + // ends WITHOUT a `sendMessage {sessionId}` report no longer drops the + // child's answer on the floor — the child's final ordinary output is + // delivered to the parent as the report, explicitly marked inferred. + // Must run while this turn's activeTurnCallMeta is still installed (the + // reply authorizes and hop-charges off it); contained so it can never + // fail the completed turn. + try { + this.maybeInferParentReply(key, agentId, msg, callMeta, p) + } catch (err) { + this.log.error(`inferred parent reply failed for ${key}: ${formatErr(err)}`) + } } catch (err) { // The turn failed before yielding a clean stop — the agent couldn't start (spawn // failure / ACP handshake), or the prompt itself rejected. Without surfacing diff --git a/packages/daemon/test/inferred-parent-reply.test.ts b/packages/daemon/test/inferred-parent-reply.test.ts new file mode 100644 index 000000000..2f0c3844d --- /dev/null +++ b/packages/daemon/test/inferred-parent-reply.test.ts @@ -0,0 +1,159 @@ +/** + * The #800 inferred reply — a headless needsReply child's answer is never + * silently dropped (the delegate-and-forward red pin, flipped). + * + * Measured motivation (webchat night-collection, #941/#905 validation): a COLD + * needsReply child mostly answers its delegation as its ordinary assistant + * response — a correct answer delivered to nobody — and the parent is never + * woken again. The mechanism fix: when a delegation turn ends cleanly without + * a `sendMessage {sessionId}` report, the daemon delivers the child's final + * output to the parent as the report, explicitly marked `[inferred reply]`. + */ +import { describe, expect, it, vi } from 'vitest' +import { Daemon } from '../src/daemon.js' +import type { MessageAgentReq } from '../src/mcp/ops.js' +import { sessionKey } from '../src/store/local-store.js' +import { fakeCpClient, scaffold, seedCallPolicy, settle } from './webchat-continuation-fixture.js' +import { callDaemonTool, daemonMcpBinding } from '../../../evals/games/mcp-client.js' + +const WAIT = { timeout: 10_000 } +const CALLER = 'bot-parent' +const CHILD = 'bot-child' + +function callReq(over: Partial = {}): MessageAgentReq { + return { + callerAgentId: CALLER, + platform: 'webchat', + callerChannel: 'wc-parent-1', + callerThread: '100.1', + toAgentId: CHILD, + text: 'What is 2 + 40? Reply with just the number.', + channel: 'wc-parent-1', + thread: '100.1', + postless: true, + needsReply: true, + ...over + } +} + +/** Boot a daemon whose CHILD host behaves per `childReply`, with the CALLER's + * session row seeded (acpSessionId minted) so needsReply has an origin. */ +async function boot(childReply: (text: string, chunk: (t: string) => void) => Promise | string) { + const prompts = new Map([ + [CALLER, []], + [CHILD, []] + ]) + const bindings = new Map() + let sessions = 0 + const factory = (agent: { id: string }, onUpdate: (sid: string, u: unknown) => void) => ({ + start: vi.fn(async () => {}), + newSession: vi.fn(async (_cwd: string, mcpServers?: unknown) => { + const sid = `acp-${agent.id}-${++sessions}` + const binding = daemonMcpBinding(mcpServers) + if (binding) bindings.set(sid, binding) + return sid + }), + hasSession: vi.fn(() => true), + prompt: vi.fn(async (sid: string, blocks: { text?: string }[]) => { + const text = blocks.map((b) => b.text ?? '').join('\n') + prompts.get(agent.id)!.push(text) + const chunk = (t: string) => + onUpdate(sid, { sessionUpdate: 'agent_message_chunk', content: { type: 'text', text: t } }) + if (agent.id === CHILD) { + chunk(await childReply(text, chunk)) + } else { + chunk('parent acknowledges.') + } + return { stopReason: 'end_turn' } + }), + cancel: vi.fn(async () => {}), + stop: vi.fn(async () => {}) + }) + const daemon = new Daemon({ root: scaffold([CALLER, CHILD]), hostFactory: factory as never }) + await daemon.start() + ;(daemon as any).cpClient = fakeCpClient() + seedCallPolicy(daemon, [CALLER, CHILD]) + // The caller's live session (mid-turn its acpSessionId is already minted) — + // what messageAgent captures as the child's origin. + ;(daemon as any).store.upsertSession({ + key: sessionKey('webchat', 'wc-parent-1', '100.1', CALLER), + agentId: CALLER, + platform: 'webchat', + channel: 'wc-parent-1', + thread: '100.1', + acpSessionId: 'acp-parent-origin-1', + state: 'idle', + lastDeliveredTs: null, + updatedAt: Date.now() + }) + const call = (req: MessageAgentReq) => (daemon as any).messageAgent(req) as Promise<{ delivered: boolean }> + const parentPrompts = () => prompts.get(CALLER)! + return { daemon, prompts, bindings, call, parentPrompts } +} + +// The dispatch into the seeded parent row targets its ACP session by id, which +// the scripted factory does not have loaded — but SessionManager recreates it +// through the ordinary resume path, so the parent still receives the turn. + +describe('inferred parent reply (#800 mechanism fix)', () => { + it('a prose answer from a needsReply child is delivered to the parent, marked inferred', async () => { + const run = await boot(() => 'The answer is 42.') + try { + expect((await run.call(callReq())).delivered).toBe(true) + await vi.waitFor(() => expect(run.parentPrompts().length).toBeGreaterThanOrEqual(1), WAIT) + await settle() + const parentInput = run.parentPrompts().join('\n') + expect(parentInput).toContain('[inferred reply]') + expect(parentInput).toContain('The answer is 42.') + // Exactly one parent wake — the inferred delivery, nothing else. + expect(run.parentPrompts()).toHaveLength(1) + } finally { + await run.daemon.stop() + } + }, 30_000) + + it('an empty / no-response child answer becomes an explicit "finished without reporting" wake', async () => { + const run = await boot(() => 'AC_NO_RESPONSE') + try { + expect((await run.call(callReq())).delivered).toBe(true) + await vi.waitFor(() => expect(run.parentPrompts().length).toBeGreaterThanOrEqual(1), WAIT) + const parentInput = run.parentPrompts().join('\n') + expect(parentInput).toContain('[inferred reply]') + expect(parentInput).toContain('produced no final output') + } finally { + await run.daemon.stop() + } + }, 30_000) + + it('a child that sends its real report is NOT doubled by an inferred copy', async () => { + const run = await boot(async (text) => { + const sessionId = [...text.matchAll(/"sessionId":"([^"]+)"/g)].map((m) => m[1]).find((v) => !v!.startsWith('<')) + const binding = [...run.bindings.entries()].find(([sid]) => sid.includes(CHILD))?.[1] + if (!binding || !sessionId) return `cannot report: ${Boolean(binding)}/${sessionId}` + const result = await callDaemonTool(binding, 'sendMessage', { sessionId, message: 'REAL-REPORT: 42.' }) + return result.ok ? 'reported.' : `report failed: ${result.error}` + }) + try { + expect((await run.call(callReq())).delivered).toBe(true) + await vi.waitFor(() => expect(run.parentPrompts().join('\n')).toContain('REAL-REPORT: 42.'), WAIT) + await settle() + const parentInput = run.parentPrompts().join('\n') + expect(parentInput).not.toContain('[inferred reply]') + expect(run.parentPrompts()).toHaveLength(1) + } finally { + await run.daemon.stop() + } + }, 30_000) + + it('a plain call without needsReply never infers', async () => { + const run = await boot(() => 'Some ordinary answer.') + try { + expect((await run.call(callReq({ needsReply: false }))).delivered).toBe(true) + await vi.waitFor(() => expect(run.prompts.get(CHILD)!.length).toBeGreaterThanOrEqual(1), WAIT) + await settle() + expect(run.parentPrompts()).toHaveLength(0) + } finally { + await run.daemon.stop() + } + }, 30_000) +})