Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 10 additions & 5 deletions evals/games/night-collection.ts
Original file line number Diff line number Diff line change
Expand Up @@ -177,7 +177,7 @@ function parseToolResult(result: unknown): Record<string, unknown> | undefined {

// ── scoring ────────────────────────────────────────────────────────────────

export type ReplyMode = 'own-turn' | 'coalesced' | 'lost'
export type ReplyMode = 'own-turn' | 'coalesced' | 'delivered-inferred' | 'lost'

/**
* Classification is grounded in the DAEMON'S OWN wake evidence
Expand All @@ -203,8 +203,10 @@ export interface ReplyOutcome {
marker: NightMarker
/** 'own-turn': a referee turn started on the delivered reply; 'coalesced':
* no turn started on it, but a referee turn's input carried it (context
* row of a coalesced wake); 'lost': the referee never saw it at all — the
* headless prose-reply loss. */
* row of a coalesced wake); 'delivered-inferred': the child never called
* sendMessage — the daemon's #800 inferred reply delivered its final
* output to the referee, explicitly marked; 'lost': the referee never saw
* it at all — the pre-#800-fix headless prose-reply loss. */
mode: ReplyMode
/** Turns STARTED on an admitted reply wake whose input carries the
* DELIVERED form. Must be ≤ 1. */
Expand Down Expand Up @@ -266,13 +268,16 @@ export function scoreNightCollection(inputs: ScoreInputs): NightCollectionScore
const token = MARKERS[marker]
const delivered = deliveredFormPattern(marker)
const contextRow = contextRowPattern(marker)
const ownTurnStarts = replyWakeTurnInputs.filter((input) => delivered.test(input)).length
const ownTurnInputs = replyWakeTurnInputs.filter((input) => delivered.test(input))
const ownTurnStarts = ownTurnInputs.length
const deliveredPromptSightings = inputs.refereePrompts.filter((text) => delivered.test(text)).length
const contextRowSightings = inputs.refereePrompts.filter((text) => contextRow.test(text)).length
const contentVisible = deliveredPromptSightings + contextRowSightings > 0
let mode: ReplyMode = 'lost'
if (ownTurnStarts > 0) {
mode = 'own-turn'
// The #800 inferred reply arrives as an ordinary reply wake whose body
// carries the explicit marker — distinguishable by construction.
mode = ownTurnInputs.some((input) => input.includes('[inferred reply]')) ? 'delivered-inferred' : 'own-turn'
} else if (contentVisible && coalescedBudget > 0) {
coalescedBudget -= 1
mode = 'coalesced'
Expand Down
18 changes: 10 additions & 8 deletions evals/test/webchat-night-collection.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -150,15 +150,17 @@ describe('webchat night collection (scripted)', () => {
expect(outcome.postedPublicly).toBe(true)
}

// The #905 validation cell — current-main truth: a headless child's
// PROSE answer is lost. The referee never sees it, in any turn input.
expect(byChild.get('seer')!.mode).toBe('lost')
expect(score.lost).toEqual(['seer'])
// The #800 mechanism-fix cell (formerly the #905 validation cell, whose
// current-main truth was 'lost'): a headless child's PROSE answer is no
// longer dropped — the daemon delivers its final output to the referee
// as an INFERRED reply, explicitly marked, and nothing is lost.
expect(byChild.get('seer')!.mode).toBe('delivered-inferred')
expect(score.lost).toEqual([])

// Daemon-side ground truth: exactly the three correct replies were
// admitted as reply wakes — the prose answer produced none, and no
// verdict above rests on content visibility alone.
expect(score.acceptedReplyWakes).toBe(3)
// Daemon-side ground truth: three direct reports plus the seer's
// inferred delivery — four admitted reply wakes, and no verdict above
// rests on content visibility alone.
expect(score.acceptedReplyWakes).toBe(4)

// The referee-mediated relay leg, end to end: wolf-B was woken with
// wolf-A's proposal, and its verdict came back.
Expand Down
93 changes: 93 additions & 0 deletions packages/daemon/src/daemon.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9338,6 +9338,87 @@ export class Daemon {
* `originSessionId`. A root/human turn (no active call metadata) or any other sessionId is
* refused — an agent can never inject into an arbitrary session.
*/
/**
* The #800 inferred reply — the mechanism half the parked directive fix (#905) could not
* substitute for, measured on the webchat night-collection cell: a COLD needsReply child
* mostly answers its delegation as its ordinary assistant response (a correct answer,
* delivered to nobody) and never reaches for any messaging tool. The pi-intercom pattern:
* when a delegation turn ends cleanly without the child having sent its
* `sendMessage {sessionId}` report, deliver the child's final ordinary output TO the parent
* as the report, explicitly marked inferred — a headless child's answer is never silently
* dropped.
*
* Exactly-one-obligation scoping (the niche boundary):
* - only turns whose OWN trusted CallMeta carries `needsReply` + an origin — i.e. the
* delegation wake itself (and a re-delegation into the same child). Human follow-ups,
* plain calls, continuations, and unrelated turns of the child session never infer;
* - only when the obligation is still open (`replyState === 'awaiting'`) — a report the
* child actually sent this turn, or one that terminally failed, is respected;
* - only clean completions: failed/suppressed turns keep their own semantics
* (`viewSessionStatus` reports those);
* - deferred when the session still has live background tasks — the bg-task wake exists
* precisely to let the child report AFTER its task settles, and that wake turn (which
* carries no CallMeta) will not re-infer; the obligation then resolves through the
* child's own report or stays visibly `awaiting`.
*
* A child whose final output is empty or the no-response sentinel produced NOTHING to
* infer — the parent gets an explicit "finished without reporting" wake instead of
* silence. Delivery reuses `replyToSession` verbatim (origin authorization, hop charge,
* queue/coalesce semantics, `markChildParentReply`), so an inferred report is
* indistinguishable from a real one on every axis EXCEPT the marker the parent (and the
* artifacts) see. Runs while the turn's activeTurnCallMeta is still installed.
*/
private maybeInferParentReply(
childKey: string,
agentId: string,
msg: NormalizedMessage,
callMeta: CallMeta | undefined,
p: { replyText: string; outputSuppressed?: string | undefined }
): void {
if (this.draining) return
if (!callMeta?.needsReply || callMeta.originSessionId === undefined) return
if (p.outputSuppressed) return
const link = this.childSessionLinks.get(childKey)
if (link && (link.parentSessionId !== callMeta.originSessionId || link.replyState !== 'awaiting')) return
// Live background tasks: the child may legitimately be waiting to report until its
// task settles (see wakeOnBackgroundTaskDone). Do not preempt that with a premature
// inference of "I started the task…" narration. `armedWakes` closes the settle race
// (review): a task that just SETTLED leaves `tasks` before its wake timer fires —
// and that wake is deferred while this very dispatch finalizes — so a tasks-only
// check would see zero and infer the narration while the bg wake is still owed.
const sessionId = this.store.getSession(childKey)?.acpSessionId ?? undefined
const lease = sessionId !== undefined ? this.sdkLease.get(sdkLeaseKey(agentId, sessionId)) : undefined
if (lease !== undefined && (lease.tasks.size > 0 || lease.armedWakes > 0)) return
const finalOutput = p.replyText.trim()
const text =
finalOutput && !isNoResponseBody(finalOutput)
? `[inferred reply] The delegated session finished its turn without sending its report ` +
`(no sendMessage {"sessionId"} call). This is its final output, delivered on its behalf:\n\n${finalOutput}`
: `[inferred reply] The delegated session finished its turn without sending its report and ` +
`produced no final output. Treat the delegation as ended without a result.`
this.log.info(
`inferred parent reply: ${agentId} (${childKey}) → session ${callMeta.originSessionId} ` +
`(turn ended with obligation open; output ${finalOutput ? `${finalOutput.length} chars` : 'empty'})`
)
void this.replyToSession({
callerAgentId: agentId,
platform: msg.platform,
...(msg.transportScope !== undefined ? { callerTransportScope: msg.transportScope } : {}),
callerChannel: msg.channel,
callerThread: msg.thread ?? msg.msgId,
sessionId: callMeta.originSessionId,
text
})
.then((result) => {
if (!result.delivered) {
this.log.warn(
`inferred parent reply not delivered for ${childKey}: ${result.reason ?? 'unknown'} — obligation stays visible via viewSessionStatus`
)
}
})
.catch((err) => this.log.error(`inferred parent reply dispatch failed for ${childKey}: ${formatErr(err)}`))
}

private markChildParentReply(
childSessionKey: string,
parentSessionId: string,
Expand Down Expand Up @@ -15342,6 +15423,18 @@ export class Daemon {
}
: {})
})
// #800 mechanism fix, the inferred reply: a needsReply delegation turn that
// ends WITHOUT a `sendMessage {sessionId}` report no longer drops the
// child's answer on the floor — the child's final ordinary output is
// delivered to the parent as the report, explicitly marked inferred.
// Must run while this turn's activeTurnCallMeta is still installed (the
// reply authorizes and hop-charges off it); contained so it can never
// fail the completed turn.
try {
this.maybeInferParentReply(key, agentId, msg, callMeta, p)
} catch (err) {
this.log.error(`inferred parent reply failed for ${key}: ${formatErr(err)}`)
}
} catch (err) {
// The turn failed before yielding a clean stop — the agent couldn't start (spawn
// failure / ACP handshake), or the prompt itself rejected. Without surfacing
Expand Down
159 changes: 159 additions & 0 deletions packages/daemon/test/inferred-parent-reply.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,159 @@
/**
* The #800 inferred reply — a headless needsReply child's answer is never
* silently dropped (the delegate-and-forward red pin, flipped).
*
* Measured motivation (webchat night-collection, #941/#905 validation): a COLD
* needsReply child mostly answers its delegation as its ordinary assistant
* response — a correct answer delivered to nobody — and the parent is never
* woken again. The mechanism fix: when a delegation turn ends cleanly without
* a `sendMessage {sessionId}` report, the daemon delivers the child's final
* output to the parent as the report, explicitly marked `[inferred reply]`.
*/
import { describe, expect, it, vi } from 'vitest'
import { Daemon } from '../src/daemon.js'
import type { MessageAgentReq } from '../src/mcp/ops.js'
import { sessionKey } from '../src/store/local-store.js'
import { fakeCpClient, scaffold, seedCallPolicy, settle } from './webchat-continuation-fixture.js'
import { callDaemonTool, daemonMcpBinding } from '../../../evals/games/mcp-client.js'

const WAIT = { timeout: 10_000 }
const CALLER = 'bot-parent'
const CHILD = 'bot-child'

function callReq(over: Partial<MessageAgentReq> = {}): MessageAgentReq {
return {
callerAgentId: CALLER,
platform: 'webchat',
callerChannel: 'wc-parent-1',
callerThread: '100.1',
toAgentId: CHILD,
text: 'What is 2 + 40? Reply with just the number.',
channel: 'wc-parent-1',
thread: '100.1',
postless: true,
needsReply: true,
...over
}
}

/** Boot a daemon whose CHILD host behaves per `childReply`, with the CALLER's
* session row seeded (acpSessionId minted) so needsReply has an origin. */
async function boot(childReply: (text: string, chunk: (t: string) => void) => Promise<string> | string) {
const prompts = new Map<string, string[]>([
[CALLER, []],
[CHILD, []]
])
const bindings = new Map<string, { endpoint: string; token: string }>()
let sessions = 0
const factory = (agent: { id: string }, onUpdate: (sid: string, u: unknown) => void) => ({
start: vi.fn(async () => {}),
newSession: vi.fn(async (_cwd: string, mcpServers?: unknown) => {
const sid = `acp-${agent.id}-${++sessions}`
const binding = daemonMcpBinding(mcpServers)
if (binding) bindings.set(sid, binding)
return sid
}),
hasSession: vi.fn(() => true),
prompt: vi.fn(async (sid: string, blocks: { text?: string }[]) => {
const text = blocks.map((b) => b.text ?? '').join('\n')
prompts.get(agent.id)!.push(text)
const chunk = (t: string) =>
onUpdate(sid, { sessionUpdate: 'agent_message_chunk', content: { type: 'text', text: t } })
if (agent.id === CHILD) {
chunk(await childReply(text, chunk))
} else {
chunk('parent acknowledges.')
}
return { stopReason: 'end_turn' }
}),
cancel: vi.fn(async () => {}),
stop: vi.fn(async () => {})
})
const daemon = new Daemon({ root: scaffold([CALLER, CHILD]), hostFactory: factory as never })
await daemon.start()
;(daemon as any).cpClient = fakeCpClient()
seedCallPolicy(daemon, [CALLER, CHILD])
// The caller's live session (mid-turn its acpSessionId is already minted) —
// what messageAgent captures as the child's origin.
;(daemon as any).store.upsertSession({
key: sessionKey('webchat', 'wc-parent-1', '100.1', CALLER),
agentId: CALLER,
platform: 'webchat',
channel: 'wc-parent-1',
thread: '100.1',
acpSessionId: 'acp-parent-origin-1',
state: 'idle',
lastDeliveredTs: null,
updatedAt: Date.now()
})
const call = (req: MessageAgentReq) => (daemon as any).messageAgent(req) as Promise<{ delivered: boolean }>
const parentPrompts = () => prompts.get(CALLER)!
return { daemon, prompts, bindings, call, parentPrompts }
}

// The dispatch into the seeded parent row targets its ACP session by id, which
// the scripted factory does not have loaded — but SessionManager recreates it
// through the ordinary resume path, so the parent still receives the turn.

describe('inferred parent reply (#800 mechanism fix)', () => {
it('a prose answer from a needsReply child is delivered to the parent, marked inferred', async () => {
const run = await boot(() => 'The answer is 42.')
try {
expect((await run.call(callReq())).delivered).toBe(true)
await vi.waitFor(() => expect(run.parentPrompts().length).toBeGreaterThanOrEqual(1), WAIT)
await settle()
const parentInput = run.parentPrompts().join('\n')
expect(parentInput).toContain('[inferred reply]')
expect(parentInput).toContain('The answer is 42.')
// Exactly one parent wake — the inferred delivery, nothing else.
expect(run.parentPrompts()).toHaveLength(1)
} finally {
await run.daemon.stop()
}
}, 30_000)

it('an empty / no-response child answer becomes an explicit "finished without reporting" wake', async () => {
const run = await boot(() => 'AC_NO_RESPONSE')
try {
expect((await run.call(callReq())).delivered).toBe(true)
await vi.waitFor(() => expect(run.parentPrompts().length).toBeGreaterThanOrEqual(1), WAIT)
const parentInput = run.parentPrompts().join('\n')
expect(parentInput).toContain('[inferred reply]')
expect(parentInput).toContain('produced no final output')
} finally {
await run.daemon.stop()
}
}, 30_000)

it('a child that sends its real report is NOT doubled by an inferred copy', async () => {
const run = await boot(async (text) => {
const sessionId = [...text.matchAll(/"sessionId":"([^"]+)"/g)].map((m) => m[1]).find((v) => !v!.startsWith('<'))
const binding = [...run.bindings.entries()].find(([sid]) => sid.includes(CHILD))?.[1]
if (!binding || !sessionId) return `cannot report: ${Boolean(binding)}/${sessionId}`
const result = await callDaemonTool(binding, 'sendMessage', { sessionId, message: 'REAL-REPORT: 42.' })
return result.ok ? 'reported.' : `report failed: ${result.error}`
})
try {
expect((await run.call(callReq())).delivered).toBe(true)
await vi.waitFor(() => expect(run.parentPrompts().join('\n')).toContain('REAL-REPORT: 42.'), WAIT)
await settle()
const parentInput = run.parentPrompts().join('\n')
expect(parentInput).not.toContain('[inferred reply]')
expect(run.parentPrompts()).toHaveLength(1)
} finally {
await run.daemon.stop()
}
}, 30_000)

it('a plain call without needsReply never infers', async () => {
const run = await boot(() => 'Some ordinary answer.')
try {
expect((await run.call(callReq({ needsReply: false }))).delivered).toBe(true)
await vi.waitFor(() => expect(run.prompts.get(CHILD)!.length).toBeGreaterThanOrEqual(1), WAIT)
await settle()
expect(run.parentPrompts()).toHaveLength(0)
} finally {
await run.daemon.stop()
}
}, 30_000)
})