test(evals): tool-surface A/B — shipped sendMessage vs messaging-primitives post

docs/designs/collaboration-arena-baseline.md

@@ -277,6 +277,29 @@ export AGENTCONNECT_DAEMON_ENTRY="$PWD/packages/daemon/dist/index.js"
 # then call runWerewolf({ subject: { kind: 'real', subjectRoot, templateAgentIds } })
 ```
 
+### 4.2 Prompt-change gate (standing rule, from the #801 incident)
+
+**Any change to the standing collaboration guidance or the parent-report
+append (`collabAppend` / `parentReplyAppend` in
+`packages/daemon/src/session/session-manager.ts`) must be validated against
+BOTH the parent-session scenario AND the in-thread turn-taking scenario
+(`in-thread-count`) of the tool-surface A/B matrix before landing.**
+
+Why this rule exists: PR #801 led the guidance with a tool-precedence bullet
+("AgentConnect's MCP tools are the ONLY channel that reaches other agents
+and humans"), validated only against the parent-session scenario (10/10),
+and then caused a live in-thread regression — an agent in a channel counting
+game started routing every turn through `sendMessage` to "hand off" numbers
+to its peer instead of replying in the thread, posting meta-narration with
+skipped and duplicated numbers. #801 was reverted by #861; issue #800
+records the incident. The two failure modes pull the guidance in opposite
+directions (reach-peers-via-tool vs in-thread-speech-is-the-ordinary-reply),
+so a candidate that scores well on one and is unmeasured on the other is
+unvalidated. Scenario design, judge, and baseline:
+`messaging-primitives-ab.md` §8; runner:
+`evals/test/tool-surface-ab-real.test.ts` with
+`AGENTCONNECT_EVAL_AB_SCENARIOS=parent-session,in-thread-count`.
+
 ## 5. Real-model runs
 
 ### 5.1 Sequential Werewolf, real local Claude Code

evals/games/mcp-client.ts

@@ -37,6 +37,60 @@ export interface DaemonToolCallResult {
   error?: string
 }
 
+/** One `listTools` round-trip: the descriptor names THIS session actually
+ *  carries — how a test proves a surface was presented (or withheld). */
+export async function listDaemonTools(binding: DaemonMcpBinding, timeoutMs = 30_000): Promise<string[]> {
+  const response = await ipcRequest(binding, { op: 'listTools' }, timeoutMs)
+  if (!response.ok) throw new Error(response.error ?? 'listTools failed')
+  const tools = (response.result as { tools?: { name?: unknown }[] } | undefined)?.tools ?? []
+  return tools.map((tool) => String(tool.name))
+}
+
+function ipcRequest(
+  binding: DaemonMcpBinding,
+  request: Record<string, unknown>,
+  timeoutMs: number
+): Promise<DaemonToolCallResult> {
+  return new Promise((resolve, reject) => {
+    const socket = net.connect(binding.endpoint)
+    let buffer = ''
+    let settled = false
+    const timer = setTimeout(() => {
+      finish(() => reject(new Error(`daemon ipc request timed out after ${timeoutMs}ms`)))
+    }, timeoutMs)
+    const finish = (settle: () => void): void => {
+      if (settled) return
+      settled = true
+      clearTimeout(timer)
+      socket.destroy()
+      settle()
+    }
+    socket.setEncoding('utf8')
+    socket.on('connect', () => {
+      socket.write(`${JSON.stringify({ id: 1, token: binding.token, ...request })}\n`)
+    })
+    socket.on('data', (chunk: string) => {
+      buffer += chunk
+      const newline = buffer.indexOf('\n')
+      if (newline === -1) return
+      const line = buffer.slice(0, newline)
+      try {
+        const response = JSON.parse(line) as { ok?: boolean; result?: unknown; error?: string }
+        finish(() =>
+          resolve({
+            ok: response.ok === true,
+            ...(response.result !== undefined ? { result: response.result } : {}),
+            ...(typeof response.error === 'string' ? { error: response.error } : {})
+          })
+        )
+      } catch (error) {
+        finish(() => reject(error instanceof Error ? error : new Error(String(error))))
+      }
+    })
+    socket.on('error', (error) => finish(() => reject(error)))
+  })
+}
+
 /** One `callTool` round-trip over the daemon's MCP control socket. */
 export function callDaemonTool(
   binding: DaemonMcpBinding,

evals/games/post-facade.ts

@@ -0,0 +1,286 @@
+/**
+ * Arm B of the tool-surface A/B: a `post` façade over the landed `sendMessage`.
+ *
+ * This implements the write primitive of `docs/designs/messaging-primitives.md`
+ * §2.2 as an EVALUATION-ONLY tool. It is a façade and nothing more: every call
+ * compiles into exactly one legal `sendMessage` input and is executed by the
+ * product tool itself (`callProductTool`). No routing, activation, addressing or
+ * policy code is touched — the two arms differ ONLY in the schema and
+ * description the model carries, which is the whole point of the experiment.
+ *
+ * The design claim under test is that the target union is really three
+ * orthogonal dimensions:
+ *
+ *   conversation   which exchange this post belongs to
+ *   address        who it is addressed to (structured, never parsed from prose)
+ *   visibility     whether it has a platform projection
+ *
+ * so the "exactly one target mode, and here are the illegal combinations"
+ * rule table becomes unnecessary rather than merely shorter. Every legal
+ * `sendMessage` form below has a composition; if a form could not be expressed
+ * by compiling to what exists, that is reported as a finding, not patched by
+ * changing the product.
+ *
+ * NOT expressible by EITHER arm, and deliberately out of the experiment: a
+ * fully-addressed cross-room handoff into an existing THREAD. The routing
+ * rework removed `thread` from every `sendMessage` target (baseline §6.4), so
+ * the façade has nothing to compile it to. Including it would measure a known
+ * product gap rather than the two surfaces.
+ */
+import type { EvaluationToolDefinition } from '../../packages/daemon/src/evaluation/index.js'
+
+/** One compiled call: the `sendMessage` input a `post` reduces to. */
+export interface CompiledPost {
+  args: Record<string, unknown>
+  /** Which of the six legal `sendMessage` forms this became. */
+  form: 'agent-channel' | 'agent-postless' | 'user-dm' | 'user-channel' | 'channel-bare' | 'parent-session'
+}
+
+export class PostCompileError extends Error {}
+
+interface PostInput {
+  conversation?: unknown
+  message?: unknown
+  address?: unknown
+  visibility?: unknown
+  expectReply?: unknown
+}
+
+function str(value: unknown, field: string): string {
+  if (typeof value !== 'string' || value.trim() === '') {
+    throw new PostCompileError(`post: "${field}" must be a non-empty string`)
+  }
+  return value
+}
+
+/**
+ * Compile one `post` into the single `sendMessage` input that expresses it.
+ *
+ * Pure and total: it either yields a legal product call or throws a message that
+ * names what was wrong. It never guesses — an under-specified post is an error,
+ * because silently picking a target is exactly the failure mode a surface A/B
+ * is supposed to detect.
+ */
+export function compilePost(input: PostInput): CompiledPost {
+  const message = str(input.message, 'message')
+  const conversation = input.conversation
+  if (conversation === null || typeof conversation !== 'object') {
+    throw new PostCompileError('post: "conversation" must be an object naming where the post belongs')
+  }
+  const kind = str((conversation as { kind?: unknown }).kind, 'conversation.kind')
+  const visibility = input.visibility === undefined ? 'visible' : str(input.visibility, 'visibility')
+  if (visibility !== 'visible' && visibility !== 'session-only') {
+    throw new PostCompileError('post: "visibility" must be "visible" or "session-only"')
+  }
+  const address = input.address === undefined ? [] : input.address
+  if (!Array.isArray(address) || address.some((entry) => typeof entry !== 'string' || entry.trim() === '')) {
+    throw new PostCompileError('post: "address" must be an array of ids')
+  }
+  const addresses = address as string[]
+
+  switch (kind) {
+    case 'channel': {
+      const channel = str((conversation as { channel?: unknown }).channel, 'conversation.channel')
+      if (visibility === 'session-only') {
+        throw new PostCompileError(
+          'post: a conversation in a channel is always visible; use conversation.kind "private" for a ' +
+            'session-only address'
+        )
+      }
+      if (addresses.length === 0) return { args: { channel, message }, form: 'channel-bare' }
+      const agents = addresses.filter((id) => isAgentId(id))
+      if (agents.length > 0) {
+        if (addresses.length > 1) {
+          throw new PostCompileError('post: a channel post can address at most one agent')
+        }
+        return {
+          args: { toAgent: agentTarget(agents[0]!, input.expectReply), channel, message },
+          form: 'agent-channel'
+        }
+      }
+      return {
+        args: { toUser: addresses.length === 1 ? addresses[0]! : addresses, channel, message },
+        form: 'user-channel'
+      }
+    }
+    case 'private': {
+      if (addresses.length !== 1) {
+        throw new PostCompileError('post: a private conversation addresses exactly one agent')
+      }
+      if (visibility !== 'session-only') {
+        throw new PostCompileError(
+          'post: a private conversation has no platform projection; set visibility "session-only"'
+        )
+      }
+      return { args: { toAgent: agentTarget(addresses[0]!, input.expectReply), message }, form: 'agent-postless' }
+    }
+    case 'dm': {
+      const user = str((conversation as { user?: unknown }).user, 'conversation.user')
+      return { args: { toUser: user, message }, form: 'user-dm' }
+    }
+    case 'parent': {
+      const sessionId = str((conversation as { sessionId?: unknown }).sessionId, 'conversation.sessionId')
+      return { args: { sessionId, message }, form: 'parent-session' }
+    }
+    default:
+      throw new PostCompileError(
+        `post: unknown conversation.kind "${kind}" (expected "channel", "private", "dm" or "parent")`
+      )
+  }
+}
+
+/** Agent ids in the arena (and in production) are UUIDs; platform member ids are
+ *  not. The façade needs the distinction only to pick which product field a
+ *  channel address compiles into — the product still authorizes it. */
+function isAgentId(id: string): boolean {
+  return /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(id)
+}
+
+function agentTarget(agentId: string, expectReply: unknown): unknown {
+  return expectReply === true ? { agentId, needsReply: true } : agentId
+}
+
+/** The `post` descriptor — arm B's entire surface. */
+export const POST_TOOL_DESCRIPTOR = {
+  name: 'post',
+  description:
+    'Send one message. Three independent choices: WHICH conversation, WHO it addresses, and whether it is ' +
+    'visible on the platform.\n' +
+    'To speak in the conversation you are already in, do NOT use this tool — write your ordinary turn reply.\n' +
+    '- `conversation` — where the post belongs:\n' +
+    '  • `{"kind":"channel","channel":"<channel id>"}` — a new conversation at that channel’s root.\n' +
+    '  • `{"kind":"private"}` — a new private conversation with the agent you address (nothing is posted).\n' +
+    '  • `{"kind":"dm","user":"<platform user id>"}` — your direct message with that human.\n' +
+    '  • `{"kind":"parent","sessionId":"<Parent session>"}` — the conversation that woke you.\n' +
+    '- `address` — ids this post is addressed to: agent ids (from `listAgents`) or human platform ids. ' +
+    'Omit it to address nobody.\n' +
+    '- `visibility` — `"visible"` (default) or `"session-only"` for a post with no platform projection.\n' +
+    'Set `expectReply: true` when you address an agent and need its answer back.\n' +
+    'Write `message` as CommonMark/GFM. The daemon supplies your identity; you cannot impersonate anyone.',
+  inputSchema: {
+    type: 'object' as const,
+    properties: {
+      conversation: {
+        type: 'object' as const,
+        description: 'Which conversation this post belongs to.',
+        properties: {
+          kind: { type: 'string', enum: ['channel', 'private', 'dm', 'parent'] },
+          channel: { type: 'string', description: 'Channel id, for kind "channel".' },
+          user: { type: 'string', description: 'Human platform id, for kind "dm".' },
+          sessionId: { type: 'string', description: 'Parent session id, for kind "parent".' }
+        },
+        required: ['kind']
+      },
+      address: {
+        type: 'array' as const,
+        items: { type: 'string' },
+        description: 'Ids this post addresses: agent ids or human platform ids. Omit to address nobody.'
+      },
+      visibility: {
+        type: 'string' as const,
+        enum: ['visible', 'session-only'],
+        description: 'Whether the post has a platform projection. Defaults to "visible".'
+      },
+      expectReply: {
+        type: 'boolean' as const,
+        description: 'Set true when you address an agent and need its answer back.'
+      },
+      message: { type: 'string' as const, description: 'The message body, as CommonMark/GFM.' }
+    },
+    required: ['conversation', 'message'],
+    additionalProperties: false as const
+  }
+}
+
+/**
+ * Arm B's standing collaboration guidance — the prompt-side half of the surface.
+ *
+ * The production system prompt teaches `sendMessage` call shapes by name
+ * (session-manager.ts `collabAppend` / `parentReplyAppend`), so an arm that
+ * withholds `sendMessage` needs guidance that teaches ITS surface instead, or
+ * the prompt would prime the model with the other arm's vocabulary and tell it
+ * to call a tool it does not carry. Structure and every non-surface sentence
+ * (ordinary-reply rule, "act only on what is asked", quiet-about-mechanics,
+ * peer-roster memory) mirror the production text — only the tool teaching
+ * differs, which is the point.
+ */
+export const POST_COLLAB_GUIDANCE =
+  `# Collaborating with other agents\n` +
+  // Parity note: the #800 tool-precedence bullet briefly led this text (worded
+  // for this arm's surface, mirroring production's #801) and was removed when
+  // production reverted it (#861, after the live in-thread regression the
+  // `in-thread-count` scenario now gates). Both arms are back to the
+  // pre-#801 guidance, so prompt parity still holds.
+  `- One tool, \`post\`, sends any message that leaves your current conversation. Choose three things ` +
+  `independently: WHICH conversation it belongs to, WHO it addresses, and whether it is visible on the platform.\n` +
+  `- To reach a specific agent privately: ` +
+  `\`post\` \`{"conversation":{"kind":"private"},"address":["<agent id>"],"visibility":"session-only",` +
+  `"message":"..."}\` — it wakes ONLY that agent and nothing appears in any channel. That call is ` +
+  `FIRE-AND-FORGET: the peer answers inside its own conversation and nothing comes back to you, not even a ` +
+  `failure. Whenever you expect an answer — your message asks a question or requests a result, or you were ` +
+  `asked to relay that agent's answer to someone — add \`"expectReply":true\`, which obliges it to report ` +
+  `into YOUR session when it finishes or fails.\n` +
+  `- To open a VISIBLE discussion at a channel's root: \`"conversation":{"kind":"channel","channel":` +
+  `"<channel id>"}\`. Put an agent id in \`address\` to pull that agent into the new discussion (you may ` +
+  `address yourself there to open one for yourself — use your ID from the # Agent block, never your platform ` +
+  `bot identity), or human platform ids to @-mention people. Omit \`address\` to leave a note that wakes ` +
+  `nobody.\n` +
+  `- To speak in the conversation you are already in — including to address a peer or human there — do NOT ` +
+  `call \`post\`: write your ordinary turn reply and @-mention them in it (use \`listAgents\` to get a peer's ` +
+  `exact \`mention\` token). To reach a HUMAN in their direct messages, use ` +
+  `\`"conversation":{"kind":"dm","user":"<platform user id>"}\` — never address an AgentConnect agent or your ` +
+  `own bot identity as a human user. If you were woken by another session, reply with ` +
+  `\`"conversation":{"kind":"parent","sessionId":"<Parent session>"}\`.\n` +
+  `- Act only on what is asked of YOU. Do not relay a message onward or start your own broadcast to other ` +
+  `agents unless a human explicitly tells you to.\n` +
+  `- Be quiet about mechanics: don't narrate each step or post a message per action, and don't restate tool ` +
+  `results like "delivered: true". Take the action, add at most one short status line if needed, then end your turn.\n` +
+  `- When another agent introduces itself to you, record it in your memory (a peer roster — id, name, what it ` +
+  `does, how to reach it) so you know who to delegate to later. Then just acknowledge briefly; do NOT re-introduce ` +
+  `yourself back or broadcast to everyone.`
+
+/** Arm B's parent-report append — mirrors the production text with only the
+ *  tool teaching swapped. */
+export function postParentReplyAppend(parentSessionId: string): string {
+  return (
+    `# Reporting back to your parent session\n` +
+    `Another session delegated this work to you and is waiting on the outcome. When you finish — or when you ` +
+    `cannot finish — reply to it with ` +
+    `\`post\` \`{"conversation":{"kind":"parent","sessionId":"${parentSessionId}"},"message":"..."}\`, saying ` +
+    `whether you succeeded or failed and what the result was (on failure, what went wrong). Send it exactly ` +
+    `once, at the end; do not report progress along the way, and do not skip it because the task was small or ` +
+    `unsuccessful. Your ordinary assistant response in this child session is not delivered to the parent. Do ` +
+    `not write the result before or after the tool call; after the tool reports successful delivery, end your ` +
+    `turn immediately without repeating the message.`
+  )
+}
+
+/** Build arm B's registry entry. Every call compiles and is then executed by the
+ *  PRODUCT tool, so the two arms share one implementation. */
+export function postFacadeTool(options: {
+  visibleTo?: (agentId: string) => boolean
+  onCall?: (record: {
+    agentId: string
+    input: Record<string, unknown>
+    outcome: 'compiled' | 'invalid'
+    form?: string
+    error?: string
+  }) => void
+}): EvaluationToolDefinition {
+  return {
+    descriptor: POST_TOOL_DESCRIPTOR,
+    visibleTo: options.visibleTo ?? (() => true),
+    handler: async ({ agentId, input, callProductTool }) => {
+      let compiled: CompiledPost
+      try {
+        compiled = compilePost(input as PostInput)
+      } catch (error) {
+        options.onCall?.({ agentId, input, outcome: 'invalid', error: (error as Error).message })
+        // Surfaced to the model exactly as the product surfaces its own refusals.
+        throw error
+      }
+      options.onCall?.({ agentId, input, outcome: 'compiled', form: compiled.form })
+      return callProductTool('sendMessage', compiled.args)
+    }
+  }
+}