From 8783bbb426b80b41cfe8aa561636a045154f9f70 Mon Sep 17 00:00:00 2001
From: Poytr1 <pooytr1@gmail.com>
Date: Wed, 5 Aug 2026 21:51:51 +0800
Subject: [PATCH 01/15] =?UTF-8?q?feat(evals):=20add=20the=20`post`=20fa?=
 =?UTF-8?q?=C3=A7ade=20and=20measure=20the=20static=20cost=20of=20both=20t?=
 =?UTF-8?q?ool=20surfaces?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Groundwork for the A/B comparing the landed `sendMessage` surface against the
messaging-primitives `post` primitive. This commit lands the façade, its
contract tests, and the credential-free static measurement; the behavioral
scenarios follow.

The façade is a façade and nothing else. Every `post` compiles into exactly ONE
legal `sendMessage` input and is then executed BY the product tool, so the two
arms share one implementation and differ only in the schema and description the
model carries. No routing, activation, addressing or policy code is touched.

Two small evaluation-only seams make that possible:

- `McpControlServer.executeProductTool` lets an evaluation-registry tool run a
  product tool on the same trusted token-bound SessionContext. It grants no
  capability the caller did not already have, and re-entry is safe because the
  registry is consulted by exact name and a product tool never matches it.
- `hideProductTools` withholds named product descriptors for a run, so an A/B
  arm presents exactly one surface for a capability. A withheld tool stays fully
  executable — which is precisely how the façade compiles down to it.

All six legal forms compile: agent+channel, bare channel, postless agent call,
DM, channel-root user mentions, and the parent-session reply. The façade refuses
rather than guesses when a post is under-specified, and refuses the combinations
the product has no form for instead of inventing one.

Deliberately NOT expressible by either arm, and excluded from the experiment: a
fully-addressed cross-room handoff into an existing THREAD. The routing rework
removed `thread` from every target (baseline §6.4), so including it would
measure a known product gap rather than the two surfaces.

STATIC COST, measured from the real descriptors rather than from prose:

  sendMessage   2602 description + 7353 schema = 9955 chars (~2489 tokens)
  post          1046 description + 1025 schema = 2071 chars (~518 tokens)

a 4.8x smaller surface the model must carry on every turn. Token figures are a
tokenizer-free approximation and are labelled as such.

8 contract tests, credential-free.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
---
 evals/games/post-facade.ts                    | 223 ++++++++++++++++++
 evals/test/post-facade.test.ts                | 178 ++++++++++++++
 packages/daemon/src/daemon.ts                 |  13 +-
 packages/daemon/src/evaluation/environment.ts |  13 +
 packages/daemon/src/mcp/control-server.ts     |  17 ++
 5 files changed, 443 insertions(+), 1 deletion(-)
 create mode 100644 evals/games/post-facade.ts
 create mode 100644 evals/test/post-facade.test.ts

diff --git a/evals/games/post-facade.ts b/evals/games/post-facade.ts
new file mode 100644
index 000000000..cb2eb19be
--- /dev/null
+++ b/evals/games/post-facade.ts
@@ -0,0 +1,223 @@
+/**
+ * Arm B of the tool-surface A/B: a `post` façade over the landed `sendMessage`.
+ *
+ * This implements the write primitive of `docs/designs/messaging-primitives.md`
+ * §2.2 as an EVALUATION-ONLY tool. It is a façade and nothing more: every call
+ * compiles into exactly one legal `sendMessage` input and is executed by the
+ * product tool itself (`callProductTool`). No routing, activation, addressing or
+ * policy code is touched — the two arms differ ONLY in the schema and
+ * description the model carries, which is the whole point of the experiment.
+ *
+ * The design claim under test is that the target union is really three
+ * orthogonal dimensions:
+ *
+ *   conversation   which exchange this post belongs to
+ *   address        who it is addressed to (structured, never parsed from prose)
+ *   visibility     whether it has a platform projection
+ *
+ * so the "exactly one target mode, and here are the illegal combinations"
+ * rule table becomes unnecessary rather than merely shorter. Every legal
+ * `sendMessage` form below has a composition; if a form could not be expressed
+ * by compiling to what exists, that is reported as a finding, not patched by
+ * changing the product.
+ *
+ * NOT expressible by EITHER arm, and deliberately out of the experiment: a
+ * fully-addressed cross-room handoff into an existing THREAD. The routing
+ * rework removed `thread` from every `sendMessage` target (baseline §6.4), so
+ * the façade has nothing to compile it to. Including it would measure a known
+ * product gap rather than the two surfaces.
+ */
+import type { EvaluationToolDefinition } from '../../packages/daemon/src/evaluation/index.js'
+
+/** One compiled call: the `sendMessage` input a `post` reduces to. */
+export interface CompiledPost {
+  args: Record<string, unknown>
+  /** Which of the six legal `sendMessage` forms this became. */
+  form: 'agent-channel' | 'agent-postless' | 'user-dm' | 'user-channel' | 'channel-bare' | 'parent-session'
+}
+
+export class PostCompileError extends Error {}
+
+interface PostInput {
+  conversation?: unknown
+  message?: unknown
+  address?: unknown
+  visibility?: unknown
+  expectReply?: unknown
+}
+
+function str(value: unknown, field: string): string {
+  if (typeof value !== 'string' || value.trim() === '') {
+    throw new PostCompileError(`post: "${field}" must be a non-empty string`)
+  }
+  return value
+}
+
+/**
+ * Compile one `post` into the single `sendMessage` input that expresses it.
+ *
+ * Pure and total: it either yields a legal product call or throws a message that
+ * names what was wrong. It never guesses — an under-specified post is an error,
+ * because silently picking a target is exactly the failure mode a surface A/B
+ * is supposed to detect.
+ */
+export function compilePost(input: PostInput): CompiledPost {
+  const message = str(input.message, 'message')
+  const conversation = input.conversation
+  if (conversation === null || typeof conversation !== 'object') {
+    throw new PostCompileError('post: "conversation" must be an object naming where the post belongs')
+  }
+  const kind = str((conversation as { kind?: unknown }).kind, 'conversation.kind')
+  const visibility = input.visibility === undefined ? 'visible' : str(input.visibility, 'visibility')
+  if (visibility !== 'visible' && visibility !== 'session-only') {
+    throw new PostCompileError('post: "visibility" must be "visible" or "session-only"')
+  }
+  const address = input.address === undefined ? [] : input.address
+  if (!Array.isArray(address) || address.some((entry) => typeof entry !== 'string' || entry.trim() === '')) {
+    throw new PostCompileError('post: "address" must be an array of ids')
+  }
+  const addresses = address as string[]
+
+  switch (kind) {
+    case 'channel': {
+      const channel = str((conversation as { channel?: unknown }).channel, 'conversation.channel')
+      if (visibility === 'session-only') {
+        throw new PostCompileError(
+          'post: a conversation in a channel is always visible; use conversation.kind "private" for a ' +
+            'session-only address'
+        )
+      }
+      if (addresses.length === 0) return { args: { channel, message }, form: 'channel-bare' }
+      const agents = addresses.filter((id) => isAgentId(id))
+      if (agents.length > 0) {
+        if (addresses.length > 1) {
+          throw new PostCompileError('post: a channel post can address at most one agent')
+        }
+        return {
+          args: { toAgent: agentTarget(agents[0]!, input.expectReply), channel, message },
+          form: 'agent-channel'
+        }
+      }
+      return {
+        args: { toUser: addresses.length === 1 ? addresses[0]! : addresses, channel, message },
+        form: 'user-channel'
+      }
+    }
+    case 'private': {
+      if (addresses.length !== 1) {
+        throw new PostCompileError('post: a private conversation addresses exactly one agent')
+      }
+      if (visibility !== 'session-only') {
+        throw new PostCompileError(
+          'post: a private conversation has no platform projection; set visibility "session-only"'
+        )
+      }
+      return { args: { toAgent: agentTarget(addresses[0]!, input.expectReply), message }, form: 'agent-postless' }
+    }
+    case 'dm': {
+      const user = str((conversation as { user?: unknown }).user, 'conversation.user')
+      return { args: { toUser: user, message }, form: 'user-dm' }
+    }
+    case 'parent': {
+      const sessionId = str((conversation as { sessionId?: unknown }).sessionId, 'conversation.sessionId')
+      return { args: { sessionId, message }, form: 'parent-session' }
+    }
+    default:
+      throw new PostCompileError(
+        `post: unknown conversation.kind "${kind}" (expected "channel", "private", "dm" or "parent")`
+      )
+  }
+}
+
+/** Agent ids in the arena (and in production) are UUIDs; platform member ids are
+ *  not. The façade needs the distinction only to pick which product field a
+ *  channel address compiles into — the product still authorizes it. */
+function isAgentId(id: string): boolean {
+  return /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(id)
+}
+
+function agentTarget(agentId: string, expectReply: unknown): unknown {
+  return expectReply === true ? { agentId, needsReply: true } : agentId
+}
+
+/** The `post` descriptor — arm B's entire surface. */
+export const POST_TOOL_DESCRIPTOR = {
+  name: 'post',
+  description:
+    'Send one message. Three independent choices: WHICH conversation, WHO it addresses, and whether it is ' +
+    'visible on the platform.\n' +
+    'To speak in the conversation you are already in, do NOT use this tool — write your ordinary turn reply.\n' +
+    '- `conversation` — where the post belongs:\n' +
+    '  • `{"kind":"channel","channel":"<channel id>"}` — a new conversation at that channel’s root.\n' +
+    '  • `{"kind":"private"}` — a new private conversation with the agent you address (nothing is posted).\n' +
+    '  • `{"kind":"dm","user":"<platform user id>"}` — your direct message with that human.\n' +
+    '  • `{"kind":"parent","sessionId":"<Parent session>"}` — the conversation that woke you.\n' +
+    '- `address` — ids this post is addressed to: agent ids (from `listAgents`) or human platform ids. ' +
+    'Omit it to address nobody.\n' +
+    '- `visibility` — `"visible"` (default) or `"session-only"` for a post with no platform projection.\n' +
+    'Set `expectReply: true` when you address an agent and need its answer back.\n' +
+    'Write `message` as CommonMark/GFM. The daemon supplies your identity; you cannot impersonate anyone.',
+  inputSchema: {
+    type: 'object' as const,
+    properties: {
+      conversation: {
+        type: 'object' as const,
+        description: 'Which conversation this post belongs to.',
+        properties: {
+          kind: { type: 'string', enum: ['channel', 'private', 'dm', 'parent'] },
+          channel: { type: 'string', description: 'Channel id, for kind "channel".' },
+          user: { type: 'string', description: 'Human platform id, for kind "dm".' },
+          sessionId: { type: 'string', description: 'Parent session id, for kind "parent".' }
+        },
+        required: ['kind']
+      },
+      address: {
+        type: 'array' as const,
+        items: { type: 'string' },
+        description: 'Ids this post addresses: agent ids or human platform ids. Omit to address nobody.'
+      },
+      visibility: {
+        type: 'string' as const,
+        enum: ['visible', 'session-only'],
+        description: 'Whether the post has a platform projection. Defaults to "visible".'
+      },
+      expectReply: {
+        type: 'boolean' as const,
+        description: 'Set true when you address an agent and need its answer back.'
+      },
+      message: { type: 'string' as const, description: 'The message body, as CommonMark/GFM.' }
+    },
+    required: ['conversation', 'message'],
+    additionalProperties: false as const
+  }
+}
+
+/** Build arm B's registry entry. Every call compiles and is then executed by the
+ *  PRODUCT tool, so the two arms share one implementation. */
+export function postFacadeTool(options: {
+  visibleTo?: (agentId: string) => boolean
+  onCall?: (record: {
+    agentId: string
+    input: Record<string, unknown>
+    outcome: 'compiled' | 'invalid'
+    form?: string
+    error?: string
+  }) => void
+}): EvaluationToolDefinition {
+  return {
+    descriptor: POST_TOOL_DESCRIPTOR,
+    visibleTo: options.visibleTo ?? (() => true),
+    handler: async ({ agentId, input, callProductTool }) => {
+      let compiled: CompiledPost
+      try {
+        compiled = compilePost(input as PostInput)
+      } catch (error) {
+        options.onCall?.({ agentId, input, outcome: 'invalid', error: (error as Error).message })
+        // Surfaced to the model exactly as the product surfaces its own refusals.
+        throw error
+      }
+      options.onCall?.({ agentId, input, outcome: 'compiled', form: compiled.form })
+      return callProductTool('sendMessage', compiled.args)
+    }
+  }
+}
diff --git a/evals/test/post-facade.test.ts b/evals/test/post-facade.test.ts
new file mode 100644
index 000000000..ee16369d1
--- /dev/null
+++ b/evals/test/post-facade.test.ts
@@ -0,0 +1,178 @@
+import { describe, expect, it } from 'vitest'
+import { toolsForIntegrations } from '../../packages/daemon/src/mcp/tools.js'
+import { IntegrationSchema } from '../../packages/daemon/src/agents/agent-schema.js'
+import { POST_TOOL_DESCRIPTOR, PostCompileError, compilePost, postFacadeTool } from '../games/post-facade.js'
+
+/**
+ * Arm B's contract, credential-free: the façade must compile every legal
+ * `sendMessage` form and refuse the rest, and the STATIC cost of each surface
+ * must be measurable without a model. Both are prerequisites for the A/B being
+ * a fair comparison rather than two different implementations.
+ */
+
+const AGENT = '4722901a-eec4-466d-8128-174123af6af0'
+const OTHER_AGENT = '1a6646ba-b35b-44fe-9ea5-0c8baeba7f7a'
+
+describe('post façade — compiles to exactly one legal sendMessage form', () => {
+  it('covers all six legal forms of the landed surface', () => {
+    // Case 1: wake one agent AND post a visible root in a channel.
+    expect(compilePost({ conversation: { kind: 'channel', channel: 'C1' }, address: [AGENT], message: 'hi' })).toEqual({
+      form: 'agent-channel',
+      args: { toAgent: AGENT, channel: 'C1', message: 'hi' }
+    })
+    // Case 2: bare visible root, nobody woken, nobody mentioned.
+    expect(compilePost({ conversation: { kind: 'channel', channel: 'C1' }, message: 'hi' })).toEqual({
+      form: 'channel-bare',
+      args: { channel: 'C1', message: 'hi' }
+    })
+    // Postless peer wake: a private conversation has no platform projection.
+    expect(
+      compilePost({ conversation: { kind: 'private' }, address: [AGENT], visibility: 'session-only', message: 'hi' })
+    ).toEqual({ form: 'agent-postless', args: { toAgent: AGENT, message: 'hi' } })
+    // DM to a human.
+    expect(compilePost({ conversation: { kind: 'dm', user: 'U1' }, message: 'hi' })).toEqual({
+      form: 'user-dm',
+      args: { toUser: 'U1', message: 'hi' }
+    })
+    // Channel root addressing humans — one id and many.
+    expect(compilePost({ conversation: { kind: 'channel', channel: 'C1' }, address: ['U1'], message: 'hi' })).toEqual({
+      form: 'user-channel',
+      args: { toUser: 'U1', channel: 'C1', message: 'hi' }
+    })
+    expect(
+      compilePost({ conversation: { kind: 'channel', channel: 'C1' }, address: ['U1', 'U2'], message: 'hi' })
+    ).toEqual({ form: 'user-channel', args: { toUser: ['U1', 'U2'], channel: 'C1', message: 'hi' } })
+    // Session-only reply into the parent conversation.
+    expect(compilePost({ conversation: { kind: 'parent', sessionId: 'S1' }, message: 'hi' })).toEqual({
+      form: 'parent-session',
+      args: { sessionId: 'S1', message: 'hi' }
+    })
+  })
+
+  it('carries needsReply through as an orthogonal flag', () => {
+    expect(
+      compilePost({
+        conversation: { kind: 'private' },
+        address: [AGENT],
+        visibility: 'session-only',
+        expectReply: true,
+        message: 'q'
+      }).args
+    ).toEqual({ toAgent: { agentId: AGENT, needsReply: true }, message: 'q' })
+  })
+
+  it('refuses an under-specified post instead of guessing a target', () => {
+    // Guessing is the exact failure a surface comparison must not hide.
+    expect(() => compilePost({ message: 'hi' })).toThrow(PostCompileError)
+    expect(() => compilePost({ conversation: { kind: 'channel' }, message: 'hi' })).toThrow(/conversation\.channel/)
+    expect(() => compilePost({ conversation: { kind: 'dm' }, message: 'hi' })).toThrow(/conversation\.user/)
+    expect(() => compilePost({ conversation: { kind: 'parent' }, message: 'hi' })).toThrow(/conversation\.sessionId/)
+    expect(() => compilePost({ conversation: { kind: 'nope' }, message: 'hi' })).toThrow(/unknown conversation\.kind/)
+    expect(() => compilePost({ conversation: { kind: 'channel', channel: 'C1' } })).toThrow(/"message"/)
+  })
+
+  it('refuses combinations the product has no form for, rather than inventing one', () => {
+    // A channel post is always visible; session-only lives on `private`.
+    expect(() =>
+      compilePost({ conversation: { kind: 'channel', channel: 'C1' }, visibility: 'session-only', message: 'hi' })
+    ).toThrow(/always visible/)
+    // A private conversation is exactly one agent, and has no visible form.
+    expect(() => compilePost({ conversation: { kind: 'private' }, message: 'hi' })).toThrow(/exactly one agent/)
+    expect(() =>
+      compilePost({ conversation: { kind: 'private' }, address: [AGENT], visibility: 'visible', message: 'hi' })
+    ).toThrow(/session-only/)
+    // The product's channel form wakes at most one agent.
+    expect(() =>
+      compilePost({ conversation: { kind: 'channel', channel: 'C1' }, address: [AGENT, OTHER_AGENT], message: 'hi' })
+    ).toThrow(/at most one agent/)
+  })
+
+  it('executes through the PRODUCT tool, never its own implementation', async () => {
+    const calls: { name: string; args: Record<string, unknown> }[] = []
+    const records: { outcome: string; form?: string }[] = []
+    const tool = postFacadeTool({ onCall: (record) => records.push({ outcome: record.outcome, form: record.form }) })
+    await tool.handler({
+      runId: 'r',
+      agentId: AGENT,
+      sessionContext: {} as never,
+      input: { conversation: { kind: 'channel', channel: 'C1' }, message: 'hi' },
+      callProductTool: async (name, args) => {
+        calls.push({ name, args })
+        return { ok: true }
+      }
+    })
+    // The façade adds a schema, not a second implementation.
+    expect(calls).toEqual([{ name: 'sendMessage', args: { channel: 'C1', message: 'hi' } }])
+    expect(records).toEqual([{ outcome: 'compiled', form: 'channel-bare' }])
+  })
+
+  it('reports an invalid call without reaching the product tool', async () => {
+    const calls: string[] = []
+    const records: { outcome: string; error?: string }[] = []
+    const tool = postFacadeTool({ onCall: (record) => records.push({ outcome: record.outcome, error: record.error }) })
+    await expect(
+      tool.handler({
+        runId: 'r',
+        agentId: AGENT,
+        sessionContext: {} as never,
+        input: { message: 'hi' },
+        callProductTool: async (name) => {
+          calls.push(name)
+          return {}
+        }
+      })
+    ).rejects.toThrow(PostCompileError)
+    expect(calls).toEqual([])
+    expect(records[0]!.outcome).toBe('invalid')
+  })
+})
+
+describe('static cost of each tool surface', () => {
+  /** Characters of schema + description the model must carry for one tool. */
+  function staticCost(descriptor: { name: string; description?: string; inputSchema?: unknown }) {
+    const description = descriptor.description ?? ''
+    const schema = JSON.stringify(descriptor.inputSchema ?? {})
+    return {
+      descriptionChars: description.length,
+      schemaChars: schema.length,
+      totalChars: description.length + schema.length,
+      // A rough, tokenizer-free estimate. Reported as approximate on purpose.
+      approxTokens: Math.round((description.length + schema.length) / 4)
+    }
+  }
+
+  function sendMessageDescriptor() {
+    const integration = IntegrationSchema.parse({
+      id: 'i1',
+      platform: 'slack',
+      core: { mode: 'direct', bindRules: [] },
+      config: { botToken: 'xoxb-x', appToken: 'xapp-x' }
+    })
+    const tool = toolsForIntegrations([integration], { collaboration: true }).find((t) => t.name === 'sendMessage')
+    if (!tool) throw new Error('sendMessage descriptor not found')
+    return tool
+  }
+
+  it('measures both surfaces from the real descriptors, not from prose', () => {
+    const a = staticCost(sendMessageDescriptor())
+    const b = staticCost(POST_TOOL_DESCRIPTOR)
+    // Pin the shape of the measurement, not the exact numbers: the assertion is
+    // that both are measurable and that the façade is not accidentally larger.
+    expect(a.totalChars).toBeGreaterThan(1000)
+    expect(b.totalChars).toBeGreaterThan(500)
+    expect(b.totalChars).toBeLessThan(a.totalChars)
+    // Report them so a run of this test records the numbers.
+    console.log(`static cost — sendMessage: ${JSON.stringify(a)}  post: ${JSON.stringify(b)}`)
+  })
+
+  it('the landed surface really does enumerate more forms than the façade', () => {
+    const description = sendMessageDescriptor().description ?? ''
+    // Arm A's description spells out target modes and their illegal pairings;
+    // arm B's spells out three independent dimensions. Counting the literal
+    // JSON form templates is the closest objective proxy.
+    const armAForms = (description.match(/`\{"/g) ?? []).length
+    const armBForms = (POST_TOOL_DESCRIPTOR.description.match(/`\{"/g) ?? []).length
+    expect(armAForms).toBeGreaterThanOrEqual(6)
+    expect(armBForms).toBeLessThan(armAForms)
+  })
+})
diff --git a/packages/daemon/src/daemon.ts b/packages/daemon/src/daemon.ts
index 9b32bfce6..69900cdd4 100644
--- a/packages/daemon/src/daemon.ts
+++ b/packages/daemon/src/daemon.ts
@@ -2879,7 +2879,10 @@ export class Daemon {
           runId: this.opts.evaluation?.runId ?? 'evaluation',
           agentId: ctx.agentId,
           sessionContext: ctx,
-          input: args
+          input: args,
+          // A façade tool compiles down to the real product tool on the SAME
+          // trusted context; it can reach nothing the caller could not.
+          callProductTool: (productName, productArgs) => this.mcp.executeProductTool(ctx, productName, productArgs)
         })
         return { result }
       },
@@ -3093,6 +3096,14 @@ export class Daemon {
         // Collaboration Arena §6: game-owned structured action tools, appended
         // AFTER the product tools (collision-checked at startup) and filtered
         // by per-agent visibility (e.g. only living players see `vote`).
+        // Evaluation-only surface selection: withhold named product descriptors
+        // so an A/B arm presents exactly one surface for a capability. The tool
+        // itself stays executable (a façade compiles down to it).
+        const hidden = this.opts.evaluation?.environment?.hideProductTools
+        if (hidden?.length) {
+          const withheld = new Set(hidden)
+          tools = tools.filter((tool) => !withheld.has(tool.name))
+        }
         const evaluationTools = this.opts.evaluation?.environment?.tools
         if (evaluationTools?.length) {
           tools.push(...evaluationTools.filter((definition) => definition.visibleTo(agent.id)).map((d) => d.descriptor))
diff --git a/packages/daemon/src/evaluation/environment.ts b/packages/daemon/src/evaluation/environment.ts
index 51d7ed9d6..26f6368a5 100644
--- a/packages/daemon/src/evaluation/environment.ts
+++ b/packages/daemon/src/evaluation/environment.ts
@@ -64,6 +64,12 @@ export interface EvaluationToolDefinition {
     agentId: string
     sessionContext: SessionContext
     input: Record<string, unknown>
+    /** Run a PRODUCT tool on the same trusted SessionContext. This is what lets
+     *  an evaluation FAÇADE — a different schema over an existing capability —
+     *  compile down to the real implementation instead of re-implementing it,
+     *  which is the only way an A/B of two tool surfaces compares like with
+     *  like. It grants no capability the caller did not already have. */
+    callProductTool(name: string, args: Record<string, unknown>): Promise<unknown>
   }): Promise<unknown>
 }
 
@@ -77,6 +83,13 @@ export interface DaemonEvaluationEnvironment {
   collaborationRoutes: CollabRoutesSnapshot
   /** §6 evaluation tool registry — game-owned structured action tools. */
   tools?: readonly EvaluationToolDefinition[]
+  /** Product tool names to WITHHOLD from the session tool set for this run.
+   *  Evaluation-only, and it exists for one reason: an A/B of two tool surfaces
+   *  for the same capability is only a comparison if each arm presents one of
+   *  them. Withholding a descriptor changes nothing about what the daemon will
+   *  execute — a hidden tool remains fully functional if something calls it,
+   *  which is exactly how a façade compiles down to it. */
+  hideProductTools?: readonly string[]
 }
 
 // ─── §4 ingress payloads ────────────────────────────────────────────────────
diff --git a/packages/daemon/src/mcp/control-server.ts b/packages/daemon/src/mcp/control-server.ts
index 30c936c53..e7a09522f 100644
--- a/packages/daemon/src/mcp/control-server.ts
+++ b/packages/daemon/src/mcp/control-server.ts
@@ -92,6 +92,23 @@ export class McpControlServer {
     socket.on('close', () => this.conns.delete(socket))
   }
 
+  /**
+   * Run a PRODUCT tool on behalf of an evaluation-registry tool
+   * (collaboration-arena.md §6). This exists for one purpose: an evaluation
+   * façade that presents a different SCHEMA for an existing capability has to
+   * compile down to that capability's real implementation, not re-implement it,
+   * or an A/B of the two surfaces would not be comparing like with like.
+   *
+   * It is evaluation-only plumbing and changes no routing, activation or policy:
+   * the tool it runs is the same one the model could have called directly, on
+   * the same trusted token-bound `SessionContext`. Re-entry is safe because the
+   * evaluation registry is consulted by exact name and a product tool never
+   * matches it.
+   */
+  async executeProductTool(ctx: SessionContext, name: string, args: Record<string, unknown>): Promise<unknown> {
+    return executeTool(ctx, name, args, this.deps)
+  }
+
   private async handle(req: IpcRequest, socket: net.Socket): Promise<void> {
     const reply = (res: IpcResponse) => {
       if (!socket.destroyed) socket.write(encodeFrame(res))

From b0565aaa2b693c50e838081b732c9d75761be595 Mon Sep 17 00:00:00 2001
From: Poytr1 <pooytr1@gmail.com>
Date: Thu, 6 Aug 2026 15:17:20 +0800
Subject: [PATCH 02/15] feat(evals): add the A/B scenario matrix and its
 credential-free metric extraction
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Four scenarios of the reduced matrix, a send-form classifier shared by both
arms, and trial metric extraction from the recorded ACP event stream.

The classifier is what makes the arms comparable: arm A's raw sendMessage args
and arm B's compiled args score through one vocabulary, so 'addressed it
correctly, first try' means the same thing on both sides. A wrong-but-accepted
form counts as a failure to complete, not a success — the product accepting a
call says nothing about whether it did the task.

10 contract tests. One of them already earned its keep: it caught the first
scenario's task text using the word 'conversation', which is an arm-B field
name and would have biased the comparison toward the façade.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
---
 evals/games/tool-surface-ab.ts     | 181 +++++++++++++++++++++++++++++
 evals/test/tool-surface-ab.test.ts | 158 +++++++++++++++++++++++++
 2 files changed, 339 insertions(+)
 create mode 100644 evals/games/tool-surface-ab.ts
 create mode 100644 evals/test/tool-surface-ab.test.ts

diff --git a/evals/games/tool-surface-ab.ts b/evals/games/tool-surface-ab.ts
new file mode 100644
index 000000000..4f93517db
--- /dev/null
+++ b/evals/games/tool-surface-ab.ts
@@ -0,0 +1,181 @@
+/**
+ * Tool-surface A/B: the landed `sendMessage` against the `post` façade.
+ *
+ * The two arms are identical in every respect except the tool surface the model
+ * carries — same model, same topology, same seeds, same task text. Arm A gets
+ * the product `sendMessage`; arm B gets `post` (§ post-facade.ts) with
+ * `sendMessage` withheld from the descriptor list. Because the façade compiles
+ * down to and is executed by `sendMessage`, both arms exercise one
+ * implementation, so any difference is attributable to the surface.
+ *
+ * WHAT IS MEASURED, and why this shape. Every task here is one explicit send
+ * whose correct product form is known in advance. So each tool call can be
+ * classified into the same six-form vocabulary for both arms, which makes
+ * "did the agent address this correctly, first try" comparable rather than
+ * arm-specific. The tasks describe the GOAL and never name a tool, a field or a
+ * form — naming them would test instruction-following, not the surface.
+ */
+import type { EvaluationToolDefinition } from '../../packages/daemon/src/evaluation/index.js'
+
+export type SendForm =
+  'agent-channel' | 'agent-postless' | 'user-dm' | 'user-channel' | 'channel-bare' | 'parent-session' | 'unclassifiable'
+
+/** The four scenarios of the reduced matrix, each a single explicit send. */
+export interface AbScenario {
+  id: string
+  /** The product form a correct attempt must produce. */
+  expected: SendForm
+  /** Task text, delivered as trusted referee control. Names no tool and no field. */
+  instruction(ids: { peerAgentId: string; channel: string; humanUserId: string }): string
+  /** Scenario 4 needs a real parent session, created by a scripted caller. */
+  needsCaller?: boolean
+}
+
+export const AB_SCENARIOS: AbScenario[] = [
+  {
+    id: 'agent-channel',
+    expected: 'agent-channel',
+    instruction: ({ peerAgentId, channel }) =>
+      `Open a fresh discussion in channel ${channel} that the people there can see, and pull agent ` +
+      `${peerAgentId} into that same discussion so it replies in the same place. Say: "status check please".`
+  },
+  {
+    id: 'channel-bare',
+    expected: 'channel-bare',
+    instruction: ({ channel }) =>
+      `Publish the announcement "deploy finished" so it is visible in channel ${channel}. Nobody should be woken ` +
+      `up by it and nobody should be notified — it is a notice for people to read later.`
+  },
+  {
+    id: 'agent-postless',
+    expected: 'agent-postless',
+    instruction: ({ peerAgentId }) =>
+      `Ask agent ${peerAgentId} privately for its current status, and make sure its answer comes back to you. ` +
+      `Nothing at all may become visible in any channel — this exchange must leave no trace anyone else can read.`
+  },
+  {
+    id: 'parent-session',
+    expected: 'parent-session',
+    needsCaller: true,
+    instruction: () =>
+      `Answer the question you were just asked, sending your answer back to whoever asked it so it reaches them ` +
+      `directly. Do not publish it anywhere public.`
+  }
+]
+
+/** Classify one attempted send into the shared form vocabulary. Used for BOTH
+ *  arms: arm A's raw `sendMessage` args and arm B's compiled args. */
+export function classifySendForm(args: Record<string, unknown> | undefined): SendForm {
+  if (!args) return 'unclassifiable'
+  const hasChannel = typeof args.channel === 'string' && args.channel !== ''
+  if (args.sessionId !== undefined) return 'parent-session'
+  if (args.toAgent !== undefined) return hasChannel ? 'agent-channel' : 'agent-postless'
+  if (args.toUser !== undefined) return hasChannel ? 'user-channel' : 'user-dm'
+  if (hasChannel) return 'channel-bare'
+  return 'unclassifiable'
+}
+
+/** One attempted tool call, as reconstructed from the ACP event stream. */
+export interface AbAttempt {
+  tool: string
+  args?: Record<string, unknown>
+  form: SendForm
+  failed: boolean
+  error?: string
+}
+
+export interface AbTrialMetrics {
+  attempts: AbAttempt[]
+  /** Calls the surface itself refused: schema violation or illegal combination. */
+  invalidCalls: number
+  /** Did the FIRST attempt produce the expected form and not fail? */
+  firstAttemptSuccess: boolean
+  /** Did any attempt eventually produce the expected form and not fail? */
+  completed: boolean
+  /** Attempts needed to reach the first correct, non-failing call (0 if never). */
+  attemptsToSuccess: number
+  toolCalls: number
+  totalTokens: number
+  latencyMs: number
+}
+
+interface AcpToolEvent {
+  toolCallId?: string
+  title?: string
+  status?: string
+  rawInput?: unknown
+  content?: unknown
+  _meta?: { claudeCode?: { toolName?: string } }
+  sessionUpdate?: string
+}
+
+/**
+ * Reconstruct one trial's attempts from the recorded evaluation events.
+ *
+ * ACP reports a tool call across several updates (pending → args → result), so
+ * attempts are folded by `toolCallId` and only the final state of each is
+ * scored. A call is `failed` when its terminal status says so — that is how
+ * both a schema violation and a product refusal surface, which is exactly the
+ * comprehensibility signal.
+ */
+export function extractTrialMetrics(
+  events: { type: string; data: Record<string, unknown> }[],
+  options: { toolName: string; expected: SendForm; latencyMs: number }
+): AbTrialMetrics {
+  const byId = new Map<string, AbAttempt>()
+  const order: string[] = []
+  let totalTokens = 0
+  for (const event of events) {
+    if (event.type === 'turn.completed') {
+      const usage = event.data.usage as { totalTokens?: unknown } | undefined
+      if (usage && typeof usage.totalTokens === 'number') totalTokens += usage.totalTokens
+      continue
+    }
+    if (event.type !== 'acp.update') continue
+    const update = event.data.update as AcpToolEvent | undefined
+    if (!update) continue
+    if (update.sessionUpdate !== 'tool_call' && update.sessionUpdate !== 'tool_call_update') continue
+    const name = update._meta?.claudeCode?.toolName ?? update.title
+    const id = update.toolCallId
+    if (typeof id !== 'string') continue
+    // Only the surface under test counts as an attempt.
+    const isSubject = typeof name === 'string' && name.toLowerCase().includes(options.toolName.toLowerCase())
+    if (!isSubject && !byId.has(id)) continue
+    const existing = byId.get(id)
+    if (!existing) {
+      byId.set(id, { tool: String(name), form: 'unclassifiable', failed: false })
+      order.push(id)
+    }
+    const attempt = byId.get(id)!
+    if (update.rawInput && typeof update.rawInput === 'object' && Object.keys(update.rawInput).length > 0) {
+      attempt.args = update.rawInput as Record<string, unknown>
+      attempt.form = classifySendForm(attempt.args)
+    }
+    if (update.status === 'failed') {
+      attempt.failed = true
+      const text = JSON.stringify(update.content ?? '')
+      attempt.error = text.slice(0, 300)
+    }
+    if (update.status === 'completed') attempt.failed = false
+  }
+  const attempts = order.map((id) => byId.get(id)!)
+  const successIndex = attempts.findIndex((attempt) => !attempt.failed && attempt.form === options.expected)
+  return {
+    attempts,
+    invalidCalls: attempts.filter((attempt) => attempt.failed).length,
+    firstAttemptSuccess: successIndex === 0,
+    completed: successIndex >= 0,
+    attemptsToSuccess: successIndex >= 0 ? successIndex + 1 : 0,
+    toolCalls: attempts.length,
+    totalTokens,
+    latencyMs: options.latencyMs
+  }
+}
+
+/** Arm B's registry: the façade, with `sendMessage` withheld. */
+export function armBTools(facade: EvaluationToolDefinition): {
+  tools: EvaluationToolDefinition[]
+  hideProductTools: string[]
+} {
+  return { tools: [facade], hideProductTools: ['sendMessage'] }
+}
diff --git a/evals/test/tool-surface-ab.test.ts b/evals/test/tool-surface-ab.test.ts
new file mode 100644
index 000000000..e33e605ea
--- /dev/null
+++ b/evals/test/tool-surface-ab.test.ts
@@ -0,0 +1,158 @@
+import { describe, expect, it } from 'vitest'
+import { AB_SCENARIOS, classifySendForm, extractTrialMetrics } from '../games/tool-surface-ab.js'
+
+/**
+ * The A/B's measurement apparatus, tested without model credentials. If the
+ * classifier or the metric extraction is wrong, every number in the write-up is
+ * wrong — so both are pinned here rather than trusted.
+ */
+
+function toolCall(id: string, name: string, fields: Record<string, unknown> = {}) {
+  return {
+    type: 'acp.update',
+    data: { update: { sessionUpdate: 'tool_call', toolCallId: id, title: name, ...fields } }
+  }
+}
+function toolUpdate(id: string, name: string, fields: Record<string, unknown>) {
+  return {
+    type: 'acp.update',
+    data: { update: { sessionUpdate: 'tool_call_update', toolCallId: id, title: name, ...fields } }
+  }
+}
+
+describe('send-form classifier — one vocabulary for both arms', () => {
+  it('maps every legal product shape to its form', () => {
+    expect(classifySendForm({ toAgent: 'a', channel: 'C' })).toBe('agent-channel')
+    expect(classifySendForm({ toAgent: 'a' })).toBe('agent-postless')
+    expect(classifySendForm({ toAgent: { agentId: 'a', needsReply: true } })).toBe('agent-postless')
+    expect(classifySendForm({ toUser: 'U', channel: 'C' })).toBe('user-channel')
+    expect(classifySendForm({ toUser: 'U' })).toBe('user-dm')
+    expect(classifySendForm({ channel: 'C' })).toBe('channel-bare')
+    expect(classifySendForm({ sessionId: 'S' })).toBe('parent-session')
+  })
+
+  it('refuses to classify a shape that names no target', () => {
+    expect(classifySendForm({ message: 'hi' })).toBe('unclassifiable')
+    expect(classifySendForm(undefined)).toBe('unclassifiable')
+    expect(classifySendForm({ channel: '' })).toBe('unclassifiable')
+  })
+
+  it('is symmetric: an arm-B compiled call scores exactly like the arm-A call it becomes', () => {
+    // This is the property that makes the two arms comparable at all.
+    const compiled = { toAgent: 'a', channel: 'C', message: 'x' }
+    expect(classifySendForm(compiled)).toBe(classifySendForm({ toAgent: 'a', channel: 'C', message: 'x' }))
+  })
+})
+
+describe('trial metric extraction', () => {
+  it('scores a clean first-attempt success', () => {
+    const metrics = extractTrialMetrics(
+      [
+        toolCall('t1', 'sendMessage'),
+        toolUpdate('t1', 'sendMessage', { rawInput: { toAgent: 'a', channel: 'C', message: 'x' } }),
+        toolUpdate('t1', 'sendMessage', { status: 'completed' }),
+        { type: 'turn.completed', data: { usage: { totalTokens: 1200 } } }
+      ],
+      { toolName: 'sendMessage', expected: 'agent-channel', latencyMs: 5000 }
+    )
+    expect(metrics).toMatchObject({
+      firstAttemptSuccess: true,
+      completed: true,
+      attemptsToSuccess: 1,
+      toolCalls: 1,
+      invalidCalls: 0,
+      totalTokens: 1200
+    })
+  })
+
+  it('counts a refused call and the self-correction that follows it', () => {
+    // The comprehensibility signal: one rejection, then a corrected retry.
+    const metrics = extractTrialMetrics(
+      [
+        toolCall('t1', 'sendMessage'),
+        toolUpdate('t1', 'sendMessage', { rawInput: { toAgent: 'a', toUser: 'U', message: 'x' } }),
+        toolUpdate('t1', 'sendMessage', { status: 'failed', content: 'exactly one target mode' }),
+        toolCall('t2', 'sendMessage'),
+        toolUpdate('t2', 'sendMessage', { rawInput: { toAgent: 'a', channel: 'C', message: 'x' } }),
+        toolUpdate('t2', 'sendMessage', { status: 'completed' })
+      ],
+      { toolName: 'sendMessage', expected: 'agent-channel', latencyMs: 9000 }
+    )
+    expect(metrics.toolCalls).toBe(2)
+    expect(metrics.invalidCalls).toBe(1)
+    expect(metrics.firstAttemptSuccess).toBe(false)
+    expect(metrics.completed).toBe(true)
+    expect(metrics.attemptsToSuccess).toBe(2)
+    expect(String(metrics.attempts[0]!.error)).toContain('exactly one target mode')
+  })
+
+  it('scores a wrong-but-accepted form as a failure to complete, not a success', () => {
+    // Posting at a channel root when a postless call was required is accepted by
+    // the product and still wrong for the task — the metric must not reward it.
+    const metrics = extractTrialMetrics(
+      [
+        toolCall('t1', 'sendMessage'),
+        toolUpdate('t1', 'sendMessage', { rawInput: { toAgent: 'a', channel: 'C', message: 'x' } }),
+        toolUpdate('t1', 'sendMessage', { status: 'completed' })
+      ],
+      { toolName: 'sendMessage', expected: 'agent-postless', latencyMs: 4000 }
+    )
+    expect(metrics.invalidCalls).toBe(0)
+    expect(metrics.firstAttemptSuccess).toBe(false)
+    expect(metrics.completed).toBe(false)
+    expect(metrics.attemptsToSuccess).toBe(0)
+  })
+
+  it('ignores tool calls that are not the surface under test', () => {
+    const metrics = extractTrialMetrics(
+      [
+        toolCall('t0', 'listAgents'),
+        toolUpdate('t0', 'listAgents', { status: 'completed' }),
+        toolCall('t1', 'post'),
+        toolUpdate('t1', 'post', { rawInput: { conversation: { kind: 'channel', channel: 'C' }, message: 'x' } }),
+        toolUpdate('t1', 'post', { status: 'completed' })
+      ],
+      { toolName: 'post', expected: 'channel-bare', latencyMs: 3000 }
+    )
+    expect(metrics.toolCalls).toBe(1)
+    // Arm B's raw input is the façade shape, so it classifies through the same
+    // vocabulary only after compilation — an uncompiled façade call is not a form.
+    expect(metrics.attempts[0]!.tool).toBe('post')
+  })
+
+  it('sums tokens across every turn of the trial', () => {
+    const metrics = extractTrialMetrics(
+      [
+        { type: 'turn.completed', data: { usage: { totalTokens: 500 } } },
+        { type: 'turn.completed', data: { usage: { totalTokens: 700 } } }
+      ],
+      { toolName: 'sendMessage', expected: 'channel-bare', latencyMs: 1 }
+    )
+    expect(metrics.totalTokens).toBe(1200)
+  })
+})
+
+describe('the reduced scenario matrix', () => {
+  it('is four scenarios, each with a known-correct product form', () => {
+    expect(AB_SCENARIOS).toHaveLength(4)
+    expect(AB_SCENARIOS.map((scenario) => scenario.id)).toEqual([
+      'agent-channel',
+      'channel-bare',
+      'agent-postless',
+      'parent-session'
+    ])
+    for (const scenario of AB_SCENARIOS) expect(scenario.expected).not.toBe('unclassifiable')
+  })
+
+  it('never names a tool, a field or a form in the task text', () => {
+    // Naming them would test instruction-following instead of the surface.
+    const ids = { peerAgentId: 'PEER', channel: 'CHAN', humanUserId: 'UHUMAN' }
+    const banned = ['sendMessage', 'post(', 'toAgent', 'toUser', 'sessionId', 'conversation', 'visibility', 'address']
+    for (const scenario of AB_SCENARIOS) {
+      const text = scenario.instruction(ids)
+      for (const token of banned) {
+        expect(text, `${scenario.id} leaks "${token}"`).not.toContain(token)
+      }
+    }
+  })
+})

From 1e9b930a320ab827a5f7e546390d34e4eeb4a954 Mon Sep 17 00:00:00 2001
From: Poytr1 <pooytr1@gmail.com>
Date: Sun, 9 Aug 2026 11:14:39 +0800
Subject: [PATCH 03/15] =?UTF-8?q?feat(evals):=20arm-parity=20guidance=20se?=
 =?UTF-8?q?am=20=E2=80=94=20each=20A/B=20arm's=20prompt=20teaches=20its=20?=
 =?UTF-8?q?own=20surface?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The standing collaboration guidance and the parent-report append teach
`sendMessage` call shapes by name, so an arm that withholds `sendMessage`
would carry a system prompt describing a tool it does not have — priming it
with the other arm's vocabulary. This is the prompt-side complement of
`hideProductTools`: the evaluation environment may supply replacement
texts, production never sets them, and everything else in the prompt stays
byte-identical.

Arm B's texts mirror the production structure sentence-for-sentence outside
the tool teaching (pinned by tests), and the parent append names the exact
`post` parent form with the real session id.

Also: give the metric extractor per-component token sums (cache traffic
dominates local runs, so input+output is reported beside the total), an
arm-specific classifier hook (arm B classifies the COMPILED product args),
a turn count, and give scenario 4 a concrete relayed question so the
parent-session wake carries a real task.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 evals/games/post-facade.ts                    | 58 ++++++++++++
 evals/games/tool-surface-ab.ts                | 66 +++++++++++--
 evals/test/post-facade.test.ts                | 38 +++++++-
 packages/daemon/src/daemon.ts                 |  6 ++
 packages/daemon/src/evaluation/environment.ts | 12 +++
 .../daemon/src/session/session-manager.ts     | 92 +++++++++++--------
 6 files changed, 223 insertions(+), 49 deletions(-)

diff --git a/evals/games/post-facade.ts b/evals/games/post-facade.ts
index cb2eb19be..f4e4228b5 100644
--- a/evals/games/post-facade.ts
+++ b/evals/games/post-facade.ts
@@ -192,6 +192,64 @@ export const POST_TOOL_DESCRIPTOR = {
   }
 }
 
+/**
+ * Arm B's standing collaboration guidance — the prompt-side half of the surface.
+ *
+ * The production system prompt teaches `sendMessage` call shapes by name
+ * (session-manager.ts `collabAppend` / `parentReplyAppend`), so an arm that
+ * withholds `sendMessage` needs guidance that teaches ITS surface instead, or
+ * the prompt would prime the model with the other arm's vocabulary and tell it
+ * to call a tool it does not carry. Structure and every non-surface sentence
+ * (ordinary-reply rule, "act only on what is asked", quiet-about-mechanics,
+ * peer-roster memory) mirror the production text — only the tool teaching
+ * differs, which is the point.
+ */
+export const POST_COLLAB_GUIDANCE =
+  `# Collaborating with other agents\n` +
+  `- One tool, \`post\`, sends any message that leaves your current conversation. Choose three things ` +
+  `independently: WHICH conversation it belongs to, WHO it addresses, and whether it is visible on the platform.\n` +
+  `- To reach a specific agent privately: ` +
+  `\`post\` \`{"conversation":{"kind":"private"},"address":["<agent id>"],"visibility":"session-only",` +
+  `"message":"..."}\` — it wakes ONLY that agent and nothing appears in any channel. That call is ` +
+  `FIRE-AND-FORGET: the peer answers inside its own conversation and nothing comes back to you, not even a ` +
+  `failure. Whenever you expect an answer — your message asks a question or requests a result, or you were ` +
+  `asked to relay that agent's answer to someone — add \`"expectReply":true\`, which obliges it to report ` +
+  `into YOUR session when it finishes or fails.\n` +
+  `- To open a VISIBLE discussion at a channel's root: \`"conversation":{"kind":"channel","channel":` +
+  `"<channel id>"}\`. Put an agent id in \`address\` to pull that agent into the new discussion (you may ` +
+  `address yourself there to open one for yourself — use your ID from the # Agent block, never your platform ` +
+  `bot identity), or human platform ids to @-mention people. Omit \`address\` to leave a note that wakes ` +
+  `nobody.\n` +
+  `- To speak in the conversation you are already in — including to address a peer or human there — do NOT ` +
+  `call \`post\`: write your ordinary turn reply and @-mention them in it (use \`listAgents\` to get a peer's ` +
+  `exact \`mention\` token). To reach a HUMAN in their direct messages, use ` +
+  `\`"conversation":{"kind":"dm","user":"<platform user id>"}\` — never address an AgentConnect agent or your ` +
+  `own bot identity as a human user. If you were woken by another session, reply with ` +
+  `\`"conversation":{"kind":"parent","sessionId":"<Parent session>"}\`.\n` +
+  `- Act only on what is asked of YOU. Do not relay a message onward or start your own broadcast to other ` +
+  `agents unless a human explicitly tells you to.\n` +
+  `- Be quiet about mechanics: don't narrate each step or post a message per action, and don't restate tool ` +
+  `results like "delivered: true". Take the action, add at most one short status line if needed, then end your turn.\n` +
+  `- When another agent introduces itself to you, record it in your memory (a peer roster — id, name, what it ` +
+  `does, how to reach it) so you know who to delegate to later. Then just acknowledge briefly; do NOT re-introduce ` +
+  `yourself back or broadcast to everyone.`
+
+/** Arm B's parent-report append — mirrors the production text with only the
+ *  tool teaching swapped. */
+export function postParentReplyAppend(parentSessionId: string): string {
+  return (
+    `# Reporting back to your parent session\n` +
+    `Another session delegated this work to you and is waiting on the outcome. When you finish — or when you ` +
+    `cannot finish — reply to it with ` +
+    `\`post\` \`{"conversation":{"kind":"parent","sessionId":"${parentSessionId}"},"message":"..."}\`, saying ` +
+    `whether you succeeded or failed and what the result was (on failure, what went wrong). Send it exactly ` +
+    `once, at the end; do not report progress along the way, and do not skip it because the task was small or ` +
+    `unsuccessful. Your ordinary assistant response in this child session is not delivered to the parent. Do ` +
+    `not write the result before or after the tool call; after the tool reports successful delivery, end your ` +
+    `turn immediately without repeating the message.`
+  )
+}
+
 /** Build arm B's registry entry. Every call compiles and is then executed by the
  *  PRODUCT tool, so the two arms share one implementation. */
 export function postFacadeTool(options: {
diff --git a/evals/games/tool-surface-ab.ts b/evals/games/tool-surface-ab.ts
index 4f93517db..dbaa7eeca 100644
--- a/evals/games/tool-surface-ab.ts
+++ b/evals/games/tool-surface-ab.ts
@@ -57,9 +57,11 @@ export const AB_SCENARIOS: AbScenario[] = [
     id: 'parent-session',
     expected: 'parent-session',
     needsCaller: true,
+    // Relayed to the subject INSIDE a needsReply wake by the caller agent, so
+    // the subject really does have a parent session to answer into.
     instruction: () =>
-      `Answer the question you were just asked, sending your answer back to whoever asked it so it reaches them ` +
-      `directly. Do not publish it anywhere public.`
+      `What is the sum of 17 and 25? Work it out and get your answer back to whoever is asking you, so it ` +
+      `reaches them directly. Do not publish the answer anywhere public.`
   }
 ]
 
@@ -84,6 +86,14 @@ export interface AbAttempt {
   error?: string
 }
 
+export interface AbTokenBreakdown {
+  total: number
+  input: number
+  output: number
+  cacheRead: number
+  cacheWrite: number
+}
+
 export interface AbTrialMetrics {
   attempts: AbAttempt[]
   /** Calls the surface itself refused: schema violation or illegal combination. */
@@ -96,6 +106,10 @@ export interface AbTrialMetrics {
   attemptsToSuccess: number
   toolCalls: number
   totalTokens: number
+  /** Component sums over the same turns as `totalTokens`. Cache traffic
+   *  dominates a local run, so input+output is reported alongside the total. */
+  tokens: AbTokenBreakdown
+  turns: number
   latencyMs: number
 }
 
@@ -120,15 +134,34 @@ interface AcpToolEvent {
  */
 export function extractTrialMetrics(
   events: { type: string; data: Record<string, unknown> }[],
-  options: { toolName: string; expected: SendForm; latencyMs: number }
+  options: {
+    toolName: string
+    expected: SendForm
+    latencyMs: number
+    /** Arm-specific bridge from raw tool input to the shared form vocabulary.
+     *  Arm A classifies the product args directly (default); arm B compiles the
+     *  façade input first, so both arms are scored on the SAME product shapes. */
+    classify?: (args: Record<string, unknown> | undefined) => SendForm
+  }
 ): AbTrialMetrics {
+  const classify = options.classify ?? classifySendForm
   const byId = new Map<string, AbAttempt>()
   const order: string[] = []
-  let totalTokens = 0
+  const tokens: AbTokenBreakdown = { total: 0, input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }
+  let turns = 0
   for (const event of events) {
     if (event.type === 'turn.completed') {
-      const usage = event.data.usage as { totalTokens?: unknown } | undefined
-      if (usage && typeof usage.totalTokens === 'number') totalTokens += usage.totalTokens
+      turns += 1
+      const usage = event.data.usage as Record<string, unknown> | undefined
+      const add = (key: keyof AbTokenBreakdown, field: string) => {
+        const value = usage?.[field]
+        if (typeof value === 'number' && Number.isFinite(value)) tokens[key] += value
+      }
+      add('total', 'totalTokens')
+      add('input', 'inputTokens')
+      add('output', 'outputTokens')
+      add('cacheRead', 'cachedReadTokens')
+      add('cacheWrite', 'cachedWriteTokens')
       continue
     }
     if (event.type !== 'acp.update') continue
@@ -149,7 +182,7 @@ export function extractTrialMetrics(
     const attempt = byId.get(id)!
     if (update.rawInput && typeof update.rawInput === 'object' && Object.keys(update.rawInput).length > 0) {
       attempt.args = update.rawInput as Record<string, unknown>
-      attempt.form = classifySendForm(attempt.args)
+      attempt.form = classify(attempt.args)
     }
     if (update.status === 'failed') {
       attempt.failed = true
@@ -167,11 +200,28 @@ export function extractTrialMetrics(
     completed: successIndex >= 0,
     attemptsToSuccess: successIndex >= 0 ? successIndex + 1 : 0,
     toolCalls: attempts.length,
-    totalTokens,
+    totalTokens: tokens.total,
+    tokens,
+    turns,
     latencyMs: options.latencyMs
   }
 }
 
+/** Arm B's classifier: compile the façade input, then classify the product args
+ *  it becomes — the symmetry that makes the two arms score identically. An
+ *  input the façade refuses names no legal form. */
+export function classifyPostForm(
+  compile: (input: Record<string, unknown>) => { args: Record<string, unknown> },
+  args: Record<string, unknown> | undefined
+): SendForm {
+  if (!args) return 'unclassifiable'
+  try {
+    return classifySendForm(compile(args).args)
+  } catch {
+    return 'unclassifiable'
+  }
+}
+
 /** Arm B's registry: the façade, with `sendMessage` withheld. */
 export function armBTools(facade: EvaluationToolDefinition): {
   tools: EvaluationToolDefinition[]
diff --git a/evals/test/post-facade.test.ts b/evals/test/post-facade.test.ts
index ee16369d1..71af8830d 100644
--- a/evals/test/post-facade.test.ts
+++ b/evals/test/post-facade.test.ts
@@ -1,7 +1,14 @@
 import { describe, expect, it } from 'vitest'
 import { toolsForIntegrations } from '../../packages/daemon/src/mcp/tools.js'
 import { IntegrationSchema } from '../../packages/daemon/src/agents/agent-schema.js'
-import { POST_TOOL_DESCRIPTOR, PostCompileError, compilePost, postFacadeTool } from '../games/post-facade.js'
+import {
+  POST_COLLAB_GUIDANCE,
+  POST_TOOL_DESCRIPTOR,
+  PostCompileError,
+  compilePost,
+  postFacadeTool,
+  postParentReplyAppend
+} from '../games/post-facade.js'
 
 /**
  * Arm B's contract, credential-free: the façade must compile every legal
@@ -148,7 +155,9 @@ describe('static cost of each tool surface', () => {
       core: { mode: 'direct', bindRules: [] },
       config: { botToken: 'xoxb-x', appToken: 'xapp-x' }
     })
-    const tool = toolsForIntegrations([integration], { collaboration: true }).find((t) => t.name === 'sendMessage')
+    // #761 removed the evaluation-only collaboration toggle: the collaboration
+    // surface (sendMessage included) is unconditionally present.
+    const tool = toolsForIntegrations([integration]).find((t) => t.name === 'sendMessage')
     if (!tool) throw new Error('sendMessage descriptor not found')
     return tool
   }
@@ -165,6 +174,31 @@ describe('static cost of each tool surface', () => {
     console.log(`static cost — sendMessage: ${JSON.stringify(a)}  post: ${JSON.stringify(b)}`)
   })
 
+  it("arm B's standing guidance teaches its own surface and never the other arm's", () => {
+    // The system prompt is part of a tool surface. If arm B's guidance named
+    // `sendMessage` or its fields, the arm would be primed with vocabulary for
+    // a tool it does not carry — a measured confound, not a hypothetical.
+    for (const text of [POST_COLLAB_GUIDANCE, postParentReplyAppend('S1')]) {
+      for (const token of ['sendMessage', 'toAgent', 'toUser', 'needsReply']) {
+        expect(text, `arm B guidance leaks "${token}"`).not.toContain(token)
+      }
+      expect(text).toContain('post')
+    }
+    expect(postParentReplyAppend('S1')).toContain('"sessionId":"S1"')
+    expect(postParentReplyAppend('S1')).toContain('"kind":"parent"')
+  })
+
+  it("arm B's guidance keeps the non-surface behavioral rules of the production text verbatim", () => {
+    // Only the tool teaching may differ between the arms' prompts.
+    for (const sentence of [
+      'Act only on what is asked of YOU.',
+      "Be quiet about mechanics: don't narrate each step",
+      'When another agent introduces itself to you, record it in your memory'
+    ]) {
+      expect(POST_COLLAB_GUIDANCE).toContain(sentence)
+    }
+  })
+
   it('the landed surface really does enumerate more forms than the façade', () => {
     const description = sendMessageDescriptor().description ?? ''
     // Arm A's description spells out target modes and their illegal pairings;
diff --git a/packages/daemon/src/daemon.ts b/packages/daemon/src/daemon.ts
index 69900cdd4..0a9c367de 100644
--- a/packages/daemon/src/daemon.ts
+++ b/packages/daemon/src/daemon.ts
@@ -3065,6 +3065,12 @@ export class Daemon {
       // The session integration's own bot identity (auth.test-resolved on both
       // socket and send-only connections) for the `# Agent` Slack-identity line.
       slackBotUserIdFor: (integrationId) => this.connByIntegration.get(integrationId)?.botUserId || undefined,
+      // Evaluation-only surface-fidelity seam (see DaemonEvaluationEnvironment):
+      // an A/B arm that swaps the messaging tool surface swaps the matching
+      // guidance text with it. Absent everywhere outside evaluation runs.
+      ...(this.opts.evaluation?.environment?.collaborationGuidance
+        ? { collaborationGuidance: this.opts.evaluation.environment.collaborationGuidance }
+        : {}),
       // No runtime is whitelisted for the model-authored `setSessionTitle` fallback
       // anymore: codex-acp >= 1.1.3 emits native session_info_update titles itself
       // (issue #659), so every runtime now relies on its native ACP title path. The
diff --git a/packages/daemon/src/evaluation/environment.ts b/packages/daemon/src/evaluation/environment.ts
index 26f6368a5..c39578f0b 100644
--- a/packages/daemon/src/evaluation/environment.ts
+++ b/packages/daemon/src/evaluation/environment.ts
@@ -90,6 +90,18 @@ export interface DaemonEvaluationEnvironment {
    *  execute — a hidden tool remains fully functional if something calls it,
    *  which is exactly how a façade compiles down to it. */
   hideProductTools?: readonly string[]
+  /** EVALUATION-ONLY surface-fidelity seam, the prompt-side complement of
+   *  `hideProductTools`: the standing collaboration guidance and the
+   *  parent-report append teach `sendMessage` call shapes by name, so an arm
+   *  that withholds `sendMessage` would otherwise carry a system prompt
+   *  describing a tool it does not have — priming it with the OTHER arm's
+   *  vocabulary and sabotaging the comparison. An arm supplies texts that teach
+   *  exactly the surface it presents; everything else in the prompt stays
+   *  byte-identical. */
+  collaborationGuidance?: {
+    collabAppend?: string
+    parentReplyAppend?: (parentSessionId: string) => string
+  }
 }
 
 // ─── §4 ingress payloads ────────────────────────────────────────────────────
diff --git a/packages/daemon/src/session/session-manager.ts b/packages/daemon/src/session/session-manager.ts
index cf957230c..d825dc473 100644
--- a/packages/daemon/src/session/session-manager.ts
+++ b/packages/daemon/src/session/session-manager.ts
@@ -282,6 +282,16 @@ export class SessionManager {
       /** Whether this runtime needs AgentConnect's model-authored title fallback.
        *  Native-title runtimes (for example Claude) leave this false. */
       usesSessionTitleTool?: (agent: Agent) => boolean
+      /** EVALUATION-ONLY (daemon evaluation environment): replace the standing
+       *  collaboration guidance and the parent-report append, so an A/B arm's
+       *  system prompt teaches exactly the messaging surface that arm presents.
+       *  The prompt is part of a tool surface — an arm that withholds
+       *  `sendMessage` must not carry text describing it. Production never sets
+       *  this; everything else in the prompt stays byte-identical. */
+      collaborationGuidance?: {
+        collabAppend?: string
+        parentReplyAppend?: (parentSessionId: string) => string
+      }
       /** The runtime-definition env (daemon config `runtimes[].env`) for an agent's
        *  runtime. The spawn path detects config-file pointer-var conflicts over
        *  `{...runtimeEnv, ...agentEnv}` — supply the same base here so the
@@ -817,38 +827,41 @@ export class SessionManager {
     // reach humans, post at a channel root, or reply into a parent session. It has no
     // visible in-thread form: speaking in the current conversation is an ordinary reply.
     // `toAgent` without a `channel` is the postless, channel-invisible wake.
+    // An evaluation A/B arm that presents a different messaging surface swaps in its
+    // own guidance text (the prompt is part of the surface); production never sets it.
     const collabAppend =
+      this.deps.collaborationGuidance?.collabAppend ??
       `# Collaborating with other agents\n` +
-      `- To reach a specific agent privately, call \`sendMessage\` with ` +
-      `\`{"toAgent":"<agent id>","message":"..."}\` — it wakes ONLY that agent, delivered directly to it ` +
-      `(nothing is posted to the channel). That bare form is FIRE-AND-FORGET: the peer answers inside its own ` +
-      `conversation and nothing comes back to you, not even a failure. Whenever you expect an answer — your ` +
-      `message asks a question or requests a result, or you were asked to relay that agent's answer to someone ` +
-      `— send \`{"toAgent":{"agentId":"<agent id>","needsReply":true},"message":"..."}\` instead, which obliges ` +
-      `it to report into YOUR session when it finishes or fails. Add a \`channel\` ` +
-      `(\`{"toAgent":"<agent id>","channel":"<channel id>","message":"..."}\`, channel-root form) ` +
-      `to ALSO post a visible message at that channel's root and anchor the agent's conversation to that post. ` +
-      `That channel-root form may target YOURSELF to open and activate one new conversation there: use your own ` +
-      `ID from the # Agent block (also included by \`listAgents\`), never your platform bot identity. A direct ` +
-      `\`toAgent\` call without \`channel\` may not target yourself. ` +
-      `To speak in the conversation you are already in — including to address a peer or human there — do NOT ` +
-      `call \`sendMessage\`: write your ordinary turn reply and @-mention them in it (use \`listAgents\` to get ` +
-      `a peer's exact \`mention\` token). To reach HUMAN users elsewhere, use the \`toUser\` mode — never put ` +
-      `an AgentConnect agent or your own bot identity in \`toUser\`: ` +
-      `\`{"toUser":"<Slack user id>","message":"..."}\` DMs that person, and adding \`channel\` posts an ` +
-      `@-mention at the channel root. In that channel form, pass ` +
-      `an array such as \`"toUser":["<user id 1>","<user id 2>"]\` to @-mention multiple people in the one ` +
-      `message; arrays are never DMs. If you were woken by another ` +
-      `session, reply with \`{"sessionId":"<Parent session>","message":"..."}\`. To leave a visible note others ` +
-      `catch up on later without waking anyone, use \`{"channel":"<channel id>","message":"..."}\`. Every ` +
-      `visible \`sendMessage\` lands at a channel root and opens a new conversation there.\n` +
-      `- Act only on what is asked of YOU. Do not relay a message onward or start your own broadcast to other ` +
-      `agents unless a human explicitly tells you to.\n` +
-      `- Be quiet about mechanics: don't narrate each step or post a message per action, and don't restate tool ` +
-      `results like "delivered: true". Take the action, add at most one short status line if needed, then end your turn.\n` +
-      `- When another agent introduces itself to you, record it in your memory (a peer roster — id, name, what it ` +
-      `does, how to reach it) so you know who to delegate to later. Then just acknowledge briefly; do NOT re-introduce ` +
-      `yourself back or broadcast to everyone.`
+        `- To reach a specific agent privately, call \`sendMessage\` with ` +
+        `\`{"toAgent":"<agent id>","message":"..."}\` — it wakes ONLY that agent, delivered directly to it ` +
+        `(nothing is posted to the channel). That bare form is FIRE-AND-FORGET: the peer answers inside its own ` +
+        `conversation and nothing comes back to you, not even a failure. Whenever you expect an answer — your ` +
+        `message asks a question or requests a result, or you were asked to relay that agent's answer to someone ` +
+        `— send \`{"toAgent":{"agentId":"<agent id>","needsReply":true},"message":"..."}\` instead, which obliges ` +
+        `it to report into YOUR session when it finishes or fails. Add a \`channel\` ` +
+        `(\`{"toAgent":"<agent id>","channel":"<channel id>","message":"..."}\`, channel-root form) ` +
+        `to ALSO post a visible message at that channel's root and anchor the agent's conversation to that post. ` +
+        `That channel-root form may target YOURSELF to open and activate one new conversation there: use your own ` +
+        `ID from the # Agent block (also included by \`listAgents\`), never your platform bot identity. A direct ` +
+        `\`toAgent\` call without \`channel\` may not target yourself. ` +
+        `To speak in the conversation you are already in — including to address a peer or human there — do NOT ` +
+        `call \`sendMessage\`: write your ordinary turn reply and @-mention them in it (use \`listAgents\` to get ` +
+        `a peer's exact \`mention\` token). To reach HUMAN users elsewhere, use the \`toUser\` mode — never put ` +
+        `an AgentConnect agent or your own bot identity in \`toUser\`: ` +
+        `\`{"toUser":"<Slack user id>","message":"..."}\` DMs that person, and adding \`channel\` posts an ` +
+        `@-mention at the channel root. In that channel form, pass ` +
+        `an array such as \`"toUser":["<user id 1>","<user id 2>"]\` to @-mention multiple people in the one ` +
+        `message; arrays are never DMs. If you were woken by another ` +
+        `session, reply with \`{"sessionId":"<Parent session>","message":"..."}\`. To leave a visible note others ` +
+        `catch up on later without waking anyone, use \`{"channel":"<channel id>","message":"..."}\`. Every ` +
+        `visible \`sendMessage\` lands at a channel root and opens a new conversation there.\n` +
+        `- Act only on what is asked of YOU. Do not relay a message onward or start your own broadcast to other ` +
+        `agents unless a human explicitly tells you to.\n` +
+        `- Be quiet about mechanics: don't narrate each step or post a message per action, and don't restate tool ` +
+        `results like "delivered: true". Take the action, add at most one short status line if needed, then end your turn.\n` +
+        `- When another agent introduces itself to you, record it in your memory (a peer roster — id, name, what it ` +
+        `does, how to reach it) so you know who to delegate to later. Then just acknowledge briefly; do NOT re-introduce ` +
+        `yourself back or broadcast to everyone.`
 
     // The parent asked to be told how this session ends (`toAgent.needsReply`). Standing, not a
     // user turn — the obligation outlives the waking turn, so it belongs beside the collaboration
@@ -856,15 +869,16 @@ export class SessionManager {
     // scoped to a terminal report: nothing here asks for progress narration, which would turn every
     // delegated task into channel chatter.
     const parentReplyAppend = needsReplyToParent
-      ? `# Reporting back to your parent session\n` +
-        `Another session delegated this work to you and is waiting on the outcome. When you finish — or when you ` +
-        `cannot finish — reply to it with ` +
-        `\`sendMessage\` \`{"sessionId":"${effectiveOriginSessionId}","message":"..."}\`, saying whether you ` +
-        `succeeded or failed and what the result was (on failure, what went wrong). Send it exactly once, at the ` +
-        `end; do not report progress along the way, and do not skip it because the task was small or unsuccessful. ` +
-        `Your ordinary assistant response in this child session is not delivered to the parent. Do not write the ` +
-        `result before or after the tool call; after the tool reports successful delivery, end your turn immediately ` +
-        `without repeating the message.`
+      ? (this.deps.collaborationGuidance?.parentReplyAppend?.(effectiveOriginSessionId!) ??
+        `# Reporting back to your parent session\n` +
+          `Another session delegated this work to you and is waiting on the outcome. When you finish — or when you ` +
+          `cannot finish — reply to it with ` +
+          `\`sendMessage\` \`{"sessionId":"${effectiveOriginSessionId}","message":"..."}\`, saying whether you ` +
+          `succeeded or failed and what the result was (on failure, what went wrong). Send it exactly once, at the ` +
+          `end; do not report progress along the way, and do not skip it because the task was small or unsuccessful. ` +
+          `Your ordinary assistant response in this child session is not delivered to the parent. Do not write the ` +
+          `result before or after the tool call; after the tool reports successful delivery, end your turn immediately ` +
+          `without repeating the message.`)
       : ''
 
     // Standing response-choice rule for EVERY agent session and delivery scenario. Direct

From 9bea1b36e160e310f9fa30888b09f8d5068d652c Mon Sep 17 00:00:00 2001
From: Poytr1 <pooytr1@gmail.com>
Date: Sun, 9 Aug 2026 11:18:14 +0800
Subject: [PATCH 04/15] =?UTF-8?q?feat(evals):=20A/B=20fixture=20=E2=80=94?=
 =?UTF-8?q?=20one=20environment,=20two=20surfaces,=20contract-proven=20arm?=
 =?UTF-8?q?=20parity?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

AbFixture boots the real daemon against the three-room A/B topology
(briefing / peer-briefing / plaza) with production mention-gated routing and
full platform-echo fidelity. The `arm` option changes ONLY the messaging
surface: arm A is the shipped `sendMessage`; arm B carries the `post`
façade with `sendMessage` withheld and the arm-B guidance texts — while
every façade call still executes through the product tool on the same
trusted session context.

Credential-free contract tests read the daemon's own control socket and the
world's delivered effects to pin the fairness preconditions:

- arm A sessions list `sendMessage` (no `post`) and carry the production
  guidance;
- arm B sessions list `post` (no `sendMessage`), their prompt teaches the
  post surface with no `sendMessage` text anywhere, and a compiled call
  produces a real world-authorized delivery;
- a needsReply wake under arm B installs the `post`-flavored parent-report
  append on the child, and the postless exchange projects nothing into any
  channel.

Plus listDaemonTools on the scripted-host IPC client so a test can prove
what a session's descriptor list actually contained.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 evals/games/mcp-client.ts                  |  54 ++++
 evals/games/tool-surface-ab-fixture.ts     | 318 +++++++++++++++++++++
 evals/test/tool-surface-ab-fixture.test.ts | 197 +++++++++++++
 3 files changed, 569 insertions(+)
 create mode 100644 evals/games/tool-surface-ab-fixture.ts
 create mode 100644 evals/test/tool-surface-ab-fixture.test.ts

diff --git a/evals/games/mcp-client.ts b/evals/games/mcp-client.ts
index b64d0759c..497c77199 100644
--- a/evals/games/mcp-client.ts
+++ b/evals/games/mcp-client.ts
@@ -37,6 +37,60 @@ export interface DaemonToolCallResult {
   error?: string
 }
 
+/** One `listTools` round-trip: the descriptor names THIS session actually
+ *  carries — how a test proves a surface was presented (or withheld). */
+export async function listDaemonTools(binding: DaemonMcpBinding, timeoutMs = 30_000): Promise<string[]> {
+  const response = await ipcRequest(binding, { op: 'listTools' }, timeoutMs)
+  if (!response.ok) throw new Error(response.error ?? 'listTools failed')
+  const tools = (response.result as { tools?: { name?: unknown }[] } | undefined)?.tools ?? []
+  return tools.map((tool) => String(tool.name))
+}
+
+function ipcRequest(
+  binding: DaemonMcpBinding,
+  request: Record<string, unknown>,
+  timeoutMs: number
+): Promise<DaemonToolCallResult> {
+  return new Promise((resolve, reject) => {
+    const socket = net.connect(binding.endpoint)
+    let buffer = ''
+    let settled = false
+    const timer = setTimeout(() => {
+      finish(() => reject(new Error(`daemon ipc request timed out after ${timeoutMs}ms`)))
+    }, timeoutMs)
+    const finish = (settle: () => void): void => {
+      if (settled) return
+      settled = true
+      clearTimeout(timer)
+      socket.destroy()
+      settle()
+    }
+    socket.setEncoding('utf8')
+    socket.on('connect', () => {
+      socket.write(`${JSON.stringify({ id: 1, token: binding.token, ...request })}\n`)
+    })
+    socket.on('data', (chunk: string) => {
+      buffer += chunk
+      const newline = buffer.indexOf('\n')
+      if (newline === -1) return
+      const line = buffer.slice(0, newline)
+      try {
+        const response = JSON.parse(line) as { ok?: boolean; result?: unknown; error?: string }
+        finish(() =>
+          resolve({
+            ok: response.ok === true,
+            ...(response.result !== undefined ? { result: response.result } : {}),
+            ...(typeof response.error === 'string' ? { error: response.error } : {})
+          })
+        )
+      } catch (error) {
+        finish(() => reject(error instanceof Error ? error : new Error(String(error))))
+      }
+    })
+    socket.on('error', (error) => finish(() => reject(error)))
+  })
+}
+
 /** One `callTool` round-trip over the daemon's MCP control socket. */
 export function callDaemonTool(
   binding: DaemonMcpBinding,
diff --git a/evals/games/tool-surface-ab-fixture.ts b/evals/games/tool-surface-ab-fixture.ts
new file mode 100644
index 000000000..1992f0d4e
--- /dev/null
+++ b/evals/games/tool-surface-ab-fixture.ts
@@ -0,0 +1,318 @@
+/**
+ * The tool-surface A/B's runnable environment — one fixture, two arms.
+ *
+ * Boots a REAL daemon against the three-room A/B topology (see `abManifest`)
+ * with production mention-gated routing and full platform-echo fidelity. The
+ * ONLY thing the `arm` option changes is the messaging surface the sessions
+ * carry:
+ *
+ *   arm A  the product `sendMessage`, production guidance text — as shipped;
+ *   arm B  the `post` façade (post-facade.ts), `sendMessage` withheld from the
+ *          descriptor list, and the arm-B guidance texts — while every façade
+ *          call still EXECUTES through the product `sendMessage` on the same
+ *          trusted session context.
+ *
+ * Everything else — model, topology, seeds, routing, activation, policy — is
+ * byte-identical across arms, which is what makes a measured difference
+ * attributable to the surface.
+ *
+ * The subject seam is the arena's own (`GameSubjectSpec`): scripted hosts for
+ * credential-free contract tests, or a real-runtime subject template for the
+ * behavioral runs.
+ */
+import { readFileSync, writeFileSync } from 'node:fs'
+import { join } from 'node:path'
+import { SLACK_RESPONSE_FINAL_EVENT_TAG } from '../../packages/message/src/index.js'
+import type {
+  DeliveryHandle,
+  EvaluationEvent,
+  RecordedOutboundEffect
+} from '../../packages/daemon/src/evaluation/index.js'
+import { DaemonEvaluationHarness } from '../../packages/daemon/src/evaluation/index.js'
+import { POST_COLLAB_GUIDANCE, postFacadeTool, postParentReplyAppend } from './post-facade.js'
+import { prepareGameSubject, preflightRealSubject, type GameSubjectSpec } from './subject.js'
+import { compileTopology } from './topology.js'
+import type { CompiledRoom, CompiledTopology, GameTopologyManifest } from './types.js'
+import { ArenaWorld } from './world.js'
+
+export type AbArm = 'A' | 'B'
+
+/** One recorded façade call (arm B only): what the model asked for and what it
+ *  compiled to. The qualitative "HOW was the surface misused" evidence. */
+export interface PostCallRecord {
+  agentId: string
+  input: Record<string, unknown>
+  outcome: 'compiled' | 'invalid'
+  form?: string
+  error?: string
+}
+
+/**
+ * The A/B topology. `briefing` is where the subject receives its task (subject
+ * alone, so the task text stays out of every measured room), `plaza` is the
+ * shared target channel the scenarios point at, and `peer-briefing` is where
+ * scenario 4's caller receives the instruction to delegate. One Slack
+ * integration per agent reaches all of its rooms, exactly as in production.
+ */
+export function abManifest(seed: number): GameTopologyManifest {
+  return {
+    game: 'tool-surface-ab',
+    seed,
+    agents: [{ id: 'runner' }, { id: 'peer' }],
+    rooms: [
+      { id: 'briefing', platform: 'slack', members: ['runner'] },
+      { id: 'peer-briefing', platform: 'slack', members: ['peer'] },
+      { id: 'plaza', platform: 'slack', members: ['runner', 'peer'] }
+    ]
+  }
+}
+
+export interface AbFixtureOptions {
+  seed: number
+  arm: AbArm
+  /** Who plays: scripted hosts (contract tests) or a real subject template. */
+  subject: GameSubjectSpec
+  /** Scripted-host seam, passed through to the daemon (subject kind 'scripted'). */
+  hostFactory?: ConstructorParameters<typeof DaemonEvaluationHarness>[0]['hostFactory']
+  /** Patch the prepared agents' `description` (a real peer gets its persona by
+   *  CONFIGURATION, never by scripting). Keyed by agent alias. */
+  agentDescriptions?: Record<string, string>
+}
+
+export class AbFixture {
+  readonly topology: CompiledTopology
+  readonly world: ArenaWorld
+  readonly arm: AbArm
+  readonly secrets: readonly string[]
+  /** Shared with the façade's onCall hook — records appear here live (arm B). */
+  readonly facadeCalls: PostCallRecord[]
+  private readonly harness: DaemonEvaluationHarness
+  private readonly subjectCleanup: () => void
+  private readonly echoHandles: DeliveryHandle[] = []
+  private readonly threadByMessageId = new Map<string, string>()
+
+  private constructor(args: {
+    topology: CompiledTopology
+    world: ArenaWorld
+    arm: AbArm
+    harness: DaemonEvaluationHarness
+    secrets: readonly string[]
+    facadeCalls: PostCallRecord[]
+    subjectCleanup: () => void
+  }) {
+    this.topology = args.topology
+    this.world = args.world
+    this.arm = args.arm
+    this.harness = args.harness
+    this.secrets = args.secrets
+    this.facadeCalls = args.facadeCalls
+    this.subjectCleanup = args.subjectCleanup
+  }
+
+  static async start(options: AbFixtureOptions): Promise<AbFixture> {
+    const topology = compileTopology(abManifest(options.seed))
+    const world = new ArenaWorld(topology)
+    // Production shared-channel convention: activation needs a mention or
+    // thread affinity — the same rung the scenarios' correct calls rely on.
+    const base = world.buildEnvironment({ bindMatch: 'mention' })
+    const facadeCalls: PostCallRecord[] = []
+    const environment =
+      options.arm === 'B'
+        ? {
+            ...base,
+            tools: [postFacadeTool({ onCall: (record) => facadeCalls.push(record) })],
+            hideProductTools: ['sendMessage'],
+            collaborationGuidance: {
+              collabAppend: POST_COLLAB_GUIDANCE,
+              parentReplyAppend: postParentReplyAppend
+            }
+          }
+        : base
+    const subject = prepareGameSubject(topology, options.subject)
+    try {
+      if (options.agentDescriptions) {
+        for (const [alias, description] of Object.entries(options.agentDescriptions)) {
+          const agent = topology.agents.find((candidate) => candidate.alias === alias)
+          if (!agent) throw new Error(`agentDescriptions names unknown alias "${alias}"`)
+          const path = join(subject.root, 'agents', agent.agentId, 'agent.json')
+          const record = JSON.parse(readFileSync(path, 'utf8')) as Record<string, unknown>
+          writeFileSync(path, `${JSON.stringify({ ...record, description }, null, 2)}\n`)
+        }
+      }
+      if (options.subject.kind === 'real') await preflightRealSubject(subject.root)
+    } catch (error) {
+      subject.cleanup()
+      throw error
+    }
+    const harness = new DaemonEvaluationHarness({
+      root: subject.root,
+      environment,
+      runId: `ab-${options.seed}-${options.arm}`,
+      capabilityProfile: { memory: 'off' },
+      secrets: subject.secrets,
+      ...(options.hostFactory ? { hostFactory: options.hostFactory } : {})
+    })
+    const fixture = new AbFixture({
+      topology,
+      world,
+      arm: options.arm,
+      harness,
+      secrets: subject.secrets,
+      facadeCalls,
+      subjectCleanup: subject.cleanup
+    })
+    world.onDelivered((effect) => fixture.echoDeliveredPost(effect))
+    await harness.start()
+    return fixture
+  }
+
+  room(alias: string): CompiledRoom {
+    const room = this.topology.rooms.find((candidate) => candidate.alias === alias)
+    if (!room) throw new Error(`unknown room alias "${alias}"`)
+    return room
+  }
+
+  agentId(alias: string): string {
+    const agent = this.topology.agents.find((candidate) => candidate.alias === alias)
+    if (!agent) throw new Error(`unknown agent alias "${alias}"`)
+    return agent.agentId
+  }
+
+  botUserId(alias: string): string {
+    const integration = this.topology.integrations.find((candidate) => candidate.agentAlias === alias)
+    if (!integration) throw new Error(`unknown agent alias "${alias}"`)
+    return integration.botUserId
+  }
+
+  /** Production Slack echo, generalized to every room: a delivered agent post
+   *  fans back to the OTHER member integrations of that room as real platform
+   *  ingress. Whether an echo activates anyone stays the daemon's decision. */
+  private echoDeliveredPost(effect: RecordedOutboundEffect): void {
+    if (effect.status !== 'delivered' || effect.agentId === undefined) return
+    if (effect.kind !== 'reply' && effect.kind !== 'finalize') return
+    const room = this.topology.rooms.find((candidate) => candidate.channel === effect.channel)
+    if (!room) return
+    const botUserId = this.world.botUserIdFor(effect.integrationId)
+    if (botUserId === undefined || effect.messageId === undefined) return
+    const appId = this.world.botAppIdFor(effect.integrationId)
+    let thread: string
+    let ingressEventTag: string | undefined
+    if (effect.kind === 'reply') {
+      thread = effect.thread ?? effect.messageId
+      this.threadByMessageId.set(effect.messageId, thread)
+    } else {
+      thread = this.threadByMessageId.get(effect.messageId) ?? effect.thread ?? effect.messageId
+      ingressEventTag = SLACK_RESPONSE_FINAL_EVENT_TAG
+    }
+    const mentions = [...effect.text.matchAll(/<@([A-Z0-9]+)>/g)].map((match) => match[1]!)
+    const authorAgentId = effect.identity?.agentAuthorId ?? effect.agentId
+    const claim =
+      effect.response !== undefined
+        ? {
+            authorAgentId,
+            responseId: effect.response.responseId,
+            deliveryState: effect.response.deliveryState,
+            hopCount: effect.response.hopCount,
+            mentionedAgentIds: effect.response.mentionedAgentIds,
+            ...(effect.response.agentCallDeliveryId !== undefined
+              ? { agentCallDeliveryId: effect.response.agentCallDeliveryId }
+              : {})
+          }
+        : undefined
+    for (const integrationId of room.memberIntegrationIds) {
+      if (integrationId === effect.integrationId) continue
+      this.echoHandles.push(
+        this.harness.inject({
+          integrationId,
+          payload: {
+            channel: room.channel,
+            thread,
+            messageId: effect.messageId,
+            ...(ingressEventTag !== undefined ? { ingressEventTag } : {}),
+            text: effect.text,
+            sender: { id: botUserId, isBot: true, ...(appId !== undefined ? { appId } : {}) },
+            ...(mentions.length > 0 ? { mentions } : {}),
+            ...(claim !== undefined ? { agentAuthorship: claim } : {})
+          }
+        })
+      )
+    }
+  }
+
+  /** Inject one HUMAN platform message into a room, fanned to every member
+   *  integration (the same channel:ts each dedicated Slack app receives). */
+  injectHuman(
+    roomAlias: string,
+    text: string,
+    options: { mentions?: string[]; sender?: string } = {}
+  ): { messageId: string; handles: DeliveryHandle[] } {
+    const room = this.room(roomAlias)
+    const messageId = this.world.mintMessageId('slack')
+    this.world.registerRoomMessage(room.channel, messageId)
+    this.world.recordThreadMessage(room.channel, messageId, {
+      ts: messageId,
+      text,
+      sender: options.sender ?? 'W-HUMAN',
+      isBot: false
+    })
+    const handles = room.memberIntegrationIds.map((integrationId) =>
+      this.harness.inject({
+        integrationId,
+        payload: {
+          channel: room.channel,
+          thread: messageId,
+          messageId,
+          text,
+          sender: { id: options.sender ?? 'W-HUMAN', isBot: false },
+          ...(options.mentions !== undefined ? { mentions: options.mentions } : {})
+        }
+      })
+    )
+    return { messageId, handles }
+  }
+
+  /** Settle everything in flight: injected handles, echo cascades generation by
+   *  generation, then daemon idleness (which covers agent-call child turns). */
+  async settle(handles: DeliveryHandle[] = [], timeoutMs = 120_000): Promise<void> {
+    const deadline = Date.now() + timeoutMs
+    const remaining = () => Math.max(1, deadline - Date.now())
+    let pending = [...handles, ...this.echoHandles.splice(0)]
+    let generations = 0
+    while (pending.length > 0 && generations < 32) {
+      generations += 1
+      await Promise.all(pending.map((handle) => handle.completion))
+      pending = this.echoHandles.splice(0)
+    }
+    await this.harness.waitUntilIdle(remaining())
+    pending = this.echoHandles.splice(0)
+    while (pending.length > 0 && generations < 32) {
+      generations += 1
+      await Promise.all(pending.map((handle) => handle.completion))
+      await this.harness.waitUntilIdle(remaining())
+      pending = this.echoHandles.splice(0)
+    }
+  }
+
+  events(): readonly EvaluationEvent[] {
+    return this.harness.events()
+  }
+
+  /** The event subset attributable to ONE agent — what scopes the A/B metrics
+   *  to the subject instead of averaging the peer's turns into them. */
+  eventsOf(agentAlias: string): EvaluationEvent[] {
+    const agentId = this.agentId(agentAlias)
+    return this.events().filter((event) => event.agentId === agentId)
+  }
+
+  eventCollector() {
+    return this.harness.eventCollector()
+  }
+
+  async stop(): Promise<void> {
+    try {
+      await this.harness.stop()
+    } finally {
+      this.subjectCleanup()
+    }
+  }
+}
diff --git a/evals/test/tool-surface-ab-fixture.test.ts b/evals/test/tool-surface-ab-fixture.test.ts
new file mode 100644
index 000000000..cc8636eaf
--- /dev/null
+++ b/evals/test/tool-surface-ab-fixture.test.ts
@@ -0,0 +1,197 @@
+/**
+ * A/B fixture contract, credential-free: each arm must present EXACTLY its own
+ * surface — descriptors and prompt alike — while both execute through the one
+ * product implementation. This is the fairness precondition of the behavioral
+ * A/B: if arm B's sessions still listed `sendMessage`, or its prompt still
+ * taught `sendMessage` call shapes, a behavioral difference would measure a
+ * mixed surface rather than the design under test.
+ *
+ * Scripted hosts, real daemon: the assertions read the daemon's own control
+ * socket (listTools), the session's actual prompt text, and the world's
+ * delivered effects — never fixture internals.
+ */
+import { afterEach, describe, expect, it } from 'vitest'
+import { callDaemonTool, daemonMcpBinding, listDaemonTools, type DaemonMcpBinding } from '../games/mcp-client.js'
+import { AbFixture } from '../games/tool-surface-ab-fixture.js'
+
+let fixture: AbFixture | undefined
+
+afterEach(async () => {
+  await fixture?.stop()
+  fixture = undefined
+})
+
+interface CapturedSession {
+  agentAlias: string
+  prompts: string[]
+  binding?: DaemonMcpBinding
+}
+
+/** Scripted host seam that records every prompt and the session's MCP binding,
+ *  and runs an optional per-turn script with real daemon-tool access. */
+function capturingHostFactory(
+  fixtureAliasOf: (agentId: string) => string,
+  captured: CapturedSession[],
+  script?: (context: {
+    agentAlias: string
+    text: string
+    callTool: (name: string, args: Record<string, unknown>) => ReturnType<typeof callDaemonTool>
+    reply: (text: string) => void
+  }) => Promise<void> | void
+) {
+  return ((agent: { id: string }, onUpdate: (sessionId: string, update: unknown) => void) => {
+    let sessions = 0
+    const byId = new Map<string, CapturedSession>()
+    return {
+      start: async () => {},
+      newSession: async (_cwd: string, mcpServers?: unknown) => {
+        const sessionId = `ab-${agent.id.slice(0, 8)}-${(sessions += 1)}`
+        const record: CapturedSession = { agentAlias: fixtureAliasOf(agent.id), prompts: [] }
+        const binding = daemonMcpBinding(mcpServers)
+        if (binding) record.binding = binding
+        byId.set(sessionId, record)
+        captured.push(record)
+        return sessionId
+      },
+      hasSession: () => true,
+      modelOptions: () => ({ current: 'scripted-ab', models: ['scripted-ab'] }),
+      prompt: async (sessionId: string, blocks: { text?: string }[]) => {
+        const record = byId.get(sessionId)!
+        const text = blocks.map((block) => block.text ?? '').join('\n')
+        record.prompts.push(text)
+        let replied = false
+        const reply = (value: string) => {
+          replied = true
+          onUpdate(sessionId, { sessionUpdate: 'agent_message_chunk', content: { type: 'text', text: value } })
+        }
+        if (script) {
+          await script({
+            agentAlias: record.agentAlias,
+            text,
+            callTool: (name, args) => {
+              if (!record.binding) throw new Error('session has no daemon tool binding')
+              return callDaemonTool(record.binding, name, args)
+            },
+            reply
+          })
+        }
+        if (!replied) {
+          onUpdate(sessionId, { sessionUpdate: 'agent_message_chunk', content: { type: 'text', text: 'ok' } })
+        }
+        return { stopReason: 'end_turn' }
+      },
+      cancel: async () => {},
+      stop: async () => {}
+    }
+  }) as never
+}
+
+async function startArm(
+  arm: 'A' | 'B',
+  captured: CapturedSession[],
+  script?: Parameters<typeof capturingHostFactory>[2]
+) {
+  let aliasOf: (agentId: string) => string = () => 'unknown'
+  const started = await AbFixture.start({
+    seed: 4242,
+    arm,
+    subject: { kind: 'scripted' },
+    hostFactory: capturingHostFactory((agentId) => aliasOf(agentId), captured, script)
+  })
+  aliasOf = (agentId) => started.topology.agents.find((agent) => agent.agentId === agentId)?.alias ?? agentId
+  return started
+}
+
+describe('tool-surface A/B fixture — each arm presents exactly one surface', () => {
+  it('arm A: the session carries `sendMessage` and the production guidance, and no `post`', async () => {
+    const captured: CapturedSession[] = []
+    fixture = await startArm('A', captured)
+    const kick = fixture.injectHuman('briefing', `<@${fixture.botUserId('runner')}> hello`, {
+      mentions: [fixture.botUserId('runner')]
+    })
+    await fixture.settle(kick.handles)
+    const runner = captured.find((session) => session.agentAlias === 'runner')
+    expect(runner, 'runner session was created').toBeDefined()
+    expect(runner!.binding, 'runner session has daemon tools').toBeDefined()
+    const tools = await listDaemonTools(runner!.binding!)
+    expect(tools).toContain('sendMessage')
+    expect(tools).not.toContain('post')
+    const prompt = runner!.prompts.join('\n')
+    expect(prompt).toContain('# Collaborating with other agents')
+    expect(prompt).toContain('sendMessage')
+    expect(prompt).not.toContain('"kind":"channel"')
+  })
+
+  it('arm B: `post` replaces `sendMessage` in descriptors AND in the prompt, and still executes through the product', async () => {
+    const captured: CapturedSession[] = []
+    let listed: string[] = []
+    let postResult: Awaited<ReturnType<typeof callDaemonTool>> | undefined
+    const plazaChannel = () => fixture!.room('plaza').channel
+    fixture = await startArm('B', captured, async (context) => {
+      if (context.agentAlias !== 'runner' || postResult !== undefined) return
+      const runner = captured.find((session) => session.agentAlias === 'runner')!
+      listed = await listDaemonTools(runner.binding!)
+      postResult = await context.callTool('post', {
+        conversation: { kind: 'channel', channel: plazaChannel() },
+        message: 'deploy finished'
+      })
+      context.reply('posted')
+    })
+    const kick = fixture.injectHuman('briefing', `<@${fixture.botUserId('runner')}> hello`, {
+      mentions: [fixture.botUserId('runner')]
+    })
+    await fixture.settle(kick.handles)
+
+    // The descriptor list: one surface, not two.
+    expect(listed).toContain('post')
+    expect(listed).not.toContain('sendMessage')
+    // The prompt: arm B's guidance, no `sendMessage` teaching anywhere.
+    const runner = captured.find((session) => session.agentAlias === 'runner')!
+    const prompt = runner.prompts.join('\n')
+    expect(prompt).toContain('# Collaborating with other agents')
+    expect(prompt).toContain('`post`')
+    expect(prompt).not.toContain('sendMessage')
+    // The execution: the façade compiled and the PRODUCT delivered a real,
+    // world-authorized channel post (§7.2 path, not a shortcut).
+    expect(postResult?.ok, JSON.stringify(postResult)).toBe(true)
+    expect(fixture.facadeCalls).toEqual([expect.objectContaining({ outcome: 'compiled', form: 'channel-bare' })])
+    const delivered = fixture.world
+      .allEffects()
+      .filter((effect) => effect.status === 'delivered' && effect.channel === fixture!.room('plaza').channel)
+    expect(delivered.some((effect) => effect.text.includes('deploy finished'))).toBe(true)
+  })
+
+  it("arm B: a parent-session wake's report-back append teaches the `post` parent form", async () => {
+    const captured: CapturedSession[] = []
+    let wakeResult: Awaited<ReturnType<typeof callDaemonTool>> | undefined
+    fixture = await startArm('B', captured, async (context) => {
+      if (context.agentAlias === 'runner' && wakeResult === undefined) {
+        wakeResult = await context.callTool('post', {
+          conversation: { kind: 'private' },
+          address: [fixture!.agentId('peer')],
+          visibility: 'session-only',
+          expectReply: true,
+          message: 'what is your status?'
+        })
+        context.reply('asked')
+      }
+    })
+    const kick = fixture.injectHuman('briefing', `<@${fixture.botUserId('runner')}> hello`, {
+      mentions: [fixture.botUserId('runner')]
+    })
+    await fixture.settle(kick.handles)
+    expect(wakeResult?.ok, JSON.stringify(wakeResult)).toBe(true)
+    const peer = captured.find((session) => session.agentAlias === 'peer')
+    expect(peer, 'the postless wake created a peer session').toBeDefined()
+    const prompt = peer!.prompts.join('\n')
+    expect(prompt).toContain('# Reporting back to your parent session')
+    expect(prompt).toContain('"kind":"parent"')
+    expect(prompt).not.toContain('sendMessage')
+    // And nothing about the EXCHANGE reached any channel: the runner's ordinary
+    // turn reply in its own briefing thread is legitimate visible speech, but
+    // the postless wake and its question must have no platform projection.
+    const delivered = fixture.world.allEffects().filter((effect) => effect.status === 'delivered')
+    expect(delivered.filter((effect) => effect.channel !== fixture!.room('briefing').channel)).toEqual([])
+    expect(delivered.some((effect) => effect.text.includes('what is your status?'))).toBe(false)
+  })
+})

From aef812c527344d44482816d845ce25248e27b6d7 Mon Sep 17 00:00:00 2001
From: Poytr1 <pooytr1@gmail.com>
Date: Sun, 9 Aug 2026 11:21:54 +0800
Subject: [PATCH 05/15] =?UTF-8?q?feat(evals):=20the=20behavioral=20A/B=20d?=
 =?UTF-8?q?river=20=E2=80=94=204=20scenarios=20x=202=20arms=20x=20N=20tria?=
 =?UTF-8?q?ls,=20daemon-judged?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

evals/test/tool-surface-ab-real.test.ts runs the pre-registered matrix
against a real ACP subject template (env-gated, never in a CI gate). Every
score comes from the daemon's own records: an attempt satisfies a scenario
only when its EXECUTED product form (arm B scored on what its call compiled
to) names the right target ids, AND the world's effects show the intended
delivery — a delivered post in the target channel, the addressed agent's
real activation, the postless ask leaking nowhere, the parent actually
woken by the child's answer.

Scenario 4's parent session is real: the peer is told to delegate the
quoted request with an answer-back obligation, and a trial where the peer
never delegates is recorded invalid, not scored against either surface.
Provider failures and timeouts likewise invalidate the trial rather than
an arm. Token consumption is read from turn.completed usage, scoped to the
subject and reported for the whole run, with per-component sums. Arm order
is counterbalanced per (scenario, trial); artifacts (events.jsonl,
world-events.jsonl, trial.json, summary.json) are written per run,
redacted with the subject template's secret set.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 evals/test/tool-surface-ab-real.test.ts | 422 ++++++++++++++++++++++++
 1 file changed, 422 insertions(+)
 create mode 100644 evals/test/tool-surface-ab-real.test.ts

diff --git a/evals/test/tool-surface-ab-real.test.ts b/evals/test/tool-surface-ab-real.test.ts
new file mode 100644
index 000000000..7949bebd4
--- /dev/null
+++ b/evals/test/tool-surface-ab-real.test.ts
@@ -0,0 +1,422 @@
+/**
+ * Tool-surface A/B — the behavioral half, against a real ACP runtime.
+ *
+ * The credential-free half (`tool-surface-ab.test.ts`, `post-facade.test.ts`,
+ * `tool-surface-ab-fixture.test.ts`, all in the CI gates) pins the apparatus:
+ * the façade's compilation, the shared classifier, and the arm-parity
+ * preconditions. This file runs the pre-registered 4×2×3 matrix — four send
+ * scenarios, two surfaces, three trials — and is deliberately NOT in any CI
+ * gate: it needs a real runtime and provider credentials, and a model result
+ * is a rate over trials, never a single pass/fail (collaboration-arena.md §8.1).
+ *
+ * Success is judged from the DAEMON's own records, never the model's claims:
+ * the executed product-form of each attempt (arm B scored on what its call
+ * COMPILED to), the world's delivered/rejected effects, and the peer's actual
+ * activations. Tokens come from the daemon's `turn.completed` usage events,
+ * scoped to the subject agent and also reported for the whole run.
+ *
+ * Pre-registered expectations (held to in the write-up): a clear arm-B win on
+ * static descriptor cost and on invalid-call rate — the latter partly BY
+ * CONSTRUCTION, since arm B cannot even express most illegal combinations —
+ * and little or no difference on success or efficiency. n=3 per cell screens
+ * for large effects only.
+ *
+ * Run:
+ *   pnpm --filter @agentconnect.md/daemon build
+ *   export AGENTCONNECT_DAEMON_ENTRY="$PWD/packages/daemon/dist/index.js"
+ *   export AGENTCONNECT_EVAL_SUBJECT_ROOT=/absolute/path/to/subject
+ *   export AGENTCONNECT_EVAL_GAME_TEMPLATE_AGENTS=<template-agent-id>
+ *   npx vitest run evals/test/tool-surface-ab-real.test.ts
+ *
+ * Optional: AGENTCONNECT_EVAL_TRIALS (default 3), AGENTCONNECT_EVAL_AB_SCENARIOS
+ * / AGENTCONNECT_EVAL_AB_ARMS (csv filters), AGENTCONNECT_EVAL_TRIAL_BUDGET_MS.
+ */
+import { mkdirSync } from 'node:fs'
+import { join } from 'node:path'
+import { afterAll, afterEach, describe, expect, it } from 'vitest'
+import { atomicWrite, redactEvaluationValue } from '../../packages/daemon/src/evaluation/index.js'
+import { compilePost } from '../games/post-facade.js'
+import {
+  AB_SCENARIOS,
+  classifyPostForm,
+  extractTrialMetrics,
+  type AbScenario,
+  type AbTrialMetrics,
+  type SendForm
+} from '../games/tool-surface-ab.js'
+import { AbFixture, type AbArm } from '../games/tool-surface-ab-fixture.js'
+
+const subjectRoot = process.env.AGENTCONNECT_EVAL_SUBJECT_ROOT?.trim()
+const templateAgents = (process.env.AGENTCONNECT_EVAL_GAME_TEMPLATE_AGENTS ?? '')
+  .split(',')
+  .map((entry) => entry.trim())
+  .filter(Boolean)
+const configured = Boolean(subjectRoot) && templateAgents.length > 0
+const TRIALS = Number(process.env.AGENTCONNECT_EVAL_TRIALS ?? '3')
+const TRIAL_BUDGET_MS = Number(process.env.AGENTCONNECT_EVAL_TRIAL_BUDGET_MS ?? '420000')
+const ARTIFACT_DIR = join(process.cwd(), '.artifacts', 'evaluation', 'tool-surface-ab')
+const scenarioFilter = (process.env.AGENTCONNECT_EVAL_AB_SCENARIOS ?? '')
+  .split(',')
+  .map((entry) => entry.trim())
+  .filter(Boolean)
+const armFilter = (process.env.AGENTCONNECT_EVAL_AB_ARMS ?? '')
+  .split(',')
+  .map((entry) => entry.trim())
+  .filter(Boolean) as AbArm[]
+
+const scenarios = AB_SCENARIOS.filter((scenario) => scenarioFilter.length === 0 || scenarioFilter.includes(scenario.id))
+const arms: AbArm[] = (['A', 'B'] as const).filter((arm) => armFilter.length === 0 || armFilter.includes(arm))
+
+interface AbRunRecord {
+  scenario: string
+  arm: AbArm
+  trial: number
+  seed: number
+  /** 'ok' — a scoreable trial; 'invalid' — infra/peer failure, measured nothing. */
+  status: 'ok' | 'invalid'
+  invalidReason?: string
+  /** Overall: some attempt executed the expected product form at the right
+   *  target AND the daemon's effects show the intended delivery. */
+  success: boolean
+  /** The FIRST attempt already satisfied the full check (no retry loop). */
+  firstAttemptSuccess: boolean
+  attemptsToSuccess: number
+  toolCalls: number
+  invalidCalls: number
+  /** Sub-flag for the ask scenarios: was the answer-obligation flag set? */
+  expectReplySet?: boolean
+  subjectTokens: { total: number; input: number; output: number; cacheRead: number; cacheWrite: number }
+  runTokens: { total: number; input: number; output: number; cacheRead: number; cacheWrite: number }
+  subjectTurns: number
+  runTurns: number
+  latencyMs: number
+  /** Verbatim attempts on the subject surface, plus any call the model tried
+   *  to make on the OTHER arm's surface — the qualitative misuse evidence. */
+  attempts: unknown[]
+  crossSurfaceAttempts: unknown[]
+  notes: string[]
+}
+
+let fixture: AbFixture | undefined
+const results: AbRunRecord[] = []
+
+afterEach(async () => {
+  await fixture?.stop()
+  fixture = undefined
+})
+
+/** The product-level args an attempt EXECUTED as: arm A's raw input, or what
+ *  the arm-B façade compiled its input into. */
+function productArgs(arm: AbArm, args: Record<string, unknown> | undefined): Record<string, unknown> | undefined {
+  if (!args) return undefined
+  if (arm === 'A') return args
+  try {
+    return compilePost(args).args
+  } catch {
+    return undefined
+  }
+}
+
+function toAgentIdOf(args: Record<string, unknown> | undefined): string | undefined {
+  const target = args?.toAgent
+  if (typeof target === 'string') return target
+  if (target && typeof target === 'object') return (target as { agentId?: string }).agentId
+  return undefined
+}
+
+function needsReplyOf(args: Record<string, unknown> | undefined): boolean {
+  const target = args?.toAgent
+  return typeof target === 'object' && target !== null && (target as { needsReply?: unknown }).needsReply === true
+}
+
+/** Does this attempt fully satisfy the scenario — right form AND right ids? */
+function attemptSatisfies(
+  scenario: AbScenario,
+  arm: AbArm,
+  attempt: { form: SendForm; failed: boolean; args?: Record<string, unknown> },
+  ids: { peerAgentId: string; channel: string }
+): boolean {
+  if (attempt.failed || attempt.form !== scenario.expected) return false
+  const args = productArgs(arm, attempt.args)
+  if (!args) return false
+  switch (scenario.expected) {
+    case 'agent-channel':
+      return args.channel === ids.channel && toAgentIdOf(args) === ids.peerAgentId
+    case 'channel-bare':
+      return args.channel === ids.channel
+    case 'agent-postless':
+      return toAgentIdOf(args) === ids.peerAgentId && args.channel === undefined
+    case 'parent-session':
+      return typeof args.sessionId === 'string' && args.sessionId.length > 0
+    default:
+      return false
+  }
+}
+
+async function runTrial(scenario: AbScenario, arm: AbArm, trial: number): Promise<AbRunRecord> {
+  const scenarioIndex = AB_SCENARIOS.findIndex((candidate) => candidate.id === scenario.id)
+  const seed = 5000 + scenarioIndex * 100 + trial
+  fixture = await AbFixture.start({
+    seed,
+    arm,
+    subject: { kind: 'real', subjectRoot: subjectRoot!, templateAgentIds: templateAgents }
+  })
+  const runnerId = fixture.agentId('runner')
+  const peerId = fixture.agentId('peer')
+  const plaza = fixture.room('plaza')
+  const ids = { peerAgentId: peerId, channel: plaza.channel, humanUserId: 'W-HUMAN' }
+  const instruction = scenario.instruction(ids)
+  const notes: string[] = []
+
+  const kickoff = scenario.needsCaller
+    ? // Scenario 4: a real parent session. The PEER is told to delegate the
+      // quoted request to the subject and to require the answer back; the
+      // subject's scored behavior is what its child session then does.
+      fixture.injectHuman(
+        'peer-briefing',
+        `<@${fixture.botUserId('peer')}> Ask agent ${runnerId} for help with a small task. You must require ` +
+          `that its answer comes back to you — not fire-and-forget — and you must pass the request through ` +
+          `word-for-word, exactly as quoted, adding nothing: "${instruction}"`,
+        { mentions: [fixture.botUserId('peer')] }
+      )
+    : fixture.injectHuman('briefing', `<@${fixture.botUserId('runner')}> ${instruction}`, {
+        mentions: [fixture.botUserId('runner')]
+      })
+  const startedAt = Date.now()
+  await fixture.settle(kickoff.handles, TRIAL_BUDGET_MS)
+  const latencyMs = Date.now() - startedAt
+
+  const runnerEvents = fixture.eventsOf('runner')
+  const allEvents = [...fixture.events()]
+  const toolName = arm === 'A' ? 'sendMessage' : 'post'
+  const asExtractorEvents = (events: { type: string; data: Record<string, unknown> }[]) => events
+  const metrics: AbTrialMetrics = extractTrialMetrics(asExtractorEvents(runnerEvents as never), {
+    toolName,
+    expected: scenario.expected,
+    latencyMs,
+    ...(arm === 'B'
+      ? { classify: (args: Record<string, unknown> | undefined) => classifyPostForm(compilePost, args) }
+      : {})
+  })
+  const runMetrics = extractTrialMetrics(asExtractorEvents(allEvents as never), {
+    toolName,
+    expected: scenario.expected,
+    latencyMs
+  })
+
+  // ── validity: infra failures measure nothing about the surface ──
+  const providerFailure = allEvents.some(
+    (event) =>
+      event.type === 'turn.timed_out' ||
+      (event.type === 'turn.failed' &&
+        (event.data.code === 'provider_auth_required' || event.data.code === 'provider_quota_exhausted'))
+  )
+  let invalidReason: string | undefined
+  if (providerFailure) invalidReason = 'provider failure or turn timeout'
+  if (scenario.needsCaller) {
+    const delegated = runnerEvents.some(
+      (event) => event.type === 'turn.started' && String(event.data.input ?? '').includes('sum of 17 and 25')
+    )
+    if (!delegated) invalidReason = 'the caller never delegated the request to the subject'
+  }
+
+  // ── the full success check: form + ids (per attempt), then daemon effects ──
+  const satisfying = metrics.attempts.map((attempt) => attemptSatisfies(scenario, arm, attempt, ids))
+  const successIndex = satisfying.findIndex(Boolean)
+  const deliveredInPlaza = fixture.world
+    .allEffects()
+    .some((effect) => effect.status === 'delivered' && effect.channel === plaza.channel && effect.agentId === runnerId)
+  const peerActivated = allEvents.some((event) => event.type === 'turn.started' && event.agentId === peerId)
+  const instructionLeakedToPlaza = fixture.world
+    .allEffects()
+    .some(
+      (effect) =>
+        effect.status === 'delivered' && effect.channel === plaza.channel && effect.text.includes('current status')
+    )
+  let effectsOk: boolean
+  switch (scenario.id) {
+    case 'agent-channel':
+      effectsOk = deliveredInPlaza && peerActivated
+      if (!deliveredInPlaza) notes.push('no delivered post by the subject in the target channel')
+      if (!peerActivated) notes.push('the addressed agent was never activated')
+      break
+    case 'channel-bare':
+      effectsOk = deliveredInPlaza
+      if (!deliveredInPlaza) notes.push('no delivered post by the subject in the target channel')
+      break
+    case 'agent-postless':
+      effectsOk = peerActivated && !instructionLeakedToPlaza
+      if (!peerActivated) notes.push('the asked agent was never activated')
+      if (instructionLeakedToPlaza) notes.push('the private ask leaked into the shared channel')
+      break
+    case 'parent-session': {
+      // The parent (peer) must actually be woken by the reply: a later peer
+      // turn whose input carries the answer.
+      const parentGotAnswer = allEvents.some(
+        (event) =>
+          event.type === 'turn.started' && event.agentId === peerId && String(event.data.input ?? '').includes('42')
+      )
+      effectsOk = parentGotAnswer
+      if (!parentGotAnswer) notes.push("the parent session never received the child's answer")
+      break
+    }
+    default:
+      effectsOk = false
+  }
+
+  const success = successIndex >= 0 && effectsOk
+  const firstAttemptSuccess = satisfying[0] === true && effectsOk
+
+  // Ask scenarios: was the answer-obligation flag set on the satisfying call?
+  let expectReplySet: boolean | undefined
+  if (scenario.id === 'agent-postless' && successIndex >= 0) {
+    expectReplySet = needsReplyOf(productArgs(arm, metrics.attempts[successIndex]!.args))
+  }
+
+  // Cross-surface attempts: the model reaching for the OTHER arm's tool.
+  const otherName = arm === 'A' ? 'post' : 'sendMessage'
+  const crossSurfaceAttempts = runnerEvents
+    .filter((event) => event.type === 'acp.update')
+    .map((event) => event.data.update as { sessionUpdate?: string; title?: string; rawInput?: unknown } | undefined)
+    .filter(
+      (update) =>
+        update?.sessionUpdate === 'tool_call' &&
+        typeof update.title === 'string' &&
+        update.title.toLowerCase().includes(otherName.toLowerCase())
+    )
+  if (crossSurfaceAttempts.length > 0) notes.push(`subject attempted the other arm's tool ${otherName}`)
+
+  const record: AbRunRecord = {
+    scenario: scenario.id,
+    arm,
+    trial,
+    seed,
+    status: invalidReason ? 'invalid' : 'ok',
+    ...(invalidReason ? { invalidReason } : {}),
+    success,
+    firstAttemptSuccess,
+    attemptsToSuccess: successIndex >= 0 ? successIndex + 1 : 0,
+    toolCalls: metrics.toolCalls,
+    invalidCalls: metrics.invalidCalls,
+    ...(expectReplySet !== undefined ? { expectReplySet } : {}),
+    subjectTokens: metrics.tokens,
+    runTokens: runMetrics.tokens,
+    subjectTurns: metrics.turns,
+    runTurns: runMetrics.turns,
+    latencyMs,
+    attempts: metrics.attempts as unknown[],
+    crossSurfaceAttempts: crossSurfaceAttempts as unknown[],
+    notes
+  }
+
+  // ── artifacts: the daemon's own evidence, redacted, one dir per run ──
+  const dir = join(ARTIFACT_DIR, `${scenario.id}-${arm}-${trial}`)
+  mkdirSync(dir, { recursive: true, mode: 0o700 })
+  fixture.eventCollector().writeJsonl(join(dir, 'events.jsonl'))
+  const secrets = fixture.secrets
+  atomicWrite(
+    join(dir, 'world-events.jsonl'),
+    fixture.world
+      .events()
+      .map((entry) => JSON.stringify(redactEvaluationValue(entry, secrets)))
+      .join('\n') + '\n'
+  )
+  atomicWrite(
+    join(dir, 'trial.json'),
+    `${JSON.stringify(
+      redactEvaluationValue(
+        {
+          record,
+          instruction,
+          facadeCalls: fixture.facadeCalls,
+          effects: fixture.world.allEffects().map((effect) => ({
+            status: effect.status,
+            kind: effect.kind,
+            channel: effect.channel,
+            agentId: effect.agentId,
+            ...(effect.reason !== undefined ? { reason: effect.reason } : {}),
+            text: effect.text
+          }))
+        },
+        secrets
+      ),
+      null,
+      2
+    )}\n`
+  )
+  return record
+}
+
+function aggregate(records: AbRunRecord[]) {
+  const cell = (scenario: string, arm: AbArm) => {
+    const rows = records.filter((row) => row.scenario === scenario && row.arm === arm && row.status === 'ok')
+    const sum = (select: (row: AbRunRecord) => number) => rows.reduce((total, row) => total + select(row), 0)
+    const mean = (select: (row: AbRunRecord) => number) => (rows.length === 0 ? 0 : sum(select) / rows.length)
+    return {
+      trials: rows.length,
+      success: rows.filter((row) => row.success).length,
+      firstAttempt: rows.filter((row) => row.firstAttemptSuccess).length,
+      invalidCalls: sum((row) => row.invalidCalls),
+      meanToolCalls: mean((row) => row.toolCalls),
+      meanSubjectTokensTotal: Math.round(mean((row) => row.subjectTokens.total)),
+      meanSubjectTokensInOut: Math.round(mean((row) => row.subjectTokens.input + row.subjectTokens.output)),
+      meanRunTokensTotal: Math.round(mean((row) => row.runTokens.total)),
+      meanRunTokensInOut: Math.round(mean((row) => row.runTokens.input + row.runTokens.output)),
+      meanLatencyMs: Math.round(mean((row) => row.latencyMs))
+    }
+  }
+  return {
+    generatedAt: new Date().toISOString(),
+    trialsPerCell: TRIALS,
+    cells: Object.fromEntries(
+      scenarios.flatMap((scenario) => arms.map((arm) => [`${scenario.id}/${arm}`, cell(scenario.id, arm)] as const))
+    ),
+    invalidTrials: records.filter((row) => row.status === 'invalid'),
+    records
+  }
+}
+
+afterAll(() => {
+  if (results.length === 0) return
+  mkdirSync(ARTIFACT_DIR, { recursive: true, mode: 0o700 })
+  const summary = aggregate(results)
+  atomicWrite(join(ARTIFACT_DIR, 'summary.json'), `${JSON.stringify(summary, null, 2)}\n`)
+  console.log(JSON.stringify(summary.cells, null, 2))
+})
+
+describe.skipIf(!configured)('tool-surface A/B against a real ACP runtime', () => {
+  for (const [scenarioIndex, scenario] of scenarios.entries()) {
+    for (let trial = 1; trial <= TRIALS; trial += 1) {
+      // Counterbalance arm order per (scenario, trial) so neither surface
+      // systematically runs first within a pair.
+      const ordered = (scenarioIndex + trial) % 2 === 0 ? [...arms] : [...arms].reverse()
+      for (const arm of ordered) {
+        it(
+          `${scenario.id} arm ${arm} trial ${trial}`,
+          async () => {
+            const record = await runTrial(scenario, arm, trial)
+            results.push(record)
+            // A model result is reported, never asserted; only an unusable
+            // trial (infra) is surfaced — and even that only as a soft note.
+            if (record.status === 'invalid') {
+              console.warn(`INVALID trial ${scenario.id}/${arm}/${trial}: ${record.invalidReason}`)
+            }
+            expect(true).toBe(true)
+          },
+          TRIAL_BUDGET_MS + 60_000
+        )
+      }
+    }
+  }
+
+  it('produced at least one scoreable trial per cell', () => {
+    for (const scenario of scenarios) {
+      for (const arm of arms) {
+        const ok = results.filter(
+          (row) => row.scenario === scenario.id && row.arm === arm && row.status === 'ok'
+        ).length
+        expect(ok, `${scenario.id}/${arm} has no scoreable trial`).toBeGreaterThan(0)
+      }
+    }
+  })
+})

From 62611a5424c0700404fbb344c45fd76c09d64ef0 Mon Sep 17 00:00:00 2001
From: Poytr1 <pooytr1@gmail.com>
Date: Sun, 9 Aug 2026 11:49:15 +0800
Subject: [PATCH 06/15] =?UTF-8?q?docs(design):=20the=20tool-surface=20A/B?=
 =?UTF-8?q?=20write-up=20=E2=80=94=20method,=20fidelity,=20measured=20stat?=
 =?UTF-8?q?ic=20costs?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

docs/designs/messaging-primitives-ab.md records the full pre-registered
protocol (matrix, scenarios, judging, validity), the fairness seams and
their contract coverage, the honesty constraints, and the measured static
costs: sendMessage 10,024 descriptor chars (~2,506 tokens) vs post 2,071
(~518) — 4.8x — and 12,742 vs 4,465 chars (2.9x) including the standing
guidance each arm's sessions actually carry (measured from real injected
prompts, logged by the fixture contract test).

The behavioral table's run protocol is in the doc; the 24-run matrix is
implemented and blocked only on re-authenticating the local Claude Code
CLI (its OAuth refresh token expired 2026-08-09 — the 2026-08-08 baseline
re-run caught its last hours). Linkage to the primitives proposal doc is
noted but deliberately not added while #551 is an open PR.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 docs/designs/messaging-primitives-ab.md    | 214 +++++++++++++++++++++
 evals/test/tool-surface-ab-fixture.test.ts |  16 ++
 2 files changed, 230 insertions(+)
 create mode 100644 docs/designs/messaging-primitives-ab.md

diff --git a/docs/designs/messaging-primitives-ab.md b/docs/designs/messaging-primitives-ab.md
new file mode 100644
index 000000000..700d90438
--- /dev/null
+++ b/docs/designs/messaging-primitives-ab.md
@@ -0,0 +1,214 @@
+# Tool-surface A/B: `sendMessage` (shipped) vs `post` (messaging primitives)
+
+Status: apparatus landed and contract-proven; static costs measured; the
+24-run behavioral matrix is implemented and one command from producing its
+table (see §6 — the run is currently blocked on local runtime credentials,
+not on anything in this repository).
+
+The question under test, verbatim from the request that started this work:
+**how much does the primitives design improve success rate and total token
+consumption?** The primitives design is the `post` write primitive of the
+messaging-primitives proposal (PR #551); the baseline is the `sendMessage`
+surface as shipped. When that proposal lands on `main`, its doc and this one
+should link to each other (deliberately not done while #551 is an open PR).
+
+## 1. The two arms
+
+| Arm | Surface the model carries                         | Implementation that executes |
+| --- | ------------------------------------------------- | ---------------------------- |
+| A   | product `sendMessage`, production guidance text   | product `sendMessage`        |
+| B   | `post` (3 orthogonal params), arm-B guidance text | product `sendMessage`        |
+
+Arm B is a **façade** (`evals/games/post-facade.ts`): every `post` call
+compiles into exactly one legal `sendMessage` input and is executed by the
+product tool on the same trusted session context. No routing, activation,
+addressing or policy code differs between arms — a measured difference is
+attributable to the _surface_, not to a second implementation.
+
+Three evaluation-only seams make the comparison fair, and each is
+contract-tested in the CI gates:
+
+- **`executeProductTool`** (`packages/daemon/src/mcp/control-server.ts`) — an
+  evaluation-registry tool may run a product tool on the caller's own trusted
+  `SessionContext`. Grants nothing the caller didn't have.
+- **`hideProductTools`** (`packages/daemon/src/daemon.ts`) — withhold named
+  product descriptors for a run so each arm presents exactly one surface for
+  the capability. A withheld tool stays fully executable, which is precisely
+  how the façade compiles down to it.
+- **`collaborationGuidance`** (`packages/daemon/src/session/session-manager.ts`)
+  — the prompt-side complement of `hideProductTools`. The standing
+  collaboration guidance and the parent-report append teach `sendMessage`
+  call shapes _by name_; without this seam, arm B would carry a system prompt
+  describing a tool it does not have — priming it with arm A's vocabulary and
+  telling it to call a tool absent from its list. Arm B's texts mirror the
+  production structure sentence-for-sentence outside the tool teaching
+  (pinned by tests in `evals/test/post-facade.test.ts`).
+
+`evals/test/tool-surface-ab-fixture.test.ts` proves the composed result
+against a real daemon: arm A sessions list `sendMessage` (no `post`) and
+carry the production guidance; arm B sessions list `post` (no `sendMessage`),
+their prompt contains no `sendMessage` text anywhere, a compiled call
+produces a real world-authorized delivery, and a `needsReply` wake installs
+the `post`-flavored parent-report append on the child.
+
+## 2. Method (pre-registered)
+
+**Matrix**: 4 scenarios × 2 arms × 3 trials = 24 runs, local Claude Code over
+ACP (`claude-acp`, model pinned `sonnet`, `permissionMode: default`, memory
+off), driven by `evals/test/tool-surface-ab-real.test.ts`. Arm order is
+counterbalanced per (scenario, trial); topology/ids are seed-deterministic
+and identical across the two arms of a pair.
+
+**Scenarios** (`evals/games/tool-surface-ab.ts`): each is one explicit send
+whose correct product form is known in advance, so every attempt classifies
+into one six-form vocabulary for both arms. The task text names the GOAL and
+never a tool, field, or form — a banned-vocabulary test enforces this (it
+already caught the word "conversation" priming arm B once). Scored forms:
+
+1. **agent-channel** — reach a specific agent visibly in a channel.
+2. **channel-bare** — post an announcement at a channel root, waking nobody.
+3. **agent-postless** — ask an agent privately, answer required back, no
+   platform trace.
+4. **parent-session** — woken by a real parent session (the peer agent is
+   instructed to delegate a quoted question with an answer-back obligation),
+   reply into that session.
+
+**Topology**: `briefing` (subject only — instructions arrive here, outside
+every measured room), `plaza` (subject + peer, the target channel),
+`peer-briefing` (peer only, scenario 4's kickoff). Production mention-gated
+routing; full platform-echo fidelity (`evals/games/tool-surface-ab-fixture.ts`).
+
+**Judging** — from the daemon's records, never the model's claims. An attempt
+satisfies a scenario only when its _executed_ product form (arm B is scored
+on what its call **compiled to**) names the right target ids, AND the world's
+effects show the intended delivery: a delivered post in the target channel, a
+real activation of the addressed agent, the postless ask leaking into no
+channel, the parent actually woken by the child's answer.
+
+**Metrics per run**: success; first-attempt success; attempts-to-success;
+tool calls on the subject surface; invalid/rejected calls; token consumption
+from `turn.completed` usage events (total and input/output/cache-read/
+cache-write components; cache traffic dominates local runs, so input+output
+is reported beside the raw total), scoped to the subject agent and also for
+the whole run; wall time; verbatim attempts plus any call on the _other_
+arm's tool (qualitative misuse evidence).
+
+**Validity**: provider failures and turn timeouts invalidate a trial (they
+measure infrastructure, not a surface). A scenario-4 trial where the peer
+never delegates is invalid — it conditioned on the caller, not the subject.
+
+## 3. Fidelity notes and limits
+
+- **A hidden tool is unlisted, not disabled.** `hideProductTools` removes the
+  descriptor; the daemon still executes the tool if called. A model cannot
+  normally call an unadvertised MCP tool, so in practice arm B cannot reach
+  `sendMessage` — but this is a property of the runtime's tool dispatch, not
+  a daemon-side ban, and the driver records any cross-surface attempt.
+- **Not expressible by either arm** and excluded: a fully-addressed
+  cross-room handoff into an existing THREAD (the routing rework removed
+  `thread` from every `sendMessage` target). Including it would measure a
+  known product gap, not the surfaces.
+- **Invalid-call rate is partly structural.** Arm B cannot even _express_
+  most of arm A's illegal combinations (its remaining illegal combos are
+  refused by the façade with named errors). A lower arm-B invalid rate is
+  therefore expected **by construction** and is only weak evidence of
+  comprehensibility.
+- **Scenario 4 uses a real caller.** The peer's delegation itself runs on the
+  arm's surface; its failures invalidate the trial rather than scoring
+  against either arm, and are reported.
+- **n = 3 per cell screens for large effects only.** A marginal difference is
+  noise and is reported as such; a null result is not proven equivalence.
+
+## 4. Static cost — measured, not estimated
+
+Measured from the real descriptors and from the guidance text a real daemon
+injected into a session prompt (`evals/test/post-facade.test.ts`,
+`evals/test/tool-surface-ab-fixture.test.ts`, on `main` @ `70d58cd1` +
+this branch; token figures are a chars/4 approximation and labelled as such):
+
+| Surface component                | Arm A (`sendMessage`) | Arm B (`post`) | Ratio    |
+| -------------------------------- | --------------------- | -------------- | -------- |
+| Tool description (chars)         | 2,617                 | 1,046          | 2.5×     |
+| Tool input schema (chars)        | 7,407                 | 1,025          | 7.2×     |
+| Descriptor total (chars)         | **10,024**            | **2,071**      | **4.8×** |
+| Descriptor (≈ tokens)            | ~2,506                | ~518           |          |
+| Standing guidance (chars)        | 2,718                 | 2,394          | 1.1×     |
+| **Combined per session (chars)** | **12,742**            | **4,465**      | **2.9×** |
+| Combined (≈ tokens)              | ~3,186                | ~1,116         |          |
+
+The descriptor is carried by **every turn** of every session; the guidance is
+standing session context. On a cache-warm local run most of this cost lands
+in cache reads rather than fresh input, which is why the behavioral table
+reports token components, not just totals.
+
+Literal JSON form templates enumerated by the description: arm A ≥ 6 (plus
+its illegal-combination rule table); arm B 4 conversation kinds with no rule
+table — the design claim is that the orthogonal split makes the rule table
+unnecessary rather than shorter.
+
+## 5. Pre-registration
+
+Expected before any behavioral run, held to afterwards:
+
+1. **Clear arm-B win on static cost** — confirmed above (4.8× descriptor,
+   2.9× combined).
+2. **Lower arm-B invalid-call rate** — expected partly by construction (§3).
+3. **Little or no difference in task success or efficiency** (tool calls,
+   tokens net of the static gap) — sonnet-class models handle either surface
+   in these single-send scenarios; the primitives' value case is the removal
+   of the illegal-combination space and the smaller carried surface, not a
+   success-rate jump on well-specified tasks.
+4. Anything beyond ±1 trial per cell on success, or a >2× token difference
+   net of cache, would _exceed_ this pre-registration and warrants scrutiny
+   (and more trials) before belief.
+
+## 6. Behavioral results
+
+**Run status (2026-08-09): blocked on local runtime credentials.** The
+harness itself is verified to the provider boundary: the smoke trial booted
+the real daemon, materialized the real subject, launched the local
+`claude-acp` adapter, created the ACP session, and dispatched the kickoff
+turn — which failed with `provider_auth_required` ("OAuth session expired
+and could not be refreshed"). The local Claude Code CLI's OAuth refresh
+token expired on 2026-08-09 (the 2026-08-08 arena baseline re-run caught its
+last hours of validity), and re-authentication is an interactive login only
+the operator can perform.
+
+To produce the table (after `claude /login` on the host):
+
+```bash
+pnpm --filter @agentconnect.md/daemon build
+export AGENTCONNECT_DAEMON_ENTRY="$PWD/packages/daemon/dist/index.js"
+export AGENTCONNECT_EVAL_SUBJECT_ROOT=/absolute/path/to/subject   # §4.1 template shape, runtime id claude-acp
+export AGENTCONNECT_EVAL_GAME_TEMPLATE_AGENTS=<template-agent-id>
+npx vitest run evals/test/tool-surface-ab-real.test.ts
+```
+
+Per-run artifacts land in `.artifacts/evaluation/tool-surface-ab/<scenario>-<arm>-<trial>/`
+(events.jsonl, world-events.jsonl, trial.json) with an aggregated
+`summary.json`; the table below is its rendering.
+
+| Scenario       | Arm | Success | First-attempt | Invalid calls | Mean tool calls | Mean subject tokens (total / in+out) | Mean wall time |
+| -------------- | --- | ------- | ------------- | ------------- | --------------- | ------------------------------------ | -------------- |
+| agent-channel  | A   | – /3    | – /3          | –             | –               | –                                    | –              |
+| agent-channel  | B   | – /3    | – /3          | –             | –               | –                                    | –              |
+| channel-bare   | A   | – /3    | – /3          | –             | –               | –                                    | –              |
+| channel-bare   | B   | – /3    | – /3          | –             | –               | –                                    | –              |
+| agent-postless | A   | – /3    | – /3          | –             | –               | –                                    | –              |
+| agent-postless | B   | – /3    | – /3          | –             | –               | –                                    | –              |
+| parent-session | A   | – /3    | – /3          | –             | –               | –                                    | –              |
+| parent-session | B   | – /3    | – /3          | –             | –               | –                                    | –              |
+
+## 7. Conclusion so far — the honest answer to the question
+
+On **token consumption**, the measurable half is already answered: the
+primitives surface is 4.8× smaller as a tool descriptor and 2.9× smaller
+including the standing guidance — roughly **2,000 fewer descriptor-tokens
+carried on every turn** of every session (mostly as cache traffic on a warm
+local run; as fresh input on cold sessions and non-caching deployments).
+
+On **success rate**, no behavioral claim is made yet: the pre-registered
+expectation is little or no difference at n=3 on these well-specified
+single-send tasks, with arm B's advantage expected in the invalid-call
+column — and partly by construction there. The 24-run matrix is implemented,
+contract-tested, and one credential refresh away from filling §6.
diff --git a/evals/test/tool-surface-ab-fixture.test.ts b/evals/test/tool-surface-ab-fixture.test.ts
index cc8636eaf..f20ba70c8 100644
--- a/evals/test/tool-surface-ab-fixture.test.ts
+++ b/evals/test/tool-surface-ab-fixture.test.ts
@@ -27,6 +27,16 @@ interface CapturedSession {
   binding?: DaemonMcpBinding
 }
 
+/** The collaboration-guidance section of a session prompt, measured from what
+ *  the daemon actually injected — the prompt-side static cost of a surface. */
+function guidanceSection(prompt: string): string {
+  const start = prompt.indexOf('# Collaborating with other agents')
+  if (start < 0) return ''
+  const rest = prompt.slice(start)
+  const next = rest.indexOf('\n# ', 1)
+  return next > 0 ? rest.slice(0, next) : rest
+}
+
 /** Scripted host seam that records every prompt and the session's MCP binding,
  *  and runs an optional per-turn script with real daemon-tool access. */
 function capturingHostFactory(
@@ -120,6 +130,10 @@ describe('tool-surface A/B fixture — each arm presents exactly one surface', (
     expect(prompt).toContain('# Collaborating with other agents')
     expect(prompt).toContain('sendMessage')
     expect(prompt).not.toContain('"kind":"channel"')
+    const section = guidanceSection(prompt)
+    console.log(
+      `guidance cost — arm A (sendMessage): ${section.length} chars (~${Math.round(section.length / 4)} tokens)`
+    )
   })
 
   it('arm B: `post` replaces `sendMessage` in descriptors AND in the prompt, and still executes through the product', async () => {
@@ -151,6 +165,8 @@ describe('tool-surface A/B fixture — each arm presents exactly one surface', (
     expect(prompt).toContain('# Collaborating with other agents')
     expect(prompt).toContain('`post`')
     expect(prompt).not.toContain('sendMessage')
+    const section = guidanceSection(prompt)
+    console.log(`guidance cost — arm B (post): ${section.length} chars (~${Math.round(section.length / 4)} tokens)`)
     // The execution: the façade compiled and the PRODUCT delivered a real,
     // world-authorized channel post (§7.2 path, not a shortcut).
     expect(postResult?.ok, JSON.stringify(postResult)).toBe(true)

From 5cde23e5e3a26e44602b012ed3b18973f357a942 Mon Sep 17 00:00:00 2001
From: Poytr1 <pooytr1@gmail.com>
Date: Sun, 9 Aug 2026 12:24:33 +0800
Subject: [PATCH 07/15] =?UTF-8?q?fix(evals):=20grant=20=C2=A76=20evaluatio?=
 =?UTF-8?q?n-registry=20tools=20the=20system-tool=20permission=20auto-allo?=
 =?UTF-8?q?w?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Measured in the tool-surface A/B's first live trial: arm B's correct
`post` call hung on an ACP permission request until the trial budget
burned — the daemon auto-allows its OWN MCP tools (ALL_TOOL_NAMES) but an
evaluation-registry tool served by the same trusted server fell through to
the interactive policy, and the arena has no human to tap the card. That
is a guaranteed stall AND a fairness bug: arm A's product tool was
auto-allowed while arm B's surface required an unanswerable approval.

The FQNs (mcp__agentconnect__<name> / mcp.agentconnect.<name>) are minted
where the registry is installed, and resolveAcpPermission grants them with
reason 'evaluation_game_tool' through the same fail-safe matcher rungs as
isBuiltinSystemTool (exported as matchesToolPermissionFqns, unit-tested).
Empty outside evaluation runs — production behavior unchanged.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 packages/daemon/src/daemon.ts                 | 40 +++++++++++++++++--
 .../test/daemon-permission-autoallow.test.ts  | 32 ++++++++++++++-
 2 files changed, 68 insertions(+), 4 deletions(-)

diff --git a/packages/daemon/src/daemon.ts b/packages/daemon/src/daemon.ts
index 0a9c367de..e8b761688 100644
--- a/packages/daemon/src/daemon.ts
+++ b/packages/daemon/src/daemon.ts
@@ -621,6 +621,18 @@ export function isBuiltinSystemTool(
   return ids.some((id) => typeof id === 'string' && containsBuiltinToolFqn(id))
 }
 
+/** Does a permission request name one of the given adapter-flattened MCP tool
+ *  identities? The §6 evaluation-registry grant uses this with the FQNs minted
+ *  at environment install — same rungs as {@link isBuiltinSystemTool}
+ *  (title / kind / toolCallId, id-suffixed variants included), same
+ *  fail-safe: no match ⇒ the interactive policy still applies. */
+export function matchesToolPermissionFqns(params: RequestPermissionRequest, fqns: ReadonlySet<string>): boolean {
+  if (fqns.size === 0) return false
+  const tc = params.toolCall
+  const ids = [tc?.title, tc?.kind, tc?.toolCallId]
+  return ids.some((id) => typeof id === 'string' && (fqns.has(id) || [...fqns].some((fqn) => id.includes(fqn))))
+}
+
 /** Codex ACP carries MCP approval through form elicitation when the client supports it. */
 export function isBuiltinSystemToolElicitation(
   params: CreateElicitationRequest,
@@ -1921,6 +1933,10 @@ export class Daemon {
    *  integration, but are EXCLUDED from physical platform reconcile so the daemon
    *  never opens (or evicts) a real connection for a virtual transport. */
   private evaluationIntegrationIds = new Set<string>()
+  /** ACP identities (`mcp__agentconnect__<name>` / `mcp.agentconnect.<name>`) of
+   *  the §6 evaluation-registry tools — populated at environment install and
+   *  granted the same auto-allow as the daemon's own system tools. */
+  private readonly evaluationToolPermissionFqns = new Set<string>()
   // agentId → the in-flight (or resolved) host-startup promise. Resolves to the
   // STARTED host (startHostWithRetry may build several across retries — the last,
   // successful one wins). `.has()` doubles as "is this agent starting / started?".
@@ -2277,6 +2293,16 @@ export class Daemon {
         if (productNames.has(name)) throw new Error(`evaluation tool "${name}" shadows a product tool`)
         if (seen.has(name)) throw new Error(`duplicate evaluation tool "${name}"`)
         seen.add(name)
+        // §6 game tools carry the same system-tool permission grant as the
+        // product tools they sit beside: they are served by the same trusted
+        // daemon MCP server, and a real ACP subject must be able to CALL a
+        // game action without a human approver in the loop — the arena has no
+        // one to tap a card, so an interactive prompt is a guaranteed hang
+        // (measured: the tool-surface A/B's arm-B calls burned their whole
+        // trial budget on an unanswerable approval while arm A's product tool
+        // was auto-allowed — a fairness bug, not just a stall).
+        this.evaluationToolPermissionFqns.add(`mcp__${RESERVED_MCP_SERVER_NAME}__${name}`)
+        this.evaluationToolPermissionFqns.add(`mcp.${RESERVED_MCP_SERVER_NAME}.${name}`)
       }
     }
     this.log.info(
@@ -15133,16 +15159,24 @@ export class Daemon {
     // have to approve them per call. Auto-allow without rendering a card. Non-system tools
     // (incl. the runtime's dangerous built-ins) fall through to the interactive policy below.
     const p = this.pending.get(pendingTurnKey(agentId, sessionId))
-    if (isBuiltinSystemTool(params, p?.builtinSystemToolCallIds)) {
+    // §6 evaluation-registry tools share the grant: same trusted MCP server, and
+    // the arena has no human to answer an interactive card (see the FQN-set
+    // population in installEvaluationEnvironment). Empty outside evaluation runs.
+    const grantReason = isBuiltinSystemTool(params, p?.builtinSystemToolCallIds)
+      ? 'agentconnect_system_tool'
+      : matchesToolPermissionFqns(params, this.evaluationToolPermissionFqns)
+        ? 'evaluation_game_tool'
+        : undefined
+    if (grantReason) {
       const allow = params.options.find((o) => o.kind === 'allow_always' || o.kind === 'allow_once')
       if (allow) {
-        this.permissionEvaluationDetails.set(evaluationParams, { reason: 'agentconnect_system_tool' })
+        this.permissionEvaluationDetails.set(evaluationParams, { reason: grantReason })
         this.emitEvaluation({
           type: 'permission.auto_allowed',
           agentId,
           sessionId,
           ...(p?.evaluationTurnId ? { turnId: p.evaluationTurnId } : {}),
-          data: { reason: 'agentconnect_system_tool', optionId: allow.optionId }
+          data: { reason: grantReason, optionId: allow.optionId }
         })
         return { outcome: { outcome: 'selected', optionId: allow.optionId } }
       }
diff --git a/packages/daemon/test/daemon-permission-autoallow.test.ts b/packages/daemon/test/daemon-permission-autoallow.test.ts
index 43d345a42..14dfa1d33 100644
--- a/packages/daemon/test/daemon-permission-autoallow.test.ts
+++ b/packages/daemon/test/daemon-permission-autoallow.test.ts
@@ -1,6 +1,12 @@
 import { describe, it, expect, vi } from 'vitest'
 import type { CreateElicitationRequest, RequestPermissionRequest } from '@agentclientprotocol/sdk'
-import { Daemon, noneSuppressedApprovalSurface, isBuiltinSystemTool, isBuiltinSystemToolCall } from '../src/daemon.js'
+import {
+  Daemon,
+  noneSuppressedApprovalSurface,
+  isBuiltinSystemTool,
+  isBuiltinSystemToolCall,
+  matchesToolPermissionFqns
+} from '../src/daemon.js'
 import { ALL_TOOL_NAMES } from '../src/mcp/tools.js'
 
 /**
@@ -300,3 +306,27 @@ describe('built-in MCP approvals use one policy on both ACP paths', () => {
     ).resolves.toBeUndefined()
   })
 })
+
+describe('matchesToolPermissionFqns — the §6 evaluation-registry grant predicate', () => {
+  // Why this grant exists: a real ACP subject must be able to CALL a game
+  // action tool without a human approver — the arena has no one to tap a
+  // card, so an interactive prompt is a guaranteed hang. Measured in the
+  // tool-surface A/B: arm B's `post` calls burned their whole trial budget
+  // on an unanswerable approval while arm A's product tool was auto-allowed,
+  // which is a fairness bug on top of a stall.
+  const fqns = new Set(['mcp__agentconnect__post', 'mcp.agentconnect.post'])
+
+  it('matches the evaluation tool FQN wherever the runtime puts it', () => {
+    expect(matchesToolPermissionFqns(req({ title: 'mcp__agentconnect__post' }), fqns)).toBe(true)
+    expect(matchesToolPermissionFqns(req({ kind: 'mcp__agentconnect__post' }), fqns)).toBe(true)
+    expect(matchesToolPermissionFqns(req({ toolCallId: 'mcp__agentconnect__post-7' }), fqns)).toBe(true)
+    expect(matchesToolPermissionFqns(req({ title: 'mcp.agentconnect.post' }), fqns)).toBe(true)
+  })
+
+  it('fail-safe: unknown identities and empty registries still card', () => {
+    expect(matchesToolPermissionFqns(req({ title: 'mcp__agentconnect__vote' }), fqns)).toBe(false)
+    expect(matchesToolPermissionFqns(req({ title: 'mcp__othersrv__post' }), fqns)).toBe(false)
+    expect(matchesToolPermissionFqns(req({ title: 'Bash' }), fqns)).toBe(false)
+    expect(matchesToolPermissionFqns(req({ title: 'mcp__agentconnect__post' }), new Set())).toBe(false)
+  })
+})

From d0d2983008ad75752a28e4dec2bce0fd59f2ec25 Mon Sep 17 00:00:00 2001
From: Poytr1 <pooytr1@gmail.com>
Date: Sun, 9 Aug 2026 17:17:28 +0800
Subject: [PATCH 08/15] fix(evals): any failed or timed-out turn invalidates an
 A/B trial
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Measured: a subscription session limit surfaces as a generic turn_failed
RequestError plus an apologetic delivered reply — the old provider-code
allowlist scored those runs as ordinary behavioral trials with zero
attempts. This experiment is a single explicit send, so unlike a long
arena game it can never legitimately absorb a failed turn: invalidate on
any turn.failed / turn.timed_out and carry the code in the reason.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 evals/test/tool-surface-ab-real.test.ts | 19 ++++++++++++-------
 1 file changed, 12 insertions(+), 7 deletions(-)

diff --git a/evals/test/tool-surface-ab-real.test.ts b/evals/test/tool-surface-ab-real.test.ts
index 7949bebd4..8a06a5421 100644
--- a/evals/test/tool-surface-ab-real.test.ts
+++ b/evals/test/tool-surface-ab-real.test.ts
@@ -205,14 +205,19 @@ async function runTrial(scenario: AbScenario, arm: AbArm, trial: number): Promis
   })
 
   // ── validity: infra failures measure nothing about the surface ──
-  const providerFailure = allEvents.some(
-    (event) =>
-      event.type === 'turn.timed_out' ||
-      (event.type === 'turn.failed' &&
-        (event.data.code === 'provider_auth_required' || event.data.code === 'provider_quota_exhausted'))
-  )
+  // ANY failed or timed-out turn invalidates: unlike a long arena game that can
+  // absorb one failed turn and still complete, this experiment is a single
+  // explicit send — a failed turn always poisons the measurement. Measured
+  // examples that must not score as behavior: an expired provider OAuth
+  // (provider_auth_required) and a subscription session limit, which surfaces
+  // as a generic turn_failed RequestError plus an apologetic delivered reply.
+  const failedTurn = allEvents.find((event) => event.type === 'turn.failed' || event.type === 'turn.timed_out')
   let invalidReason: string | undefined
-  if (providerFailure) invalidReason = 'provider failure or turn timeout'
+  if (failedTurn) {
+    invalidReason = `turn ${failedTurn.type === 'turn.timed_out' ? 'timed out' : 'failed'} (${String(
+      failedTurn.data.code ?? 'unknown'
+    )})`
+  }
   if (scenario.needsCaller) {
     const delegated = runnerEvents.some(
       (event) => event.type === 'turn.started' && String(event.data.input ?? '').includes('sum of 17 and 25')

From 49e47197e022774df5870e22b1cd0e247fc2f615 Mon Sep 17 00:00:00 2001
From: Poytr1 <pooytr1@gmail.com>
Date: Sun, 9 Aug 2026 17:54:23 +0800
Subject: [PATCH 09/15] =?UTF-8?q?docs(design):=20record=20the=2024-run=20b?=
 =?UTF-8?q?ehavioral=20results=20=E2=80=94=20identical=20success,=2028-46%?=
 =?UTF-8?q?=20fewer=20tokens=20under=20post?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The full pre-registered matrix ran 2026-08-09 (local Claude Code over ACP,
sonnet, 24/24 valid trials). Success 11/12 vs 11/12 and first-attempt 9/12
vs 9/12 — identical, as pre-registered. Token consumption: 28-46% fewer
total subject tokens and 47-75% fewer uncached in+out under the primitives
surface wherever the session carries the full surface, wall time 21-64%
lower; child sessions equal. Pre-registration #2 REFUTED in direction: arm
B produced the only 2 invalid calls (both actionable facade refusals the
model repaired); arm A emitted zero illegal combinations in 24 runs.

Shared scenario-4 finding, both arms identically: the child session answers
its parent through Claude Code's own built-in SendMessage tool (a name
collision with the product tool), so the answer never reaches the daemon —
0/6 first-attempt on parent-session across arms, 2/3 eventual success each.
A product problem upstream of either surface.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 docs/designs/messaging-primitives-ab.md | 149 ++++++++++++++++--------
 1 file changed, 98 insertions(+), 51 deletions(-)

diff --git a/docs/designs/messaging-primitives-ab.md b/docs/designs/messaging-primitives-ab.md
index 700d90438..5b89784b2 100644
--- a/docs/designs/messaging-primitives-ab.md
+++ b/docs/designs/messaging-primitives-ab.md
@@ -1,9 +1,9 @@
 # Tool-surface A/B: `sendMessage` (shipped) vs `post` (messaging primitives)
 
-Status: apparatus landed and contract-proven; static costs measured; the
-24-run behavioral matrix is implemented and one command from producing its
-table (see §6 — the run is currently blocked on local runtime credentials,
-not on anything in this repository).
+Status: **complete.** Apparatus landed and contract-proven; static costs
+measured; the full 24-run behavioral matrix ran on 2026-08-09 (local Claude
+Code over ACP, model `sonnet`) with 24/24 valid trials — results in §6,
+conclusion in §7.
 
 The question under test, verbatim from the request that started this work:
 **how much does the primitives design improve success rate and total token
@@ -162,53 +162,100 @@ Expected before any behavioral run, held to afterwards:
    net of cache, would _exceed_ this pre-registration and warrants scrutiny
    (and more trials) before belief.
 
-## 6. Behavioral results
-
-**Run status (2026-08-09): blocked on local runtime credentials.** The
-harness itself is verified to the provider boundary: the smoke trial booted
-the real daemon, materialized the real subject, launched the local
-`claude-acp` adapter, created the ACP session, and dispatched the kickoff
-turn — which failed with `provider_auth_required` ("OAuth session expired
-and could not be refreshed"). The local Claude Code CLI's OAuth refresh
-token expired on 2026-08-09 (the 2026-08-08 arena baseline re-run caught its
-last hours of validity), and re-authentication is an interactive login only
-the operator can perform.
-
-To produce the table (after `claude /login` on the host):
-
-```bash
-pnpm --filter @agentconnect.md/daemon build
-export AGENTCONNECT_DAEMON_ENTRY="$PWD/packages/daemon/dist/index.js"
-export AGENTCONNECT_EVAL_SUBJECT_ROOT=/absolute/path/to/subject   # §4.1 template shape, runtime id claude-acp
-export AGENTCONNECT_EVAL_GAME_TEMPLATE_AGENTS=<template-agent-id>
-npx vitest run evals/test/tool-surface-ab-real.test.ts
-```
-
-Per-run artifacts land in `.artifacts/evaluation/tool-surface-ab/<scenario>-<arm>-<trial>/`
-(events.jsonl, world-events.jsonl, trial.json) with an aggregated
-`summary.json`; the table below is its rendering.
+## 6. Behavioral results (2026-08-09, 24/24 valid trials)
+
+Run: local Claude Code over ACP (`claude-acp` 0.64.0 launched via `node`,
+model `sonnet`, `permissionMode: default`, memory off), 4 scenarios × 2 arms
+× 3 trials, counterbalanced arm order, sequential on one machine. Two
+harness defects were found and fixed by the first live trials before the
+scored run (both are commits on this branch): evaluation-registry tools
+lacked the system-tool permission auto-allow (arm B's calls hung on an
+unanswerable approval card — a fairness bug), and a subscription session
+limit could score as a behavioral trial (validity now rejects any failed
+turn). Per-run artifacts:
+`.artifacts/evaluation/tool-surface-ab/<scenario>-<arm>-<trial>/`
+(events.jsonl, world-events.jsonl, trial.json) plus `summary-merged.json`;
+copies under `~/arena-runs/ab-2026-08-09/` on the measurement host.
 
 | Scenario       | Arm | Success | First-attempt | Invalid calls | Mean tool calls | Mean subject tokens (total / in+out) | Mean wall time |
 | -------------- | --- | ------- | ------------- | ------------- | --------------- | ------------------------------------ | -------------- |
-| agent-channel  | A   | – /3    | – /3          | –             | –               | –                                    | –              |
-| agent-channel  | B   | – /3    | – /3          | –             | –               | –                                    | –              |
-| channel-bare   | A   | – /3    | – /3          | –             | –               | –                                    | –              |
-| channel-bare   | B   | – /3    | – /3          | –             | –               | –                                    | –              |
-| agent-postless | A   | – /3    | – /3          | –             | –               | –                                    | –              |
-| agent-postless | B   | – /3    | – /3          | –             | –               | –                                    | –              |
-| parent-session | A   | – /3    | – /3          | –             | –               | –                                    | –              |
-| parent-session | B   | – /3    | – /3          | –             | –               | –                                    | –              |
-
-## 7. Conclusion so far — the honest answer to the question
-
-On **token consumption**, the measurable half is already answered: the
-primitives surface is 4.8× smaller as a tool descriptor and 2.9× smaller
-including the standing guidance — roughly **2,000 fewer descriptor-tokens
-carried on every turn** of every session (mostly as cache traffic on a warm
-local run; as fresh input on cold sessions and non-caching deployments).
-
-On **success rate**, no behavioral claim is made yet: the pre-registered
-expectation is little or no difference at n=3 on these well-specified
-single-send tasks, with arm B's advantage expected in the invalid-call
-column — and partly by construction there. The 24-run matrix is implemented,
-contract-tested, and one credential refresh away from filling §6.
+| agent-channel  | A   | 3/3     | 3/3           | 0             | 1.0             | 203,578 / 915                        | 33.2s          |
+| agent-channel  | B   | 3/3     | 3/3           | 0             | 1.0             | 125,247 / 412                        | 22.7s          |
+| channel-bare   | A   | 3/3     | 3/3           | 0             | 1.0             | 173,850 / 782                        | 18.0s          |
+| channel-bare   | B   | 3/3     | 3/3           | 0             | 1.0             | 124,990 / 406                        | 14.3s          |
+| agent-postless | A   | 3/3     | 3/3           | 0             | 1.0             | 261,008 / 2,350                      | 186.7s         |
+| agent-postless | B   | 3/3     | 3/3           | 0             | 1.0             | 140,889 / 598                        | 66.9s          |
+| parent-session | A   | 2/3     | 0/3           | 0             | 2.33            | 48,812 / 286                         | 77.6s          |
+| parent-session | B   | 2/3     | 0/3           | 2             | 3.0             | 50,263 / 273                         | 98.1s          |
+
+Totals: success **11/12 vs 11/12**, first-attempt **9/12 vs 9/12** —
+identical. Subject tokens are the sum over the subject agent's completed
+turns (total includes cache read/write; in+out is uncached input plus
+output). Cache traffic dominates: e.g. agent-channel arm A ≈ 186.6k cache
+read + 16.1k cache write vs arm B ≈ 102.2k + 22.6k. Whole-run totals
+(subject + peer) show the same direction, e.g. agent-postless 338.2k (A) vs
+217.5k (B).
+
+### What actually happened, qualitatively
+
+- **Scenarios 1–3 were a clean sweep for both arms**: every trial, both
+  surfaces, one correct call on the first attempt — including the postless
+  ask, where both arms set the reply obligation (`needsReply` /
+  `expectReply`) in 3/3 trials.
+- **Scenario 4 (reply to your parent session) broke both arms identically.**
+  In trial 1 of BOTH arms the child answered through **Claude Code's own
+  built-in `SendMessage` tool** (`{to, recipient, summary, …}` — the
+  runtime's native inter-agent tool, a name-collision hazard with the
+  product's `sendMessage`), so the answer never reached the daemon and the
+  parent never got it: 0/1 in each arm. In the remaining trials both arms
+  eventually made the right parent-form call but shotgunned extra routes
+  around it (a postless call to the peer's agent id, arm B also a DM to a
+  bot user id — refused `unknown_channel` — and a channel post addressing
+  two recipients — refused by the façade's "at most one agent"). First
+  attempt was wrong in 6/6 scored parent-session trials across arms.
+- **The two invalid calls belong to arm B** (the DM-to-a-bot and the
+  two-recipient channel post above). Arm A produced zero invalid calls: its
+  wrong attempts were either the runtime's native tool (not the surface) or
+  legal-but-unnecessary product forms the daemon accepted.
+
+## 7. Conclusion — the answer to the question
+
+**Success rate: no improvement, and none was expected.** 11/12 vs 11/12
+overall, 9/12 vs 9/12 first-attempt — identical, exactly the pre-registered
+expectation for well-specified single-send tasks on a sonnet-class model.
+The one shared failure mode (the child session reaching for the runtime's
+native `SendMessage` instead of the platform surface) is a product finding
+that neither surface design fixes.
+
+**Token consumption: a consistent, large arm-B win in the parent-facing
+scenarios.** Where the session carries the full surface (scenarios 1–3),
+the primitives arm consumed **28–46% fewer total subject tokens**
+(125.2k vs 203.6k; 125.0k vs 173.9k; 140.9k vs 261.0k) and **47–75% fewer
+uncached input+output tokens**, with wall time 21–64% lower. In the child
+sessions of scenario 4 the arms were equal (≈49–50k). The static gap
+(~2,000 descriptor tokens per request) explains only part of this; the rest
+is behavioral — under the bigger surface the model generated ~2× the output
+and re-read its (larger) cached prompt across more loop steps. Honest
+caveat: that behavioral component is an observation at n=3, not a
+guaranteed mechanism, and local cache pricing makes the _billable_ gap
+deployment-dependent.
+
+**Against the pre-registration:** #1 confirmed (static cost, 4.8×/2.9×).
+#2 **refuted in direction** — arm B had MORE invalid calls (2 vs 0), not
+fewer: the façade refuses combinations the model then repairs, while arm
+A's model simply never emitted an illegal product combination in these 24
+runs (its errors routed around the surface instead). At n=3 per cell this
+is anecdote-grade, but it must be said: the "invalid-call win" this
+experiment pre-registered for the primitives did not appear. #3 confirmed
+(no success-rate difference). #4: the token effect in scenarios 1–3
+approaches but does not exceed the 2× totals threshold (in+out does exceed
+it) — treat the efficiency magnitude as promising, not proven.
+
+**Net:** the primitives design does not change whether a sonnet-class agent
+delivers a well-specified message (it delivers it either way), but it
+delivers the same outcome measurably cheaper and faster wherever the full
+surface is carried, and its structural inability to express most illegal
+combinations manifested as _actionable refusals_ rather than fewer errors.
+The scenario-4 findings — the native-tool name collision and the
+route-shotgunning child — are product problems upstream of either surface
+and worth fixing regardless of which surface ships.

From 7168426344289847a514696419fccadef5632aa0 Mon Sep 17 00:00:00 2001
From: Poytr1 <pooytr1@gmail.com>
Date: Mon, 10 Aug 2026 10:54:07 +0800
Subject: [PATCH 10/15] test(evals): arm-B parity for the #800 tool-precedence
 bullet
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The production guidance now leads with the tool-precedence rule (the
runtime's built-in SendMessage silently swallows parent reports — issue
800). Arm B carries the equivalent bullet worded for its surface, so the
A/B keeps comparing surfaces rather than one arm's extra warning.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 evals/games/post-facade.ts | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/evals/games/post-facade.ts b/evals/games/post-facade.ts
index f4e4228b5..60fd53839 100644
--- a/evals/games/post-facade.ts
+++ b/evals/games/post-facade.ts
@@ -206,6 +206,13 @@ export const POST_TOOL_DESCRIPTOR = {
  */
 export const POST_COLLAB_GUIDANCE =
   `# Collaborating with other agents\n` +
+  // Arm parity with the production precedence bullet (issue #800): the runtime's
+  // built-in messaging tools are a hazard for BOTH surfaces, so both arms carry
+  // the equivalent warning — worded for the surface each arm actually has.
+  `- AgentConnect's tools (the \`agentconnect\` MCP server, e.g. \`mcp__agentconnect__post\`) are the ` +
+  `ONLY channel that reaches other agents and humans here. Your runtime may offer built-in tools with similar ` +
+  `purposes (e.g. a bare \`SendMessage\`) — those do NOT reach AgentConnect and anything sent through them is ` +
+  `lost. Never use them for messaging, reporting back, or collaboration.\n` +
   `- One tool, \`post\`, sends any message that leaves your current conversation. Choose three things ` +
   `independently: WHICH conversation it belongs to, WHO it addresses, and whether it is visible on the platform.\n` +
   `- To reach a specific agent privately: ` +

From 6c28d3a00f99e0b26c664317f088808a830b3c7c Mon Sep 17 00:00:00 2001
From: Poytr1 <pooytr1@gmail.com>
Date: Mon, 10 Aug 2026 11:21:06 +0800
Subject: [PATCH 11/15] =?UTF-8?q?fix(evals):=20address=20review=20?=
 =?UTF-8?q?=E2=80=94=20gate=20the=20A/B=20contracts=20in=20CI,=20restore?=
 =?UTF-8?q?=20prompt=20parity,=20tighten=20the=20channel-bare=20judge?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Review findings on the exact head, all fixed:

1. The façade/classifier/fixture contract files were not in either CI
   contract command's explicit path list — added all three to
   eval:collab:contracts (now 18 files / 138 tests).
2. Prompt parity: merged main, which now carries the #800 tool-precedence
   bullet (#801), so BOTH arms' guidance includes it again. The doc's
   static table now records both revisions — the 24-run revision (parity
   held: neither arm had the bullet) and the current head (parity held:
   both do, A 3,099 / B 2,771 chars) — plus the separately measured
   bullet effect (built-in SendMessage attempts 3/6 -> 0/10, losses 0/10,
   success 10/10; issue #800).
3. channel-bare's judge now also requires that nobody was woken; the six
   recorded trials were re-verified under the stricter rule (peer ran
   zero turns in all six) and their 6/6 stands.

The substring FQN matching in matchesToolPermissionFqns is deliberate
consistency with the production isBuiltinSystemTool matcher (same
id-suffixed adapter variants), noted in the review reply.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 docs/designs/messaging-primitives-ab.md | 38 +++++++++++++++++--------
 evals/test/tool-surface-ab-real.test.ts |  5 +++-
 package.json                            |  2 +-
 3 files changed, 31 insertions(+), 14 deletions(-)

diff --git a/docs/designs/messaging-primitives-ab.md b/docs/designs/messaging-primitives-ab.md
index 5b89784b2..f6ba4ccac 100644
--- a/docs/designs/messaging-primitives-ab.md
+++ b/docs/designs/messaging-primitives-ab.md
@@ -123,18 +123,22 @@ never delegates is invalid — it conditioned on the caller, not the subject.
 
 Measured from the real descriptors and from the guidance text a real daemon
 injected into a session prompt (`evals/test/post-facade.test.ts`,
-`evals/test/tool-surface-ab-fixture.test.ts`, on `main` @ `70d58cd1` +
-this branch; token figures are a chars/4 approximation and labelled as such):
-
-| Surface component                | Arm A (`sendMessage`) | Arm B (`post`) | Ratio    |
-| -------------------------------- | --------------------- | -------------- | -------- |
-| Tool description (chars)         | 2,617                 | 1,046          | 2.5×     |
-| Tool input schema (chars)        | 7,407                 | 1,025          | 7.2×     |
-| Descriptor total (chars)         | **10,024**            | **2,071**      | **4.8×** |
-| Descriptor (≈ tokens)            | ~2,506                | ~518           |          |
-| Standing guidance (chars)        | 2,718                 | 2,394          | 1.1×     |
-| **Combined per session (chars)** | **12,742**            | **4,465**      | **2.9×** |
-| Combined (≈ tokens)              | ~3,186                | ~1,116         |          |
+`evals/test/tool-surface-ab-fixture.test.ts`; token figures are a chars/4
+approximation and labelled as such). Two revisions matter, because the #800
+tool-precedence bullet landed in BOTH arms' guidance after the behavioral
+runs (production via #801, arm B via this branch's parity commit) — prompt
+parity held at each revision:
+
+| Surface component                                            | Arm A (`sendMessage`) | Arm B (`post`) | Ratio    |
+| ------------------------------------------------------------ | --------------------- | -------------- | -------- |
+| Tool description (chars)                                     | 2,617                 | 1,046          | 2.5×     |
+| Tool input schema (chars)                                    | 7,407                 | 1,025          | 7.2×     |
+| Descriptor total (chars)                                     | **10,024**            | **2,071**      | **4.8×** |
+| Descriptor (≈ tokens)                                        | ~2,506                | ~518           |          |
+| Standing guidance, at the 24-run revision (chars)            | 2,718                 | 2,394          | 1.1×     |
+| Standing guidance, current head with the #800 bullet (chars) | 3,099                 | 2,771          | 1.1×     |
+| **Combined per session, current head (chars)**               | **13,123**            | **4,842**      | **2.7×** |
+| Combined, current head (≈ tokens)                            | ~3,281                | ~1,211         |          |
 
 The descriptor is carried by **every turn** of every session; the guidance is
 standing session context. On a cache-warm local run most of this cost lands
@@ -177,6 +181,16 @@ turn). Per-run artifacts:
 (events.jsonl, world-events.jsonl, trial.json) plus `summary-merged.json`;
 copies under `~/arena-runs/ab-2026-08-09/` on the measurement host.
 
+Revision notes. (a) These 24 runs predate the #800 tool-precedence bullet;
+NEITHER arm carried it, so prompt parity held. The bullet's effect was then
+measured separately (parent-session × 5 per arm, criteria pre-fixed):
+built-in `SendMessage` attempts 3/6 → 0/10, losses 2/6 → 0/10, success
+4/6 → 10/10 — recorded on issue #800 / PR #801, artifacts under
+`~/arena-runs/ab-2026-08-09/precedence-fix/`. (b) The channel-bare judge was
+later tightened to also require that nobody was woken; the six recorded
+channel-bare trials were re-verified under the stricter rule (the peer ran
+zero turns in all six) and their 6/6 stands.
+
 | Scenario       | Arm | Success | First-attempt | Invalid calls | Mean tool calls | Mean subject tokens (total / in+out) | Mean wall time |
 | -------------- | --- | ------- | ------------- | ------------- | --------------- | ------------------------------------ | -------------- |
 | agent-channel  | A   | 3/3     | 3/3           | 0             | 1.0             | 203,578 / 915                        | 33.2s          |
diff --git a/evals/test/tool-surface-ab-real.test.ts b/evals/test/tool-surface-ab-real.test.ts
index 8a06a5421..e09e51d53 100644
--- a/evals/test/tool-surface-ab-real.test.ts
+++ b/evals/test/tool-surface-ab-real.test.ts
@@ -246,8 +246,11 @@ async function runTrial(scenario: AbScenario, arm: AbArm, trial: number): Promis
       if (!peerActivated) notes.push('the addressed agent was never activated')
       break
     case 'channel-bare':
-      effectsOk = deliveredInPlaza
+      // The scenario's whole point is "visible note, nobody woken": a delivered
+      // post that ALSO activated the peer is a failure, not a success.
+      effectsOk = deliveredInPlaza && !peerActivated
       if (!deliveredInPlaza) notes.push('no delivered post by the subject in the target channel')
+      if (peerActivated) notes.push('the bare post woke the peer — the scenario requires waking nobody')
       break
     case 'agent-postless':
       effectsOk = peerActivated && !instructionLeakedToPlaza
diff --git a/package.json b/package.json
index 5ec012d97..eaaa8fead 100644
--- a/package.json
+++ b/package.json
@@ -11,7 +11,7 @@
     "eval:addons": "pnpm --filter @agentconnect.md/daemon build && node evals/run-addons.mjs",
     "eval:addons:view": "promptfoo view -n",
     "eval:collab": "pnpm --filter @agentconnect.md/daemon build && node evals/run-collaboration.mjs",
-    "eval:collab:contracts": "vitest run evals/test/routing-acceptance.test.ts evals/test/connection-surface.test.ts evals/test/virtual-connections.test.ts evals/test/world-authorization.test.ts evals/test/topology.test.ts evals/test/counting.test.ts evals/test/quota-counting.test.ts evals/test/cross-room-counting.test.ts evals/test/werewolf.test.ts evals/test/game-runner.test.ts evals/test/game-subject.test.ts evals/test/collaboration-game-provider.test.ts evals/test/game-result-assertion.test.ts packages/daemon/test/evaluation-game-ingress.test.ts packages/daemon/test/evaluation-game-tools.test.ts",
+    "eval:collab:contracts": "vitest run evals/test/routing-acceptance.test.ts evals/test/connection-surface.test.ts evals/test/virtual-connections.test.ts evals/test/world-authorization.test.ts evals/test/topology.test.ts evals/test/counting.test.ts evals/test/quota-counting.test.ts evals/test/cross-room-counting.test.ts evals/test/werewolf.test.ts evals/test/game-runner.test.ts evals/test/game-subject.test.ts evals/test/collaboration-game-provider.test.ts evals/test/game-result-assertion.test.ts evals/test/post-facade.test.ts evals/test/tool-surface-ab.test.ts evals/test/tool-surface-ab-fixture.test.ts packages/daemon/test/evaluation-game-ingress.test.ts packages/daemon/test/evaluation-game-tools.test.ts",
     "eval:collab:routing": "vitest run evals/test/routing-acceptance.test.ts evals/test/connection-surface.test.ts",
     "eval:collab:view": "promptfoo view -n",
     "eval:contracts": "vitest run packages/daemon/test/evaluation-events.test.ts packages/daemon/test/evaluation-atif.test.ts packages/daemon/test/evaluation-permission.test.ts packages/daemon/test/evaluation-runner.test.ts packages/daemon/test/daemon-evaluation.test.ts evals/test/outcome.test.ts evals/test/provider.test.ts evals/test/paired-summary.test.ts",

From 36332e0decda147155aa51aeaa3f220ad01e4659 Mon Sep 17 00:00:00 2001
From: Poytr1 <pooytr1@gmail.com>
Date: Tue, 11 Aug 2026 23:46:47 +0800
Subject: [PATCH 12/15] =?UTF-8?q?test(evals):=20add=20scenario=205=20?=
 =?UTF-8?q?=E2=80=94=20in-thread=20turn-taking,=20the=20#801=20regression?=
 =?UTF-8?q?=20gate?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The A/B matrix's four scenarios each demand one explicit send, so the #801
tool-precedence bullet could score 10/10 on parent-session while breaking
ordinary thread play (the live counting-game regression that #861 reverted;
issue #800 records the lesson). Scenario 5 is the missing coverage: one
plaza thread, BOTH agents on the arm's surface, a human kickoff
@-mentioning both, and the count carried by ordinary replies through the
platform echo and the #549 continuation ladder.

- `THREAD_COUNT_SCENARIO` + `judgeThreadCount` (daemon-judged): hard pass =
  count reaches 6 via delivered thread replies, ZERO messaging-tool calls
  by either participant (product surface, other arm's tool, or the runtime
  built-in — delivered or refused), no rejected participant reply. Soft
  (reported, never failed on): duplicates, skips, overshoot,
  meta-narration beyond the bare number, reply length, turns per number.
- Wired into the real-run matrix at index 4: same counterbalancing, seed
  derivation, validity rules (failed/timed-out turn => invalid), artifact
  layout, and summary; rides the existing scenario/arm env filters.
- Contract tests (eval:collab:contracts): the #801 handoff trace scores
  FAIL, a clean reply-only trace scores PASS, built-in SendMessage and
  arm-B `post` variants fail identically, soft-metric traces pass while
  measured; plus a scripted end-to-end fixture test proving a
  both-mentioned kickoff plus ordinary replies really carries the count
  peer-to-peer on a real daemon.
- Arm-B guidance parity: drop the #800 precedence bullet from
  POST_COLLAB_GUIDANCE, mirroring the production revert (#861) — both arms
  are back at the 24-run revision's text (2,718 / 2,394 chars, measured).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 evals/games/post-facade.ts                 |  12 +-
 evals/games/tool-surface-ab.ts             | 232 ++++++++++++++++
 evals/test/tool-surface-ab-fixture.test.ts |  57 ++++
 evals/test/tool-surface-ab-real.test.ts    | 236 +++++++++++++++-
 evals/test/tool-surface-ab.test.ts         | 303 ++++++++++++++++++++-
 5 files changed, 822 insertions(+), 18 deletions(-)

diff --git a/evals/games/post-facade.ts b/evals/games/post-facade.ts
index 60fd53839..45984d6a8 100644
--- a/evals/games/post-facade.ts
+++ b/evals/games/post-facade.ts
@@ -206,13 +206,11 @@ export const POST_TOOL_DESCRIPTOR = {
  */
 export const POST_COLLAB_GUIDANCE =
   `# Collaborating with other agents\n` +
-  // Arm parity with the production precedence bullet (issue #800): the runtime's
-  // built-in messaging tools are a hazard for BOTH surfaces, so both arms carry
-  // the equivalent warning — worded for the surface each arm actually has.
-  `- AgentConnect's tools (the \`agentconnect\` MCP server, e.g. \`mcp__agentconnect__post\`) are the ` +
-  `ONLY channel that reaches other agents and humans here. Your runtime may offer built-in tools with similar ` +
-  `purposes (e.g. a bare \`SendMessage\`) — those do NOT reach AgentConnect and anything sent through them is ` +
-  `lost. Never use them for messaging, reporting back, or collaboration.\n` +
+  // Parity note: the #800 tool-precedence bullet briefly led this text (worded
+  // for this arm's surface, mirroring production's #801) and was removed when
+  // production reverted it (#861, after the live in-thread regression the
+  // `in-thread-count` scenario now gates). Both arms are back to the
+  // pre-#801 guidance, so prompt parity still holds.
   `- One tool, \`post\`, sends any message that leaves your current conversation. Choose three things ` +
   `independently: WHICH conversation it belongs to, WHO it addresses, and whether it is visible on the platform.\n` +
   `- To reach a specific agent privately: ` +
diff --git a/evals/games/tool-surface-ab.ts b/evals/games/tool-surface-ab.ts
index dbaa7eeca..5645334a5 100644
--- a/evals/games/tool-surface-ab.ts
+++ b/evals/games/tool-surface-ab.ts
@@ -207,6 +207,238 @@ export function extractTrialMetrics(
   }
 }
 
+// ─────────────────────────────────────────────────────────────────────────────
+// Scenario 5 — in-thread turn-taking conversation (the #801 regression gate).
+//
+// The four send scenarios above each demand ONE explicit send, so a prompt
+// change that made the model over-use the messaging surface could pass the
+// whole matrix while breaking ordinary thread play. That is exactly what
+// happened with the #801 tool-precedence bullet (validated only against
+// parent-session, 10/10): in a live channel counting game the agent started
+// routing every in-thread turn through `sendMessage` to "hand off" the next
+// number to its peer, posting meta-narration into the thread with skipped and
+// duplicated numbers. #801 was reverted (#861); issue #800 records the lesson.
+//
+// This scenario is the missing coverage: one channel thread, TWO subject
+// agents on the arm's surface, a human kickoff @-mentioning both, and the
+// agents then take turns counting via ORDINARY replies (the #549 continuation
+// ladder — each delivered reply echoes back and wakes the peer). The correct
+// number of messaging-tool calls here is ZERO: in-thread speech is the
+// ordinary turn reply, by product convention (`sendMessage` deliberately has
+// no in-thread form).
+// ─────────────────────────────────────────────────────────────────────────────
+
+/** Small on purpose: enough replies to prove sustained turn-taking, cheap
+ *  enough to run 2 arms × 3 trials routinely as a prompt-change gate. */
+export const THREAD_COUNT_TARGET = 6
+
+export const THREAD_COUNT_SCENARIO = {
+  id: 'in-thread-count',
+  target: THREAD_COUNT_TARGET,
+  /** Kickoff, spoken by a HUMAN into the shared thread, @-mentioning both
+   *  participants. Names no tool, no field, no form — the same
+   *  banned-vocabulary rule as the send scenarios covers this text. */
+  instruction: (mentions: { first: string; second: string }) =>
+    `${mentions.first} ${mentions.second} Let's count together right here in this thread, taking turns. ` +
+    `Each turn is one reply in this thread containing ONLY the next number — nothing else, no commentary. ` +
+    `Start at 1. Do not repeat a number that is already in the thread, and after you contribute one, let the ` +
+    `other participant take the next one. Stop once ${THREAD_COUNT_TARGET} has appeared.`
+}
+
+/** Is this tool name a messaging tool, on ANY surface the session might carry?
+ *  Covers the product `sendMessage` (`mcp__agentconnect__sendMessage`), the
+ *  arm-B façade `post` (`mcp__agentconnect__post`), and the Claude Code
+ *  runtime's own built-in `SendMessage` (the #800 name-collision hazard).
+ *  During the in-thread game every one of them is the #801 failure mode. */
+export function isMessagingToolName(name: string): boolean {
+  const normalized = name.toLowerCase()
+  if (normalized.includes('sendmessage')) return true
+  return normalized === 'post' || normalized.endsWith('__post')
+}
+
+export interface ThreadCountEffect {
+  sequence: number
+  kind: string
+  status: string
+  channel: string
+  thread?: string
+  agentId?: string
+  text: string
+}
+
+export interface ThreadCountMessagingCall {
+  agentId?: string
+  tool: string
+  failed: boolean
+}
+
+export interface ThreadCountVerdict {
+  /** The hard verdict. Fail reasons are enumerated in `failures`. */
+  pass: boolean
+  failures: string[]
+  /** Highest number ≤ target seen in a delivered participant thread reply. */
+  reached: number
+  target: number
+  /** First integer of each delivered participant thread reply that carries
+   *  one, in delivery order — the visible count as the thread saw it. */
+  numbersPosted: number[]
+  /** HARD RULE: every messaging-tool call by any participant during the game
+   *  (any surface, delivered or refused). One is the #801 failure mode. */
+  messagingToolCalls: ThreadCountMessagingCall[]
+  /** Participant thread replies the world refused to deliver. */
+  lostMessages: number
+  // ── soft metrics: reported, never failed on ──
+  duplicates: number
+  skips: number
+  /** Numbers posted beyond the stop target. */
+  overshoot: number
+  /** Delivered participant replies in the thread (numbered or not). */
+  replies: number
+  /** Replies that are just the number (markdown emphasis/punctuation allowed). */
+  bareNumberReplies: number
+  /** Replies carrying a number plus prose — the meta-narration signal
+   *  ("Handing off for 5"-style). */
+  metaNarrationReplies: number
+  meanReplyChars: number
+  /** Participant completed turns per counted number. */
+  turnsPerNumber: number
+}
+
+interface ThreadCountJudgeOptions {
+  target: number
+  /** The two subject agents' ids. */
+  participants: readonly string[]
+  channel: string
+  /** Root message id of the kickoff thread. */
+  thread: string
+  /** The world's recorded outbound effects, in sequence order. */
+  effects: readonly ThreadCountEffect[]
+  /** The daemon's evaluation events (all agents). */
+  events: readonly { type: string; agentId?: string; data: Record<string, unknown> }[]
+}
+
+/**
+ * Judge one in-thread turn-taking trial from the daemon's records, never the
+ * models' claims — same philosophy as the send scenarios.
+ *
+ * Hard pass: the count reached the target via ordinary delivered thread
+ * replies, ZERO messaging-tool calls by any participant during the game (a
+ * call with a legitimate non-thread purpose has no reason to occur in this
+ * scenario, so the rule stays simple: any messaging-tool call = fail), and no
+ * participant reply was lost (rejected by the world).
+ *
+ * Soft (reported, not failed on): duplicated and skipped numbers, overshoot
+ * past the stop target, meta-narration beyond the bare number, reply length,
+ * turns per number.
+ */
+export function judgeThreadCount(options: ThreadCountJudgeOptions): ThreadCountVerdict {
+  const participants = new Set(options.participants)
+
+  // ── messaging-tool calls, folded by toolCallId across ACP updates ──
+  const callById = new Map<string, ThreadCountMessagingCall>()
+  const callOrder: string[] = []
+  let participantTurns = 0
+  for (const event of options.events) {
+    if (event.agentId === undefined || !participants.has(event.agentId)) continue
+    if (event.type === 'turn.completed') {
+      participantTurns += 1
+      continue
+    }
+    if (event.type !== 'acp.update') continue
+    const update = event.data.update as
+      | {
+          sessionUpdate?: string
+          toolCallId?: string
+          title?: string
+          status?: string
+          _meta?: { claudeCode?: { toolName?: string } }
+        }
+      | undefined
+    if (!update) continue
+    if (update.sessionUpdate !== 'tool_call' && update.sessionUpdate !== 'tool_call_update') continue
+    const id = update.toolCallId
+    if (typeof id !== 'string') continue
+    const name = update._meta?.claudeCode?.toolName ?? update.title
+    const existing = callById.get(id)
+    if (!existing) {
+      if (typeof name !== 'string' || !isMessagingToolName(name)) continue
+      callById.set(id, { agentId: event.agentId, tool: name, failed: false })
+      callOrder.push(id)
+    }
+    const call = callById.get(id)!
+    if (update.status === 'failed') call.failed = true
+    if (update.status === 'completed') call.failed = false
+  }
+  const messagingToolCalls = callOrder.map((id) => callById.get(id)!)
+
+  // ── the visible thread: delivered participant replies, in order ──
+  const participantThreadEffects = options.effects.filter(
+    (effect) =>
+      effect.kind === 'reply' &&
+      effect.agentId !== undefined &&
+      participants.has(effect.agentId) &&
+      effect.channel === options.channel &&
+      (effect.thread === undefined || effect.thread === options.thread)
+  )
+  const delivered = participantThreadEffects.filter((effect) => effect.status === 'delivered')
+  const lostMessages = participantThreadEffects.filter((effect) => effect.status === 'rejected').length
+
+  const numbersPosted: number[] = []
+  let bareNumberReplies = 0
+  let metaNarrationReplies = 0
+  let replyChars = 0
+  for (const effect of delivered) {
+    // Digits inside platform mention tokens (`<@W123…>`) are not count signal.
+    const text = effect.text.replace(/<@[^>]+>/g, '').trim()
+    replyChars += text.length
+    const match = /-?\d+/.exec(text)
+    if (!match) continue
+    numbersPosted.push(Number(match[0]))
+    // Bare = the number alone, allowing markdown emphasis and punctuation.
+    if (/^[*_`~\s]*-?\d+[*_`~\s.!]*$/.test(text)) bareNumberReplies += 1
+    else metaNarrationReplies += 1
+  }
+
+  const occurrences = new Map<number, number>()
+  for (const value of numbersPosted) occurrences.set(value, (occurrences.get(value) ?? 0) + 1)
+  const reached = Math.max(0, ...numbersPosted.filter((value) => value >= 1 && value <= options.target))
+  let duplicates = 0
+  let skips = 0
+  for (let value = 1; value <= reached; value += 1) {
+    const count = occurrences.get(value) ?? 0
+    if (count === 0) skips += 1
+    else duplicates += count - 1
+  }
+  const overshoot = numbersPosted.filter((value) => value > options.target).length
+
+  const failures: string[] = []
+  if (reached < options.target) {
+    failures.push(`the count reached ${reached} of ${options.target} via ordinary thread replies`)
+  }
+  for (const call of messagingToolCalls) {
+    failures.push(`participant ${call.agentId ?? 'unknown'} called messaging tool "${call.tool}" during the game`)
+  }
+  if (lostMessages > 0) failures.push(`${lostMessages} participant thread repl(ies) were rejected, not delivered`)
+
+  return {
+    pass: failures.length === 0,
+    failures,
+    reached,
+    target: options.target,
+    numbersPosted,
+    messagingToolCalls,
+    lostMessages,
+    duplicates,
+    skips,
+    overshoot,
+    replies: delivered.length,
+    bareNumberReplies,
+    metaNarrationReplies,
+    meanReplyChars: delivered.length === 0 ? 0 : Number((replyChars / delivered.length).toFixed(1)),
+    turnsPerNumber: options.target === 0 ? 0 : Number((participantTurns / options.target).toFixed(2))
+  }
+}
+
 /** Arm B's classifier: compile the façade input, then classify the product args
  *  it becomes — the symmetry that makes the two arms score identically. An
  *  input the façade refuses names no legal form. */
diff --git a/evals/test/tool-surface-ab-fixture.test.ts b/evals/test/tool-surface-ab-fixture.test.ts
index f20ba70c8..7cc5a466a 100644
--- a/evals/test/tool-surface-ab-fixture.test.ts
+++ b/evals/test/tool-surface-ab-fixture.test.ts
@@ -12,6 +12,7 @@
  */
 import { afterEach, describe, expect, it } from 'vitest'
 import { callDaemonTool, daemonMcpBinding, listDaemonTools, type DaemonMcpBinding } from '../games/mcp-client.js'
+import { THREAD_COUNT_SCENARIO, judgeThreadCount } from '../games/tool-surface-ab.js'
 import { AbFixture } from '../games/tool-surface-ab-fixture.js'
 
 let fixture: AbFixture | undefined
@@ -210,4 +211,60 @@ describe('tool-surface A/B fixture — each arm presents exactly one surface', (
     expect(delivered.filter((effect) => effect.channel !== fixture!.room('briefing').channel)).toEqual([])
     expect(delivered.some((effect) => effect.text.includes('what is your status?'))).toBe(false)
   })
+
+  it('scenario 5: a both-mentioned kickoff plus ordinary replies carries the count — and the judge passes it', async () => {
+    // The in-thread turn-taking scenario's transport contract, credential-free:
+    // one plaza thread, both agents woken by the kickoff, and each delivered
+    // ordinary reply echoing back to wake the peer (#549 continuation) with NO
+    // messaging-tool call anywhere. If this wiring were broken, a real-model
+    // run would score the harness, not the prompt — exactly the class of
+    // silent fault the arena's scripted gates exist to catch.
+    const captured: CapturedSession[] = []
+    const target = THREAD_COUNT_SCENARIO.target
+    let next = 1
+    fixture = await startArm('A', captured, async (context) => {
+      // A deterministic well-behaved player: contribute the next number as an
+      // ORDINARY reply; after the target, acknowledge completion once.
+      if (next <= target) {
+        context.reply(String(next))
+        next += 1
+      } else {
+        context.reply('the count is complete')
+      }
+    })
+    const kickoff = fixture.injectHuman(
+      'plaza',
+      THREAD_COUNT_SCENARIO.instruction({
+        first: `<@${fixture.botUserId('runner')}>`,
+        second: `<@${fixture.botUserId('peer')}>`
+      }),
+      { mentions: [fixture.botUserId('runner'), fixture.botUserId('peer')] }
+    )
+    await fixture.settle(kickoff.handles)
+    const runnerId = fixture.agentId('runner')
+    const peerId = fixture.agentId('peer')
+    const verdict = judgeThreadCount({
+      target,
+      participants: [runnerId, peerId],
+      channel: fixture.room('plaza').channel,
+      thread: kickoff.messageId,
+      effects: fixture.world.allEffects(),
+      events: [...fixture.events()] as never
+    })
+    expect(verdict.failures).toEqual([])
+    expect(verdict.pass).toBe(true)
+    expect(verdict.reached).toBe(target)
+    expect(verdict.messagingToolCalls).toEqual([])
+    expect(verdict.lostMessages).toBe(0)
+    // BOTH participants contributed — the echo really woke the peer; a run
+    // where one agent counts alone would pass the count but not this pin.
+    const contributors = new Set(
+      fixture.world
+        .allEffects()
+        .filter((effect) => effect.status === 'delivered' && effect.kind === 'reply' && effect.agentId !== undefined)
+        .map((effect) => effect.agentId)
+    )
+    expect(contributors.has(runnerId)).toBe(true)
+    expect(contributors.has(peerId)).toBe(true)
+  })
 })
diff --git a/evals/test/tool-surface-ab-real.test.ts b/evals/test/tool-surface-ab-real.test.ts
index e09e51d53..c21b398d2 100644
--- a/evals/test/tool-surface-ab-real.test.ts
+++ b/evals/test/tool-surface-ab-real.test.ts
@@ -4,8 +4,9 @@
  * The credential-free half (`tool-surface-ab.test.ts`, `post-facade.test.ts`,
  * `tool-surface-ab-fixture.test.ts`, all in the CI gates) pins the apparatus:
  * the façade's compilation, the shared classifier, and the arm-parity
- * preconditions. This file runs the pre-registered 4×2×3 matrix — four send
- * scenarios, two surfaces, three trials — and is deliberately NOT in any CI
+ * preconditions. This file runs the pre-registered matrix — four send
+ * scenarios plus the in-thread turn-taking scenario (the #801 regression
+ * gate), two surfaces, three trials each — and is deliberately NOT in any CI
  * gate: it needs a real runtime and provider credentials, and a model result
  * is a rate over trials, never a single pass/fail (collaboration-arena.md §8.1).
  *
@@ -38,11 +39,14 @@ import { atomicWrite, redactEvaluationValue } from '../../packages/daemon/src/ev
 import { compilePost } from '../games/post-facade.js'
 import {
   AB_SCENARIOS,
+  THREAD_COUNT_SCENARIO,
   classifyPostForm,
   extractTrialMetrics,
+  judgeThreadCount,
   type AbScenario,
   type AbTrialMetrics,
-  type SendForm
+  type SendForm,
+  type ThreadCountVerdict
 } from '../games/tool-surface-ab.js'
 import { AbFixture, type AbArm } from '../games/tool-surface-ab-fixture.js'
 
@@ -66,6 +70,11 @@ const armFilter = (process.env.AGENTCONNECT_EVAL_AB_ARMS ?? '')
 
 const scenarios = AB_SCENARIOS.filter((scenario) => scenarioFilter.length === 0 || scenarioFilter.includes(scenario.id))
 const arms: AbArm[] = (['A', 'B'] as const).filter((arm) => armFilter.length === 0 || armFilter.includes(arm))
+// Scenario 5 (in-thread turn-taking) rides the same filters; its index in the
+// full matrix follows the four send scenarios, which keeps counterbalancing
+// and seed derivation consistent with them.
+const runThreadCount = scenarioFilter.length === 0 || scenarioFilter.includes(THREAD_COUNT_SCENARIO.id)
+const THREAD_COUNT_SCENARIO_INDEX = AB_SCENARIOS.length
 
 interface AbRunRecord {
   scenario: string
@@ -97,8 +106,38 @@ interface AbRunRecord {
   notes: string[]
 }
 
+interface TokenBreakdown {
+  total: number
+  input: number
+  output: number
+  cacheRead: number
+  cacheWrite: number
+}
+
+/** One scenario-5 run: the daemon-judged verdict plus the run economics. Both
+ *  agents are subjects here, so tokens are reported per participant and for
+ *  the whole run — there is no single "subject agent" to scope to. */
+interface ThreadCountRunRecord {
+  scenario: typeof THREAD_COUNT_SCENARIO.id
+  arm: AbArm
+  trial: number
+  seed: number
+  status: 'ok' | 'invalid'
+  invalidReason?: string
+  verdict: ThreadCountVerdict
+  runnerTokens: TokenBreakdown
+  peerTokens: TokenBreakdown
+  runTokens: TokenBreakdown
+  runnerTurns: number
+  peerTurns: number
+  runTurns: number
+  latencyMs: number
+  notes: string[]
+}
+
 let fixture: AbFixture | undefined
 const results: AbRunRecord[] = []
+const threadCountResults: ThreadCountRunRecord[] = []
 
 afterEach(async () => {
   await fixture?.stop()
@@ -355,6 +394,129 @@ async function runTrial(scenario: AbScenario, arm: AbArm, trial: number): Promis
   return record
 }
 
+/**
+ * Scenario 5 — in-thread turn-taking (the #801 regression gate). One plaza
+ * thread, BOTH agents on the arm's surface, a human kickoff @-mentioning both;
+ * the agents continue via ordinary replies (each delivered reply echoes back
+ * and wakes the peer through the #549 continuation ladder). Judged by
+ * `judgeThreadCount` from the daemon's records: the count must reach the
+ * target through delivered thread replies with ZERO messaging-tool calls by
+ * either participant and no lost replies.
+ */
+async function runThreadCountTrial(arm: AbArm, trial: number): Promise<ThreadCountRunRecord> {
+  const seed = 5000 + THREAD_COUNT_SCENARIO_INDEX * 100 + trial
+  fixture = await AbFixture.start({
+    seed,
+    arm,
+    subject: { kind: 'real', subjectRoot: subjectRoot!, templateAgentIds: templateAgents }
+  })
+  const runnerId = fixture.agentId('runner')
+  const peerId = fixture.agentId('peer')
+  const plaza = fixture.room('plaza')
+  const instruction = THREAD_COUNT_SCENARIO.instruction({
+    first: `<@${fixture.botUserId('runner')}>`,
+    second: `<@${fixture.botUserId('peer')}>`
+  })
+  const notes: string[] = []
+
+  const kickoff = fixture.injectHuman('plaza', instruction, {
+    mentions: [fixture.botUserId('runner'), fixture.botUserId('peer')]
+  })
+  const startedAt = Date.now()
+  await fixture.settle(kickoff.handles, TRIAL_BUDGET_MS)
+  const latencyMs = Date.now() - startedAt
+
+  const allEvents = [...fixture.events()]
+  const toolName = arm === 'A' ? 'sendMessage' : 'post'
+
+  // ── validity: infra failures measure nothing about the prompt/surface ──
+  const failedTurn = allEvents.find((event) => event.type === 'turn.failed' || event.type === 'turn.timed_out')
+  let invalidReason: string | undefined
+  if (failedTurn) {
+    invalidReason = `turn ${failedTurn.type === 'turn.timed_out' ? 'timed out' : 'failed'} (${String(
+      failedTurn.data.code ?? 'unknown'
+    )})`
+  }
+  const anyParticipantTurn = allEvents.some(
+    (event) => event.type === 'turn.started' && (event.agentId === runnerId || event.agentId === peerId)
+  )
+  if (!anyParticipantTurn) invalidReason ??= 'the kickoff never activated either participant'
+
+  const verdict = judgeThreadCount({
+    target: THREAD_COUNT_SCENARIO.target,
+    participants: [runnerId, peerId],
+    channel: plaza.channel,
+    thread: kickoff.messageId,
+    effects: fixture.world.allEffects(),
+    events: allEvents as never
+  })
+  for (const failure of verdict.failures) notes.push(failure)
+
+  // Token/turn economics per participant and for the whole run. The extractor
+  // is reused for its usage folding only; scenario 5 has no expected form.
+  const tokenMetrics = (events: { type: string; data: Record<string, unknown> }[]) =>
+    extractTrialMetrics(events, { toolName, expected: 'unclassifiable', latencyMs })
+  const runnerMetrics = tokenMetrics(fixture.eventsOf('runner') as never)
+  const peerMetrics = tokenMetrics(fixture.eventsOf('peer') as never)
+  const runMetrics = tokenMetrics(allEvents as never)
+
+  const record: ThreadCountRunRecord = {
+    scenario: THREAD_COUNT_SCENARIO.id,
+    arm,
+    trial,
+    seed,
+    status: invalidReason ? 'invalid' : 'ok',
+    ...(invalidReason ? { invalidReason } : {}),
+    verdict,
+    runnerTokens: runnerMetrics.tokens,
+    peerTokens: peerMetrics.tokens,
+    runTokens: runMetrics.tokens,
+    runnerTurns: runnerMetrics.turns,
+    peerTurns: peerMetrics.turns,
+    runTurns: runMetrics.turns,
+    latencyMs,
+    notes
+  }
+
+  // ── artifacts: same layout as the send scenarios, one dir per run ──
+  const dir = join(ARTIFACT_DIR, `${THREAD_COUNT_SCENARIO.id}-${arm}-${trial}`)
+  mkdirSync(dir, { recursive: true, mode: 0o700 })
+  fixture.eventCollector().writeJsonl(join(dir, 'events.jsonl'))
+  const secrets = fixture.secrets
+  atomicWrite(
+    join(dir, 'world-events.jsonl'),
+    fixture.world
+      .events()
+      .map((entry) => JSON.stringify(redactEvaluationValue(entry, secrets)))
+      .join('\n') + '\n'
+  )
+  atomicWrite(
+    join(dir, 'trial.json'),
+    `${JSON.stringify(
+      redactEvaluationValue(
+        {
+          record,
+          instruction,
+          facadeCalls: fixture.facadeCalls,
+          effects: fixture.world.allEffects().map((effect) => ({
+            status: effect.status,
+            kind: effect.kind,
+            channel: effect.channel,
+            thread: effect.thread,
+            agentId: effect.agentId,
+            ...(effect.reason !== undefined ? { reason: effect.reason } : {}),
+            text: effect.text
+          }))
+        },
+        secrets
+      ),
+      null,
+      2
+    )}\n`
+  )
+  return record
+}
+
 function aggregate(records: AbRunRecord[]) {
   const cell = (scenario: string, arm: AbArm) => {
     const rows = records.filter((row) => row.scenario === scenario && row.arm === arm && row.status === 'ok')
@@ -373,23 +535,52 @@ function aggregate(records: AbRunRecord[]) {
       meanLatencyMs: Math.round(mean((row) => row.latencyMs))
     }
   }
+  const threadCell = (arm: AbArm) => {
+    const rows = threadCountResults.filter((row) => row.arm === arm && row.status === 'ok')
+    const mean = (select: (row: ThreadCountRunRecord) => number) =>
+      rows.length === 0 ? 0 : rows.reduce((total, row) => total + select(row), 0) / rows.length
+    return {
+      trials: rows.length,
+      pass: rows.filter((row) => row.verdict.pass).length,
+      messagingToolCalls: rows.reduce((total, row) => total + row.verdict.messagingToolCalls.length, 0),
+      lostMessages: rows.reduce((total, row) => total + row.verdict.lostMessages, 0),
+      meanReached: Number(mean((row) => row.verdict.reached).toFixed(2)),
+      duplicates: rows.reduce((total, row) => total + row.verdict.duplicates, 0),
+      skips: rows.reduce((total, row) => total + row.verdict.skips, 0),
+      overshoot: rows.reduce((total, row) => total + row.verdict.overshoot, 0),
+      meanBareNumberReplies: Number(mean((row) => row.verdict.bareNumberReplies).toFixed(2)),
+      meanMetaNarrationReplies: Number(mean((row) => row.verdict.metaNarrationReplies).toFixed(2)),
+      meanReplyChars: Number(mean((row) => row.verdict.meanReplyChars).toFixed(1)),
+      meanTurnsPerNumber: Number(mean((row) => row.verdict.turnsPerNumber).toFixed(2)),
+      meanRunTokensTotal: Math.round(mean((row) => row.runTokens.total)),
+      meanRunTokensInOut: Math.round(mean((row) => row.runTokens.input + row.runTokens.output)),
+      meanLatencyMs: Math.round(mean((row) => row.latencyMs))
+    }
+  }
   return {
     generatedAt: new Date().toISOString(),
     trialsPerCell: TRIALS,
     cells: Object.fromEntries(
       scenarios.flatMap((scenario) => arms.map((arm) => [`${scenario.id}/${arm}`, cell(scenario.id, arm)] as const))
     ),
-    invalidTrials: records.filter((row) => row.status === 'invalid'),
-    records
+    threadCountCells: runThreadCount
+      ? Object.fromEntries(arms.map((arm) => [`${THREAD_COUNT_SCENARIO.id}/${arm}`, threadCell(arm)] as const))
+      : {},
+    invalidTrials: [
+      ...records.filter((row) => row.status === 'invalid'),
+      ...threadCountResults.filter((row) => row.status === 'invalid')
+    ],
+    records,
+    threadCountRecords: threadCountResults
   }
 }
 
 afterAll(() => {
-  if (results.length === 0) return
+  if (results.length === 0 && threadCountResults.length === 0) return
   mkdirSync(ARTIFACT_DIR, { recursive: true, mode: 0o700 })
   const summary = aggregate(results)
   atomicWrite(join(ARTIFACT_DIR, 'summary.json'), `${JSON.stringify(summary, null, 2)}\n`)
-  console.log(JSON.stringify(summary.cells, null, 2))
+  console.log(JSON.stringify({ ...summary.cells, ...summary.threadCountCells }, null, 2))
 })
 
 describe.skipIf(!configured)('tool-surface A/B against a real ACP runtime', () => {
@@ -417,6 +608,31 @@ describe.skipIf(!configured)('tool-surface A/B against a real ACP runtime', () =
     }
   }
 
+  // Scenario 5: in-thread turn-taking, same counterbalancing rule at its
+  // matrix index. A model result is reported, never asserted (a hard-fail
+  // verdict here IS a result — the prompt-change gate reads the summary).
+  if (runThreadCount) {
+    for (let trial = 1; trial <= TRIALS; trial += 1) {
+      const ordered = (THREAD_COUNT_SCENARIO_INDEX + trial) % 2 === 0 ? [...arms] : [...arms].reverse()
+      for (const arm of ordered) {
+        it(
+          `${THREAD_COUNT_SCENARIO.id} arm ${arm} trial ${trial}`,
+          async () => {
+            const record = await runThreadCountTrial(arm, trial)
+            threadCountResults.push(record)
+            if (record.status === 'invalid') {
+              console.warn(`INVALID trial ${THREAD_COUNT_SCENARIO.id}/${arm}/${trial}: ${record.invalidReason}`)
+            } else if (!record.verdict.pass) {
+              console.warn(`FAIL ${THREAD_COUNT_SCENARIO.id}/${arm}/${trial}: ${record.verdict.failures.join('; ')}`)
+            }
+            expect(true).toBe(true)
+          },
+          TRIAL_BUDGET_MS + 60_000
+        )
+      }
+    }
+  }
+
   it('produced at least one scoreable trial per cell', () => {
     for (const scenario of scenarios) {
       for (const arm of arms) {
@@ -426,5 +642,11 @@ describe.skipIf(!configured)('tool-surface A/B against a real ACP runtime', () =
         expect(ok, `${scenario.id}/${arm} has no scoreable trial`).toBeGreaterThan(0)
       }
     }
+    if (runThreadCount) {
+      for (const arm of arms) {
+        const ok = threadCountResults.filter((row) => row.arm === arm && row.status === 'ok').length
+        expect(ok, `${THREAD_COUNT_SCENARIO.id}/${arm} has no scoreable trial`).toBeGreaterThan(0)
+      }
+    }
   })
 })
diff --git a/evals/test/tool-surface-ab.test.ts b/evals/test/tool-surface-ab.test.ts
index e33e605ea..d3ef60c0c 100644
--- a/evals/test/tool-surface-ab.test.ts
+++ b/evals/test/tool-surface-ab.test.ts
@@ -1,5 +1,13 @@
 import { describe, expect, it } from 'vitest'
-import { AB_SCENARIOS, classifySendForm, extractTrialMetrics } from '../games/tool-surface-ab.js'
+import {
+  AB_SCENARIOS,
+  THREAD_COUNT_SCENARIO,
+  classifySendForm,
+  extractTrialMetrics,
+  isMessagingToolName,
+  judgeThreadCount,
+  type ThreadCountEffect
+} from '../games/tool-surface-ab.js'
 
 /**
  * The A/B's measurement apparatus, tested without model credentials. If the
@@ -146,13 +154,300 @@ describe('the reduced scenario matrix', () => {
 
   it('never names a tool, a field or a form in the task text', () => {
     // Naming them would test instruction-following instead of the surface.
+    // The rule covers scenario 5's kickoff too: the in-thread game must be
+    // won by the STANDING guidance alone, never by the task text steering
+    // the model toward or away from a tool.
     const ids = { peerAgentId: 'PEER', channel: 'CHAN', humanUserId: 'UHUMAN' }
     const banned = ['sendMessage', 'post(', 'toAgent', 'toUser', 'sessionId', 'conversation', 'visibility', 'address']
-    for (const scenario of AB_SCENARIOS) {
-      const text = scenario.instruction(ids)
+    const texts = [
+      ...AB_SCENARIOS.map((scenario) => [scenario.id, scenario.instruction(ids)] as const),
+      [THREAD_COUNT_SCENARIO.id, THREAD_COUNT_SCENARIO.instruction({ first: '<@B1>', second: '<@B2>' })] as const
+    ]
+    for (const [id, text] of texts) {
       for (const token of banned) {
-        expect(text, `${scenario.id} leaks "${token}"`).not.toContain(token)
+        expect(text, `${id} leaks "${token}"`).not.toContain(token)
       }
     }
   })
+
+  it('includes the in-thread turn-taking scenario with a small stop target', () => {
+    expect(THREAD_COUNT_SCENARIO.id).toBe('in-thread-count')
+    expect(THREAD_COUNT_SCENARIO.target).toBe(6)
+    const text = THREAD_COUNT_SCENARIO.instruction({ first: '<@B1>', second: '<@B2>' })
+    // The kickoff @-mentions both participants and states the stop target.
+    expect(text).toContain('<@B1>')
+    expect(text).toContain('<@B2>')
+    expect(text).toContain('6')
+  })
+})
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Scenario 5's judge. The incident it gates (#801 → revert #861): a prompt
+// change validated only against parent-session made an agent route in-thread
+// turns through `sendMessage`, posting meta-narration with skipped/duplicated
+// numbers. The judge must score that trace FAIL from the daemon's records, and
+// a clean reply-only trace PASS.
+// ─────────────────────────────────────────────────────────────────────────────
+
+const RUNNER = 'agent-runner'
+const PEER = 'agent-peer'
+const CHANNEL = 'C-PLAZA'
+const THREAD = 'T-ROOT'
+
+function reply(sequence: number, agentId: string, text: string, status = 'delivered'): ThreadCountEffect {
+  return { sequence, kind: 'reply', status, channel: CHANNEL, thread: THREAD, agentId, text }
+}
+
+/** A completed messaging-tool call as the ACP stream records it. */
+function messagingCall(agentId: string, id: string, name: string, viaMeta = false) {
+  const update = viaMeta
+    ? { sessionUpdate: 'tool_call', toolCallId: id, title: 'Send a message', _meta: { claudeCode: { toolName: name } } }
+    : { sessionUpdate: 'tool_call', toolCallId: id, title: name }
+  return [
+    { type: 'acp.update', agentId, data: { update } },
+    {
+      type: 'acp.update',
+      agentId,
+      data: { update: { sessionUpdate: 'tool_call_update', toolCallId: id, status: 'completed' } }
+    }
+  ]
+}
+
+function turns(agentId: string, count: number) {
+  return Array.from({ length: count }, () => ({ type: 'turn.completed', agentId, data: {} }))
+}
+
+/** Alternating bare-number replies 1..target — the clean game. */
+function cleanReplies(target: number): ThreadCountEffect[] {
+  return Array.from({ length: target }, (_, index) =>
+    reply(index + 1, index % 2 === 0 ? RUNNER : PEER, String(index + 1))
+  )
+}
+
+describe('messaging-tool name matcher', () => {
+  it('matches every messaging surface a session might carry, and nothing else', () => {
+    expect(isMessagingToolName('sendMessage')).toBe(true)
+    expect(isMessagingToolName('mcp__agentconnect__sendMessage')).toBe(true)
+    expect(isMessagingToolName('SendMessage')).toBe(true) // Claude Code built-in
+    expect(isMessagingToolName('post')).toBe(true)
+    expect(isMessagingToolName('mcp__agentconnect__post')).toBe(true)
+    expect(isMessagingToolName('listAgents')).toBe(false)
+    expect(isMessagingToolName('setSessionTitle')).toBe(false)
+    expect(isMessagingToolName('compost')).toBe(false)
+    expect(isMessagingToolName('postpone')).toBe(false)
+  })
+})
+
+describe('in-thread turn-taking judge — hard rules', () => {
+  it('passes a clean reply-only game', () => {
+    const verdict = judgeThreadCount({
+      target: 6,
+      participants: [RUNNER, PEER],
+      channel: CHANNEL,
+      thread: THREAD,
+      effects: cleanReplies(6),
+      events: [...turns(RUNNER, 3), ...turns(PEER, 3)]
+    })
+    expect(verdict.pass).toBe(true)
+    expect(verdict.failures).toEqual([])
+    expect(verdict.reached).toBe(6)
+    expect(verdict.numbersPosted).toEqual([1, 2, 3, 4, 5, 6])
+    expect(verdict.messagingToolCalls).toEqual([])
+    expect(verdict.lostMessages).toBe(0)
+    expect(verdict.duplicates).toBe(0)
+    expect(verdict.skips).toBe(0)
+    expect(verdict.bareNumberReplies).toBe(6)
+    expect(verdict.metaNarrationReplies).toBe(0)
+    expect(verdict.turnsPerNumber).toBe(1)
+  })
+
+  it('fails the #801 trace: a sendMessage "handoff" during the game', () => {
+    // The recorded live regression: the agent posts meta-narration in-thread
+    // and routes the actual number through the messaging tool.
+    const effects = [
+      reply(1, RUNNER, '1'),
+      reply(2, PEER, '2'),
+      reply(3, RUNNER, '3'),
+      reply(4, PEER, '4'),
+      reply(5, RUNNER, 'Handing off for 5 / 已把 5 交给 test2'),
+      reply(6, PEER, '5'),
+      reply(7, RUNNER, '6')
+    ]
+    const verdict = judgeThreadCount({
+      target: 6,
+      participants: [RUNNER, PEER],
+      channel: CHANNEL,
+      thread: THREAD,
+      effects,
+      events: [...messagingCall(RUNNER, 't1', 'mcp__agentconnect__sendMessage'), ...turns(RUNNER, 4), ...turns(PEER, 3)]
+    })
+    expect(verdict.pass).toBe(false)
+    expect(verdict.failures.some((failure) => failure.includes('mcp__agentconnect__sendMessage'))).toBe(true)
+    expect(verdict.messagingToolCalls).toEqual([
+      { agentId: RUNNER, tool: 'mcp__agentconnect__sendMessage', failed: false }
+    ])
+    // The meta-narration is measured even though the tool call already fails it.
+    expect(verdict.metaNarrationReplies).toBe(1)
+  })
+
+  it('fails on the runtime built-in SendMessage too (the #800 collision, via _meta)', () => {
+    const verdict = judgeThreadCount({
+      target: 6,
+      participants: [RUNNER, PEER],
+      channel: CHANNEL,
+      thread: THREAD,
+      effects: cleanReplies(6),
+      events: [...messagingCall(PEER, 't9', 'SendMessage', true), ...turns(RUNNER, 3), ...turns(PEER, 3)]
+    })
+    expect(verdict.pass).toBe(false)
+    expect(verdict.messagingToolCalls).toEqual([{ agentId: PEER, tool: 'SendMessage', failed: false }])
+  })
+
+  it("fails on arm B's `post` façade the same way — the rule is surface-neutral", () => {
+    const verdict = judgeThreadCount({
+      target: 6,
+      participants: [RUNNER, PEER],
+      channel: CHANNEL,
+      thread: THREAD,
+      effects: cleanReplies(6),
+      events: [...messagingCall(RUNNER, 't2', 'mcp__agentconnect__post'), ...turns(RUNNER, 3), ...turns(PEER, 3)]
+    })
+    expect(verdict.pass).toBe(false)
+    expect(verdict.messagingToolCalls[0]!.tool).toBe('mcp__agentconnect__post')
+  })
+
+  it('counts even a REFUSED messaging call — the reflex is the failure, not the delivery', () => {
+    const events = [
+      {
+        type: 'acp.update',
+        agentId: RUNNER,
+        data: { update: { sessionUpdate: 'tool_call', toolCallId: 'tf', title: 'sendMessage' } }
+      },
+      {
+        type: 'acp.update',
+        agentId: RUNNER,
+        data: { update: { sessionUpdate: 'tool_call_update', toolCallId: 'tf', status: 'failed', content: 'refused' } }
+      },
+      ...turns(RUNNER, 3),
+      ...turns(PEER, 3)
+    ]
+    const verdict = judgeThreadCount({
+      target: 6,
+      participants: [RUNNER, PEER],
+      channel: CHANNEL,
+      thread: THREAD,
+      effects: cleanReplies(6),
+      events
+    })
+    expect(verdict.pass).toBe(false)
+    expect(verdict.messagingToolCalls).toEqual([{ agentId: RUNNER, tool: 'sendMessage', failed: true }])
+  })
+
+  it('ignores non-messaging tools and non-participant events', () => {
+    const events = [
+      ...messagingCall(RUNNER, 't3', 'listAgents'),
+      ...messagingCall('someone-else', 't4', 'sendMessage'),
+      ...turns(RUNNER, 3),
+      ...turns(PEER, 3)
+    ]
+    const verdict = judgeThreadCount({
+      target: 6,
+      participants: [RUNNER, PEER],
+      channel: CHANNEL,
+      thread: THREAD,
+      effects: cleanReplies(6),
+      events
+    })
+    expect(verdict.pass).toBe(true)
+    expect(verdict.messagingToolCalls).toEqual([])
+  })
+
+  it('fails when the count never reaches the target', () => {
+    const verdict = judgeThreadCount({
+      target: 6,
+      participants: [RUNNER, PEER],
+      channel: CHANNEL,
+      thread: THREAD,
+      effects: cleanReplies(4),
+      events: [...turns(RUNNER, 2), ...turns(PEER, 2)]
+    })
+    expect(verdict.pass).toBe(false)
+    expect(verdict.reached).toBe(4)
+    expect(verdict.failures[0]).toContain('reached 4 of 6')
+  })
+
+  it('fails when a participant reply was rejected (a lost message)', () => {
+    const effects = [...cleanReplies(6), reply(7, PEER, 'and this one never landed', 'rejected')]
+    const verdict = judgeThreadCount({
+      target: 6,
+      participants: [RUNNER, PEER],
+      channel: CHANNEL,
+      thread: THREAD,
+      effects,
+      events: [...turns(RUNNER, 3), ...turns(PEER, 4)]
+    })
+    expect(verdict.pass).toBe(false)
+    expect(verdict.lostMessages).toBe(1)
+  })
+})
+
+describe('in-thread turn-taking judge — soft metrics never fail a trial', () => {
+  it('reports duplicates and skips while the trial still passes', () => {
+    // 4 was skipped, 2 was duplicated, and 6 appeared: hard criteria hold.
+    const effects = [
+      reply(1, RUNNER, '1'),
+      reply(2, PEER, '2'),
+      reply(3, RUNNER, '2'),
+      reply(4, PEER, '3'),
+      reply(5, RUNNER, '5'),
+      reply(6, PEER, '6')
+    ]
+    const verdict = judgeThreadCount({
+      target: 6,
+      participants: [RUNNER, PEER],
+      channel: CHANNEL,
+      thread: THREAD,
+      effects,
+      events: [...turns(RUNNER, 3), ...turns(PEER, 3)]
+    })
+    expect(verdict.pass).toBe(true)
+    expect(verdict.duplicates).toBe(1)
+    expect(verdict.skips).toBe(1)
+  })
+
+  it('measures meta-narration and overshoot without failing on them', () => {
+    const effects = [
+      ...cleanReplies(5),
+      reply(6, PEER, 'And now **6** — the count is complete!'),
+      reply(7, RUNNER, '7')
+    ]
+    const verdict = judgeThreadCount({
+      target: 6,
+      participants: [RUNNER, PEER],
+      channel: CHANNEL,
+      thread: THREAD,
+      effects,
+      events: [...turns(RUNNER, 4), ...turns(PEER, 3)]
+    })
+    expect(verdict.pass).toBe(true)
+    expect(verdict.metaNarrationReplies).toBe(1)
+    expect(verdict.overshoot).toBe(1)
+    expect(verdict.bareNumberReplies).toBe(6) // 1..5 plus the bare "7"
+    expect(verdict.meanReplyChars).toBeGreaterThan(1)
+    expect(verdict.turnsPerNumber).toBeCloseTo(7 / 6, 2)
+  })
+
+  it('does not count digits inside mention tokens as count signal', () => {
+    const effects = [...cleanReplies(6), reply(7, PEER, '<@W123456> the count is complete')]
+    const verdict = judgeThreadCount({
+      target: 6,
+      participants: [RUNNER, PEER],
+      channel: CHANNEL,
+      thread: THREAD,
+      effects,
+      events: [...turns(RUNNER, 3), ...turns(PEER, 4)]
+    })
+    expect(verdict.pass).toBe(true)
+    expect(verdict.numbersPosted).toEqual([1, 2, 3, 4, 5, 6])
+  })
 })

From db42907a5b72dde3502a6879da0d81e1fc018dec Mon Sep 17 00:00:00 2001
From: Poytr1 <pooytr1@gmail.com>
Date: Tue, 11 Aug 2026 23:53:11 +0800
Subject: [PATCH 13/15] docs(design): record scenario 5, its 6/6 baseline, and
 the prompt-change gate
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

messaging-primitives-ab.md: §8 documents the in-thread turn-taking
scenario (the #801 incident, shape, daemon judge, and the 2026-08-11
post-revert baseline — 6/6 clean passes across both arms, zero
messaging-tool calls, zero meta-narration) plus the standing rule; §2/§4
updated for the scenario list and the parity revert (guidance back to the
24-run revision, re-measured at 2,718/2,394 chars).

collaboration-arena-baseline.md: §4.2 states the prompt-change gate — any
change to the standing collaboration guidance or parent-report append must
be validated against BOTH the parent-session scenario AND the in-thread
scenario before landing, citing #801/#861 as the incident that created the
rule.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 docs/designs/collaboration-arena-baseline.md |  23 +++
 docs/designs/messaging-primitives-ab.md      | 153 +++++++++++++++++--
 2 files changed, 160 insertions(+), 16 deletions(-)

diff --git a/docs/designs/collaboration-arena-baseline.md b/docs/designs/collaboration-arena-baseline.md
index f33a8c992..20826bbd9 100644
--- a/docs/designs/collaboration-arena-baseline.md
+++ b/docs/designs/collaboration-arena-baseline.md
@@ -277,6 +277,29 @@ export AGENTCONNECT_DAEMON_ENTRY="$PWD/packages/daemon/dist/index.js"
 # then call runWerewolf({ subject: { kind: 'real', subjectRoot, templateAgentIds } })
 ```
 
+### 4.2 Prompt-change gate (standing rule, from the #801 incident)
+
+**Any change to the standing collaboration guidance or the parent-report
+append (`collabAppend` / `parentReplyAppend` in
+`packages/daemon/src/session/session-manager.ts`) must be validated against
+BOTH the parent-session scenario AND the in-thread turn-taking scenario
+(`in-thread-count`) of the tool-surface A/B matrix before landing.**
+
+Why this rule exists: PR #801 led the guidance with a tool-precedence bullet
+("AgentConnect's MCP tools are the ONLY channel that reaches other agents
+and humans"), validated only against the parent-session scenario (10/10),
+and then caused a live in-thread regression — an agent in a channel counting
+game started routing every turn through `sendMessage` to "hand off" numbers
+to its peer instead of replying in the thread, posting meta-narration with
+skipped and duplicated numbers. #801 was reverted by #861; issue #800
+records the incident. The two failure modes pull the guidance in opposite
+directions (reach-peers-via-tool vs in-thread-speech-is-the-ordinary-reply),
+so a candidate that scores well on one and is unmeasured on the other is
+unvalidated. Scenario design, judge, and baseline:
+`messaging-primitives-ab.md` §8; runner:
+`evals/test/tool-surface-ab-real.test.ts` with
+`AGENTCONNECT_EVAL_AB_SCENARIOS=parent-session,in-thread-count`.
+
 ## 5. Real-model runs
 
 ### 5.1 Sequential Werewolf, real local Claude Code
diff --git a/docs/designs/messaging-primitives-ab.md b/docs/designs/messaging-primitives-ab.md
index f6ba4ccac..11541923e 100644
--- a/docs/designs/messaging-primitives-ab.md
+++ b/docs/designs/messaging-primitives-ab.md
@@ -3,7 +3,10 @@
 Status: **complete.** Apparatus landed and contract-proven; static costs
 measured; the full 24-run behavioral matrix ran on 2026-08-09 (local Claude
 Code over ACP, model `sonnet`) with 24/24 valid trials — results in §6,
-conclusion in §7.
+conclusion in §7. **2026-08-11:** the matrix gained scenario 5 (in-thread
+turn-taking, the #801 regression gate — §8) after a prompt change validated
+only against parent-session caused a live in-thread regression; §8 also
+records the standing prompt-change gate that incident created.
 
 The question under test, verbatim from the request that started this work:
 **how much does the primitives design improve success rate and total token
@@ -72,6 +75,13 @@ already caught the word "conversation" priming arm B once). Scored forms:
 4. **parent-session** — woken by a real parent session (the peer agent is
    instructed to delegate a quoted question with an answer-back obligation),
    reply into that session.
+5. **in-thread-count** — added 2026-08-11 (§8): NOT a send scenario. One
+   plaza thread, BOTH agents on the arm's surface, a human kickoff
+   @-mentioning both; the agents take turns counting to 6 via ordinary
+   replies (each delivered reply echoes back and wakes the peer through the
+   #549 continuation ladder). The correct number of messaging-tool calls is
+   ZERO — in-thread speech is the ordinary turn reply, by product
+   convention. The same banned-vocabulary rule covers its kickoff text.
 
 **Topology**: `briefing` (subject only — instructions arrive here, outside
 every measured room), `plaza` (subject + peer, the target channel),
@@ -124,21 +134,23 @@ never delegates is invalid — it conditioned on the caller, not the subject.
 Measured from the real descriptors and from the guidance text a real daemon
 injected into a session prompt (`evals/test/post-facade.test.ts`,
 `evals/test/tool-surface-ab-fixture.test.ts`; token figures are a chars/4
-approximation and labelled as such). Two revisions matter, because the #800
-tool-precedence bullet landed in BOTH arms' guidance after the behavioral
-runs (production via #801, arm B via this branch's parity commit) — prompt
-parity held at each revision:
-
-| Surface component                                            | Arm A (`sendMessage`) | Arm B (`post`) | Ratio    |
-| ------------------------------------------------------------ | --------------------- | -------------- | -------- |
-| Tool description (chars)                                     | 2,617                 | 1,046          | 2.5×     |
-| Tool input schema (chars)                                    | 7,407                 | 1,025          | 7.2×     |
-| Descriptor total (chars)                                     | **10,024**            | **2,071**      | **4.8×** |
-| Descriptor (≈ tokens)                                        | ~2,506                | ~518           |          |
-| Standing guidance, at the 24-run revision (chars)            | 2,718                 | 2,394          | 1.1×     |
-| Standing guidance, current head with the #800 bullet (chars) | 3,099                 | 2,771          | 1.1×     |
-| **Combined per session, current head (chars)**               | **13,123**            | **4,842**      | **2.7×** |
-| Combined, current head (≈ tokens)                            | ~3,281                | ~1,211         |          |
+approximation and labelled as such). Revision history: the #800
+tool-precedence bullet briefly landed in BOTH arms' guidance after the
+behavioral runs (production via #801, arm B via this branch's parity
+commit, ~+380 chars per arm) and was then removed from both when production
+reverted it (#861, after the live in-thread regression §8 gates) — prompt
+parity held at every revision, and the current head is back at the 24-run
+revision's guidance:
+
+| Surface component                                | Arm A (`sendMessage`) | Arm B (`post`) | Ratio    |
+| ------------------------------------------------ | --------------------- | -------------- | -------- |
+| Tool description (chars)                         | 2,617                 | 1,046          | 2.5×     |
+| Tool input schema (chars)                        | 7,407                 | 1,025          | 7.2×     |
+| Descriptor total (chars)                         | **10,024**            | **2,071**      | **4.8×** |
+| Descriptor (≈ tokens)                            | ~2,506                | ~518           |          |
+| Standing guidance, current head = 24-run (chars) | 2,718                 | 2,394          | 1.1×     |
+| **Combined per session, current head (chars)**   | **12,742**            | **4,465**      | **2.9×** |
+| Combined, current head (≈ tokens)                | ~3,186                | ~1,116         |          |
 
 The descriptor is carried by **every turn** of every session; the guidance is
 standing session context. On a cache-warm local run most of this cost lands
@@ -273,3 +285,112 @@ combinations manifested as _actionable refusals_ rather than fewer errors.
 The scenario-4 findings — the native-tool name collision and the
 route-shotgunning child — are product problems upstream of either surface
 and worth fixing regardless of which surface ships.
+
+## 8. Scenario 5: in-thread turn-taking — the #801 regression gate
+
+### 8.1 The incident that created it
+
+The #800 name-collision finding (§6) was fixed by a prompt-side precedence
+bullet (#801): "AgentConnect's MCP tools are the ONLY channel that reaches
+other agents and humans here." It was validated against the parent-session
+scenario only — 10/10, up from 4/6 — and merged. It then caused a **live
+in-thread regression**: in a real Slack thread counting game the agent
+stopped replying with numbers and instead routed every turn through
+`sendMessage` to "hand off" the next number to its peer, posting only
+meta-narration into the thread ("Handing off for 5 / 已把 5 交给 test2")
+with skipped and duplicated numbers in the visible count. The bullet's
+"ONLY channel" over-generalized: the product convention is that
+current-thread communication IS the ordinary reply (`sendMessage`
+deliberately has no in-thread form), and the bullet taught the model that
+plain replies reach nobody. #801 was reverted (#861); issue #800 is
+reopened and records the lesson: **the matrix had no in-thread conversation
+scenario, so a prompt change could pass the whole matrix while breaking
+ordinary thread play.** Scenario 5 is that missing coverage.
+
+### 8.2 Shape
+
+One `plaza` thread; **both** agents are subjects running the arm's surface;
+a human kickoff @-mentions both: take turns counting from 1, reply with
+just the next number, stop at 6 (small on purpose — cheap enough to run as
+a routine gate). The kickoff names no tool, field, or form (the
+banned-vocabulary test covers it). The agents then continue via ordinary
+replies: each delivered reply echoes back as real platform ingress and
+wakes the peer through the #549 continuation ladder — the mechanics the
+live counting game used. The topology, seeds, routing, echo, and validity
+rules (any failed/timed-out turn ⇒ invalid trial) are the matrix's own;
+counterbalancing places the scenario at matrix index 4.
+
+### 8.3 Judge (`judgeThreadCount`, contract-tested in the CI gate)
+
+Daemon-judged, same philosophy as the send scenarios — score what the
+system recorded, never what a model claims:
+
+- **Hard pass**: the count reaches the target via delivered ordinary thread
+  replies; **ZERO messaging-tool calls by either participant during the
+  game** — the product surface (`sendMessage`/`post`), the other arm's
+  tool, and the runtime's built-in `SendMessage` all count, delivered or
+  refused, because a messaging call has no legitimate purpose in this
+  scenario (any such call IS the #801 failure mode); and no participant
+  reply rejected by the world (a lost message).
+- **Soft metrics** (reported, never failed on): duplicated and skipped
+  numbers, overshoot past the stop target, meta-narration beyond the bare
+  number (replies carrying a number plus prose; mean reply length vs the
+  expected 1 char), turns per number, tokens per participant and per run.
+
+The judge is pinned by credential-free contract tests in
+`evals/test/tool-surface-ab.test.ts` (the `eval:collab:contracts` gate): a
+scripted trace replaying the #801 handoff pattern must score FAIL, a clean
+reply-only trace must score PASS, the built-in-`SendMessage` and arm-B
+`post` variants must FAIL identically, and soft-metric traces must pass
+while being measured. `evals/test/tool-surface-ab-fixture.test.ts` proves
+the transport end-to-end against a real daemon with scripted hosts: a
+both-mentioned kickoff plus ordinary replies really carries the count
+peer-to-peer through the echo, with both participants contributing, and
+the judge passes it.
+
+### 8.4 Baseline results (2026-08-11, post-revert prompt, 2 arms × 3 trials)
+
+Run: local Claude Code over ACP (`claude-agent-acp` 0.64.0 launched via
+`node`, model `sonnet`, `permissionMode: default`, memory off), 2 arms × 3
+trials, counterbalanced arm order, sequential on one machine, 6/6 valid.
+This baselines the scenario on the CURRENT (post-revert, pre-#801-identical)
+guidance — the expectation was that both arms pass cleanly, since the
+pre-#801 text never caused the in-thread failure, but it was measured
+rather than assumed. Artifacts:
+`.artifacts/evaluation/tool-surface-ab/in-thread-count-<arm>-<trial>/`;
+copies under `~/arena-runs/ab-2026-08-11-in-thread-count/` on the
+measurement host.
+
+| Arm | Trial | Verdict  | Visible count | Messaging calls | Lost | Dup / skip / overshoot | Bare / meta replies | Turns per number | Run tokens (total / in+out) | Wall  |
+| --- | ----- | -------- | ------------- | --------------- | ---- | ---------------------- | ------------------- | ---------------- | --------------------------- | ----- |
+| A   | 1     | **PASS** | 1–6 in order  | 0               | 0    | 0 / 0 / 0              | 6 / 0               | 1.17             | 295,219 / 214               | 46.7s |
+| A   | 2     | **PASS** | 1–6 in order  | 0               | 0    | 0 / 0 / 0              | 6 / 0               | 1.17             | 294,154 / 114               | 42.5s |
+| A   | 3     | **PASS** | 1–6 in order  | 0               | 0    | 0 / 0 / 0              | 6 / 0               | 1.17             | 302,221 / 880               | 60.3s |
+| B   | 1     | **PASS** | 1–6 in order  | 0               | 0    | 0 / 0 / 0              | 6 / 0               | 1.17             | 298,318 / 325               | 44.3s |
+| B   | 2     | **PASS** | 1–6 in order  | 0               | 0    | 0 / 0 / 0              | 6 / 0               | 1.17             | 317,230 / 734               | 51.6s |
+| B   | 3     | **PASS** | 1–6 in order  | 0               | 0    | 0 / 0 / 0              | 6 / 0               | 1.17             | 299,571 / 200               | 66.8s |
+
+**6/6 clean sweep, both arms.** Every trial produced exactly the six bare
+numbers 1–6 in order (mean reply length 1 char — zero meta-narration), via
+7 participant turns (both agents woken by the kickoff, then five
+echo-driven continuation turns), with zero messaging-tool calls of any
+kind — product surface, other arm's tool, or the runtime built-in — zero
+lost replies, and zero duplicates, skips, or overshoot. This is the
+pre-#801 guidance behaving exactly as the live product did before the
+regression, now pinned as the gate's baseline: a future guidance candidate
+that scores below 3/3 per arm here is a regression against this table, no
+matter what it scores on parent-session.
+
+### 8.5 The standing prompt-change gate
+
+**Any change to the standing collaboration guidance or the parent-report
+append (`collabAppend` / `parentReplyAppend` in
+`packages/daemon/src/session/session-manager.ts`) must be validated against
+BOTH the parent-session scenario AND this in-thread scenario before
+landing.** #801 is the incident that created this rule: a prompt fix
+measured only on the report-back path traded a silent parent-report loss
+for a visible conversation regression, because the two failure modes pull
+the guidance in opposite directions ("use the tool to reach peers" vs
+"in-thread speech is the ordinary reply"). A candidate rewrite that scores
+well on one and is unmeasured on the other is unvalidated. The same gate is
+recorded in `collaboration-arena-baseline.md` §4.2.

From cb89aaeeeaa2f1f91c555e8eb2b9f4a0d3fc4324 Mon Sep 17 00:00:00 2001
From: Poytr1 <pooytr1@gmail.com>
Date: Wed, 12 Aug 2026 00:02:44 +0800
Subject: [PATCH 14/15] fix(evals): close two false-pass paths in the in-thread
 judge (review)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- Match messaging tools by name segment, so the daemon's dotted ACP
  identity (`mcp.agentconnect.post`) fails the hard rule exactly like
  `post` / `mcp__agentconnect__post`, while `compost`/`postpone` still
  never match.
- Require a STRICT thread match on counted replies: a participant reply
  effect with no `thread` (or another thread) is a channel-root post in a
  different conversation — numbers landed there (where a messaging-tool
  detour would put them) no longer satisfy the count. The 6/6 baseline is
  unaffected: every recorded reply carries the kickoff thread id.

Both pinned by new contract tests in the eval:collab:contracts gate.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 evals/games/tool-surface-ab.ts     | 21 +++++++++++++-----
 evals/test/tool-surface-ab.test.ts | 34 ++++++++++++++++++++++++++++++
 2 files changed, 50 insertions(+), 5 deletions(-)

diff --git a/evals/games/tool-surface-ab.ts b/evals/games/tool-surface-ab.ts
index 5645334a5..25880f205 100644
--- a/evals/games/tool-surface-ab.ts
+++ b/evals/games/tool-surface-ab.ts
@@ -247,13 +247,21 @@ export const THREAD_COUNT_SCENARIO = {
 
 /** Is this tool name a messaging tool, on ANY surface the session might carry?
  *  Covers the product `sendMessage` (`mcp__agentconnect__sendMessage`), the
- *  arm-B façade `post` (`mcp__agentconnect__post`), and the Claude Code
- *  runtime's own built-in `SendMessage` (the #800 name-collision hazard).
- *  During the in-thread game every one of them is the #801 failure mode. */
+ *  arm-B façade `post` under EVERY runtime-assigned ACP identity — bare
+ *  `post`, underscore-flattened `mcp__agentconnect__post`, and the dotted
+ *  `mcp.agentconnect.post` the daemon equally supports (daemon.ts FQN
+ *  matching) — and the Claude Code runtime's own built-in `SendMessage` (the
+ *  #800 name-collision hazard). During the in-thread game every one of them
+ *  is the #801 failure mode. Matching is by name SEGMENT, so `compost` or
+ *  `postpone` never match while any separator spelling of `post` does. */
 export function isMessagingToolName(name: string): boolean {
   const normalized = name.toLowerCase()
   if (normalized.includes('sendmessage')) return true
-  return normalized === 'post' || normalized.endsWith('__post')
+  const lastSegment = normalized
+    .split(/[^a-z0-9]+/)
+    .filter(Boolean)
+    .at(-1)
+  return lastSegment === 'post'
 }
 
 export interface ThreadCountEffect {
@@ -372,13 +380,16 @@ export function judgeThreadCount(options: ThreadCountJudgeOptions): ThreadCountV
   const messagingToolCalls = callOrder.map((id) => callById.get(id)!)
 
   // ── the visible thread: delivered participant replies, in order ──
+  // STRICT thread match: a reply effect with no `thread` (or another thread)
+  // is a channel-root post opening a DIFFERENT conversation — counting it
+  // would let numbers posted outside the game thread pass the count.
   const participantThreadEffects = options.effects.filter(
     (effect) =>
       effect.kind === 'reply' &&
       effect.agentId !== undefined &&
       participants.has(effect.agentId) &&
       effect.channel === options.channel &&
-      (effect.thread === undefined || effect.thread === options.thread)
+      effect.thread === options.thread
   )
   const delivered = participantThreadEffects.filter((effect) => effect.status === 'delivered')
   const lostMessages = participantThreadEffects.filter((effect) => effect.status === 'rejected').length
diff --git a/evals/test/tool-surface-ab.test.ts b/evals/test/tool-surface-ab.test.ts
index d3ef60c0c..84c0e7ebc 100644
--- a/evals/test/tool-surface-ab.test.ts
+++ b/evals/test/tool-surface-ab.test.ts
@@ -231,6 +231,9 @@ describe('messaging-tool name matcher', () => {
     expect(isMessagingToolName('SendMessage')).toBe(true) // Claude Code built-in
     expect(isMessagingToolName('post')).toBe(true)
     expect(isMessagingToolName('mcp__agentconnect__post')).toBe(true)
+    // The daemon supports the dotted ACP identity too (daemon.ts FQN matching):
+    // a call under that spelling must not slip past the hard rule.
+    expect(isMessagingToolName('mcp.agentconnect.post')).toBe(true)
     expect(isMessagingToolName('listAgents')).toBe(false)
     expect(isMessagingToolName('setSessionTitle')).toBe(false)
     expect(isMessagingToolName('compost')).toBe(false)
@@ -376,6 +379,37 @@ describe('in-thread turn-taking judge — hard rules', () => {
     expect(verdict.failures[0]).toContain('reached 4 of 6')
   })
 
+  it('fails on the dotted ACP identity of the façade too', () => {
+    const verdict = judgeThreadCount({
+      target: 6,
+      participants: [RUNNER, PEER],
+      channel: CHANNEL,
+      thread: THREAD,
+      effects: cleanReplies(6),
+      events: [...messagingCall(PEER, 't8', 'mcp.agentconnect.post'), ...turns(RUNNER, 3), ...turns(PEER, 3)]
+    })
+    expect(verdict.pass).toBe(false)
+    expect(verdict.messagingToolCalls[0]!.tool).toBe('mcp.agentconnect.post')
+  })
+
+  it('does not count numbers posted OUTSIDE the game thread toward the count', () => {
+    // A reply effect with no `thread` (or a different one) is a channel-root
+    // post opening a different conversation — exactly where a messaging-tool
+    // detour would land the numbers. The count must not be satisfiable there.
+    const offThread = cleanReplies(6).map((effect, index) => (index >= 3 ? { ...effect, thread: undefined } : effect))
+    const verdict = judgeThreadCount({
+      target: 6,
+      participants: [RUNNER, PEER],
+      channel: CHANNEL,
+      thread: THREAD,
+      effects: offThread,
+      events: [...turns(RUNNER, 3), ...turns(PEER, 3)]
+    })
+    expect(verdict.pass).toBe(false)
+    expect(verdict.reached).toBe(3)
+    expect(verdict.numbersPosted).toEqual([1, 2, 3])
+  })
+
   it('fails when a participant reply was rejected (a lost message)', () => {
     const effects = [...cleanReplies(6), reply(7, PEER, 'and this one never landed', 'rejected')]
     const verdict = judgeThreadCount({

From 87dd69cd305902a6318eba804a21d672b6989256 Mon Sep 17 00:00:00 2001
From: Poytr1 <pooytr1@gmail.com>
Date: Wed, 12 Aug 2026 00:07:05 +0800
Subject: [PATCH 15/15] fix(evals): match invocation-id-suffixed messaging FQNs
 in the in-thread judge (review)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Some ACP adapters suffix an opaque invocation id to the flattened MCP name
(daemon.ts containsBuiltinToolFqn, e.g. `mcp__agentconnect__post-42`);
the last-segment matcher missed those, leaving a false-pass spelling.
Match `post` as a BOUNDED separator-delimited segment instead — every
underscore/dotted/suffixed spelling fails the hard rule, `compost` and
`postpone` still never match — and pin the suffixed spellings in both the
matcher and judge contracts.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 evals/games/tool-surface-ab.ts     | 14 ++++++-------
 evals/test/tool-surface-ab.test.ts | 32 ++++++++++++++++++++----------
 2 files changed, 28 insertions(+), 18 deletions(-)

diff --git a/evals/games/tool-surface-ab.ts b/evals/games/tool-surface-ab.ts
index 25880f205..9008218b9 100644
--- a/evals/games/tool-surface-ab.ts
+++ b/evals/games/tool-surface-ab.ts
@@ -252,16 +252,16 @@ export const THREAD_COUNT_SCENARIO = {
  *  `mcp.agentconnect.post` the daemon equally supports (daemon.ts FQN
  *  matching) — and the Claude Code runtime's own built-in `SendMessage` (the
  *  #800 name-collision hazard). During the in-thread game every one of them
- *  is the #801 failure mode. Matching is by name SEGMENT, so `compost` or
- *  `postpone` never match while any separator spelling of `post` does. */
+ *  is the #801 failure mode. Matching is by BOUNDED name segment: some ACP
+ *  adapters suffix an opaque invocation id to the flattened FQN (daemon.ts
+ *  `containsBuiltinToolFqn`, e.g. `mcp__agentconnect__post-42`), so `post`
+ *  must match wherever it appears as its own separator-delimited segment —
+ *  while `compost`/`postpone` never do. The gate's bias is deliberate: an
+ *  over-match makes a reviewable FAIL, an under-match a silent false PASS. */
 export function isMessagingToolName(name: string): boolean {
   const normalized = name.toLowerCase()
   if (normalized.includes('sendmessage')) return true
-  const lastSegment = normalized
-    .split(/[^a-z0-9]+/)
-    .filter(Boolean)
-    .at(-1)
-  return lastSegment === 'post'
+  return /(^|[^a-z0-9])post([^a-z0-9]|$)/.test(normalized)
 }
 
 export interface ThreadCountEffect {
diff --git a/evals/test/tool-surface-ab.test.ts b/evals/test/tool-surface-ab.test.ts
index 84c0e7ebc..0b2c1cfa2 100644
--- a/evals/test/tool-surface-ab.test.ts
+++ b/evals/test/tool-surface-ab.test.ts
@@ -234,6 +234,11 @@ describe('messaging-tool name matcher', () => {
     // The daemon supports the dotted ACP identity too (daemon.ts FQN matching):
     // a call under that spelling must not slip past the hard rule.
     expect(isMessagingToolName('mcp.agentconnect.post')).toBe(true)
+    // ...and adapters may suffix an opaque invocation id to the flattened FQN
+    // (daemon.ts containsBuiltinToolFqn) — the suffixed spellings must match.
+    expect(isMessagingToolName('mcp__agentconnect__post-42')).toBe(true)
+    expect(isMessagingToolName('mcp.agentconnect.post-42')).toBe(true)
+    expect(isMessagingToolName('mcp__agentconnect__sendMessage-42')).toBe(true)
     expect(isMessagingToolName('listAgents')).toBe(false)
     expect(isMessagingToolName('setSessionTitle')).toBe(false)
     expect(isMessagingToolName('compost')).toBe(false)
@@ -379,17 +384,22 @@ describe('in-thread turn-taking judge — hard rules', () => {
     expect(verdict.failures[0]).toContain('reached 4 of 6')
   })
 
-  it('fails on the dotted ACP identity of the façade too', () => {
-    const verdict = judgeThreadCount({
-      target: 6,
-      participants: [RUNNER, PEER],
-      channel: CHANNEL,
-      thread: THREAD,
-      effects: cleanReplies(6),
-      events: [...messagingCall(PEER, 't8', 'mcp.agentconnect.post'), ...turns(RUNNER, 3), ...turns(PEER, 3)]
-    })
-    expect(verdict.pass).toBe(false)
-    expect(verdict.messagingToolCalls[0]!.tool).toBe('mcp.agentconnect.post')
+  it('fails on the dotted and invocation-id-suffixed ACP identities of the façade too', () => {
+    // Adapters legitimately spell the same tool `mcp.agentconnect.post` or
+    // suffix an opaque invocation id (`mcp__agentconnect__post-42`); every
+    // spelling must fail the hard rule identically.
+    for (const spelling of ['mcp.agentconnect.post', 'mcp__agentconnect__post-42', 'mcp.agentconnect.post-42']) {
+      const verdict = judgeThreadCount({
+        target: 6,
+        participants: [RUNNER, PEER],
+        channel: CHANNEL,
+        thread: THREAD,
+        effects: cleanReplies(6),
+        events: [...messagingCall(PEER, 't8', spelling), ...turns(RUNNER, 3), ...turns(PEER, 3)]
+      })
+      expect(verdict.pass, `${spelling} must fail the hard rule`).toBe(false)
+      expect(verdict.messagingToolCalls[0]!.tool).toBe(spelling)
+    }
   })
 
   it('does not count numbers posted OUTSIDE the game thread toward the count', () => {