diff --git a/evals/games/platform-echo.ts b/evals/games/platform-echo.ts index 717b37d00..225b57ab2 100644 --- a/evals/games/platform-echo.ts +++ b/evals/games/platform-echo.ts @@ -32,6 +32,9 @@ export interface PlatformEchoOutcome { admitted: boolean reason?: string fromAlias: string + /** The room this echo travelled through — one loop-guard circuit per room, so + * a member of two rooms is charged against two independent budgets. */ + room: string } export interface PlatformEchoOptions { @@ -149,7 +152,8 @@ export class PlatformEcho { ...(ingressEventTag !== undefined ? { ingressEventTag } : {}), admitted: admission.admitted, ...(admission.admitted ? {} : { reason: admission.reason }), - fromAlias + fromAlias, + room: this.room.alias }) }) .catch(() => {}) diff --git a/evals/games/werewolf.ts b/evals/games/werewolf.ts index 5ef463eab..2595e6236 100644 --- a/evals/games/werewolf.ts +++ b/evals/games/werewolf.ts @@ -179,7 +179,9 @@ export class WerewolfGame implements CollaborationGameWorld { private started = false private terminalReason: string | undefined private winner: 'village' | 'werewolves' | undefined - private readonly pendingWaves: GameWave[] = [] + /** Waves the referee has DECIDED on but not yet handed to the runner, kept as + * thunks — see {@link roomBroadcast} for why nothing may be minted early. */ + private readonly pendingWaves: (() => GameWave)[] = [] private readonly actions: RecordedAction[] = [] private nightKill: { target: string; sequence: number } | undefined private nightProtect: string | undefined @@ -263,7 +265,7 @@ export class WerewolfGame implements CollaborationGameWorld { /** Record what the daemon did with one peer wake-up. Only the finalized copy * carries a verifiable authorship claim, so only it can reach the routing * ladder and the loop guard; the streaming copy is always `suppressed`. */ - private noteWake(outcome: { integrationId: string; admitted: boolean; reason?: string }): void { + private noteWake(outcome: { integrationId: string; admitted: boolean; reason?: string; room: string }): void { const player = [...this.players.values()].find((candidate) => candidate.integrationId === outcome.integrationId) if (!player) return const entry = this.wakes.get(player.alias) ?? { admitted: 0, gated: 0, suppressed: 0 } @@ -280,6 +282,7 @@ export class WerewolfGame implements CollaborationGameWorld { round: this.round, phase: this.phase, agentAlias: player.alias, + roomId: outcome.room, admitted: outcome.admitted, ...(outcome.reason !== undefined ? { reason: outcome.reason } : {}), atMs: Date.now() @@ -501,62 +504,81 @@ export class WerewolfGame implements CollaborationGameWorld { // ── referee delivery helpers ────────────────────────────────────────────── - private roomBroadcast(room: CompiledRoom, text: string): GameWave { - const messageId = this.world.mintMessageId(room.platform) - this.world.registerRoomMessage(room.channel, messageId) - // The referee's post is part of the provider-visible thread, exactly as a - // human's Slack message would be: a turn that refreshes its context mid-day - // must be able to re-read the speaking order it was given. - this.world.recordThreadMessage(room.channel, room.thread, { - ts: messageId, - text, - sender: this.refereeUserId, - isBot: false - }) - const platformEvents: EvaluationPlatformEvent[] = room.memberIntegrationIds.map((integrationId) => ({ - integrationId, - payload: { + /** + * A referee room post, DEFERRED: calling this only decides the message, and + * the returned thunk publishes it. + * + * The referee's post is part of the provider-visible thread, exactly as a + * human's Slack message would be — a turn that refreshes its context mid-day + * must be able to re-read the speaking order it was given. That is precisely + * why it may not become visible early: the referee decides a phase message + * from inside `applyEffects`, while peer turns woken by the effect that ended + * the phase are still open, but its ingress is only injected with the NEXT + * wave. Writing the thread row at decision time let those open turns refresh + * onto a message the daemon had never delivered — acting on a phase change + * that never passed the loop guard, so the trusted-human turn that resets the + * automatic budget was skipped and the reply landed on the peer circuit + * instead. Publishing at emit time makes the thread row, the world event, and + * the ingress one atomic step. + */ + private roomBroadcast(room: CompiledRoom, text: string): () => GameWave { + return () => { + const messageId = this.world.mintMessageId(room.platform) + this.world.registerRoomMessage(room.channel, messageId) + this.world.recordThreadMessage(room.channel, room.thread, { + ts: messageId, + text, + sender: this.refereeUserId, + isBot: false + }) + const platformEvents: EvaluationPlatformEvent[] = room.memberIntegrationIds.map((integrationId) => ({ + integrationId, + payload: { + channel: room.channel, + thread: room.thread, + messageId, + text, + sender: { id: this.refereeUserId, isBot: false } + } + })) + this.world.appendEvent({ + type: 'referee.room_event', + origin: 'referee', + roomId: room.alias, channel: room.channel, - thread: room.thread, messageId, - text, - sender: { id: this.refereeUserId, isBot: false } - } - })) - this.world.appendEvent({ - type: 'referee.room_event', - origin: 'referee', - roomId: room.alias, - channel: room.channel, - messageId, - text - }) - return { platformEvents, refereeEvents: [] } + text + }) + return { platformEvents, refereeEvents: [] } + } } /** Trusted private control (§4.2): pre-addressed, excluded from ingress - * scoring, tagged origin:'referee'. */ - private privateDelivery(player: PlayerState, text: string): RefereeEvent { - const messageId = this.world.mintMessageId(player.dm.platform) - this.world.registerRoomMessage(player.dm.channel, messageId) - this.world.appendEvent({ - type: 'referee.private_event', - origin: 'referee', - toAlias: player.alias, - channel: player.dm.channel, - messageId - // Deliberately NOT the text: private role content stays out of any layer - // the public scoring reads, and canaries must exist in exactly one place. - }) - return { - targetAgentId: player.agentId, - platform: player.dm.platform as RefereeEvent['platform'], - integrationId: player.integrationId, - channel: player.dm.channel, - thread: player.dm.thread, - messageId, - text, - isDm: true + * scoring, tagged origin:'referee'. Deferred like {@link roomBroadcast} so a + * wave's ids and world events are minted in delivery order. */ + private privateDelivery(player: PlayerState, text: string): () => RefereeEvent { + return () => { + const messageId = this.world.mintMessageId(player.dm.platform) + this.world.registerRoomMessage(player.dm.channel, messageId) + this.world.appendEvent({ + type: 'referee.private_event', + origin: 'referee', + toAlias: player.alias, + channel: player.dm.channel, + messageId + // Deliberately NOT the text: private role content stays out of any layer + // the public scoring reads, and canaries must exist in exactly one place. + }) + return { + targetAgentId: player.agentId, + platform: player.dm.platform as RefereeEvent['platform'], + integrationId: player.integrationId, + channel: player.dm.channel, + thread: player.dm.thread, + messageId, + text, + isDm: true + } } } @@ -595,20 +617,20 @@ export class WerewolfGame implements CollaborationGameWorld { .filter((player) => player.role !== 'werewolf') .map((player) => player.alias) const denWolves = this.wolves().filter((wolf) => wolf.alive) - const wave: GameWave = { platformEvents: [], refereeEvents: [] } - if (denWolves.length > 0) { - const denPrompt = this.roomBroadcast( - this.wolfDen, - `NIGHT ${this.round}. Wolves: talk here and agree on tonight's victim. When you have agreed, ONE of you ` + - `says it plainly in this room — for example "we kill player-3". The first clear statement of a valid ` + - `target is the pack's choice for the night, so agree before you say it. ` + - `Targets: ${wolfTargets.join(', ')}.` - ) - wave.platformEvents.push(...denPrompt.platformEvents) - } + const denPrompt = + denWolves.length > 0 + ? this.roomBroadcast( + this.wolfDen, + `NIGHT ${this.round}. Wolves: talk here and agree on tonight's victim. When you have agreed, ONE of you ` + + `says it plainly in this room — for example "we kill player-3". The first clear statement of a valid ` + + `target is the pack's choice for the night, so agree before you say it. ` + + `Targets: ${wolfTargets.join(', ')}.` + ) + : undefined + const privates: (() => RefereeEvent)[] = [] const seer = this.living().find((player) => player.role === 'seer') if (seer) { - wave.refereeEvents.push( + privates.push( this.privateDelivery( seer, `NIGHT ${this.round}. Reply here naming the ONE living player you inspect tonight ` + @@ -618,7 +640,7 @@ export class WerewolfGame implements CollaborationGameWorld { } const doctor = this.living().find((player) => player.role === 'doctor') if (doctor) { - wave.refereeEvents.push( + privates.push( this.privateDelivery( doctor, `NIGHT ${this.round}. Reply here naming the ONE living player you protect tonight ` + @@ -626,7 +648,10 @@ export class WerewolfGame implements CollaborationGameWorld { ) ) } - this.pendingWaves.push(wave) + this.pendingWaves.push(() => ({ + platformEvents: denPrompt ? denPrompt().platformEvents : [], + refereeEvents: privates.map((deliver) => deliver()) + })) } private resolveNightAndQueueDay(): void { @@ -686,13 +711,13 @@ export class WerewolfGame implements CollaborationGameWorld { this.phase = 'day' this.dayStage = 'discussion' this.dayVotes.clear() - const wave: GameWave = { platformEvents: [], refereeEvents: [] } + const privates: (() => RefereeEvent)[] = [] // The seer's result is private control, delivered alongside the public day. if (this.nightInspect) { const seer = this.players.get(this.nightInspect.seer) const target = this.players.get(this.nightInspect.target) if (seer?.alive && target) { - wave.refereeEvents.push( + privates.push( this.privateDelivery( seer, `Inspection result: ${target.alias} is ${target.role === 'werewolf' ? 'a werewolf' : 'not a werewolf'}.` @@ -734,8 +759,10 @@ export class WerewolfGame implements CollaborationGameWorld { `two sentences, and never use an @-mention. If it is not your turn yet, or you have already spoken, ` + `say nothing at all. The referee will ask for votes once the last speaker has finished.` ) - wave.platformEvents.push(...dayPrompt.platformEvents) - this.pendingWaves.push(wave) + this.pendingWaves.push(() => { + const refereeEvents = privates.map((deliver) => deliver()) + return { platformEvents: dayPrompt().platformEvents, refereeEvents } + }) } /** One delivered public-room speech during the sequential discussion. */ @@ -936,9 +963,9 @@ export class WerewolfGame implements CollaborationGameWorld { ) // Night 1 follows immediately after roles are delivered. this.queueNight() - return { platformEvents: opening.platformEvents, refereeEvents: roleDeliveries } + return { platformEvents: opening().platformEvents, refereeEvents: roleDeliveries.map((deliver) => deliver()) } } - if (this.pendingWaves.length > 0) return this.pendingWaves.shift()! + if (this.pendingWaves.length > 0) return this.pendingWaves.shift()!() // Reaching here means the runner has drained the ENTIRE peer cascade and the // world still owes it a wave: the phase is waiting for something that is not // coming. Close the phase out on the evidence rather than stalling the run — @@ -956,7 +983,7 @@ export class WerewolfGame implements CollaborationGameWorld { if (this.dayStage === 'discussion') this.closeDiscussionAndQueueVote() else this.resolveDay() } - return this.pendingWaves.shift() ?? { platformEvents: [], refereeEvents: [] } + return this.pendingWaves.shift()?.() ?? { platformEvents: [], refereeEvents: [] } } drainOutboundEffects(): readonly RecordedOutboundEffect[] { diff --git a/evals/test/werewolf.test.ts b/evals/test/werewolf.test.ts index cc92b23a8..08c4d063a 100644 --- a/evals/test/werewolf.test.ts +++ b/evals/test/werewolf.test.ts @@ -2,6 +2,7 @@ import { mkdtempSync, readFileSync, rmSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' import { afterAll, describe, expect, it } from 'vitest' +import { MAX_AUTOMATIC_TURNS_PER_WINDOW } from '../../packages/daemon/src/daemon/loop-guard-scope.js' import { runWerewolf, werewolfDmRoomAlias, werewolfManifest } from '../games/engine.js' import { compileTopology } from '../games/topology.js' import { WerewolfGame, assignWerewolfRoles } from '../games/werewolf.js' @@ -441,17 +442,14 @@ describe('werewolf multi-round play — night → sequential day → vote → re expect(result.verdict.invariants.privateLeaks).toBe(0) }, 300_000) - it('SCRIPTED BOUNDARY: a seven-player game exhausts the budget inside one 60s window', async () => { - // Actions are messages now, so a day costs the room its discussion AND its - // votes — roughly double the traffic the tool path charged. A scripted game - // finishes in about two seconds, so every round lands inside ONE 60s - // loop-guard window and the budget never refreshes: circuits latch at - // exactly MAX_AUTOMATIC_TURNS_PER_WINDOW and the later rounds are empty. - // - // This is a property of scripted SPEED, not of the design: the same table - // with real models takes ~90s for round 1 alone, the window rolls, and the - // game completes with zero gated wakes (baseline §5.4). Pinned so a change - // in either direction is visible. + it('SCRIPTED BOUNDARY: a seven-seat order fits the automatic budget, so nothing ever latches', async () => { + // The complement of the twelve-seat test below, and the reason the boundary + // is about ORDER LENGTH rather than wall-clock speed: a phase charges each + // player one automatic turn per peer that speaks in it, and the referee's + // own phase post is a trusted human turn that resets the automatic counter. + // Seven seats therefore keep every phase under MAX_AUTOMATIC_TURNS_PER_WINDOW + // no matter how fast the run is, and the game plays to a rule with no gated + // wake at all. Pinned so a change in either direction is visible. const result = await runWerewolf({ seed: 42, playerCount: 7, @@ -460,22 +458,65 @@ describe('werewolf multi-round play — night → sequential day → vote → re timeoutMs: 300_000 }) expect(result.status).toBe('passed') + expect(result.verdict.terminalReason).toBe('completed') const wakes = result.verdict.outcome.peerWakes as Record - const latched = result.verdict.outcome.loopGuardLatched as string[] - expect(latched.length).toBeGreaterThan(0) - expect(result.verdict.metrics.peerWakesGated).toBeGreaterThan(0) - // Every latched non-wolf holds exactly one circuit and stops at the budget. - const roles = result.verdict.outcome.roles as Record - for (const alias of latched.filter((name) => roles[name] !== 'werewolf')) { - expect(wakes[alias]!.admitted).toBe(8) - expect(wakes[alias]!.gated).toBeGreaterThan(0) + expect(result.verdict.outcome.loopGuardLatched).toEqual([]) + expect(result.verdict.metrics.peerWakesGated).toBe(0) + for (const entry of Object.values(wakes)) expect(entry.gated).toBe(0) + + const events = worldEvents(result.paths.worldEvents) + // Per ROOM — one loop-guard circuit each — no player is charged as many as + // MAX_AUTOMATIC_TURNS_PER_WINDOW automatic wakes between two referee posts. + // That headroom is the whole claim: it is why nothing above latches. + const sinceReferee = new Map>() + for (const event of events) { + const room = String(event.roomId) + if (event.type === 'referee.room_event') { + sinceReferee.get(room)?.clear() + continue + } + if (event.type !== 'peer.wake' || event.admitted !== true) continue + const charged = sinceReferee.get(room) ?? new Map() + const alias = String(event.agentAlias) + const count = (charged.get(alias) ?? 0) + 1 + charged.set(alias, count) + sinceReferee.set(room, charged) + expect(count, `${alias} took ${count} automatic wakes on one ${room} budget`).toBeLessThan( + MAX_AUTOMATIC_TURNS_PER_WINDOW + ) } - // The wolves hold TWO circuits — the public room and the den — so they - // absorb more before latching. That is the den echo being real ingress. - for (const alias of Object.keys(roles).filter((name) => roles[name] === 'werewolf')) { - expect(wakes[alias]!.admitted).toBeGreaterThan(8) + + // The wolves hold TWO circuits — the public room and the den — so only they + // are charged in the den at all. That is the den echo being real ingress. + const roles = result.verdict.outcome.roles as Record + const denWakes = new Set( + events + .filter((event) => event.type === 'peer.wake' && event.roomId === 'wolf-den') + .map((event) => String(event.agentAlias)) + ) + expect([...denWakes].sort()).toEqual( + Object.keys(roles) + .filter((alias) => roles[alias] === 'werewolf') + .sort() + ) + + // A phase reaches a player ONLY as delivered ingress: every speech follows a + // DAY post and every vote follows a VOTE post. A referee message that became + // visible to an open turn before the daemon delivered it would show up here + // as a vote cast while the room was still on the DAY prompt. + let lastPrompt = '' + for (const event of events) { + if (event.type === 'referee.room_event' && event.roomId === 'village-square') { + lastPrompt = String(event.text) + continue + } + if (event.type !== 'platform.echo' || event.roomId !== 'village-square') continue + if (event.deliveryState !== 'final') continue + const expected = /^I vote for /.test(String(event.text)) ? /^VOTE \d+\./ : /^DAY \d+\./ + expect(lastPrompt, `"${String(event.text)}" answered "${lastPrompt.slice(0, 20)}"`).toMatch(expected) } - // And the game still ends on a rule, with no invariant touched. + + // And the game ends on a rule, with no invariant touched. expect(result.verdict.invariants).toMatchObject({ privateLeaks: 0, attemptedUnauthorizedEffects: 0 }) }, 300_000) }) @@ -488,17 +529,15 @@ describe('werewolf day phase — natural sequential discussion driven by peer me const days = result.verdict.outcome.dayDiscussions as DayRecord[] expect(days.length).toBeGreaterThanOrEqual(1) - // The FIRST day is the one with a full budget behind it: every speech lands - // on the announced order, exactly once each, in order. Later scripted days - // run on an exhausted window (see the SCRIPTED BOUNDARY test), so they are - // allowed to stall — but nothing may ever speak OUT of order. - expect(days[0]!.outcome).toBe('order_complete') - expect(days[0]!.spoke).toEqual(days[0]!.order) - expect(days[0]!.neverSpoke).toEqual([]) + // This table's order is shorter than the automatic budget and every phase + // refreshes it (see the SCRIPTED BOUNDARY test), so EVERY day completes: + // each speech lands on the announced order, exactly once each, in order. for (const day of days) { + expect(day.outcome).toBe('order_complete') + expect(day.spoke).toEqual(day.order) + expect(day.neverSpoke).toEqual([]) expect(day.outOfOrder).toEqual([]) expect(day.reachedVote).toBe(true) - expect(day.spoke).toEqual(day.order.slice(0, day.spoke.length)) } const events = worldEvents(result.paths.worldEvents)