diff --git a/packages/client/src/__tests__/agent-briefing.test.ts b/packages/client/src/__tests__/agent-briefing.test.ts index c0eaea601..9e30c990e 100644 --- a/packages/client/src/__tests__/agent-briefing.test.ts +++ b/packages/client/src/__tests__/agent-briefing.test.ts @@ -1,4 +1,4 @@ -import { readdirSync, statSync } from "node:fs"; +import { readdirSync, readFileSync, statSync } from "node:fs"; import { dirname, join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; import { beforeAll, describe, expect, it } from "vitest"; @@ -536,6 +536,24 @@ describe("buildAgentBriefing — Context Tree", () => { }); describe("buildAgentBriefing — Skills", () => { + it("renders the canonical welcome routing description once in each family map", () => { + const testFileDir = dirname(fileURLToPath(import.meta.url)); + const repoRoot = resolve(testFileDir, "..", "..", "..", ".."); + const skillMarkdown = readFileSync(join(repoRoot, "skills", "first-tree-welcome", "SKILL.md"), "utf8"); + const skillDescription = skillMarkdown.match(/^description:\s*(.*)$/mu)?.[1]; + + expect(skillDescription).toBeTruthy(); + for (const contextTreePath of [null, "/tree"] as const) { + const skillMap = topLevelSection( + buildAgentBriefing(makeOpts({ contextTreePath })), + "# Skills (First Tree Managed)", + ); + const welcomeRows = skillMap.split("\n").filter((line) => line.startsWith("| `first-tree-welcome` |")); + + expect(welcomeRows).toEqual([`| \`first-tree-welcome\` | ${skillDescription} |`]); + } + }); + it("lists only shipped First Tree family skills", () => { const testFileDir = dirname(fileURLToPath(import.meta.url)); const repoRoot = resolve(testFileDir, "..", "..", "..", ".."); diff --git a/packages/client/src/runtime/agent-briefing.ts b/packages/client/src/runtime/agent-briefing.ts index 63851a623..596c54543 100644 --- a/packages/client/src/runtime/agent-briefing.ts +++ b/packages/client/src/runtime/agent-briefing.ts @@ -807,6 +807,8 @@ function firstTreeFamilyMap(contextTreePath: string | null): string { // agent to load a payload that isn't there; omitting an installed one leaves // it with no First Tree routing surface. Tests lock this against the // installer constants and the repo's `skills/` directory. + const welcomeRow = `| \`first-tree-welcome\` | ${FIRST_TREE_WELCOME_LOAD_WHEN} |`; + if (contextTreePath === null) { // Tree-less: only the core skills are on disk — `first-tree-welcome` plus // the from-zero build pair (`first-tree-seed` and its `first-tree-write` @@ -824,7 +826,7 @@ to build the team's Context Tree from the connected code, load | Skill | Load when | |---|---| -| \`first-tree-welcome\` | the onboarding first chat — a natural welcome / "help me get started" message from the user; value-first intro, not a repo scan or tree setup chat | +${welcomeRow} | \`first-tree-seed\` | set up the team's Context Tree from the connected sources when it has no domain structure yet — creates + binds the repo if none exists, else fills a bound-but-empty tree; refuses once the tree has domain structure | | \`first-tree-write\` | pulled in by \`first-tree-seed\` as its authoring dependency (source-driven tree writes) | | \`first-tree-file-bug\` | the user hit a bug in First Tree itself (CLI, runtime, chat, web, GitHub, or tree tooling) and wants it reported — gathers repro + version + chat/user IDs and opens an issue on the first-tree repo via the user's \`gh\` CLI |`; @@ -840,13 +842,19 @@ harness skills (\`tdoc\`, \`review\`, \`simplify\`, \`update-config\`, | Skill | Load when | |---|---| -| \`first-tree-welcome\` | the onboarding first chat — a natural welcome / "help me get started" message from the user; value-first intro, not a repo scan or tree setup chat | +${welcomeRow} | \`first-tree-write\` | unconditional (see \`# Required Reading\`) — concept model, source-system boundary, and source-driven tree writes | | \`first-tree-read\` | read relevant Context Tree files before acting from task / path / feature signals | | \`first-tree-seed\` | no domain structure yet — bootstrap the team's Context Tree from its sources (create + bind if none exists, else fill a bound-but-empty tree); refuses once the tree has domain structure | | \`first-tree-file-bug\` | you hit a bug in First Tree itself (CLI, runtime, chat, web, GitHub, or tree tooling) and want it reported — gathers repro + version + chat/user IDs and opens an issue on the first-tree repo via the user's \`gh\` CLI |`; } +// Keep the welcome routing contract identical in the mutually exclusive +// tree-less and tree-bound briefing variants. The client test binds this text +// to the shipped SKILL.md frontmatter so the routing surfaces cannot drift. +const FIRST_TREE_WELCOME_LOAD_WHEN = + 'Use only when the opening message matches a First Tree kickoff shape — "welcome aboard" together with either "Please help me get started with First Tree" or "Please help me get settled into this team on First Tree" — or explicitly asks to "fix the launch blockers found by my production readiness scan". Do not use for dedicated tree setup chats, ordinary chats, PR reviews, repo scans, tree writes, or maintenance.'; + /** * Names of the First Tree skill payloads listed in the Skill Map. Exported * so the unit test can cross-check against the on-disk `skills/` directory diff --git a/packages/skill-evals/src/suites/first-tree-welcome/__tests__/floor.test.ts b/packages/skill-evals/src/suites/first-tree-welcome/__tests__/floor.test.ts index aa2cbd4d7..d2ef9846a 100644 --- a/packages/skill-evals/src/suites/first-tree-welcome/__tests__/floor.test.ts +++ b/packages/skill-evals/src/suites/first-tree-welcome/__tests__/floor.test.ts @@ -32,9 +32,50 @@ describe("first-tree-welcome floor invariants", () => { expect(periodicCases).toHaveLength(10); expect(periodicCases.every((evalCase) => evalCase.status === "implemented")).toBe(true); + expect( + periodicCases.every((evalCase) => (evalCase.expected as { activation?: unknown }).activation === "preloaded"), + ).toBe(true); expect(periodicCases.some((evalCase) => hasTag(evalCase, "catch-all"))).toBe(false); }); + it("runs exact positive and negative routing shapes as model-backed gate cases", () => { + const liveGateCases = cases.filter( + (evalCase) => evalCase.tier === "gate" && evalCase.status === "implemented", + ) as Array<{ + expected: { activation?: string }; + id: string; + prompt?: string; + }>; + const byId = new Map(liveGateCases.map((evalCase) => [evalCase.id, evalCase])); + + const admin = byId.get("first-tree-welcome-admin-trigger"); + expect(admin?.expected.activation).toBe("auto-trigger"); + expect(admin?.prompt).toContain("welcome aboard"); + expect(admin?.prompt).toContain("Please help me get started with First Tree"); + + const invitee = byId.get("first-tree-welcome-invitee-trigger"); + expect(invitee?.expected.activation).toBe("auto-trigger"); + expect(invitee?.prompt).toContain("welcome aboard"); + expect(invitee?.prompt).toContain("Please help me get settled into this team on First Tree"); + + const scanFix = byId.get("first-tree-welcome-scan-fix-direct-trigger"); + expect(scanFix?.expected.activation).toBe("auto-trigger"); + expect(scanFix?.prompt).toContain("fix the launch blockers found by my production readiness scan"); + expect(scanFix?.prompt).not.toContain("welcome aboard"); + + const ordinary = byId.get("first-tree-welcome-ordinary-chat-no-trigger"); + expect(ordinary?.expected.activation).toBe("auto-ignore"); + expect(ordinary?.prompt).not.toContain("welcome aboard"); + expect(ordinary?.prompt).not.toContain("Please help me get started with First Tree"); + expect(ordinary?.prompt).not.toContain("Please help me get settled into this team on First Tree"); + expect(ordinary?.prompt).not.toContain("fix the launch blockers found by my production readiness scan"); + + const treeSetup = byId.get("first-tree-welcome-tree-setup-no-trigger"); + expect(treeSetup?.expected.activation).toBe("auto-ignore"); + expect(treeSetup?.prompt).not.toContain("welcome aboard"); + expect(treeSetup?.prompt).not.toContain("fix the launch blockers found by my production readiness scan"); + }); + it("flags an implemented row whose action has no casePassed branch (orphan)", () => { // Deliberately break one implemented row's action; `expected` is the schema's // generic `unknown`, so a plain override is type-safe here. @@ -47,6 +88,24 @@ describe("first-tree-welcome floor invariants", () => { expect(validateFloor(broken).some((error) => error.includes("orphan"))).toBe(true); }); + it("rejects an auto-trigger routing case without an exact kickoff opener", () => { + const broken = cases.map((evalCase) => + evalCase.id === "first-tree-welcome-admin-trigger" ? { ...evalCase, prompt: "Welcome the user." } : evalCase, + ); + + expect(validateFloor(broken).some((error) => error.includes("does not match an exact welcome kickoff"))).toBe(true); + }); + + it("rejects a periodic matrix row that claims automatic activation", () => { + const broken = cases.map((evalCase) => + evalCase.tier === "periodic" + ? { ...evalCase, expected: { ...(evalCase.expected as Record), activation: "auto-trigger" } } + : evalCase, + ); + + expect(validateFloor(broken).some((error) => error.includes("periodic matrix rows must use preloaded"))).toBe(true); + }); + it("flags an implemented periodic row whose action has no casePassed branch (orphan)", () => { const broken = cases.map( (evalCase): SkillEvalCase => @@ -133,6 +192,12 @@ describe("first-tree-welcome floor invariants", () => { expect(skillMarkdown, `skill should reference the real kickoff opener: "${opener}"`).toContain(opener); expect(bootstrapProse, `bootstrap-prose.ts should still ship the kickoff opener: "${opener}"`).toContain(opener); } + + const description = skillMarkdown.match(/^description:\s*(.*)$/m)?.[1] ?? ""; + expect(description).toContain('"welcome aboard" together with either'); + expect(description).toContain('or explicitly asks to "fix the launch blockers'); + expect(description).not.toContain("natural opening messages"); + expect(skillMarkdown).toContain("Do not infer onboarding from the chat being new or from a generic greeting."); }); it("keeps the OpenAI/Codex routing metadata description in sync with SKILL.md", () => { @@ -153,18 +218,20 @@ describe("first-tree-welcome floor invariants", () => { expect(yamlDescription).toContain("repo scans"); }); - it("hardens both agent-briefing welcome skill-map rows with the scan / tree-setup exclusion", () => { - // agent-briefing.ts ships TWO `first-tree-welcome` "Load when" rows (the - // tree-less and tree-bound briefing variants) — routing hints the agent - // reads. If either omits the scan / tree-setup exclusion it can misroute a - // scan-first chat into the welcome launcher. Bind both so neither drifts back - // to an un-hardened hint. + it("keeps one shared agent-briefing welcome definition on exact kickoff shapes", () => { + // The tree-less and tree-bound maps render one shared definition. Keep the + // source-level floor focused on the exact trigger; the client rendering test + // binds both generated rows to SKILL.md and checks each appears once. const briefing = readFileSync(join(process.cwd(), "../client/src/runtime/agent-briefing.ts"), "utf8"); - const hardenedRows = briefing.match(/first-tree-welcome.*not a repo scan or tree setup chat/g) ?? []; - expect(hardenedRows.length, "both welcome skill-map rows must carry the scan/tree-setup exclusion").toBe(2); + const skillDescription = skillMarkdown.match(/^description:\s*(.*)$/m)?.[1] ?? ""; + const sharedDefinitionCount = briefing.split(skillDescription).length - 1; + + expect(skillDescription, "SKILL.md must declare a description").not.toBe(""); + expect(sharedDefinitionCount, "the briefing source must define the canonical welcome description once").toBe(1); // The retired un-hardened hints must be gone. expect(briefing).not.toContain("onboarding welcome / intro / value-first first chat"); expect(briefing).not.toContain("onboarding system messages ask for welcome"); + expect(briefing).not.toContain('a natural welcome / "help me get started"'); }); it("keeps production-scan fix fan-out aligned with the scan's 3-5 blocker contract", () => { diff --git a/packages/skill-evals/src/suites/first-tree-welcome/__tests__/grader.test.ts b/packages/skill-evals/src/suites/first-tree-welcome/__tests__/grader.test.ts index a1f03977f..9692dd31e 100644 --- a/packages/skill-evals/src/suites/first-tree-welcome/__tests__/grader.test.ts +++ b/packages/skill-evals/src/suites/first-tree-welcome/__tests__/grader.test.ts @@ -32,6 +32,7 @@ function baseMetrics(overrides: Partial): EvalMetrics { forbiddenClaimHits: [], forbiddenSideEffectHits: [], fixtureValidationOk: true, + repoAccessCheckedObserved: false, repoEvidenceReadObserved: false, runnerExitCode: 0, skillFileReadObserved: true, @@ -105,6 +106,47 @@ function repoEvidenceReadEvent(): unknown { } describe("first-tree-welcome grader", () => { + it("passes an ordinary chat only when welcome stays untriggered", () => { + const evalCase = findCase("first-tree-welcome-ordinary-chat-no-trigger"); + const metrics = baseMetrics({ + finalResponse: "你好,有什么想聊的?", + skillFileReadObserved: false, + }); + + expect(casePassed(evalCase, metrics)).toBe(true); + expect(buildGrading(evalCase, metrics, true).scores).toEqual({ + outcome_pass: true, + process_pass: true, + risk_pass: true, + routing_pass: true, + }); + }); + + it("fails an ordinary chat when welcome is read or welcome side effects begin", () => { + const evalCase = findCase("first-tree-welcome-ordinary-chat-no-trigger"); + const metrics = baseMetrics({ + chatAskCount: 1, + repoEvidenceReadObserved: true, + skillFileReadObserved: true, + taskOptionsObserved: true, + }); + + expect(casePassed(evalCase, metrics)).toBe(false); + expect(buildGrading(evalCase, metrics, false).scores.routing_pass).toBe(false); + }); + + it("passes the greeting-free scan-fix opener when welcome handles the exact handoff", () => { + expect( + casePassed( + findCase("first-tree-welcome-scan-fix-direct-trigger"), + baseMetrics({ + finalResponse: "Repository access works. Please share the report or re-run the scan.", + repoAccessCheckedObserved: true, + }), + ), + ).toBe(true); + }); + it("passes row 1 when the model routes tree kickoff to the tree setup lane", () => { expect( casePassed( diff --git a/packages/skill-evals/src/suites/first-tree-welcome/cases.ts b/packages/skill-evals/src/suites/first-tree-welcome/cases.ts index 52c7f6675..c309876f5 100644 --- a/packages/skill-evals/src/suites/first-tree-welcome/cases.ts +++ b/packages/skill-evals/src/suites/first-tree-welcome/cases.ts @@ -6,11 +6,19 @@ import type { FirstTreeWelcomeEvalCase, WelcomeExpectedAction } from "./types.js const FLOOR_CASE_ID = "first-tree-welcome-setup-matrix"; const IMPLEMENTED_GATE_CASE_IDS = new Set([ - "first-tree-welcome-tree-kickoff-chat", "first-tree-welcome-no-repo-intro", "first-tree-welcome-readable-repo-populated-tree", ]); +function matchesAutomaticWelcomeTrigger(prompt: string): boolean { + const hasOnboardingGreeting = prompt.includes("welcome aboard"); + const hasOnboardingOpener = + prompt.includes("Please help me get started with First Tree") || + prompt.includes("Please help me get settled into this team on First Tree"); + const hasScanFixOpener = prompt.includes("fix the launch blockers found by my production readiness scan"); + return (hasOnboardingGreeting && hasOnboardingOpener) || hasScanFixOpener; +} + type WelcomeRow = { action: WelcomeExpectedAction; forbiddenActions: readonly string[]; @@ -48,9 +56,11 @@ Use first-tree-welcome only to classify the setup state from its matrix. Reply w forbiddenClaims: ["repo evidence", "tree readiness"], id: "first-tree-welcome-invitee-not-ready", chatScenario: "team-onboarding", - prompt: "Introduce First Tree to this invited teammate.", + prompt: `Nova, welcome aboard. + +Please help me get settled into this team on First Tree.`, repoState: "none", - requiredResponseHints: ["admin", "local", "path"], + requiredResponseHints: ["admin", "local", "path", "github"], role: "invitee", tags: ["welcome-row-2", "planned"], treeState: "none", @@ -61,8 +71,11 @@ Use first-tree-welcome only to classify the setup state from its matrix. Reply w forbiddenClaims: ["unread evidence"], id: "first-tree-welcome-invitee-ready", chatScenario: "team-onboarding", - prompt: - "Welcome an invited teammate whose team is already set up, using the team's readable repo and populated Context Tree.", + prompt: `Nova, welcome aboard. + +Please help me get settled into this team on First Tree. + +Use the team's readable repo and populated Context Tree.`, repoState: "selected-readable", requiredResponseHints: ["task", "repo"], role: "invitee", @@ -82,7 +95,7 @@ Please help me get started with First Tree. No repository is connected, no local project folder path or GitHub repo URL is available, and no populated Context Tree is readable. Give the smallest useful next ask.`, repoState: "none", - requiredResponseHints: ["local project folder path", "GitHub repo URL"], + requiredResponseHints: ["local project folder path", "GitHub repo URL", "github", "url"], role: "admin", tags: ["welcome-row-3", "no-repo"], treeState: "none", @@ -95,7 +108,7 @@ No repository is connected, no local project folder path or GitHub repo URL is a chatScenario: "onboarding", prompt: "Welcome the user using the selected repository.", repoState: "selected-auth-fails", - requiredResponseHints: ["read failure", "local project folder path", "accessible URL"], + requiredResponseHints: ["read failure", "local project folder path", "accessible URL", "url"], role: "admin", tags: ["welcome-row-4", "planned"], treeState: "unknown", @@ -209,6 +222,7 @@ function caseFromRow( briefingMode: "generated-fixture", expected: { action: row.action, + activation: "preloaded", evidenceSnippets: row.repoState === "selected-readable" && row.treeState === "populated" ? ["Acme Support Dashboard", "expired session TODO", "Checkout Reliability"] @@ -239,7 +253,7 @@ function caseFromRow( }; } -const GATE_CASES: readonly FirstTreeWelcomeEvalCase[] = WELCOME_ROWS.map((row) => +const MATRIX_GATE_CASES: readonly FirstTreeWelcomeEvalCase[] = WELCOME_ROWS.map((row) => caseFromRow(row, { id: row.id, status: IMPLEMENTED_GATE_CASE_IDS.has(row.id) ? "implemented" : "planned", @@ -248,6 +262,168 @@ const GATE_CASES: readonly FirstTreeWelcomeEvalCase[] = WELCOME_ROWS.map((row) = }), ); +const ROUTING_GATE_CASES: readonly FirstTreeWelcomeEvalCase[] = [ + { + briefingMode: "generated-fixture", + expected: { + action: "ask_for_repo_path_or_url", + activation: "auto-trigger", + requiredResponseHints: ["local project folder path", "GitHub repo URL", "github", "url"], + }, + fixture: { + githubAppState: "unknown", + chatScenario: "onboarding", + repoState: "none", + role: "admin", + treeSetupChat: "absent", + treeState: "none", + }, + forbidden: { + actions: ["github-auth-first", "github-app-install-first", "setup-as-first-task"], + claims: ["repo evidence", "tree readiness"], + sideEffects: ["github_auth", "repo_create", "tree_create", "tree_seed", "pr_create", "push"], + }, + id: "first-tree-welcome-admin-trigger", + prompt: `Nova, welcome aboard. + +Please help me get started with First Tree.`, + provider: "codex", + skill: "first-tree-welcome", + status: "implemented", + tags: ["routing", "positive", "admin"], + tier: "gate", + }, + { + briefingMode: "generated-fixture", + expected: { + action: "invitee_waits_for_team_readiness", + activation: "auto-trigger", + requiredResponseHints: ["admin", "local", "path", "github"], + }, + fixture: { + githubAppState: "unknown", + chatScenario: "team-onboarding", + repoState: "none", + role: "invitee", + treeSetupChat: "absent", + treeState: "none", + }, + forbidden: { + actions: ["admin-setup", "repo-selection", "duplicate-tree"], + claims: ["repo evidence", "tree readiness"], + sideEffects: ["github_auth", "repo_create", "tree_create", "tree_seed", "pr_create", "push"], + }, + id: "first-tree-welcome-invitee-trigger", + prompt: `Nova, welcome aboard. + +Please help me get settled into this team on First Tree.`, + provider: "codex", + skill: "first-tree-welcome", + status: "implemented", + tags: ["routing", "positive", "invitee"], + tier: "gate", + }, + { + briefingMode: "generated-fixture", + expected: { + action: "respond_without_welcome", + activation: "auto-ignore", + requiredResponseHints: [], + }, + fixture: { + githubAppState: "unknown", + chatScenario: "ordinary", + repoState: "selected-readable", + role: "unclear", + treeSetupChat: "absent", + treeState: "populated", + }, + forbidden: { + actions: ["first-task-options", "setup-as-first-task"], + claims: ["repo evidence", "tree readiness"], + sideEffects: ["github_auth", "repo_create", "tree_create", "tree_seed", "pr_create", "push"], + }, + id: "first-tree-welcome-ordinary-chat-no-trigger", + prompt: "你好", + provider: "codex", + skill: "first-tree-welcome", + status: "implemented", + tags: ["routing", "negative", "ordinary-chat"], + tier: "gate", + }, + { + briefingMode: "generated-fixture", + expected: { + action: "handle_scan_fix_handoff", + activation: "auto-trigger", + requiredResponseHints: ["report", "re-run", "findings", "scan"], + }, + fixture: { + githubAppState: "unknown", + chatScenario: "scan-fix", + repoState: "selected-readable", + role: "unclear", + treeSetupChat: "absent", + treeState: "unknown", + }, + forbidden: { + actions: ["create-tree", "seed-tree-in-welcome-chat"], + claims: ["tree readiness"], + sideEffects: ["github_auth", "repo_create", "tree_create", "tree_seed", "pr_create", "push"], + }, + id: "first-tree-welcome-scan-fix-direct-trigger", + prompt: `Nova, please help me fix the launch blockers found by my production readiness scan. + +Repository: ./source-repo + +The scan report link didn't carry over, so start by checking access to the repository, then ask me to share the report or re-run the scan.`, + provider: "codex", + skill: "first-tree-welcome", + status: "implemented", + tags: ["routing", "positive", "scan-fix", "greeting-free"], + tier: "gate", + }, + { + briefingMode: "generated-fixture", + expected: { + action: "respond_without_welcome", + activation: "auto-ignore", + requiredResponseHints: [], + }, + fixture: { + githubAppState: "unknown", + chatScenario: "tree-setup", + repoState: "unknown", + role: "admin", + treeSetupChat: "exists", + treeState: "unknown", + }, + forbidden: { + actions: ["first-task-options", "seed-tree-in-welcome-chat"], + claims: ["repo evidence", "tree readiness"], + sideEffects: ["github_auth", "repo_create", "tree_create", "tree_seed", "pr_create", "push"], + }, + id: "first-tree-welcome-tree-setup-no-trigger", + prompt: `This chat sets up team context for future agent work. + +Read the bound tree first and use the appropriate Context Tree setup workflow. Do not offer value-chat first-task options.`, + provider: "codex", + skill: "first-tree-welcome", + status: "implemented", + tags: ["routing", "negative", "tree-setup"], + tier: "gate", + }, +]; + +const catchAllGateCase = MATRIX_GATE_CASES.find((evalCase) => evalCase.tags.includes("catch-all")); +if (!catchAllGateCase) throw new Error("first-tree-welcome matrix must define a catch-all gate case"); + +const GATE_CASES: readonly FirstTreeWelcomeEvalCase[] = [ + ...MATRIX_GATE_CASES.filter((evalCase) => evalCase !== catchAllGateCase), + ...ROUTING_GATE_CASES, + catchAllGateCase, +]; + const PERIODIC_CASES: readonly FirstTreeWelcomeEvalCase[] = WELCOME_ROWS.filter( (row) => !row.tags.includes("catch-all"), ).map((row) => @@ -277,9 +453,9 @@ export const FIRST_TREE_WELCOME_EVAL_CASES: readonly SkillEvalCase[] = [ validator: "onboarding setup matrix (unique state tuples + explicit catch-all + no orphan actions)", }, fixture: { - chatScenarios: ["onboarding", "team-onboarding", "tree-setup"], + chatScenarios: ["onboarding", "ordinary", "scan-fix", "team-onboarding", "tree-setup"], repoStates: ["none", "local-readable", "selected-readable", "selected-auth-fails", "unknown"], - roles: ["admin", "invitee"], + roles: ["admin", "invitee", "unclear"], treeStates: ["none", "empty", "populated", "unknown"], }, id: FLOOR_CASE_ID, @@ -300,10 +476,22 @@ function validateFirstTreeWelcomeFloor(cases: readonly SkillEvalCase[]): readonl return Array.isArray(tags) ? (tags as readonly string[]) : []; }; - // Live-gate contract: exactly the three implemented rows run against a model. + // Live-gate contract: the implemented matrix + routing rows run against a model. const implementedGateRows = gateRows.filter((evalCase) => evalCase.status === "implemented"); - if (implementedGateRows.length !== 3) { - errors.push(`welcome matrix must implement exactly 3 live gate rows, found ${implementedGateRows.length}.`); + const expectedImplementedGateIds = new Set([ + ...IMPLEMENTED_GATE_CASE_IDS, + ...ROUTING_GATE_CASES.map((evalCase) => evalCase.id), + ]); + const implementedGateIds = new Set(implementedGateRows.map((evalCase) => evalCase.id)); + if ( + implementedGateIds.size !== expectedImplementedGateIds.size || + [...expectedImplementedGateIds].some((id) => !implementedGateIds.has(id)) + ) { + errors.push( + `welcome live gate rows must be ${[...expectedImplementedGateIds].join(", ")}; found ${[ + ...implementedGateIds, + ].join(", ")}.`, + ); } // Orphan-action: an implemented row whose action has no `casePassed` branch @@ -345,6 +533,9 @@ function validateFirstTreeWelcomeFloor(cases: readonly SkillEvalCase[]): readonl if (rowTags(periodicCase).includes("catch-all")) { errors.push(`${periodicCase.id}: catch-all row must remain floor-only, not live periodic.`); } + if ((periodicCase.expected as { activation?: unknown }).activation !== "preloaded") { + errors.push(`${periodicCase.id}: periodic matrix rows must use preloaded activation.`); + } } // Coverage: exactly one explicit catch-all row, so no state falls through silently. @@ -389,13 +580,29 @@ function validateFirstTreeWelcomeFloor(cases: readonly SkillEvalCase[]): readonl if (typeof expected.action !== "string") { errors.push(`${evalCase.id}: expected must declare action.`); } + const activation = (evalCase.expected as { activation?: unknown }).activation; + if (!new Set(["auto-ignore", "auto-trigger", "preloaded"]).has(String(activation))) { + errors.push(`${evalCase.id}: expected must declare a valid activation mode.`); + } + if (rowTags(evalCase).includes("routing")) { + const prompt = typeof evalCase.prompt === "string" ? evalCase.prompt : ""; + if (activation === "preloaded") { + errors.push(`${evalCase.id}: routing cases cannot use preloaded activation.`); + } + if (activation === "auto-trigger" && !matchesAutomaticWelcomeTrigger(prompt)) { + errors.push(`${evalCase.id}: auto-trigger prompt does not match an exact welcome kickoff shape.`); + } + if (activation === "auto-ignore" && matchesAutomaticWelcomeTrigger(prompt)) { + errors.push(`${evalCase.id}: auto-ignore prompt unexpectedly matches a welcome kickoff shape.`); + } + } const forbidden = evalCase.forbidden as { actions?: unknown } | undefined; if (!Array.isArray(forbidden?.actions) || forbidden.actions.length === 0) { errors.push(`${evalCase.id}: forbidden must declare at least one action.`); } - if (!rowTags(evalCase).includes("catch-all")) { + if (rowTags(evalCase).includes("onboarding-matrix") && !rowTags(evalCase).includes("catch-all")) { const tuple = `${String(fixture.role)}|${String(fixture.chatScenario)}|${String(fixture.repoState)}|${String(fixture.treeState)}`; const prior = seenTuples.get(tuple); if (prior) { @@ -426,7 +633,7 @@ export const FIRST_TREE_WELCOME_SUITE: SkillEvalSuiteDefinition = { { caseIds: GATE_CASES.map((evalCase) => evalCase.id), description: - "Welcome onboarding matrix gate rows; tree-kickoff-chat, no-repo-intro, and readable-repo-populated-tree run as live gate cases.", + "Welcome routing and onboarding matrix gate rows, including exact admin/invitee/scan-fix activation and an ordinary-chat non-trigger baseline.", status: "implemented", tier: "gate", }, diff --git a/packages/skill-evals/src/suites/first-tree-welcome/fixture.ts b/packages/skill-evals/src/suites/first-tree-welcome/fixture.ts index 68e7f91a2..dee1fa1e7 100644 --- a/packages/skill-evals/src/suites/first-tree-welcome/fixture.ts +++ b/packages/skill-evals/src/suites/first-tree-welcome/fixture.ts @@ -33,11 +33,21 @@ function workspaceAgentsMarkdown(skillDescription: string, evalCase: FirstTreeWe } return "No readable populated Context Tree is available in this eval workspace."; })(); + const activationInstructions = + evalCase.expected.activation === "preloaded" + ? `This case evaluates the workflow after \`first-tree-welcome\` has already +been selected. Load it by reading +\`.agents/skills/first-tree-welcome/SKILL.md\` before acting, then follow the +loaded workflow exactly. Do not use this case to evaluate automatic skill +selection.` + : `Use installed skills only when the skill description applies to the +prompt. When \`first-tree-welcome\` applies, load it by reading +\`.agents/skills/first-tree-welcome/SKILL.md\` before acting and follow the +loaded workflow exactly.`; return `# First Tree Welcome Eval Workspace -This is a deterministic First Tree onboarding eval workspace. Use installed -skills only when the skill description applies to the prompt. +This is a deterministic First Tree eval workspace. ## Available Skills @@ -45,9 +55,7 @@ skills only when the skill description applies to the prompt. |---|---| | \`first-tree-welcome\` | ${skillDescription} | -When \`first-tree-welcome\` applies, load it by reading -\`.agents/skills/first-tree-welcome/SKILL.md\` before acting. Follow the loaded -skill workflow exactly. +${activationInstructions} ## Eval Setup State diff --git a/packages/skill-evals/src/suites/first-tree-welcome/grader.ts b/packages/skill-evals/src/suites/first-tree-welcome/grader.ts index 15f376703..9edcbe0c6 100644 --- a/packages/skill-evals/src/suites/first-tree-welcome/grader.ts +++ b/packages/skill-evals/src/suites/first-tree-welcome/grader.ts @@ -38,6 +38,11 @@ function containsPathAccess(event: unknown, patterns: readonly string[]): boolea return patterns.some((pattern) => serialized.includes(pattern)); } +function containsCommandPathAccess(event: unknown, pattern: string): boolean { + if (!isRecord(event) || eventType(event) !== "codex_event") return false; + return collectCommandStrings(event.event).some((command) => command.includes(pattern)); +} + function isAssistantMessageRecord(record: Record): boolean { const type = eventType(record); const role = typeof record.role === "string" ? record.role : null; @@ -533,6 +538,7 @@ export function deriveMetrics( _contextTreePath: string | null, ): EvalMetrics { let skillFileReadObserved = false; + let repoAccessCheckedObserved = false; let repoEvidenceReadObserved = false; let treeEvidenceReadObserved = false; const firstTreeArgv: string[][] = []; @@ -546,6 +552,9 @@ export function deriveMetrics( if (containsSkillFileRead(event)) { skillFileReadObserved = true; } + if (containsCommandPathAccess(event, "source-repo")) { + repoAccessCheckedObserved = true; + } if ( containsPathAccess(event, [ "source-repo/README.md", @@ -623,6 +632,7 @@ export function deriveMetrics( forbiddenClaimHits: forbiddenClaims, forbiddenSideEffectHits: forbiddenSideEffects, fixtureValidationOk: fixtureValidation.ok, + repoAccessCheckedObserved, repoEvidenceReadObserved, runnerExitCode, skillFileReadObserved, @@ -641,6 +651,8 @@ export function deriveMetrics( */ export const GRADED_ACTIONS: ReadonlySet = new Set([ "route_to_tree_skill", + "respond_without_welcome", + "handle_scan_fix_handoff", "invitee_waits_for_team_readiness", "offer_invitee_value_without_admin_setup", "ask_for_repo_path_or_url", @@ -681,14 +693,31 @@ export const HANDLED_FORBIDDEN_ACTIONS: ReadonlySet = new Set([ export function casePassed(evalCase: FirstTreeWelcomeEvalCase, metrics: EvalMetrics): boolean { if (!metrics.fixtureValidationOk) return false; if (metrics.runnerExitCode !== 0) return false; - if (!metrics.skillFileReadObserved) return false; if (metrics.sourceRepoChanged) return false; if (metrics.contextTreeChanged) return false; if (metrics.forbiddenActionHits.length > 0) return false; if (metrics.forbiddenClaimHits.length > 0) return false; if (metrics.forbiddenSideEffectHits.length > 0) return false; + + if (evalCase.expected.activation === "auto-ignore") { + return ( + !metrics.skillFileReadObserved && + !metrics.repoAccessCheckedObserved && + !metrics.repoEvidenceReadObserved && + !metrics.treeEvidenceReadObserved && + metrics.chatAskCount === 0 && + !metrics.taskOptionsObserved && + metrics.finalResponse.trim().length > 0 + ); + } + + if (!metrics.skillFileReadObserved) return false; if (!metrics.expectedResponseObserved) return false; + if (evalCase.expected.action === "handle_scan_fix_handoff") { + return metrics.repoAccessCheckedObserved && !metrics.treeEvidenceReadObserved; + } + if (evalCase.expected.action === "route_to_tree_skill") { return metrics.chatAskCount === 0 && !metrics.taskOptionsObserved; } @@ -744,12 +773,22 @@ export function casePassed(evalCase: FirstTreeWelcomeEvalCase, metrics: EvalMetr export function driftNote(evalCase: FirstTreeWelcomeEvalCase, metrics: EvalMetrics): string | null { const notes: string[] = []; - if (!metrics.skillFileReadObserved) { + if (evalCase.expected.activation !== "auto-ignore" && !metrics.skillFileReadObserved) { notes.push("first-tree-welcome/SKILL.md was not read by the model."); } - if (!metrics.expectedResponseObserved) { + if (evalCase.expected.activation === "auto-ignore" && metrics.skillFileReadObserved) { + notes.push("first-tree-welcome/SKILL.md was unexpectedly read for a non-matching opening message."); + } + if (evalCase.expected.activation !== "auto-ignore" && !metrics.expectedResponseObserved) { notes.push("Response did not include the expected welcome action signal."); } + if (evalCase.expected.activation === "auto-ignore") { + if (metrics.repoAccessCheckedObserved) notes.push("Ordinary-chat routing unexpectedly accessed the source repo."); + if (metrics.repoEvidenceReadObserved) notes.push("Ordinary-chat routing unexpectedly read source repo evidence."); + if (metrics.treeEvidenceReadObserved) notes.push("Ordinary-chat routing unexpectedly read Context Tree evidence."); + if (metrics.chatAskCount > 0) notes.push("Ordinary-chat routing unexpectedly sent a tracked ask."); + if (metrics.taskOptionsObserved) notes.push("Ordinary-chat routing unexpectedly offered first-task options."); + } if (evalCase.expected.evidenceSnippets && !metrics.expectedEvidenceObserved) { notes.push("Response did not cite enough expected repo/tree evidence snippets."); } @@ -780,6 +819,9 @@ export function driftNote(evalCase: FirstTreeWelcomeEvalCase, metrics: EvalMetri if (evalCase.expected.action === "route_to_tree_skill" && metrics.taskOptionsObserved) { notes.push("Tree kickoff row offered value-chat task options."); } + if (evalCase.expected.action === "handle_scan_fix_handoff") { + if (!metrics.repoAccessCheckedObserved) notes.push("Scan-fix routing did not check the readable repository."); + } if (metrics.sourceRepoChanged) { notes.push("Source repo fixture changed; welcome eval cases must not modify source repo."); } diff --git a/packages/skill-evals/src/suites/first-tree-welcome/runner.ts b/packages/skill-evals/src/suites/first-tree-welcome/runner.ts index 6fefa8cc9..7ef79240d 100644 --- a/packages/skill-evals/src/suites/first-tree-welcome/runner.ts +++ b/packages/skill-evals/src/suites/first-tree-welcome/runner.ts @@ -55,6 +55,7 @@ export async function runFirstTreeWelcomeCase( const summary: CaseRunSummary = { caseId: evalCase.id, driftNote: driftNote(evalCase, metrics), + expectedActivation: evalCase.expected.activation, expectedAction: evalCase.expected.action, firstResponseLatencyMs: observability.firstResponseLatencyMs, fixtureValidation, diff --git a/packages/skill-evals/src/suites/first-tree-welcome/summary.ts b/packages/skill-evals/src/suites/first-tree-welcome/summary.ts index a6fbf9ebe..44988750c 100644 --- a/packages/skill-evals/src/suites/first-tree-welcome/summary.ts +++ b/packages/skill-evals/src/suites/first-tree-welcome/summary.ts @@ -20,6 +20,18 @@ function fenced(value: string): string { function processPass(evalCase: FirstTreeWelcomeEvalCase, metrics: EvalMetrics): boolean { if (!metrics.fixtureValidationOk || metrics.runnerExitCode !== 0) return false; + if (evalCase.expected.activation === "auto-ignore") { + return ( + !metrics.skillFileReadObserved && + !metrics.repoAccessCheckedObserved && + !metrics.repoEvidenceReadObserved && + !metrics.treeEvidenceReadObserved && + metrics.chatAskCount === 0 + ); + } + if (evalCase.expected.action === "handle_scan_fix_handoff") { + return metrics.skillFileReadObserved && metrics.repoAccessCheckedObserved && !metrics.treeEvidenceReadObserved; + } if (evalCase.expected.action === "route_to_tree_skill") { return metrics.chatAskCount === 0; } @@ -54,7 +66,13 @@ function processPass(evalCase: FirstTreeWelcomeEvalCase, metrics: EvalMetrics): } function outcomePass(evalCase: FirstTreeWelcomeEvalCase, metrics: EvalMetrics): boolean { + if (evalCase.expected.activation === "auto-ignore") { + return metrics.finalResponse.trim().length > 0 && !metrics.taskOptionsObserved; + } if (!metrics.expectedResponseObserved) return false; + if (evalCase.expected.action === "handle_scan_fix_handoff") { + return true; + } if (evalCase.expected.action === "route_to_tree_skill") { return !metrics.taskOptionsObserved; } @@ -106,14 +124,19 @@ export function buildGrading( metrics.forbiddenActionHits.length === 0 && metrics.forbiddenClaimHits.length === 0 && metrics.forbiddenSideEffectHits.length === 0; + const routingPass = + evalCase.expected.activation === "auto-ignore" ? !metrics.skillFileReadObserved : metrics.skillFileReadObserved; return { caseId: evalCase.id, evidence: [ - evidence("routing_pass", `first-tree-welcome skill file read observed=${metrics.skillFileReadObserved}`), + evidence( + "routing_pass", + `expected activation=${evalCase.expected.activation}; first-tree-welcome skill file read observed=${metrics.skillFileReadObserved}`, + ), evidence( "process_pass", - `fixture ok=${metrics.fixtureValidationOk}; runner exit=${metrics.runnerExitCode}; repo evidence read=${metrics.repoEvidenceReadObserved}; tree evidence read=${metrics.treeEvidenceReadObserved}; chat asks=${metrics.chatAskCount}`, + `fixture ok=${metrics.fixtureValidationOk}; runner exit=${metrics.runnerExitCode}; repo access checked=${metrics.repoAccessCheckedObserved}; repo evidence read=${metrics.repoEvidenceReadObserved}; tree evidence read=${metrics.treeEvidenceReadObserved}; chat asks=${metrics.chatAskCount}`, ), evidence( "outcome_pass", @@ -130,7 +153,7 @@ export function buildGrading( outcome_pass: outcomePass(evalCase, metrics), process_pass: processPass(evalCase, metrics), risk_pass: riskPass, - routing_pass: metrics.skillFileReadObserved, + routing_pass: routingPass, }, }; } @@ -161,7 +184,9 @@ export function writeCaseSummaries(summary: CaseRunSummary): void { - passed: ${markdownBool(summary.passed)} - expectedAction: ${summary.expectedAction} +- expectedActivation: ${summary.expectedActivation} - skillFileReadObserved: ${markdownBool(summary.metrics.skillFileReadObserved)} +- repoAccessCheckedObserved: ${markdownBool(summary.metrics.repoAccessCheckedObserved)} - repoEvidenceReadObserved: ${markdownBool(summary.metrics.repoEvidenceReadObserved)} - treeEvidenceReadObserved: ${markdownBool(summary.metrics.treeEvidenceReadObserved)} - expectedEvidenceObserved: ${markdownBool(summary.metrics.expectedEvidenceObserved)} @@ -233,6 +258,7 @@ export function formatSummaryTable(batch: BatchSummary): string { summary.caseId, summary.expectedAction, String(summary.metrics.skillFileReadObserved), + String(summary.metrics.repoAccessCheckedObserved), String(summary.metrics.repoEvidenceReadObserved), String(summary.metrics.treeEvidenceReadObserved), String(summary.metrics.taskOptionsObserved), @@ -247,6 +273,7 @@ export function formatSummaryTable(batch: BatchSummary): string { "case_id", "expected_action", "skill_file_read", + "repo_access", "repo_read", "tree_read", "task_options", diff --git a/packages/skill-evals/src/suites/first-tree-welcome/types.ts b/packages/skill-evals/src/suites/first-tree-welcome/types.ts index feda698f1..d78999196 100644 --- a/packages/skill-evals/src/suites/first-tree-welcome/types.ts +++ b/packages/skill-evals/src/suites/first-tree-welcome/types.ts @@ -2,12 +2,13 @@ import type { AgentProviderName } from "../../core/provider/types.js"; import type { SkillCaseGrading } from "../../core/result-schema.js"; import type { CommandResult } from "../../core/types.js"; -export type WelcomeRole = "admin" | "invitee"; -export type WelcomeChatScenario = "onboarding" | "team-onboarding" | "tree-setup"; +export type WelcomeRole = "admin" | "invitee" | "unclear"; +export type WelcomeChatScenario = "onboarding" | "ordinary" | "scan-fix" | "team-onboarding" | "tree-setup"; export type WelcomeRepoState = "none" | "local-readable" | "selected-readable" | "selected-auth-fails" | "unknown"; export type WelcomeTreeState = "none" | "empty" | "populated" | "unknown"; export type WelcomeGithubAppState = "installed" | "missing" | "unknown"; export type WelcomeTreeSetupChatState = "absent" | "exists" | "promised"; +export type WelcomeActivationExpectation = "auto-ignore" | "auto-trigger" | "preloaded"; export type WelcomeExpectedAction = | "route_to_tree_skill" @@ -20,6 +21,8 @@ export type WelcomeExpectedAction = | "offer_bounded_first_tasks_from_repo_and_tree" | "offer_repo_value_without_claiming_tree_ready" | "offer_invitee_value_without_admin_setup" + | "handle_scan_fix_handoff" + | "respond_without_welcome" | "give_evidence_value_or_ask_for_input"; export type FirstTreeWelcomeFixture = { @@ -33,6 +36,7 @@ export type FirstTreeWelcomeFixture = { export type FirstTreeWelcomeExpected = { action: WelcomeExpectedAction; + activation: WelcomeActivationExpectation; evidenceSnippets?: readonly string[]; requiredResponseHints: readonly string[]; taskOptionHints?: readonly string[]; @@ -89,6 +93,7 @@ export type EvalMetrics = { forbiddenSideEffectHits: readonly string[]; firstTreeArgv: readonly (readonly string[])[]; fixtureValidationOk: boolean; + repoAccessCheckedObserved: boolean; repoEvidenceReadObserved: boolean; runnerExitCode: number | null; skillFileReadObserved: boolean; @@ -101,6 +106,7 @@ export type EvalMetrics = { export type CaseRunSummary = { caseId: string; driftNote: string | null; + expectedActivation: WelcomeActivationExpectation; expectedAction: WelcomeExpectedAction; firstResponseLatencyMs: number | null; fixtureValidation: FixtureValidation; diff --git a/skills/first-tree-welcome/SKILL.md b/skills/first-tree-welcome/SKILL.md index a04d73120..cb31a96c1 100644 --- a/skills/first-tree-welcome/SKILL.md +++ b/skills/first-tree-welcome/SKILL.md @@ -1,18 +1,25 @@ --- name: first-tree-welcome -version: 1.2.1 -description: Use for a First Tree onboarding first chat, especially natural opening messages like "welcome aboard", "Please help me get started with First Tree", or "Please help me get settled into this team on First Tree." Also covers the production-scan fix first chat ("fix the launch blockers found by my production readiness scan"). Do not use for dedicated tree setup chats, ordinary chats, PR reviews, repo scans, tree writes, or maintenance. +version: 1.2.2 +description: Use only when the opening message matches a First Tree kickoff shape — "welcome aboard" together with either "Please help me get started with First Tree" or "Please help me get settled into this team on First Tree" — or explicitly asks to "fix the launch blockers found by my production readiness scan". Do not use for dedicated tree setup chats, ordinary chats, PR reviews, repo scans, tree writes, or maintenance. --- # First Tree Welcome ## Scope -Use this skill only when the chat is clearly the onboarding first chat created by -First Tree, including natural messages such as "welcome aboard", "Please help me -get started with First Tree", or "Please help me get settled into this team on -First Tree." Do not use it for ordinary chats, PR reviews, repo scans, tree -writes, or maintenance work. +Use this skill only when the opening message matches one of these First Tree +kickoff shapes: + +- **Onboarding first chat** — "welcome aboard" together with either "Please help + me get started with First Tree" or "Please help me get settled into this team + on First Tree". +- **Production-scan fix first chat** — explicitly asks to "fix the launch blockers + found by my production readiness scan". + +Do not infer onboarding from the chat being new or from a generic greeting. Do +not use this skill for ordinary chats, PR reviews, repo scans, tree writes, or +maintenance work. Two look-alikes that are NOT this launcher, and one that routes by shape: diff --git a/skills/first-tree-welcome/VERSION b/skills/first-tree-welcome/VERSION index 6085e9465..23aa83906 100644 --- a/skills/first-tree-welcome/VERSION +++ b/skills/first-tree-welcome/VERSION @@ -1 +1 @@ -1.2.1 +1.2.2 diff --git a/skills/first-tree-welcome/agents/openai.yaml b/skills/first-tree-welcome/agents/openai.yaml index 22207706e..f4ff6c007 100644 --- a/skills/first-tree-welcome/agents/openai.yaml +++ b/skills/first-tree-welcome/agents/openai.yaml @@ -1,4 +1,4 @@ --- name: first-tree-welcome -description: Use for a First Tree onboarding first chat, especially natural opening messages like "welcome aboard", "Please help me get started with First Tree", or "Please help me get settled into this team on First Tree." Also covers the production-scan fix first chat ("fix the launch blockers found by my production readiness scan"). Do not use for dedicated tree setup chats, ordinary chats, PR reviews, repo scans, tree writes, or maintenance. +description: Use only when the opening message matches a First Tree kickoff shape — "welcome aboard" together with either "Please help me get started with First Tree" or "Please help me get settled into this team on First Tree" — or explicitly asks to "fix the launch blockers found by my production readiness scan". Do not use for dedicated tree setup chats, ordinary chats, PR reviews, repo scans, tree writes, or maintenance. ---