diff --git a/bun.lock b/bun.lock index 5c1dbe904..ecb769b30 100644 --- a/bun.lock +++ b/bun.lock @@ -89,7 +89,7 @@ }, "packages/protocol": { "name": "@indexnetwork/protocol", - "version": "23.6.4", + "version": "23.7.0", "dependencies": { "@langchain/core": "1.1.48", "@langchain/langgraph": "1.3.2", diff --git a/packages/protocol/package.json b/packages/protocol/package.json index 662d9967b..fae8de39c 100644 --- a/packages/protocol/package.json +++ b/packages/protocol/package.json @@ -1,6 +1,6 @@ { "name": "@indexnetwork/protocol", - "version": "23.6.4", + "version": "23.7.0", "type": "module", "main": "./dist/index.js", "types": "./dist/index.d.ts", diff --git a/packages/protocol/src/intents/graph/intent.graph.reconcile.ts b/packages/protocol/src/intents/graph/intent.graph.reconcile.ts index 852041cbf..365250ea3 100644 --- a/packages/protocol/src/intents/graph/intent.graph.reconcile.ts +++ b/packages/protocol/src/intents/graph/intent.graph.reconcile.ts @@ -9,7 +9,7 @@ import { getAbortSignalConfig } from "../../shared/agent/model-signal.js"; import { timed } from "../../shared/observability/performance.js"; import { requestContext } from "../../shared/observability/request-context.js"; import type { DebugMetaAgent } from "../../agents/agent.module.js"; -import { buildExplicitUpdateActions, enforceIntentActionBoundary, generateIntentEmbedding, getSpecificityWarning, isVague, logger, MAX_PERMISSIBLE_ENTROPY, MIN_CLEAR_INTENT_SCORE, toSpeechActType, type IntentGraphDeps, type IntentState } from "./intent.graph.shared.js"; +import { buildExplicitUpdateActions, combineFelicityScores, enforceIntentActionBoundary, generateIntentEmbedding, getSpecificityWarning, isVague, logger, MAX_PERMISSIBLE_ENTROPY, MIN_CLEAR_INTENT_SCORE, toSpeechActType, type IntentGraphDeps, type IntentState } from "./intent.graph.shared.js"; /** @@ -99,12 +99,9 @@ export async function verificationNode(state: IntentState, deps: IntentGraphDeps }; } - // Calculate Score - const score = Math.min( - verdict.felicity_scores.authority, - verdict.felicity_scores.sincerity, - verdict.felicity_scores.clarity - ); + // Calculate Score. Authority participates only when a profile was + // supplied to judge it against. + const score = combineFelicityScores(verdict.felicity_scores, state.userProfile); // Return enriched intent return { diff --git a/packages/protocol/src/intents/graph/intent.graph.shared.ts b/packages/protocol/src/intents/graph/intent.graph.shared.ts index 802e5d8eb..e8aaef298 100644 --- a/packages/protocol/src/intents/graph/intent.graph.shared.ts +++ b/packages/protocol/src/intents/graph/intent.graph.shared.ts @@ -105,6 +105,28 @@ export const isVague = (description: string, entropy: number, clarity: number): return false; }; +/** + * Combine the felicity scores into the single number callers store. + * + * Authority is a preparatory condition: it asks whether the *speaker's profile* + * supports the speech act. When no profile was supplied the verifier has nothing + * to answer that from, so its guess is left out of the minimum rather than + * allowed to cap the result. The propose path attaches no profile on purpose — + * a signal derives only from the person's answers — so honouring that means not + * scoring them against a profile they were never asked for. + */ +export const combineFelicityScores = ( + felicityScores: { authority: number; sincerity: number; clarity: number }, + userProfile: string | undefined, +): number => { + const profileBacked = Boolean(userProfile?.trim()); + return Math.min( + ...(profileBacked ? [felicityScores.authority] : []), + felicityScores.sincerity, + felicityScores.clarity, + ); +}; + export const getSpecificityWarning = (verdict: { specificity_warning?: string | null }): string => { const warning = verdict.specificity_warning?.trim(); return warning && warning.length > 0 ? warning : DEFAULT_SPECIFICITY_WARNING; diff --git a/packages/protocol/src/intents/graph/intent.graph.ts b/packages/protocol/src/intents/graph/intent.graph.ts index fd08b9e6b..1d31370a7 100644 --- a/packages/protocol/src/intents/graph/intent.graph.ts +++ b/packages/protocol/src/intents/graph/intent.graph.ts @@ -63,12 +63,15 @@ export class IntentGraphFactory { // - UPDATE: prep → inference → reconciliation → executor → END (skips verification if no new intents) // - DELETE: prep → reconciliation → executor → END (skips inference and verification) // - PROPOSE: prep → inference → verification → END (no reconciliation/execution, no DB writes) + // A caller that supplies a stage's output skips that stage: seed + // `inferredIntents` and prep routes straight to verification. .addEdge(START, "prep") - // After prep: read mode → query; else inference or reconciler + // After prep: read mode → query; else inference, verification, or reconciler .addConditionalEdges("prep", afterPrepRoute, { query: "query", inference: "inference", + verification: "verification", reconciler: "reconciler", __end__: END, }) @@ -115,8 +118,11 @@ export function afterPrepRoute(state: IntentState): string { /** - * Determines if inference should run based on operation mode. + * Determines if inference should run based on operation mode and seeded state. * Delete operations skip inference entirely and go straight to reconciliation. + * A caller that already supplies the candidate signals skips it too: inference + * extracts candidates from messy text, so there is nothing left for it to do + * when the candidates arrive with the invocation. */ export function shouldRunInference(state: IntentState): string { if (state.operationMode === 'delete') { @@ -124,6 +130,14 @@ export function shouldRunInference(state: IntentState): string { return 'reconciler'; } + if (state.inferredIntents.length > 0) { + logger.verbose('Intents supplied by the caller - skipping inference, routing to verification', { + operationMode: state.operationMode, + seededIntentCount: state.inferredIntents.length, + }); + return 'verification'; + } + logger.verbose('Running inference', { operationMode: state.operationMode }); diff --git a/packages/protocol/src/intents/intake/intake.orchestrator.ts b/packages/protocol/src/intents/intake/intake.orchestrator.ts index 07b0dcc0f..978e591c1 100644 --- a/packages/protocol/src/intents/intake/intake.orchestrator.ts +++ b/packages/protocol/src/intents/intake/intake.orchestrator.ts @@ -75,6 +75,21 @@ const optionSchema = z.object({ description: z.string(), }); +/** + * Structurally identical to {@link optionSchema}, and deliberately a separate + * object. + * + * Reusing one zod instance for two fields of the same schema makes the JSON + * Schema converter emit the second as a `$ref` into `definitions`. Gemini + * rejects that document, so the call burned its retry and then answered from + * the fallback model instead — a 1.5s call became a 6.5s one, on a model nobody + * chose. Two instances keep both option shapes inlined. + */ +const bridgeOptionSchema = z.object({ + label: z.string().min(1), + description: z.string(), +}); + const answerFirstQuestionSchema = z.object({ missingAxis: z .enum(["purpose", "desired_attributes", "exchange", "constraint"]) @@ -86,23 +101,22 @@ const answerFirstQuestionSchema = z.object({ .min(2) .max(3) .describe("Two or three distinct choices derived only from the answered intake rounds."), + profileBridgeOption: bridgeOptionSchema + .nullable() + .describe( + "One natural profile intersection appended after the answer-grounded options, " + + "or null when no brief was supplied or the bridge would be forced. " + + "Null for every question is a correct answer.", + ), multiSelect: z.boolean(), }); -const followUpPlanSchema = z.object({ +/** Exported for the schema-shape guard in the sibling spec. */ +export const followUpPlanSchema = z.object({ questions: z.array(answerFirstQuestionSchema), plannedFollowUpCount: z.number().int().min(0), }); -const profileBridgePlanSchema = z.object({ - bridges: z.array(z.object({ - questionIndex: z.number().int().min(0), - profileBridgeOption: optionSchema - .nullable() - .describe("One natural profile intersection, or null when the bridge would be forced."), - })), -}); - const synthesisSchema = z.object({ description: z.string().min(1), lookingFor: z.string().min(1), @@ -135,14 +149,21 @@ export const FALLBACK_BRING_QUESTION: IntakePackQuestion = { multiSelect: false, }; -const PLAN_SYSTEM_PROMPT = `You plan and write answer-first follow-up intake questions for a networking product. +const FOLLOW_UP_SYSTEM_PROMPT = `You plan and write answer-first follow-up intake questions for a networking product. -You receive ONLY the answered intake rounds. Use them to choose the next missing -axis, write a standalone prompt that names the newly stated person or domain, and -create 2-3 meaningfully distinct answerGroundedOptions. Choose the most useful -unanswered axis from: purpose, desired_attributes, exchange, or constraint. Never -re-ask an axis the rounds already answer. Do not infer a professional background, -industry, capability, or commercial goal that the rounds do not state. +Each question has two parts, written in this order: the answer-grounded core, then +one optional profile bridge. Keep them separate — the brief may shape the bridge and +nothing else. + +CORE — answerGroundedOptions, from the answered rounds alone +Use the answered intake rounds to choose the next missing axis, write a standalone +prompt that names the newly stated person or domain, and create 2-3 meaningfully +distinct answerGroundedOptions. Choose the most useful unanswered axis from: +purpose, desired_attributes, exchange, or constraint. Never re-ask an axis the +rounds already answer. Do not infer a professional background, industry, +capability, or commercial goal that the rounds do not state. Write these options as +if no profile brief had been supplied: nothing in the brief may add, remove, +reword, or reclassify one of them. Every answerGroundedOption must be visibly anchored to the stated person or domain in its label or description and represent a concrete, distinct path. Generic labels @@ -151,23 +172,24 @@ not say what the user would learn, share, collaborate on, or network about. - Scuba divers: "Learn diving techniques", "Find dive buddies", "Share dive stories". - Climate founders: "Compare climate sectors", "Find climate peers", "Discuss adaptation". +BRIDGE — profileBridgeOption, at most one per question +Only after the core options are written, compare the question, its missing axis, and +those options with the profile brief, and return either one genuinely useful +profileBridgeOption or null. A bridge is useful only when it creates a natural +additional path that is not already represented; it is appended after the +answer-grounded options and never rewrites, replaces, removes, or reclassifies one +of them. Never return more than one bridge per question. When the profile theme is +unrelated, when no brief was supplied, or when the intersection would feel forced, +return null. Running club + investor may support one sponsorship bridge; scuba +divers + pianist should return null. Returning null for every question is a normal, +correct answer — never invent a bridge to avoid it. + Each option needs a short label and a one-line description. Set multiSelect true only when several options can genuinely apply together. Never expose raw JSON, IDs, or internal vocabulary. plannedFollowUpCount is the TOTAL number of follow-up questions the interview should contain, including any returned now; when the input already fixes it, echo that value unchanged.`; -const PROFILE_BRIDGE_SYSTEM_PROMPT = `You may append one optional profile bridge to each already-written signal-intake question. - -The question, missing axis, and answer-grounded options are immutable. For each -questionIndex, compare them with the profile brief and return either one genuinely -useful profileBridgeOption or null. A bridge is useful only when it creates a -natural additional path that is not already represented. Never rewrite, replace, -remove, or reclassify a core option. Never return more than one bridge per question. -When the profile theme is unrelated or the intersection would feel forced, return -null. Running club + investor may support one sponsorship bridge; scuba divers + -pianist should return null.`; - const SYNTHESIS_SYSTEM_PROMPT = `You write one clear signal for a networking product. You receive ONLY the interview: each question the person was asked and the answer @@ -193,11 +215,18 @@ export function answerLabel(answer: IntakeAnswer): string { return [...answer.selectedOptions, answer.freeText?.trim() ?? ""].filter(Boolean).join(", "); } +/** + * Clamp one generated question into a renderable one. + * + * The bridge is structurally optional: an absent, null, or unusable + * `profileBridgeOption` — and any bridge returned when no brief was supplied — + * simply leaves the question with its answer-grounded options. + */ function normalizeFollowUpQuestion( question: z.infer, - profileBridge: z.infer | null, brief: string, ): IntakePackQuestion { + const profileBridge = brief.trim() ? question.profileBridgeOption ?? null : null; const seen = new Set(); const normalizeOption = (option: { label: string; description: string }) => { const label = option.label.trim(); @@ -233,7 +262,6 @@ function normalizeFollowUpQuestion( /** Runs the two live stages of the fast intake funnel: follow-up planning and synthesis. */ export class SignalIntakeOrchestrator { private readonly plannerModel: Runnable>; - private readonly profileBridgeModel: Runnable>; private readonly synthesisModel: Runnable; /** @@ -241,13 +269,10 @@ export class SignalIntakeOrchestrator { */ constructor(models?: { planner?: Runnable>; - profileBridge?: Runnable>; synthesis?: Runnable; }) { this.plannerModel = models?.planner ?? createStructuredModel("signalIntakePack", followUpPlanSchema) as unknown as Runnable>; - this.profileBridgeModel = models?.profileBridge - ?? createStructuredModel("signalIntakePack", profileBridgePlanSchema, { name: "signal_intake_profile_bridges" }) as unknown as Runnable>; this.synthesisModel = models?.synthesis ?? createStructuredModel("signalIntakePack", synthesisSchema) as unknown as Runnable; } @@ -255,6 +280,11 @@ export class SignalIntakeOrchestrator { /** * Plan and write follow-up questions from the brief and answered rounds. * + * One call writes both halves of each question: the answer-grounded core and + * the optional profile bridge. The bridge is a nullable field rather than a + * second call, so personalization staying silent — for every question — is an + * ordinary success, never a failure to swallow. + * * @param input - Brief, answered rounds, per-call cap, and any locked plan * @returns Up to `maxFollowUps` renderable questions plus the total plan; * the static fallback question with count 1 when generation fails @@ -266,38 +296,19 @@ export class SignalIntakeOrchestrator { const lockedLine = input.plannedFollowUpCount !== undefined ? `\n\nThe interview plan is fixed at ${input.plannedFollowUpCount} follow-up question(s) in total; ${input.rounds.length - 1} already asked. Echo that count unchanged.` : ""; + const briefSection = input.brief.trim() + ? `\n\nPROFILE BRIEF (profileBridgeOption only — it may not touch an answerGroundedOption):\n${input.brief.trim()}` + : "\n\nPROFILE BRIEF: none supplied. Return profileBridgeOption null for every question."; try { const raw = await this.plannerModel.invoke([ - new SystemMessage(PLAN_SYSTEM_PROMPT), + new SystemMessage(FOLLOW_UP_SYSTEM_PROMPT), new HumanMessage( - `ANSWERED ROUNDS:\n${roundsText}\n\nWrite up to ${input.maxFollowUps} follow-up question(s).${lockedLine}`, + `ANSWERED ROUNDS:\n${roundsText}${briefSection}\n\nWrite up to ${input.maxFollowUps} follow-up question(s).${lockedLine}`, ), ]); - const coreQuestions = raw.questions.slice(0, input.maxFollowUps); - const bridgeByQuestion = new Map | null>(); - if (coreQuestions.length > 0 && input.brief.trim()) { - try { - const bridgePlan = await this.profileBridgeModel.invoke([ - new SystemMessage(PROFILE_BRIDGE_SYSTEM_PROMPT), - new HumanMessage( - `ANSWERED ROUNDS:\n${roundsText}\n\nPROFILE BRIEF:\n${input.brief}\n\nIMMUTABLE CORE QUESTIONS:\n${JSON.stringify(coreQuestions, null, 2)}\n\nReturn one bridge decision for each questionIndex.`, - ), - ]); - for (const bridge of bridgePlan.bridges) { - if ( - bridge.questionIndex < coreQuestions.length - && !bridgeByQuestion.has(bridge.questionIndex) - ) { - bridgeByQuestion.set(bridge.questionIndex, bridge.profileBridgeOption); - } - } - } catch { - // Profile personalization is optional. A bridge-model failure must not - // discard valid answer-grounded questions from the primary model. - } - } - const questions = coreQuestions.map((question, index) => - normalizeFollowUpQuestion(question, bridgeByQuestion.get(index) ?? null, input.brief)); + const questions = raw.questions + .slice(0, input.maxFollowUps) + .map((question) => normalizeFollowUpQuestion(question, input.brief)); // Backstop: while budget remains, a successful empty plan must not // silently shrink the interview; serve the static fallback instead. if (questions.length === 0 && input.maxFollowUps > 0) { diff --git a/packages/protocol/src/intents/tests/intake.orchestrator.spec.ts b/packages/protocol/src/intents/tests/intake.orchestrator.spec.ts index 2c3c97886..bcc4f6faa 100644 --- a/packages/protocol/src/intents/tests/intake.orchestrator.spec.ts +++ b/packages/protocol/src/intents/tests/intake.orchestrator.spec.ts @@ -1,6 +1,8 @@ import { describe, expect, it } from "bun:test"; -import { answerLabel, FALLBACK_BRING_QUESTION, FALLBACK_WHO_QUESTION, SignalIntakeOrchestrator } from "../intake/intake.orchestrator.js"; +import { toJsonSchema } from "@langchain/core/utils/json_schema"; + +import { answerLabel, FALLBACK_BRING_QUESTION, FALLBACK_WHO_QUESTION, followUpPlanSchema, SignalIntakeOrchestrator } from "../intake/intake.orchestrator.js"; interface Capture { prompt?: string; @@ -19,13 +21,8 @@ function stub(value: T, capture?: Capture) { } as never; } -const NO_BRIDGES = { bridges: [] }; - function plannerModels(plan: T, capture?: Capture) { - return { - planner: stub(plan, capture), - profileBridge: stub(NO_BRIDGES), - }; + return { planner: stub(plan, capture) }; } const question = { @@ -36,6 +33,7 @@ const question = { { label: "Distribution", description: "You have an audience" }, { label: "Engineering depth", description: "You can build it" }, ], + profileBridgeOption: null, multiSelect: false, }; @@ -51,31 +49,24 @@ describe("answerLabel", () => { describe("SignalIntakeOrchestrator.generateFollowUps", () => { it("assembles answer-grounded options before one optional profile bridge", async () => { - const orchestrator = new SignalIntakeOrchestrator({ - planner: stub({ - questions: [{ - missingAxis: "purpose", - title: "Purpose", - prompt: "What would make meeting scuba divers valuable to you?", - answerGroundedOptions: [ - { label: "Find dive buddies", description: "Meet people to dive with" }, - { label: "Learn from experts", description: "Meet experienced divers" }, - { label: "Marine conservation", description: "Work on ocean stewardship" }, - ], - multiSelect: false, - }], - plannedFollowUpCount: 1, - }), - profileBridge: stub({ - bridges: [{ - questionIndex: 0, - profileBridgeOption: { - label: "Underwater technology", - description: "Connect diving with your technology background", - }, - }], - }), - }); + const orchestrator = new SignalIntakeOrchestrator(plannerModels({ + questions: [{ + missingAxis: "purpose", + title: "Purpose", + prompt: "What would make meeting scuba divers valuable to you?", + answerGroundedOptions: [ + { label: "Find dive buddies", description: "Meet people to dive with" }, + { label: "Learn from experts", description: "Meet experienced divers" }, + { label: "Marine conservation", description: "Work on ocean stewardship" }, + ], + profileBridgeOption: { + label: "Underwater technology", + description: "Connect diving with your technology background", + }, + multiSelect: false, + }], + plannedFollowUpCount: 1, + })); const result = await orchestrator.generateFollowUps({ brief: "The user builds AI products.", @@ -96,13 +87,9 @@ describe("SignalIntakeOrchestrator.generateFollowUps", () => { plannedFollowUpCount: 2, }; - it("keeps the profile out of core generation and sends it only to bridge generation", async () => { - const plannerCapture: Capture = {}; - const bridgeCapture: Capture = {}; - const orchestrator = new SignalIntakeOrchestrator({ - planner: stub(plan, plannerCapture), - profileBridge: stub({ bridges: [] }, bridgeCapture), - }); + it("sends the brief in a bridge-only slot and keeps both prompts' constraints", async () => { + const capture: Capture = {}; + const orchestrator = new SignalIntakeOrchestrator(plannerModels(plan, capture)); const result = await orchestrator.generateFollowUps({ brief: "Ada builds developer tools.", @@ -112,43 +99,81 @@ describe("SignalIntakeOrchestrator.generateFollowUps", () => { expect(result.questions).toHaveLength(2); expect(result.plannedFollowUpCount).toBe(2); - expect(plannerCapture.prompt).toContain("A design partner"); - expect(plannerCapture.prompt).not.toContain("Ada builds developer tools."); - expect(plannerCapture.messages?.[0]).toContain("receive ONLY the answered intake rounds"); - expect(plannerCapture.messages?.[0]).toContain("Generic labels"); - expect(plannerCapture.messages?.[0]).toContain("Learn diving techniques"); - expect(bridgeCapture.prompt).toContain("Ada builds developer tools."); - expect(bridgeCapture.prompt).toContain("A design partner"); - expect(bridgeCapture.prompt).toContain("IMMUTABLE CORE QUESTIONS"); - expect(bridgeCapture.messages?.[0]).toContain("question, missing axis, and answer-grounded options are immutable"); + expect(capture.prompt).toContain("A design partner"); + // One call now carries the brief, so the wall is a labelled slot plus a + // prompt constraint rather than a second model that never saw the rounds. + expect(capture.prompt).toContain("PROFILE BRIEF (profileBridgeOption only"); + expect(capture.prompt).toContain("Ada builds developer tools."); + const systemPrompt = capture.messages?.[0] ?? ""; + expect(systemPrompt).toContain("from the answered rounds alone"); + expect(systemPrompt).toContain("Write these options as\nif no profile brief had been supplied"); + expect(systemPrompt).toContain("nothing in the brief may add, remove,"); + expect(systemPrompt).toContain("Generic labels"); + expect(systemPrompt).toContain("Learn diving techniques"); + // The bridge prompt's own thinking survives the merge. + expect(systemPrompt).toContain("never rewrites, replaces, removes, or reclassifies one"); + expect(systemPrompt).toContain("Never return more than one bridge per question"); + expect(systemPrompt).toContain("scuba\ndivers + pianist should return null"); }); - it("deduplicates options without letting the profile bridge displace two answer-grounded choices", async () => { - const orchestrator = new SignalIntakeOrchestrator({ - planner: stub({ + it("keeps 2-3 answer-grounded core options on every question, whatever the bridge does", async () => { + // The two-call split enforced this structurally: the bridge model received + // the core questions as immutable input. In one call it is a prompt + // constraint, so the shape is asserted here instead. + const bridges = [ + null, + { label: "Underwater technology", description: "A natural bridge" }, + // A bridge that duplicates a core label is dropped, never substituted. + { label: " find dive buddies ", description: "The same choice again" }, + ]; + for (const profileBridgeOption of bridges) { + const orchestrator = new SignalIntakeOrchestrator(plannerModels({ questions: [{ - missingAxis: "desired_attributes", - title: "Divers", - prompt: "Which scuba divers would be most useful to meet?", + missingAxis: "purpose", + title: "Purpose", + prompt: "What would make meeting scuba divers valuable to you?", answerGroundedOptions: [ - { label: "Dive buddies", description: "People to dive with" }, - { label: " dive buddies ", description: "Duplicate after trimming" }, - { label: "Experienced instructors", description: "People to learn from" }, + { label: "Find dive buddies", description: "Meet people to dive with" }, + { label: "Learn from experts", description: "Meet experienced divers" }, ], + profileBridgeOption, multiSelect: false, }], plannedFollowUpCount: 1, - }), - profileBridge: stub({ - bridges: [{ - questionIndex: 0, - profileBridgeOption: { - label: "Underwater technologists", - description: "A natural bridge to the user's background", - }, - }], - }), - }); + })); + + const result = await orchestrator.generateFollowUps({ + brief: "The user builds AI products.", + rounds: [{ prompt: "Who?", answer: { selectedOptions: [], freeText: "scuba divers" } }], + maxFollowUps: 1, + }); + + const labels = result.questions[0]?.options.map((option) => option.label) ?? []; + expect(labels.slice(0, 2)).toEqual(["Find dive buddies", "Learn from experts"]); + expect(labels.length).toBeLessThanOrEqual(3); + expect(result.questions[0]).not.toEqual(FALLBACK_BRING_QUESTION); + } + }); + + it("deduplicates options without letting the profile bridge displace two answer-grounded choices", async () => { + const orchestrator = new SignalIntakeOrchestrator(plannerModels({ + questions: [{ + missingAxis: "desired_attributes", + title: "Divers", + prompt: "Which scuba divers would be most useful to meet?", + answerGroundedOptions: [ + { label: "Dive buddies", description: "People to dive with" }, + { label: " dive buddies ", description: "Duplicate after trimming" }, + { label: "Experienced instructors", description: "People to learn from" }, + ], + profileBridgeOption: { + label: "Underwater technologists", + description: "A natural bridge to the user's background", + }, + multiSelect: false, + }], + plannedFollowUpCount: 1, + })); const result = await orchestrator.generateFollowUps({ brief: "The user builds technology products.", @@ -164,30 +189,23 @@ describe("SignalIntakeOrchestrator.generateFollowUps", () => { }); it("falls back when deduplication leaves fewer than two answer-grounded choices", async () => { - const orchestrator = new SignalIntakeOrchestrator({ - planner: stub({ - questions: [{ - missingAxis: "purpose", - title: "Purpose", - prompt: "What do you want from meeting scuba divers?", - answerGroundedOptions: [ - { label: "Dive buddies", description: "People to dive with" }, - { label: " dive buddies ", description: "The same choice" }, - ], - multiSelect: false, - }], - plannedFollowUpCount: 1, - }), - profileBridge: stub({ - bridges: [{ - questionIndex: 0, - profileBridgeOption: { - label: "Underwater technology", - description: "A profile-derived bridge", - }, - }], - }), - }); + const orchestrator = new SignalIntakeOrchestrator(plannerModels({ + questions: [{ + missingAxis: "purpose", + title: "Purpose", + prompt: "What do you want from meeting scuba divers?", + answerGroundedOptions: [ + { label: "Dive buddies", description: "People to dive with" }, + { label: " dive buddies ", description: "The same choice" }, + ], + profileBridgeOption: { + label: "Underwater technology", + description: "A profile-derived bridge", + }, + multiSelect: false, + }], + plannedFollowUpCount: 1, + })); const result = await orchestrator.generateFollowUps({ brief: "The user builds technology products.", @@ -201,22 +219,70 @@ describe("SignalIntakeOrchestrator.generateFollowUps", () => { }); }); - it("keeps valid core questions when optional bridge generation fails", async () => { + it("treats a null bridge on every question as an ordinary success, not a failure", async () => { + let calls = 0; const orchestrator = new SignalIntakeOrchestrator({ - planner: stub({ questions: [question], plannedFollowUpCount: 1 }), - profileBridge: { invoke: async () => { throw new Error("bridge model down"); } } as never, + planner: { + invoke: async () => { + calls += 1; + return { + questions: [question, { ...question, prompt: "Where should we look?" }], + plannedFollowUpCount: 2, + }; + }, + } as never, }); + const result = await orchestrator.generateFollowUps({ + brief: "Ada builds developer tools.", + rounds: [{ prompt: "Who?", answer: { selectedOptions: ["A design partner"] } }], + maxFollowUps: 2, + }); + + // Personalization is optional: silence is not an error and buys no retry. + expect(calls).toBe(1); + expect(result.questions).toHaveLength(2); + for (const served of result.questions) { + expect(served.options.map((option) => option.label)).toEqual(["Distribution", "Engineering depth"]); + } + }); + + it("omitting the bridge field entirely is as good as returning null", async () => { + const { profileBridgeOption: _omitted, ...withoutBridge } = question; + const orchestrator = new SignalIntakeOrchestrator(plannerModels({ + questions: [withoutBridge], + plannedFollowUpCount: 1, + })); + const result = await orchestrator.generateFollowUps({ brief: "Ada builds developer tools.", rounds: [{ prompt: "Who?", answer: { selectedOptions: ["A design partner"] } }], maxFollowUps: 1, }); - expect(result.questions[0]?.options.map((option) => option.label)).toEqual([ - "Distribution", - "Engineering depth", - ]); + expect(result.questions[0]?.options.map((option) => option.label)) + .toEqual(["Distribution", "Engineering depth"]); + }); + + it("drops a bridge offered when no brief was supplied", async () => { + const capture: Capture = {}; + const orchestrator = new SignalIntakeOrchestrator(plannerModels({ + questions: [{ + ...question, + profileBridgeOption: { label: "Invented bridge", description: "From a brief that does not exist" }, + }], + plannedFollowUpCount: 1, + }, capture)); + + const result = await orchestrator.generateFollowUps({ + brief: " ", + rounds: [{ prompt: "Who?", answer: { selectedOptions: ["A design partner"] } }], + maxFollowUps: 1, + }); + + expect(capture.prompt).toContain("PROFILE BRIEF: none supplied"); + expect(result.questions[0]?.options.map((option) => option.label)) + .toEqual(["Distribution", "Engineering depth"]); }); it("truncates model output to maxFollowUps", async () => { @@ -386,3 +452,19 @@ describe("static fallbacks", () => { } }); }); + +describe("the follow-up plan schema as the provider receives it", () => { + it("inlines every option shape instead of emitting a $ref", () => { + // `withStructuredOutput` converts this schema with the same converter and + // sends it as `response_format.json_schema`. Reusing one zod instance for + // two fields makes the converter emit the second as a `$ref` into + // `definitions`, which Gemini rejects: the call burns its retry and answers + // from the fallback model instead. Merging the bridge into this schema is + // exactly the change that could reintroduce that, so it is asserted here. + const json = JSON.stringify(toJsonSchema(followUpPlanSchema)); + + expect(json).not.toContain("$ref"); + expect(json).not.toContain("definitions"); + expect(json).toContain("profileBridgeOption"); + }); +}); diff --git a/packages/protocol/src/intents/tests/intent.graph.profile-blind.spec.ts b/packages/protocol/src/intents/tests/intent.graph.profile-blind.spec.ts new file mode 100644 index 000000000..816c2ce37 --- /dev/null +++ b/packages/protocol/src/intents/tests/intent.graph.profile-blind.spec.ts @@ -0,0 +1,184 @@ +/** + * The profile-blind propose path: what it skips, and what it must not score. + * + * The signal-intake funnel invokes the graph on the output of synthesis — one + * clean, self-contained, first-person signal — with no profile attached. Two + * consequences are asserted here: + * + * 1. Inference has nothing to do on that input, so a caller that supplies its + * output routes straight to verification. + * 2. `authority` asks whether the speaker's profile supports the speech act. + * With no profile it is a number guessed from nothing, so it must not cap the + * score the caller stores. + */ +import { describe, expect, it } from "bun:test"; + +import { IntentGraphFactory, shouldRunInference } from "../graph/intent.graph.js"; +import { combineFelicityScores } from "../graph/intent.graph.shared.js"; +import { verificationNode } from "../graph/intent.graph.reconcile.js"; + +import type { SemanticVerifierOutput } from "../intent.verifier.js"; +import type { IntentGraphDeps, IntentState } from "../graph/intent.graph.shared.js"; +import type { IntentGraphDatabase } from "../../shared/interfaces/database.interface.js"; + +const VERDICT: SemanticVerifierOutput = { + reasoning: "A specific search directive", + classification: "DIRECTIVE", + felicity_scores: { authority: 40, sincerity: 90, clarity: 85 }, + semantic_entropy: 0.2, + referential_anchor: null, + referential_breadth: "narrow", + missing_selectional_constraints: [], + specificity_warning: null, + flags: ["SKILL_MISMATCH"], +}; + +const SIGNAL = "I'm looking for a design partner to test my developer tooling this quarter."; + +const database = { + getProfile: async () => ({ identity: { name: "Ada" } }), + getActiveIntents: async () => [], +} as unknown as IntentGraphDatabase; + +/** Build a graph whose inferrer and verifier both count their calls. */ +function makeGraph() { + const inferrerCalls: Array = []; + const verifierCalls: Array<{ content: string; context: string }> = []; + const graph = new IntentGraphFactory(database, undefined, undefined, { + inferrer: { + invoke: async (content: string | null) => { + inferrerCalls.push(content); + return { + intents: [{ + type: "goal" as const, + description: "Something the inferrer made up", + reasoning: "Inferred", + confidence: "high" as const, + }], + }; + }, + }, + verifier: { + invoke: async (content: string, context: string) => { + verifierCalls.push({ content, context }); + return VERDICT; + }, + }, + reconciler: { invoke: async () => ({ actions: [] }) }, + }).createGraph(); + return { graph, inferrerCalls, verifierCalls }; +} + +describe("supplying a stage's output skips that stage", () => { + it("verifies the caller's signal without re-inferring it", async () => { + const { graph, inferrerCalls, verifierCalls } = makeGraph(); + + const result = await graph.invoke({ + userId: "ada", + userProfile: "", + operationMode: "propose", + inputContent: SIGNAL, + inferredIntents: [{ + type: "goal", + description: SIGNAL, + reasoning: "Synthesized from the intake interview answers.", + confidence: "high", + }], + }); + + expect(inferrerCalls).toEqual([]); + expect(verifierCalls.map((call) => call.content)).toEqual([SIGNAL]); + expect(result.verifiedIntents).toHaveLength(1); + expect(result.verifiedIntents[0]?.description).toBe(SIGNAL); + }); + + it("still runs inference when the caller supplies only raw text", async () => { + const { graph, inferrerCalls, verifierCalls } = makeGraph(); + + const result = await graph.invoke({ + userId: "ada", + userProfile: "", + operationMode: "propose", + inputContent: "erm, people to build with I guess", + }); + + expect(inferrerCalls).toEqual(["erm, people to build with I guess"]); + expect(verifierCalls.map((call) => call.content)).toEqual(["Something the inferrer made up"]); + expect(result.verifiedIntents).toHaveLength(1); + }); + + it("routes on the seed, not on the operation mode", () => { + const seeded = { + operationMode: "create", + inferredIntents: [{ type: "goal", description: SIGNAL, reasoning: "r", confidence: "high" }], + } as unknown as IntentState; + const empty = { operationMode: "create", inferredIntents: [] } as unknown as IntentState; + const deleting = { + operationMode: "delete", + inferredIntents: [{ type: "goal", description: SIGNAL, reasoning: "r", confidence: "high" }], + } as unknown as IntentState; + + expect(shouldRunInference(seeded)).toBe("verification"); + expect(shouldRunInference(empty)).toBe("inference"); + // Delete still wins: it skips inference to reach the reconciler, not verification. + expect(shouldRunInference(deleting)).toBe("reconciler"); + }); +}); + +describe("authority scores nothing when no profile was supplied", () => { + const felicityScores = { authority: 40, sincerity: 90, clarity: 85 }; + + it("leaves authority out of the minimum with no profile", () => { + for (const profile of ["", " ", undefined]) { + expect(combineFelicityScores(felicityScores, profile)).toBe(85); + } + }); + + it("keeps authority in the minimum when a profile backs it", () => { + expect(combineFelicityScores(felicityScores, "Ada is a staff engineer.")).toBe(40); + }); + + it("does not cap the stored score of a profile-blind verification", async () => { + const deps = { + verifier: { invoke: async () => VERDICT }, + } as unknown as IntentGraphDeps; + const state = { + userId: "ada", + userProfile: "", + operationMode: "propose", + inferredIntents: [{ + type: "goal", + description: SIGNAL, + reasoning: "Synthesized from the intake interview answers.", + confidence: "high", + }], + } as unknown as IntentState; + + const result = await verificationNode(state, deps); + + expect(result.verifiedIntents?.[0]?.score).toBe(85); + // The raw verdict is stored untouched; only the combination changes. + expect(result.verifiedIntents?.[0]?.verification?.felicity_scores.authority).toBe(40); + }); + + it("still lets authority cap the score when a profile was supplied", async () => { + const deps = { + verifier: { invoke: async () => VERDICT }, + } as unknown as IntentGraphDeps; + const state = { + userId: "ada", + userProfile: "Ada is a staff engineer at a large search company.", + operationMode: "create", + inferredIntents: [{ + type: "goal", + description: SIGNAL, + reasoning: "Stated by the user", + confidence: "high", + }], + } as unknown as IntentState; + + const result = await verificationNode(state, deps); + + expect(result.verifiedIntents?.[0]?.score).toBe(40); + }); +}); diff --git a/services/api/src/services/signal-intake.service.ts b/services/api/src/services/signal-intake.service.ts index a22579130..3ab580a43 100644 --- a/services/api/src/services/signal-intake.service.ts +++ b/services/api/src/services/signal-intake.service.ts @@ -506,7 +506,8 @@ export class SignalIntakeService { // ----------------------------------------------------------------------------- /** Compiled once and reused: the intent graph always runs in verification-only - * `propose` mode here, so no DB writes happen from this leg of the funnel. */ + * `propose` mode here, with inference pre-seeded, so this leg of the funnel + * costs exactly one model call and writes nothing. */ const productionIntents = new Intents({ database: intentDatabaseAdapter, embedder: new EmbedderAdapter(), @@ -516,13 +517,31 @@ const compiledIntentGraph = productionIntents.createGraph(); /** Verify-only invocation of the intent graph. The profile-graph leg is skipped * and nothing is supplied in its place: an intent derives only from the person's - * answers, so inference and verification see the synthesized signal alone. */ + * answers, so verification sees the synthesized signal alone. + * + * Inference is skipped by supplying its output: synthesis has already written + * one clean, self-contained, first-person signal, so there is no messy text left + * to pull candidates out of, and the profile-blind call could not enrich them + * either. Seeding `inferredIntents` routes prep straight to verification. The + * one thing inference also does — extracting tombstones — is not reachable from + * this path: `runSynthesis` reads only description, score, and verification, and + * propose mode never reaches the reconciler that acts on a tombstone. */ async function invokeIntentGraphProduction(input: { userId: string; inputContent: string; }): Promise<{ verifiedIntents?: Array> }> { const result = await compiledIntentGraph.invoke( - { ...input, userProfile: '', operationMode: 'propose' as const }, + { + ...input, + userProfile: '', + operationMode: 'propose' as const, + inferredIntents: [{ + type: 'goal' as const, + description: input.inputContent, + reasoning: 'Synthesized from the intake interview answers.', + confidence: 'high' as const, + }], + }, { recursionLimit: 100 }, ); return result as { verifiedIntents?: Array> };