From e89d8016964134a76cf99a41e9b8c02f8e03d5a0 Mon Sep 17 00:00:00 2001 From: Yanek Yuk Date: Fri, 21 Aug 2026 16:21:48 -0500 Subject: [PATCH] perf(intake): the funnel stops paying for work it already has MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three cuts between a person's last answer at /i/new and the proposal they see: 5 model calls per signal become 3, and the blocking stretch 3 becomes 2. The profile bridge folds into the follow-up planner as a nullable field on each question, so personalization is one prompt section rather than a second serial call — and silence on every question is an ordinary success, not a failure to swallow. Merging put optionSchema in the schema twice, which made the converter emit a $ref Gemini rejects: the call was burning its retry and answering from the fallback model. One extra zod instance fixes it, and a spec guard fails on any $ref. The graph stops re-inferring what synthesis just wrote. A caller that supplies a stage's output now skips that stage: seeded inferredIntents route prep straight to verification, and the funnel seeds the synthesized signal. Chat and MCP still infer, because they feed raw utterances in. And authority stops capping the score of signals it was never given a profile to judge. The propose path attaches no profile on purpose; honouring that means leaving authority out of the minimum rather than guessing it. Claude-Session: https://claude.ai/code/session_01Ay9d5zypTYuunAUZX5Vpsw --- bun.lock | 2 +- packages/protocol/package.json | 2 +- .../intents/graph/intent.graph.reconcile.ts | 11 +- .../src/intents/graph/intent.graph.shared.ts | 22 ++ .../src/intents/graph/intent.graph.ts | 18 +- .../src/intents/intake/intake.orchestrator.ts | 131 ++++---- .../intents/tests/intake.orchestrator.spec.ts | 280 +++++++++++------- .../tests/intent.graph.profile-blind.spec.ts | 184 ++++++++++++ .../api/src/services/signal-intake.service.ts | 25 +- 9 files changed, 502 insertions(+), 173 deletions(-) create mode 100644 packages/protocol/src/intents/tests/intent.graph.profile-blind.spec.ts diff --git a/bun.lock b/bun.lock index 5c1dbe904a..ecb769b301 100644 --- a/bun.lock +++ b/bun.lock @@ -89,7 +89,7 @@ }, "packages/protocol": { "name": "@indexnetwork/protocol", - "version": "23.6.4", + "version": "23.7.0", "dependencies": { "@langchain/core": "1.1.48", "@langchain/langgraph": "1.3.2", diff --git a/packages/protocol/package.json b/packages/protocol/package.json index 662d9967b6..fae8de39c8 100644 --- a/packages/protocol/package.json +++ b/packages/protocol/package.json @@ -1,6 +1,6 @@ { "name": "@indexnetwork/protocol", - "version": "23.6.4", + "version": "23.7.0", "type": "module", "main": "./dist/index.js", "types": "./dist/index.d.ts", diff --git a/packages/protocol/src/intents/graph/intent.graph.reconcile.ts b/packages/protocol/src/intents/graph/intent.graph.reconcile.ts index 852041cbf8..365250ea37 100644 --- a/packages/protocol/src/intents/graph/intent.graph.reconcile.ts +++ b/packages/protocol/src/intents/graph/intent.graph.reconcile.ts @@ -9,7 +9,7 @@ import { getAbortSignalConfig } from "../../shared/agent/model-signal.js"; import { timed } from "../../shared/observability/performance.js"; import { requestContext } from "../../shared/observability/request-context.js"; import type { DebugMetaAgent } from "../../agents/agent.module.js"; -import { buildExplicitUpdateActions, enforceIntentActionBoundary, generateIntentEmbedding, getSpecificityWarning, isVague, logger, MAX_PERMISSIBLE_ENTROPY, MIN_CLEAR_INTENT_SCORE, toSpeechActType, type IntentGraphDeps, type IntentState } from "./intent.graph.shared.js"; +import { buildExplicitUpdateActions, combineFelicityScores, enforceIntentActionBoundary, generateIntentEmbedding, getSpecificityWarning, isVague, logger, MAX_PERMISSIBLE_ENTROPY, MIN_CLEAR_INTENT_SCORE, toSpeechActType, type IntentGraphDeps, type IntentState } from "./intent.graph.shared.js"; /** @@ -99,12 +99,9 @@ export async function verificationNode(state: IntentState, deps: IntentGraphDeps }; } - // Calculate Score - const score = Math.min( - verdict.felicity_scores.authority, - verdict.felicity_scores.sincerity, - verdict.felicity_scores.clarity - ); + // Calculate Score. Authority participates only when a profile was + // supplied to judge it against. + const score = combineFelicityScores(verdict.felicity_scores, state.userProfile); // Return enriched intent return { diff --git a/packages/protocol/src/intents/graph/intent.graph.shared.ts b/packages/protocol/src/intents/graph/intent.graph.shared.ts index 802e5d8eb8..e8aaef298e 100644 --- a/packages/protocol/src/intents/graph/intent.graph.shared.ts +++ b/packages/protocol/src/intents/graph/intent.graph.shared.ts @@ -105,6 +105,28 @@ export const isVague = (description: string, entropy: number, clarity: number): return false; }; +/** + * Combine the felicity scores into the single number callers store. + * + * Authority is a preparatory condition: it asks whether the *speaker's profile* + * supports the speech act. When no profile was supplied the verifier has nothing + * to answer that from, so its guess is left out of the minimum rather than + * allowed to cap the result. The propose path attaches no profile on purpose — + * a signal derives only from the person's answers — so honouring that means not + * scoring them against a profile they were never asked for. + */ +export const combineFelicityScores = ( + felicityScores: { authority: number; sincerity: number; clarity: number }, + userProfile: string | undefined, +): number => { + const profileBacked = Boolean(userProfile?.trim()); + return Math.min( + ...(profileBacked ? [felicityScores.authority] : []), + felicityScores.sincerity, + felicityScores.clarity, + ); +}; + export const getSpecificityWarning = (verdict: { specificity_warning?: string | null }): string => { const warning = verdict.specificity_warning?.trim(); return warning && warning.length > 0 ? warning : DEFAULT_SPECIFICITY_WARNING; diff --git a/packages/protocol/src/intents/graph/intent.graph.ts b/packages/protocol/src/intents/graph/intent.graph.ts index fd08b9e6b7..1d31370a7f 100644 --- a/packages/protocol/src/intents/graph/intent.graph.ts +++ b/packages/protocol/src/intents/graph/intent.graph.ts @@ -63,12 +63,15 @@ export class IntentGraphFactory { // - UPDATE: prep → inference → reconciliation → executor → END (skips verification if no new intents) // - DELETE: prep → reconciliation → executor → END (skips inference and verification) // - PROPOSE: prep → inference → verification → END (no reconciliation/execution, no DB writes) + // A caller that supplies a stage's output skips that stage: seed + // `inferredIntents` and prep routes straight to verification. .addEdge(START, "prep") - // After prep: read mode → query; else inference or reconciler + // After prep: read mode → query; else inference, verification, or reconciler .addConditionalEdges("prep", afterPrepRoute, { query: "query", inference: "inference", + verification: "verification", reconciler: "reconciler", __end__: END, }) @@ -115,8 +118,11 @@ export function afterPrepRoute(state: IntentState): string { /** - * Determines if inference should run based on operation mode. + * Determines if inference should run based on operation mode and seeded state. * Delete operations skip inference entirely and go straight to reconciliation. + * A caller that already supplies the candidate signals skips it too: inference + * extracts candidates from messy text, so there is nothing left for it to do + * when the candidates arrive with the invocation. */ export function shouldRunInference(state: IntentState): string { if (state.operationMode === 'delete') { @@ -124,6 +130,14 @@ export function shouldRunInference(state: IntentState): string { return 'reconciler'; } + if (state.inferredIntents.length > 0) { + logger.verbose('Intents supplied by the caller - skipping inference, routing to verification', { + operationMode: state.operationMode, + seededIntentCount: state.inferredIntents.length, + }); + return 'verification'; + } + logger.verbose('Running inference', { operationMode: state.operationMode }); diff --git a/packages/protocol/src/intents/intake/intake.orchestrator.ts b/packages/protocol/src/intents/intake/intake.orchestrator.ts index 07b0dcc0fc..978e591c1f 100644 --- a/packages/protocol/src/intents/intake/intake.orchestrator.ts +++ b/packages/protocol/src/intents/intake/intake.orchestrator.ts @@ -75,6 +75,21 @@ const optionSchema = z.object({ description: z.string(), }); +/** + * Structurally identical to {@link optionSchema}, and deliberately a separate + * object. + * + * Reusing one zod instance for two fields of the same schema makes the JSON + * Schema converter emit the second as a `$ref` into `definitions`. Gemini + * rejects that document, so the call burned its retry and then answered from + * the fallback model instead — a 1.5s call became a 6.5s one, on a model nobody + * chose. Two instances keep both option shapes inlined. + */ +const bridgeOptionSchema = z.object({ + label: z.string().min(1), + description: z.string(), +}); + const answerFirstQuestionSchema = z.object({ missingAxis: z .enum(["purpose", "desired_attributes", "exchange", "constraint"]) @@ -86,23 +101,22 @@ const answerFirstQuestionSchema = z.object({ .min(2) .max(3) .describe("Two or three distinct choices derived only from the answered intake rounds."), + profileBridgeOption: bridgeOptionSchema + .nullable() + .describe( + "One natural profile intersection appended after the answer-grounded options, " + + "or null when no brief was supplied or the bridge would be forced. " + + "Null for every question is a correct answer.", + ), multiSelect: z.boolean(), }); -const followUpPlanSchema = z.object({ +/** Exported for the schema-shape guard in the sibling spec. */ +export const followUpPlanSchema = z.object({ questions: z.array(answerFirstQuestionSchema), plannedFollowUpCount: z.number().int().min(0), }); -const profileBridgePlanSchema = z.object({ - bridges: z.array(z.object({ - questionIndex: z.number().int().min(0), - profileBridgeOption: optionSchema - .nullable() - .describe("One natural profile intersection, or null when the bridge would be forced."), - })), -}); - const synthesisSchema = z.object({ description: z.string().min(1), lookingFor: z.string().min(1), @@ -135,14 +149,21 @@ export const FALLBACK_BRING_QUESTION: IntakePackQuestion = { multiSelect: false, }; -const PLAN_SYSTEM_PROMPT = `You plan and write answer-first follow-up intake questions for a networking product. +const FOLLOW_UP_SYSTEM_PROMPT = `You plan and write answer-first follow-up intake questions for a networking product. -You receive ONLY the answered intake rounds. Use them to choose the next missing -axis, write a standalone prompt that names the newly stated person or domain, and -create 2-3 meaningfully distinct answerGroundedOptions. Choose the most useful -unanswered axis from: purpose, desired_attributes, exchange, or constraint. Never -re-ask an axis the rounds already answer. Do not infer a professional background, -industry, capability, or commercial goal that the rounds do not state. +Each question has two parts, written in this order: the answer-grounded core, then +one optional profile bridge. Keep them separate — the brief may shape the bridge and +nothing else. + +CORE — answerGroundedOptions, from the answered rounds alone +Use the answered intake rounds to choose the next missing axis, write a standalone +prompt that names the newly stated person or domain, and create 2-3 meaningfully +distinct answerGroundedOptions. Choose the most useful unanswered axis from: +purpose, desired_attributes, exchange, or constraint. Never re-ask an axis the +rounds already answer. Do not infer a professional background, industry, +capability, or commercial goal that the rounds do not state. Write these options as +if no profile brief had been supplied: nothing in the brief may add, remove, +reword, or reclassify one of them. Every answerGroundedOption must be visibly anchored to the stated person or domain in its label or description and represent a concrete, distinct path. Generic labels @@ -151,23 +172,24 @@ not say what the user would learn, share, collaborate on, or network about. - Scuba divers: "Learn diving techniques", "Find dive buddies", "Share dive stories". - Climate founders: "Compare climate sectors", "Find climate peers", "Discuss adaptation". +BRIDGE — profileBridgeOption, at most one per question +Only after the core options are written, compare the question, its missing axis, and +those options with the profile brief, and return either one genuinely useful +profileBridgeOption or null. A bridge is useful only when it creates a natural +additional path that is not already represented; it is appended after the +answer-grounded options and never rewrites, replaces, removes, or reclassifies one +of them. Never return more than one bridge per question. When the profile theme is +unrelated, when no brief was supplied, or when the intersection would feel forced, +return null. Running club + investor may support one sponsorship bridge; scuba +divers + pianist should return null. Returning null for every question is a normal, +correct answer — never invent a bridge to avoid it. + Each option needs a short label and a one-line description. Set multiSelect true only when several options can genuinely apply together. Never expose raw JSON, IDs, or internal vocabulary. plannedFollowUpCount is the TOTAL number of follow-up questions the interview should contain, including any returned now; when the input already fixes it, echo that value unchanged.`; -const PROFILE_BRIDGE_SYSTEM_PROMPT = `You may append one optional profile bridge to each already-written signal-intake question. - -The question, missing axis, and answer-grounded options are immutable. For each -questionIndex, compare them with the profile brief and return either one genuinely -useful profileBridgeOption or null. A bridge is useful only when it creates a -natural additional path that is not already represented. Never rewrite, replace, -remove, or reclassify a core option. Never return more than one bridge per question. -When the profile theme is unrelated or the intersection would feel forced, return -null. Running club + investor may support one sponsorship bridge; scuba divers + -pianist should return null.`; - const SYNTHESIS_SYSTEM_PROMPT = `You write one clear signal for a networking product. You receive ONLY the interview: each question the person was asked and the answer @@ -193,11 +215,18 @@ export function answerLabel(answer: IntakeAnswer): string { return [...answer.selectedOptions, answer.freeText?.trim() ?? ""].filter(Boolean).join(", "); } +/** + * Clamp one generated question into a renderable one. + * + * The bridge is structurally optional: an absent, null, or unusable + * `profileBridgeOption` — and any bridge returned when no brief was supplied — + * simply leaves the question with its answer-grounded options. + */ function normalizeFollowUpQuestion( question: z.infer, - profileBridge: z.infer | null, brief: string, ): IntakePackQuestion { + const profileBridge = brief.trim() ? question.profileBridgeOption ?? null : null; const seen = new Set(); const normalizeOption = (option: { label: string; description: string }) => { const label = option.label.trim(); @@ -233,7 +262,6 @@ function normalizeFollowUpQuestion( /** Runs the two live stages of the fast intake funnel: follow-up planning and synthesis. */ export class SignalIntakeOrchestrator { private readonly plannerModel: Runnable>; - private readonly profileBridgeModel: Runnable>; private readonly synthesisModel: Runnable; /** @@ -241,13 +269,10 @@ export class SignalIntakeOrchestrator { */ constructor(models?: { planner?: Runnable>; - profileBridge?: Runnable>; synthesis?: Runnable; }) { this.plannerModel = models?.planner ?? createStructuredModel("signalIntakePack", followUpPlanSchema) as unknown as Runnable>; - this.profileBridgeModel = models?.profileBridge - ?? createStructuredModel("signalIntakePack", profileBridgePlanSchema, { name: "signal_intake_profile_bridges" }) as unknown as Runnable>; this.synthesisModel = models?.synthesis ?? createStructuredModel("signalIntakePack", synthesisSchema) as unknown as Runnable; } @@ -255,6 +280,11 @@ export class SignalIntakeOrchestrator { /** * Plan and write follow-up questions from the brief and answered rounds. * + * One call writes both halves of each question: the answer-grounded core and + * the optional profile bridge. The bridge is a nullable field rather than a + * second call, so personalization staying silent — for every question — is an + * ordinary success, never a failure to swallow. + * * @param input - Brief, answered rounds, per-call cap, and any locked plan * @returns Up to `maxFollowUps` renderable questions plus the total plan; * the static fallback question with count 1 when generation fails @@ -266,38 +296,19 @@ export class SignalIntakeOrchestrator { const lockedLine = input.plannedFollowUpCount !== undefined ? `\n\nThe interview plan is fixed at ${input.plannedFollowUpCount} follow-up question(s) in total; ${input.rounds.length - 1} already asked. Echo that count unchanged.` : ""; + const briefSection = input.brief.trim() + ? `\n\nPROFILE BRIEF (profileBridgeOption only — it may not touch an answerGroundedOption):\n${input.brief.trim()}` + : "\n\nPROFILE BRIEF: none supplied. Return profileBridgeOption null for every question."; try { const raw = await this.plannerModel.invoke([ - new SystemMessage(PLAN_SYSTEM_PROMPT), + new SystemMessage(FOLLOW_UP_SYSTEM_PROMPT), new HumanMessage( - `ANSWERED ROUNDS:\n${roundsText}\n\nWrite up to ${input.maxFollowUps} follow-up question(s).${lockedLine}`, + `ANSWERED ROUNDS:\n${roundsText}${briefSection}\n\nWrite up to ${input.maxFollowUps} follow-up question(s).${lockedLine}`, ), ]); - const coreQuestions = raw.questions.slice(0, input.maxFollowUps); - const bridgeByQuestion = new Map | null>(); - if (coreQuestions.length > 0 && input.brief.trim()) { - try { - const bridgePlan = await this.profileBridgeModel.invoke([ - new SystemMessage(PROFILE_BRIDGE_SYSTEM_PROMPT), - new HumanMessage( - `ANSWERED ROUNDS:\n${roundsText}\n\nPROFILE BRIEF:\n${input.brief}\n\nIMMUTABLE CORE QUESTIONS:\n${JSON.stringify(coreQuestions, null, 2)}\n\nReturn one bridge decision for each questionIndex.`, - ), - ]); - for (const bridge of bridgePlan.bridges) { - if ( - bridge.questionIndex < coreQuestions.length - && !bridgeByQuestion.has(bridge.questionIndex) - ) { - bridgeByQuestion.set(bridge.questionIndex, bridge.profileBridgeOption); - } - } - } catch { - // Profile personalization is optional. A bridge-model failure must not - // discard valid answer-grounded questions from the primary model. - } - } - const questions = coreQuestions.map((question, index) => - normalizeFollowUpQuestion(question, bridgeByQuestion.get(index) ?? null, input.brief)); + const questions = raw.questions + .slice(0, input.maxFollowUps) + .map((question) => normalizeFollowUpQuestion(question, input.brief)); // Backstop: while budget remains, a successful empty plan must not // silently shrink the interview; serve the static fallback instead. if (questions.length === 0 && input.maxFollowUps > 0) { diff --git a/packages/protocol/src/intents/tests/intake.orchestrator.spec.ts b/packages/protocol/src/intents/tests/intake.orchestrator.spec.ts index 2c3c97886d..bcc4f6faa3 100644 --- a/packages/protocol/src/intents/tests/intake.orchestrator.spec.ts +++ b/packages/protocol/src/intents/tests/intake.orchestrator.spec.ts @@ -1,6 +1,8 @@ import { describe, expect, it } from "bun:test"; -import { answerLabel, FALLBACK_BRING_QUESTION, FALLBACK_WHO_QUESTION, SignalIntakeOrchestrator } from "../intake/intake.orchestrator.js"; +import { toJsonSchema } from "@langchain/core/utils/json_schema"; + +import { answerLabel, FALLBACK_BRING_QUESTION, FALLBACK_WHO_QUESTION, followUpPlanSchema, SignalIntakeOrchestrator } from "../intake/intake.orchestrator.js"; interface Capture { prompt?: string; @@ -19,13 +21,8 @@ function stub(value: T, capture?: Capture) { } as never; } -const NO_BRIDGES = { bridges: [] }; - function plannerModels(plan: T, capture?: Capture) { - return { - planner: stub(plan, capture), - profileBridge: stub(NO_BRIDGES), - }; + return { planner: stub(plan, capture) }; } const question = { @@ -36,6 +33,7 @@ const question = { { label: "Distribution", description: "You have an audience" }, { label: "Engineering depth", description: "You can build it" }, ], + profileBridgeOption: null, multiSelect: false, }; @@ -51,31 +49,24 @@ describe("answerLabel", () => { describe("SignalIntakeOrchestrator.generateFollowUps", () => { it("assembles answer-grounded options before one optional profile bridge", async () => { - const orchestrator = new SignalIntakeOrchestrator({ - planner: stub({ - questions: [{ - missingAxis: "purpose", - title: "Purpose", - prompt: "What would make meeting scuba divers valuable to you?", - answerGroundedOptions: [ - { label: "Find dive buddies", description: "Meet people to dive with" }, - { label: "Learn from experts", description: "Meet experienced divers" }, - { label: "Marine conservation", description: "Work on ocean stewardship" }, - ], - multiSelect: false, - }], - plannedFollowUpCount: 1, - }), - profileBridge: stub({ - bridges: [{ - questionIndex: 0, - profileBridgeOption: { - label: "Underwater technology", - description: "Connect diving with your technology background", - }, - }], - }), - }); + const orchestrator = new SignalIntakeOrchestrator(plannerModels({ + questions: [{ + missingAxis: "purpose", + title: "Purpose", + prompt: "What would make meeting scuba divers valuable to you?", + answerGroundedOptions: [ + { label: "Find dive buddies", description: "Meet people to dive with" }, + { label: "Learn from experts", description: "Meet experienced divers" }, + { label: "Marine conservation", description: "Work on ocean stewardship" }, + ], + profileBridgeOption: { + label: "Underwater technology", + description: "Connect diving with your technology background", + }, + multiSelect: false, + }], + plannedFollowUpCount: 1, + })); const result = await orchestrator.generateFollowUps({ brief: "The user builds AI products.", @@ -96,13 +87,9 @@ describe("SignalIntakeOrchestrator.generateFollowUps", () => { plannedFollowUpCount: 2, }; - it("keeps the profile out of core generation and sends it only to bridge generation", async () => { - const plannerCapture: Capture = {}; - const bridgeCapture: Capture = {}; - const orchestrator = new SignalIntakeOrchestrator({ - planner: stub(plan, plannerCapture), - profileBridge: stub({ bridges: [] }, bridgeCapture), - }); + it("sends the brief in a bridge-only slot and keeps both prompts' constraints", async () => { + const capture: Capture = {}; + const orchestrator = new SignalIntakeOrchestrator(plannerModels(plan, capture)); const result = await orchestrator.generateFollowUps({ brief: "Ada builds developer tools.", @@ -112,43 +99,81 @@ describe("SignalIntakeOrchestrator.generateFollowUps", () => { expect(result.questions).toHaveLength(2); expect(result.plannedFollowUpCount).toBe(2); - expect(plannerCapture.prompt).toContain("A design partner"); - expect(plannerCapture.prompt).not.toContain("Ada builds developer tools."); - expect(plannerCapture.messages?.[0]).toContain("receive ONLY the answered intake rounds"); - expect(plannerCapture.messages?.[0]).toContain("Generic labels"); - expect(plannerCapture.messages?.[0]).toContain("Learn diving techniques"); - expect(bridgeCapture.prompt).toContain("Ada builds developer tools."); - expect(bridgeCapture.prompt).toContain("A design partner"); - expect(bridgeCapture.prompt).toContain("IMMUTABLE CORE QUESTIONS"); - expect(bridgeCapture.messages?.[0]).toContain("question, missing axis, and answer-grounded options are immutable"); + expect(capture.prompt).toContain("A design partner"); + // One call now carries the brief, so the wall is a labelled slot plus a + // prompt constraint rather than a second model that never saw the rounds. + expect(capture.prompt).toContain("PROFILE BRIEF (profileBridgeOption only"); + expect(capture.prompt).toContain("Ada builds developer tools."); + const systemPrompt = capture.messages?.[0] ?? ""; + expect(systemPrompt).toContain("from the answered rounds alone"); + expect(systemPrompt).toContain("Write these options as\nif no profile brief had been supplied"); + expect(systemPrompt).toContain("nothing in the brief may add, remove,"); + expect(systemPrompt).toContain("Generic labels"); + expect(systemPrompt).toContain("Learn diving techniques"); + // The bridge prompt's own thinking survives the merge. + expect(systemPrompt).toContain("never rewrites, replaces, removes, or reclassifies one"); + expect(systemPrompt).toContain("Never return more than one bridge per question"); + expect(systemPrompt).toContain("scuba\ndivers + pianist should return null"); }); - it("deduplicates options without letting the profile bridge displace two answer-grounded choices", async () => { - const orchestrator = new SignalIntakeOrchestrator({ - planner: stub({ + it("keeps 2-3 answer-grounded core options on every question, whatever the bridge does", async () => { + // The two-call split enforced this structurally: the bridge model received + // the core questions as immutable input. In one call it is a prompt + // constraint, so the shape is asserted here instead. + const bridges = [ + null, + { label: "Underwater technology", description: "A natural bridge" }, + // A bridge that duplicates a core label is dropped, never substituted. + { label: " find dive buddies ", description: "The same choice again" }, + ]; + for (const profileBridgeOption of bridges) { + const orchestrator = new SignalIntakeOrchestrator(plannerModels({ questions: [{ - missingAxis: "desired_attributes", - title: "Divers", - prompt: "Which scuba divers would be most useful to meet?", + missingAxis: "purpose", + title: "Purpose", + prompt: "What would make meeting scuba divers valuable to you?", answerGroundedOptions: [ - { label: "Dive buddies", description: "People to dive with" }, - { label: " dive buddies ", description: "Duplicate after trimming" }, - { label: "Experienced instructors", description: "People to learn from" }, + { label: "Find dive buddies", description: "Meet people to dive with" }, + { label: "Learn from experts", description: "Meet experienced divers" }, ], + profileBridgeOption, multiSelect: false, }], plannedFollowUpCount: 1, - }), - profileBridge: stub({ - bridges: [{ - questionIndex: 0, - profileBridgeOption: { - label: "Underwater technologists", - description: "A natural bridge to the user's background", - }, - }], - }), - }); + })); + + const result = await orchestrator.generateFollowUps({ + brief: "The user builds AI products.", + rounds: [{ prompt: "Who?", answer: { selectedOptions: [], freeText: "scuba divers" } }], + maxFollowUps: 1, + }); + + const labels = result.questions[0]?.options.map((option) => option.label) ?? []; + expect(labels.slice(0, 2)).toEqual(["Find dive buddies", "Learn from experts"]); + expect(labels.length).toBeLessThanOrEqual(3); + expect(result.questions[0]).not.toEqual(FALLBACK_BRING_QUESTION); + } + }); + + it("deduplicates options without letting the profile bridge displace two answer-grounded choices", async () => { + const orchestrator = new SignalIntakeOrchestrator(plannerModels({ + questions: [{ + missingAxis: "desired_attributes", + title: "Divers", + prompt: "Which scuba divers would be most useful to meet?", + answerGroundedOptions: [ + { label: "Dive buddies", description: "People to dive with" }, + { label: " dive buddies ", description: "Duplicate after trimming" }, + { label: "Experienced instructors", description: "People to learn from" }, + ], + profileBridgeOption: { + label: "Underwater technologists", + description: "A natural bridge to the user's background", + }, + multiSelect: false, + }], + plannedFollowUpCount: 1, + })); const result = await orchestrator.generateFollowUps({ brief: "The user builds technology products.", @@ -164,30 +189,23 @@ describe("SignalIntakeOrchestrator.generateFollowUps", () => { }); it("falls back when deduplication leaves fewer than two answer-grounded choices", async () => { - const orchestrator = new SignalIntakeOrchestrator({ - planner: stub({ - questions: [{ - missingAxis: "purpose", - title: "Purpose", - prompt: "What do you want from meeting scuba divers?", - answerGroundedOptions: [ - { label: "Dive buddies", description: "People to dive with" }, - { label: " dive buddies ", description: "The same choice" }, - ], - multiSelect: false, - }], - plannedFollowUpCount: 1, - }), - profileBridge: stub({ - bridges: [{ - questionIndex: 0, - profileBridgeOption: { - label: "Underwater technology", - description: "A profile-derived bridge", - }, - }], - }), - }); + const orchestrator = new SignalIntakeOrchestrator(plannerModels({ + questions: [{ + missingAxis: "purpose", + title: "Purpose", + prompt: "What do you want from meeting scuba divers?", + answerGroundedOptions: [ + { label: "Dive buddies", description: "People to dive with" }, + { label: " dive buddies ", description: "The same choice" }, + ], + profileBridgeOption: { + label: "Underwater technology", + description: "A profile-derived bridge", + }, + multiSelect: false, + }], + plannedFollowUpCount: 1, + })); const result = await orchestrator.generateFollowUps({ brief: "The user builds technology products.", @@ -201,22 +219,70 @@ describe("SignalIntakeOrchestrator.generateFollowUps", () => { }); }); - it("keeps valid core questions when optional bridge generation fails", async () => { + it("treats a null bridge on every question as an ordinary success, not a failure", async () => { + let calls = 0; const orchestrator = new SignalIntakeOrchestrator({ - planner: stub({ questions: [question], plannedFollowUpCount: 1 }), - profileBridge: { invoke: async () => { throw new Error("bridge model down"); } } as never, + planner: { + invoke: async () => { + calls += 1; + return { + questions: [question, { ...question, prompt: "Where should we look?" }], + plannedFollowUpCount: 2, + }; + }, + } as never, }); + const result = await orchestrator.generateFollowUps({ + brief: "Ada builds developer tools.", + rounds: [{ prompt: "Who?", answer: { selectedOptions: ["A design partner"] } }], + maxFollowUps: 2, + }); + + // Personalization is optional: silence is not an error and buys no retry. + expect(calls).toBe(1); + expect(result.questions).toHaveLength(2); + for (const served of result.questions) { + expect(served.options.map((option) => option.label)).toEqual(["Distribution", "Engineering depth"]); + } + }); + + it("omitting the bridge field entirely is as good as returning null", async () => { + const { profileBridgeOption: _omitted, ...withoutBridge } = question; + const orchestrator = new SignalIntakeOrchestrator(plannerModels({ + questions: [withoutBridge], + plannedFollowUpCount: 1, + })); + const result = await orchestrator.generateFollowUps({ brief: "Ada builds developer tools.", rounds: [{ prompt: "Who?", answer: { selectedOptions: ["A design partner"] } }], maxFollowUps: 1, }); - expect(result.questions[0]?.options.map((option) => option.label)).toEqual([ - "Distribution", - "Engineering depth", - ]); + expect(result.questions[0]?.options.map((option) => option.label)) + .toEqual(["Distribution", "Engineering depth"]); + }); + + it("drops a bridge offered when no brief was supplied", async () => { + const capture: Capture = {}; + const orchestrator = new SignalIntakeOrchestrator(plannerModels({ + questions: [{ + ...question, + profileBridgeOption: { label: "Invented bridge", description: "From a brief that does not exist" }, + }], + plannedFollowUpCount: 1, + }, capture)); + + const result = await orchestrator.generateFollowUps({ + brief: " ", + rounds: [{ prompt: "Who?", answer: { selectedOptions: ["A design partner"] } }], + maxFollowUps: 1, + }); + + expect(capture.prompt).toContain("PROFILE BRIEF: none supplied"); + expect(result.questions[0]?.options.map((option) => option.label)) + .toEqual(["Distribution", "Engineering depth"]); }); it("truncates model output to maxFollowUps", async () => { @@ -386,3 +452,19 @@ describe("static fallbacks", () => { } }); }); + +describe("the follow-up plan schema as the provider receives it", () => { + it("inlines every option shape instead of emitting a $ref", () => { + // `withStructuredOutput` converts this schema with the same converter and + // sends it as `response_format.json_schema`. Reusing one zod instance for + // two fields makes the converter emit the second as a `$ref` into + // `definitions`, which Gemini rejects: the call burns its retry and answers + // from the fallback model instead. Merging the bridge into this schema is + // exactly the change that could reintroduce that, so it is asserted here. + const json = JSON.stringify(toJsonSchema(followUpPlanSchema)); + + expect(json).not.toContain("$ref"); + expect(json).not.toContain("definitions"); + expect(json).toContain("profileBridgeOption"); + }); +}); diff --git a/packages/protocol/src/intents/tests/intent.graph.profile-blind.spec.ts b/packages/protocol/src/intents/tests/intent.graph.profile-blind.spec.ts new file mode 100644 index 0000000000..816c2ce374 --- /dev/null +++ b/packages/protocol/src/intents/tests/intent.graph.profile-blind.spec.ts @@ -0,0 +1,184 @@ +/** + * The profile-blind propose path: what it skips, and what it must not score. + * + * The signal-intake funnel invokes the graph on the output of synthesis — one + * clean, self-contained, first-person signal — with no profile attached. Two + * consequences are asserted here: + * + * 1. Inference has nothing to do on that input, so a caller that supplies its + * output routes straight to verification. + * 2. `authority` asks whether the speaker's profile supports the speech act. + * With no profile it is a number guessed from nothing, so it must not cap the + * score the caller stores. + */ +import { describe, expect, it } from "bun:test"; + +import { IntentGraphFactory, shouldRunInference } from "../graph/intent.graph.js"; +import { combineFelicityScores } from "../graph/intent.graph.shared.js"; +import { verificationNode } from "../graph/intent.graph.reconcile.js"; + +import type { SemanticVerifierOutput } from "../intent.verifier.js"; +import type { IntentGraphDeps, IntentState } from "../graph/intent.graph.shared.js"; +import type { IntentGraphDatabase } from "../../shared/interfaces/database.interface.js"; + +const VERDICT: SemanticVerifierOutput = { + reasoning: "A specific search directive", + classification: "DIRECTIVE", + felicity_scores: { authority: 40, sincerity: 90, clarity: 85 }, + semantic_entropy: 0.2, + referential_anchor: null, + referential_breadth: "narrow", + missing_selectional_constraints: [], + specificity_warning: null, + flags: ["SKILL_MISMATCH"], +}; + +const SIGNAL = "I'm looking for a design partner to test my developer tooling this quarter."; + +const database = { + getProfile: async () => ({ identity: { name: "Ada" } }), + getActiveIntents: async () => [], +} as unknown as IntentGraphDatabase; + +/** Build a graph whose inferrer and verifier both count their calls. */ +function makeGraph() { + const inferrerCalls: Array = []; + const verifierCalls: Array<{ content: string; context: string }> = []; + const graph = new IntentGraphFactory(database, undefined, undefined, { + inferrer: { + invoke: async (content: string | null) => { + inferrerCalls.push(content); + return { + intents: [{ + type: "goal" as const, + description: "Something the inferrer made up", + reasoning: "Inferred", + confidence: "high" as const, + }], + }; + }, + }, + verifier: { + invoke: async (content: string, context: string) => { + verifierCalls.push({ content, context }); + return VERDICT; + }, + }, + reconciler: { invoke: async () => ({ actions: [] }) }, + }).createGraph(); + return { graph, inferrerCalls, verifierCalls }; +} + +describe("supplying a stage's output skips that stage", () => { + it("verifies the caller's signal without re-inferring it", async () => { + const { graph, inferrerCalls, verifierCalls } = makeGraph(); + + const result = await graph.invoke({ + userId: "ada", + userProfile: "", + operationMode: "propose", + inputContent: SIGNAL, + inferredIntents: [{ + type: "goal", + description: SIGNAL, + reasoning: "Synthesized from the intake interview answers.", + confidence: "high", + }], + }); + + expect(inferrerCalls).toEqual([]); + expect(verifierCalls.map((call) => call.content)).toEqual([SIGNAL]); + expect(result.verifiedIntents).toHaveLength(1); + expect(result.verifiedIntents[0]?.description).toBe(SIGNAL); + }); + + it("still runs inference when the caller supplies only raw text", async () => { + const { graph, inferrerCalls, verifierCalls } = makeGraph(); + + const result = await graph.invoke({ + userId: "ada", + userProfile: "", + operationMode: "propose", + inputContent: "erm, people to build with I guess", + }); + + expect(inferrerCalls).toEqual(["erm, people to build with I guess"]); + expect(verifierCalls.map((call) => call.content)).toEqual(["Something the inferrer made up"]); + expect(result.verifiedIntents).toHaveLength(1); + }); + + it("routes on the seed, not on the operation mode", () => { + const seeded = { + operationMode: "create", + inferredIntents: [{ type: "goal", description: SIGNAL, reasoning: "r", confidence: "high" }], + } as unknown as IntentState; + const empty = { operationMode: "create", inferredIntents: [] } as unknown as IntentState; + const deleting = { + operationMode: "delete", + inferredIntents: [{ type: "goal", description: SIGNAL, reasoning: "r", confidence: "high" }], + } as unknown as IntentState; + + expect(shouldRunInference(seeded)).toBe("verification"); + expect(shouldRunInference(empty)).toBe("inference"); + // Delete still wins: it skips inference to reach the reconciler, not verification. + expect(shouldRunInference(deleting)).toBe("reconciler"); + }); +}); + +describe("authority scores nothing when no profile was supplied", () => { + const felicityScores = { authority: 40, sincerity: 90, clarity: 85 }; + + it("leaves authority out of the minimum with no profile", () => { + for (const profile of ["", " ", undefined]) { + expect(combineFelicityScores(felicityScores, profile)).toBe(85); + } + }); + + it("keeps authority in the minimum when a profile backs it", () => { + expect(combineFelicityScores(felicityScores, "Ada is a staff engineer.")).toBe(40); + }); + + it("does not cap the stored score of a profile-blind verification", async () => { + const deps = { + verifier: { invoke: async () => VERDICT }, + } as unknown as IntentGraphDeps; + const state = { + userId: "ada", + userProfile: "", + operationMode: "propose", + inferredIntents: [{ + type: "goal", + description: SIGNAL, + reasoning: "Synthesized from the intake interview answers.", + confidence: "high", + }], + } as unknown as IntentState; + + const result = await verificationNode(state, deps); + + expect(result.verifiedIntents?.[0]?.score).toBe(85); + // The raw verdict is stored untouched; only the combination changes. + expect(result.verifiedIntents?.[0]?.verification?.felicity_scores.authority).toBe(40); + }); + + it("still lets authority cap the score when a profile was supplied", async () => { + const deps = { + verifier: { invoke: async () => VERDICT }, + } as unknown as IntentGraphDeps; + const state = { + userId: "ada", + userProfile: "Ada is a staff engineer at a large search company.", + operationMode: "create", + inferredIntents: [{ + type: "goal", + description: SIGNAL, + reasoning: "Stated by the user", + confidence: "high", + }], + } as unknown as IntentState; + + const result = await verificationNode(state, deps); + + expect(result.verifiedIntents?.[0]?.score).toBe(40); + }); +}); diff --git a/services/api/src/services/signal-intake.service.ts b/services/api/src/services/signal-intake.service.ts index a225791301..3ab580a43f 100644 --- a/services/api/src/services/signal-intake.service.ts +++ b/services/api/src/services/signal-intake.service.ts @@ -506,7 +506,8 @@ export class SignalIntakeService { // ----------------------------------------------------------------------------- /** Compiled once and reused: the intent graph always runs in verification-only - * `propose` mode here, so no DB writes happen from this leg of the funnel. */ + * `propose` mode here, with inference pre-seeded, so this leg of the funnel + * costs exactly one model call and writes nothing. */ const productionIntents = new Intents({ database: intentDatabaseAdapter, embedder: new EmbedderAdapter(), @@ -516,13 +517,31 @@ const compiledIntentGraph = productionIntents.createGraph(); /** Verify-only invocation of the intent graph. The profile-graph leg is skipped * and nothing is supplied in its place: an intent derives only from the person's - * answers, so inference and verification see the synthesized signal alone. */ + * answers, so verification sees the synthesized signal alone. + * + * Inference is skipped by supplying its output: synthesis has already written + * one clean, self-contained, first-person signal, so there is no messy text left + * to pull candidates out of, and the profile-blind call could not enrich them + * either. Seeding `inferredIntents` routes prep straight to verification. The + * one thing inference also does — extracting tombstones — is not reachable from + * this path: `runSynthesis` reads only description, score, and verification, and + * propose mode never reaches the reconciler that acts on a tombstone. */ async function invokeIntentGraphProduction(input: { userId: string; inputContent: string; }): Promise<{ verifiedIntents?: Array> }> { const result = await compiledIntentGraph.invoke( - { ...input, userProfile: '', operationMode: 'propose' as const }, + { + ...input, + userProfile: '', + operationMode: 'propose' as const, + inferredIntents: [{ + type: 'goal' as const, + description: input.inputContent, + reasoning: 'Synthesized from the intake interview answers.', + confidence: 'high' as const, + }], + }, { recursionLimit: 100 }, ); return result as { verifiedIntents?: Array> };