From 8f7d6ad82c8993c5ae090fc877a72d55ddf3e636 Mon Sep 17 00:00:00 2001 From: tanglearncode Date: Sun, 9 Aug 2026 06:34:15 +0800 Subject: [PATCH] feat(context): carry the questions a context answers in its bundle MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A routing description answers "what is this?", which is not how anyone searches. People search with the words of their problem, so a context described as "billing-postgres default_pool_size regression" is invisible to someone asking why checkout was throwing 5xx. Two optional lists close that gap: the questions a context should catch, in the phrasings a user would type, and the names inside it — services, ticket ids, error strings, the words that appear in one context and nowhere else. Both are matched against and neither is ever shown, so they cost nothing in the prompt and can be kept in bulk. They live in the bundle rather than in this machine's routing cache, so a context handed to a teammate is findable by the same words on their machine as on yours. Import carries them; export already did, since it copies the manifest. Both are optional. A bundle written before this existed, or by a host that does not generate them, matches on less and works exactly as before. A save that says nothing about them leaves what is there alone, the same rule extensions already follow, so one host cannot silently strip what another wrote. Nothing generates them yet — that is the next change. This is the storage, the indexing, and the proof that it works. --- .../neatcontext/src/core/context-store.mjs | 73 +++++- .../src/core/routing-candidates.mjs | 9 +- .../neatcontext/src/core/context-store.mjs | 73 +++++- .../src/core/routing-candidates.mjs | 9 +- .../neatcontext/src/core/context-store.mjs | 73 +++++- .../src/core/routing-candidates.mjs | 9 +- .../neatcontext/src/core/context-store.mjs | 73 +++++- .../src/core/routing-candidates.mjs | 9 +- .../pi/neatcontext/src/core/context-store.mjs | 73 +++++- .../src/core/routing-candidates.mjs | 9 +- shared/core/context-store.mjs | 73 +++++- shared/core/routing-candidates.mjs | 9 +- tests/routing-candidates.test.mjs | 9 +- tests/routing-expansion.test.mjs | 210 ++++++++++++++++++ 14 files changed, 686 insertions(+), 25 deletions(-) create mode 100644 tests/routing-expansion.test.mjs diff --git a/codex-marketplace/plugins/neatcontext/src/core/context-store.mjs b/codex-marketplace/plugins/neatcontext/src/core/context-store.mjs index 665823b..9fb69f6 100644 --- a/codex-marketplace/plugins/neatcontext/src/core/context-store.mjs +++ b/codex-marketplace/plugins/neatcontext/src/core/context-store.mjs @@ -34,6 +34,11 @@ const MAX_CAPTURE_FILE_BYTES = 256 * 1024; const MAX_CAPTURE_TOTAL_BYTES = 1024 * 1024; const MAX_PROFILE_BYTES = 128 * 1024; const MAX_ROUTING_DESCRIPTION = 240; +// Matching material rather than reading material: none of this is ever shown to +// a session, so the limits are about keeping a bundle sane, not a prompt short. +const MAX_ROUTING_QUESTIONS = 20; +const MAX_ROUTING_ENTITIES = 40; +const MAX_ROUTING_TERM = 200; const UPDATE_LOCK_STALE_MS = 60_000; const SKIPPED_DIRECTORIES = new Set(["node_modules", ".git", ".svn", ".hg", "__pycache__"]); @@ -100,6 +105,11 @@ function recordFor(directory, parsed) { conversationKnowledgeFolder: knowledgeManaged ? null : path.join(directory, "knowledge"), routingDescription: typeof parsed.routingDescription === "string" ? parsed.routingDescription : "", + // Index-only matching material. It travels with the bundle so a teammate's + // copy is findable by the same words as yours, which is the whole reason it + // lives here rather than in this machine's routing cache. + routingQuestions: normalizeRoutingList(parsed.routingQuestions, MAX_ROUTING_QUESTIONS), + routingEntities: normalizeRoutingList(parsed.routingEntities, MAX_ROUTING_ENTITIES), // What this context expects to be able to reach. Read leniently: a // declaration this plugin cannot make sense of is dropped rather than // allowed to hide the profile and knowledge behind it. @@ -294,6 +304,35 @@ export async function createContext({ name, knowledgeFolder, profile, extensions }; } +// The questions a context should catch, and the names that appear in it. +// +// A description answers "what is this?", which is not how anyone searches. They +// search with the words of their problem, so these hold the other vocabulary: +// the phrasings a user would actually type, and the service names, ticket ids +// and error strings that appear in one context and nowhere else. Both are +// matched against and neither is ever displayed, which is what makes them cheap +// enough to keep in bulk. +// +// Optional, unlike the description. A bundle written before this existed, or by +// a host that does not generate them, is not broken — it just matches on less. +function normalizeRoutingList(value, limit) { + if (!Array.isArray(value)) { + return []; + } + const seen = new Set(); + const kept = []; + for (const entry of value) { + if (typeof entry !== "string") continue; + const clean = entry.trim().replace(/\s+/g, " ").slice(0, MAX_ROUTING_TERM); + const key = clean.toLowerCase(); + if (clean.length === 0 || seen.has(key)) continue; + seen.add(key); + kept.push(clean); + if (kept.length === limit) break; + } + return kept; +} + function normalizeRoutingDescription(value) { const description = (value ?? "").trim().replace(/\s+/g, " "); if (description.length === 0) { @@ -392,6 +431,8 @@ export async function createCapturedContext({ name, profile, routingDescription, + routingQuestions, + routingEntities, knowledge, extensions, capturedFrom = "conversation" @@ -399,6 +440,8 @@ export async function createCapturedContext({ const cleanName = normalizeName(name); const profileText = normalizeProfile(profile); const useWhen = normalizeRoutingDescription(routingDescription); + const questions = normalizeRoutingList(routingQuestions, MAX_ROUTING_QUESTIONS); + const entities = normalizeRoutingList(routingEntities, MAX_ROUTING_ENTITIES); const files = normalizeCaptureKnowledge(knowledge); const declarations = serializeExtensionDeclarations(extensions ?? []); await ensureUniqueName(cleanName); @@ -416,6 +459,10 @@ export async function createCapturedContext({ capturedFrom: isConversationCapture(capturedFrom) ? capturedFrom : "conversation", routingDescription: useWhen }; + // Absent rather than empty when there is nothing: a manifest should not carry + // a field that says only that a host did not fill it in. + if (questions.length > 0) record.routingQuestions = questions; + if (entities.length > 0) record.routingEntities = entities; if (declarations) record.extensions = declarations; record.updatedAt = record.createdAt; @@ -540,6 +587,8 @@ async function prepareCapturedContextUpdate({ name, profile, routingDescription, + routingQuestions, + routingEntities, knowledge, extensions }) { @@ -560,6 +609,17 @@ async function prepareCapturedContextUpdate({ requireCurrentBase(record, currentHash, baseHash); const profileText = normalizeProfile(profile); const useWhen = normalizeRoutingDescription(routingDescription); + // Same rule as extensions below: a save that says nothing about the matching + // material leaves what is there alone. A host that does not generate these + // must not silently strip what another host wrote. + const questions = + routingQuestions === undefined + ? record.routingQuestions + : normalizeRoutingList(routingQuestions, MAX_ROUTING_QUESTIONS); + const entities = + routingEntities === undefined + ? record.routingEntities + : normalizeRoutingList(routingEntities, MAX_ROUTING_ENTITIES); const files = normalizeCaptureKnowledge(knowledge); // A save that says nothing about extensions leaves them exactly as they are. // Declarations are usually added deliberately, by hand or by `extensions add`, @@ -569,13 +629,18 @@ async function prepareCapturedContextUpdate({ const currentKnowledge = await readGeneratedKnowledge(record); const changes = knowledgeChanges(currentKnowledge, files); const profileChanged = (await readProfileText(record)) !== profileText; - const routingChanged = record.routingDescription !== useWhen; + const routingChanged = + record.routingDescription !== useWhen || + JSON.stringify(record.routingQuestions) !== JSON.stringify(questions) || + JSON.stringify(record.routingEntities) !== JSON.stringify(entities); const extensionsChanged = JSON.stringify(record.extensions) !== JSON.stringify(declarations); return { record, profileText, useWhen, + questions, + entities, files, declarations, changes, @@ -681,6 +746,8 @@ export async function updateCapturedContext(capture) { revision: prepared.record.revision + 1, routingDescription: prepared.useWhen, extensions: serializeExtensionDeclarations(prepared.declarations), + routingQuestions: prepared.questions, + routingEntities: prepared.entities, updatedFrom: typeof capture.updatedFrom === "string" && capture.updatedFrom.trim().length > 0 ? capture.updatedFrom.trim() @@ -798,6 +865,10 @@ export async function importCapturedContext({ bundleFolder, name }) { name: typeof name === "string" && name.trim().length > 0 ? name : manifest.name, profile, routingDescription: manifest.routingDescription, + // The point of keeping these in the bundle: a teammate's copy is findable + // by the same words as the original, without them rediscovering any of it. + routingQuestions: manifest.routingQuestions, + routingEntities: manifest.routingEntities, knowledge, // What the bundle says it expects to reach, reduced to declarations. The // import creates no binding for any of them, so the imported context arrives diff --git a/codex-marketplace/plugins/neatcontext/src/core/routing-candidates.mjs b/codex-marketplace/plugins/neatcontext/src/core/routing-candidates.mjs index aab5808..ae3ad79 100644 --- a/codex-marketplace/plugins/neatcontext/src/core/routing-candidates.mjs +++ b/codex-marketplace/plugins/neatcontext/src/core/routing-candidates.mjs @@ -24,9 +24,10 @@ import { buildIndex, rank } from "./routing-search.mjs"; // string rather than kept as a list because the scorer counts words, and a // user who wrote two aliases meant both of them. // -// `entities` and `questions` are missing on purpose: nothing generates them -// yet. The scorer treats an absent field as absent rather than empty, so adding -// them later changes what matches without changing anything here. +// `questions` and `entities` come from the bundle rather than from this +// machine, which is what makes a context findable by the same words on every +// machine it reaches. They are matched against and never displayed, so their +// size costs nothing in the prompt. export function routingFields(context, card, files) { return { name: context.name ?? "", @@ -35,6 +36,8 @@ export function routingFields(context, card, files) { // that travelled with the bundle. description: card?.useWhen || context.routingDescription || "", aliases: (card?.aliases ?? []).join(" "), + questions: (context.routingQuestions ?? []).join(" "), + entities: (context.routingEntities ?? []).join(" "), files: files.join(" ") }; } diff --git a/plugins/claude-code/neatcontext/src/core/context-store.mjs b/plugins/claude-code/neatcontext/src/core/context-store.mjs index 665823b..9fb69f6 100644 --- a/plugins/claude-code/neatcontext/src/core/context-store.mjs +++ b/plugins/claude-code/neatcontext/src/core/context-store.mjs @@ -34,6 +34,11 @@ const MAX_CAPTURE_FILE_BYTES = 256 * 1024; const MAX_CAPTURE_TOTAL_BYTES = 1024 * 1024; const MAX_PROFILE_BYTES = 128 * 1024; const MAX_ROUTING_DESCRIPTION = 240; +// Matching material rather than reading material: none of this is ever shown to +// a session, so the limits are about keeping a bundle sane, not a prompt short. +const MAX_ROUTING_QUESTIONS = 20; +const MAX_ROUTING_ENTITIES = 40; +const MAX_ROUTING_TERM = 200; const UPDATE_LOCK_STALE_MS = 60_000; const SKIPPED_DIRECTORIES = new Set(["node_modules", ".git", ".svn", ".hg", "__pycache__"]); @@ -100,6 +105,11 @@ function recordFor(directory, parsed) { conversationKnowledgeFolder: knowledgeManaged ? null : path.join(directory, "knowledge"), routingDescription: typeof parsed.routingDescription === "string" ? parsed.routingDescription : "", + // Index-only matching material. It travels with the bundle so a teammate's + // copy is findable by the same words as yours, which is the whole reason it + // lives here rather than in this machine's routing cache. + routingQuestions: normalizeRoutingList(parsed.routingQuestions, MAX_ROUTING_QUESTIONS), + routingEntities: normalizeRoutingList(parsed.routingEntities, MAX_ROUTING_ENTITIES), // What this context expects to be able to reach. Read leniently: a // declaration this plugin cannot make sense of is dropped rather than // allowed to hide the profile and knowledge behind it. @@ -294,6 +304,35 @@ export async function createContext({ name, knowledgeFolder, profile, extensions }; } +// The questions a context should catch, and the names that appear in it. +// +// A description answers "what is this?", which is not how anyone searches. They +// search with the words of their problem, so these hold the other vocabulary: +// the phrasings a user would actually type, and the service names, ticket ids +// and error strings that appear in one context and nowhere else. Both are +// matched against and neither is ever displayed, which is what makes them cheap +// enough to keep in bulk. +// +// Optional, unlike the description. A bundle written before this existed, or by +// a host that does not generate them, is not broken — it just matches on less. +function normalizeRoutingList(value, limit) { + if (!Array.isArray(value)) { + return []; + } + const seen = new Set(); + const kept = []; + for (const entry of value) { + if (typeof entry !== "string") continue; + const clean = entry.trim().replace(/\s+/g, " ").slice(0, MAX_ROUTING_TERM); + const key = clean.toLowerCase(); + if (clean.length === 0 || seen.has(key)) continue; + seen.add(key); + kept.push(clean); + if (kept.length === limit) break; + } + return kept; +} + function normalizeRoutingDescription(value) { const description = (value ?? "").trim().replace(/\s+/g, " "); if (description.length === 0) { @@ -392,6 +431,8 @@ export async function createCapturedContext({ name, profile, routingDescription, + routingQuestions, + routingEntities, knowledge, extensions, capturedFrom = "conversation" @@ -399,6 +440,8 @@ export async function createCapturedContext({ const cleanName = normalizeName(name); const profileText = normalizeProfile(profile); const useWhen = normalizeRoutingDescription(routingDescription); + const questions = normalizeRoutingList(routingQuestions, MAX_ROUTING_QUESTIONS); + const entities = normalizeRoutingList(routingEntities, MAX_ROUTING_ENTITIES); const files = normalizeCaptureKnowledge(knowledge); const declarations = serializeExtensionDeclarations(extensions ?? []); await ensureUniqueName(cleanName); @@ -416,6 +459,10 @@ export async function createCapturedContext({ capturedFrom: isConversationCapture(capturedFrom) ? capturedFrom : "conversation", routingDescription: useWhen }; + // Absent rather than empty when there is nothing: a manifest should not carry + // a field that says only that a host did not fill it in. + if (questions.length > 0) record.routingQuestions = questions; + if (entities.length > 0) record.routingEntities = entities; if (declarations) record.extensions = declarations; record.updatedAt = record.createdAt; @@ -540,6 +587,8 @@ async function prepareCapturedContextUpdate({ name, profile, routingDescription, + routingQuestions, + routingEntities, knowledge, extensions }) { @@ -560,6 +609,17 @@ async function prepareCapturedContextUpdate({ requireCurrentBase(record, currentHash, baseHash); const profileText = normalizeProfile(profile); const useWhen = normalizeRoutingDescription(routingDescription); + // Same rule as extensions below: a save that says nothing about the matching + // material leaves what is there alone. A host that does not generate these + // must not silently strip what another host wrote. + const questions = + routingQuestions === undefined + ? record.routingQuestions + : normalizeRoutingList(routingQuestions, MAX_ROUTING_QUESTIONS); + const entities = + routingEntities === undefined + ? record.routingEntities + : normalizeRoutingList(routingEntities, MAX_ROUTING_ENTITIES); const files = normalizeCaptureKnowledge(knowledge); // A save that says nothing about extensions leaves them exactly as they are. // Declarations are usually added deliberately, by hand or by `extensions add`, @@ -569,13 +629,18 @@ async function prepareCapturedContextUpdate({ const currentKnowledge = await readGeneratedKnowledge(record); const changes = knowledgeChanges(currentKnowledge, files); const profileChanged = (await readProfileText(record)) !== profileText; - const routingChanged = record.routingDescription !== useWhen; + const routingChanged = + record.routingDescription !== useWhen || + JSON.stringify(record.routingQuestions) !== JSON.stringify(questions) || + JSON.stringify(record.routingEntities) !== JSON.stringify(entities); const extensionsChanged = JSON.stringify(record.extensions) !== JSON.stringify(declarations); return { record, profileText, useWhen, + questions, + entities, files, declarations, changes, @@ -681,6 +746,8 @@ export async function updateCapturedContext(capture) { revision: prepared.record.revision + 1, routingDescription: prepared.useWhen, extensions: serializeExtensionDeclarations(prepared.declarations), + routingQuestions: prepared.questions, + routingEntities: prepared.entities, updatedFrom: typeof capture.updatedFrom === "string" && capture.updatedFrom.trim().length > 0 ? capture.updatedFrom.trim() @@ -798,6 +865,10 @@ export async function importCapturedContext({ bundleFolder, name }) { name: typeof name === "string" && name.trim().length > 0 ? name : manifest.name, profile, routingDescription: manifest.routingDescription, + // The point of keeping these in the bundle: a teammate's copy is findable + // by the same words as the original, without them rediscovering any of it. + routingQuestions: manifest.routingQuestions, + routingEntities: manifest.routingEntities, knowledge, // What the bundle says it expects to reach, reduced to declarations. The // import creates no binding for any of them, so the imported context arrives diff --git a/plugins/claude-code/neatcontext/src/core/routing-candidates.mjs b/plugins/claude-code/neatcontext/src/core/routing-candidates.mjs index aab5808..ae3ad79 100644 --- a/plugins/claude-code/neatcontext/src/core/routing-candidates.mjs +++ b/plugins/claude-code/neatcontext/src/core/routing-candidates.mjs @@ -24,9 +24,10 @@ import { buildIndex, rank } from "./routing-search.mjs"; // string rather than kept as a list because the scorer counts words, and a // user who wrote two aliases meant both of them. // -// `entities` and `questions` are missing on purpose: nothing generates them -// yet. The scorer treats an absent field as absent rather than empty, so adding -// them later changes what matches without changing anything here. +// `questions` and `entities` come from the bundle rather than from this +// machine, which is what makes a context findable by the same words on every +// machine it reaches. They are matched against and never displayed, so their +// size costs nothing in the prompt. export function routingFields(context, card, files) { return { name: context.name ?? "", @@ -35,6 +36,8 @@ export function routingFields(context, card, files) { // that travelled with the bundle. description: card?.useWhen || context.routingDescription || "", aliases: (card?.aliases ?? []).join(" "), + questions: (context.routingQuestions ?? []).join(" "), + entities: (context.routingEntities ?? []).join(" "), files: files.join(" ") }; } diff --git a/plugins/copilot/neatcontext/src/core/context-store.mjs b/plugins/copilot/neatcontext/src/core/context-store.mjs index 665823b..9fb69f6 100644 --- a/plugins/copilot/neatcontext/src/core/context-store.mjs +++ b/plugins/copilot/neatcontext/src/core/context-store.mjs @@ -34,6 +34,11 @@ const MAX_CAPTURE_FILE_BYTES = 256 * 1024; const MAX_CAPTURE_TOTAL_BYTES = 1024 * 1024; const MAX_PROFILE_BYTES = 128 * 1024; const MAX_ROUTING_DESCRIPTION = 240; +// Matching material rather than reading material: none of this is ever shown to +// a session, so the limits are about keeping a bundle sane, not a prompt short. +const MAX_ROUTING_QUESTIONS = 20; +const MAX_ROUTING_ENTITIES = 40; +const MAX_ROUTING_TERM = 200; const UPDATE_LOCK_STALE_MS = 60_000; const SKIPPED_DIRECTORIES = new Set(["node_modules", ".git", ".svn", ".hg", "__pycache__"]); @@ -100,6 +105,11 @@ function recordFor(directory, parsed) { conversationKnowledgeFolder: knowledgeManaged ? null : path.join(directory, "knowledge"), routingDescription: typeof parsed.routingDescription === "string" ? parsed.routingDescription : "", + // Index-only matching material. It travels with the bundle so a teammate's + // copy is findable by the same words as yours, which is the whole reason it + // lives here rather than in this machine's routing cache. + routingQuestions: normalizeRoutingList(parsed.routingQuestions, MAX_ROUTING_QUESTIONS), + routingEntities: normalizeRoutingList(parsed.routingEntities, MAX_ROUTING_ENTITIES), // What this context expects to be able to reach. Read leniently: a // declaration this plugin cannot make sense of is dropped rather than // allowed to hide the profile and knowledge behind it. @@ -294,6 +304,35 @@ export async function createContext({ name, knowledgeFolder, profile, extensions }; } +// The questions a context should catch, and the names that appear in it. +// +// A description answers "what is this?", which is not how anyone searches. They +// search with the words of their problem, so these hold the other vocabulary: +// the phrasings a user would actually type, and the service names, ticket ids +// and error strings that appear in one context and nowhere else. Both are +// matched against and neither is ever displayed, which is what makes them cheap +// enough to keep in bulk. +// +// Optional, unlike the description. A bundle written before this existed, or by +// a host that does not generate them, is not broken — it just matches on less. +function normalizeRoutingList(value, limit) { + if (!Array.isArray(value)) { + return []; + } + const seen = new Set(); + const kept = []; + for (const entry of value) { + if (typeof entry !== "string") continue; + const clean = entry.trim().replace(/\s+/g, " ").slice(0, MAX_ROUTING_TERM); + const key = clean.toLowerCase(); + if (clean.length === 0 || seen.has(key)) continue; + seen.add(key); + kept.push(clean); + if (kept.length === limit) break; + } + return kept; +} + function normalizeRoutingDescription(value) { const description = (value ?? "").trim().replace(/\s+/g, " "); if (description.length === 0) { @@ -392,6 +431,8 @@ export async function createCapturedContext({ name, profile, routingDescription, + routingQuestions, + routingEntities, knowledge, extensions, capturedFrom = "conversation" @@ -399,6 +440,8 @@ export async function createCapturedContext({ const cleanName = normalizeName(name); const profileText = normalizeProfile(profile); const useWhen = normalizeRoutingDescription(routingDescription); + const questions = normalizeRoutingList(routingQuestions, MAX_ROUTING_QUESTIONS); + const entities = normalizeRoutingList(routingEntities, MAX_ROUTING_ENTITIES); const files = normalizeCaptureKnowledge(knowledge); const declarations = serializeExtensionDeclarations(extensions ?? []); await ensureUniqueName(cleanName); @@ -416,6 +459,10 @@ export async function createCapturedContext({ capturedFrom: isConversationCapture(capturedFrom) ? capturedFrom : "conversation", routingDescription: useWhen }; + // Absent rather than empty when there is nothing: a manifest should not carry + // a field that says only that a host did not fill it in. + if (questions.length > 0) record.routingQuestions = questions; + if (entities.length > 0) record.routingEntities = entities; if (declarations) record.extensions = declarations; record.updatedAt = record.createdAt; @@ -540,6 +587,8 @@ async function prepareCapturedContextUpdate({ name, profile, routingDescription, + routingQuestions, + routingEntities, knowledge, extensions }) { @@ -560,6 +609,17 @@ async function prepareCapturedContextUpdate({ requireCurrentBase(record, currentHash, baseHash); const profileText = normalizeProfile(profile); const useWhen = normalizeRoutingDescription(routingDescription); + // Same rule as extensions below: a save that says nothing about the matching + // material leaves what is there alone. A host that does not generate these + // must not silently strip what another host wrote. + const questions = + routingQuestions === undefined + ? record.routingQuestions + : normalizeRoutingList(routingQuestions, MAX_ROUTING_QUESTIONS); + const entities = + routingEntities === undefined + ? record.routingEntities + : normalizeRoutingList(routingEntities, MAX_ROUTING_ENTITIES); const files = normalizeCaptureKnowledge(knowledge); // A save that says nothing about extensions leaves them exactly as they are. // Declarations are usually added deliberately, by hand or by `extensions add`, @@ -569,13 +629,18 @@ async function prepareCapturedContextUpdate({ const currentKnowledge = await readGeneratedKnowledge(record); const changes = knowledgeChanges(currentKnowledge, files); const profileChanged = (await readProfileText(record)) !== profileText; - const routingChanged = record.routingDescription !== useWhen; + const routingChanged = + record.routingDescription !== useWhen || + JSON.stringify(record.routingQuestions) !== JSON.stringify(questions) || + JSON.stringify(record.routingEntities) !== JSON.stringify(entities); const extensionsChanged = JSON.stringify(record.extensions) !== JSON.stringify(declarations); return { record, profileText, useWhen, + questions, + entities, files, declarations, changes, @@ -681,6 +746,8 @@ export async function updateCapturedContext(capture) { revision: prepared.record.revision + 1, routingDescription: prepared.useWhen, extensions: serializeExtensionDeclarations(prepared.declarations), + routingQuestions: prepared.questions, + routingEntities: prepared.entities, updatedFrom: typeof capture.updatedFrom === "string" && capture.updatedFrom.trim().length > 0 ? capture.updatedFrom.trim() @@ -798,6 +865,10 @@ export async function importCapturedContext({ bundleFolder, name }) { name: typeof name === "string" && name.trim().length > 0 ? name : manifest.name, profile, routingDescription: manifest.routingDescription, + // The point of keeping these in the bundle: a teammate's copy is findable + // by the same words as the original, without them rediscovering any of it. + routingQuestions: manifest.routingQuestions, + routingEntities: manifest.routingEntities, knowledge, // What the bundle says it expects to reach, reduced to declarations. The // import creates no binding for any of them, so the imported context arrives diff --git a/plugins/copilot/neatcontext/src/core/routing-candidates.mjs b/plugins/copilot/neatcontext/src/core/routing-candidates.mjs index aab5808..ae3ad79 100644 --- a/plugins/copilot/neatcontext/src/core/routing-candidates.mjs +++ b/plugins/copilot/neatcontext/src/core/routing-candidates.mjs @@ -24,9 +24,10 @@ import { buildIndex, rank } from "./routing-search.mjs"; // string rather than kept as a list because the scorer counts words, and a // user who wrote two aliases meant both of them. // -// `entities` and `questions` are missing on purpose: nothing generates them -// yet. The scorer treats an absent field as absent rather than empty, so adding -// them later changes what matches without changing anything here. +// `questions` and `entities` come from the bundle rather than from this +// machine, which is what makes a context findable by the same words on every +// machine it reaches. They are matched against and never displayed, so their +// size costs nothing in the prompt. export function routingFields(context, card, files) { return { name: context.name ?? "", @@ -35,6 +36,8 @@ export function routingFields(context, card, files) { // that travelled with the bundle. description: card?.useWhen || context.routingDescription || "", aliases: (card?.aliases ?? []).join(" "), + questions: (context.routingQuestions ?? []).join(" "), + entities: (context.routingEntities ?? []).join(" "), files: files.join(" ") }; } diff --git a/plugins/kimi-code/neatcontext/src/core/context-store.mjs b/plugins/kimi-code/neatcontext/src/core/context-store.mjs index 665823b..9fb69f6 100644 --- a/plugins/kimi-code/neatcontext/src/core/context-store.mjs +++ b/plugins/kimi-code/neatcontext/src/core/context-store.mjs @@ -34,6 +34,11 @@ const MAX_CAPTURE_FILE_BYTES = 256 * 1024; const MAX_CAPTURE_TOTAL_BYTES = 1024 * 1024; const MAX_PROFILE_BYTES = 128 * 1024; const MAX_ROUTING_DESCRIPTION = 240; +// Matching material rather than reading material: none of this is ever shown to +// a session, so the limits are about keeping a bundle sane, not a prompt short. +const MAX_ROUTING_QUESTIONS = 20; +const MAX_ROUTING_ENTITIES = 40; +const MAX_ROUTING_TERM = 200; const UPDATE_LOCK_STALE_MS = 60_000; const SKIPPED_DIRECTORIES = new Set(["node_modules", ".git", ".svn", ".hg", "__pycache__"]); @@ -100,6 +105,11 @@ function recordFor(directory, parsed) { conversationKnowledgeFolder: knowledgeManaged ? null : path.join(directory, "knowledge"), routingDescription: typeof parsed.routingDescription === "string" ? parsed.routingDescription : "", + // Index-only matching material. It travels with the bundle so a teammate's + // copy is findable by the same words as yours, which is the whole reason it + // lives here rather than in this machine's routing cache. + routingQuestions: normalizeRoutingList(parsed.routingQuestions, MAX_ROUTING_QUESTIONS), + routingEntities: normalizeRoutingList(parsed.routingEntities, MAX_ROUTING_ENTITIES), // What this context expects to be able to reach. Read leniently: a // declaration this plugin cannot make sense of is dropped rather than // allowed to hide the profile and knowledge behind it. @@ -294,6 +304,35 @@ export async function createContext({ name, knowledgeFolder, profile, extensions }; } +// The questions a context should catch, and the names that appear in it. +// +// A description answers "what is this?", which is not how anyone searches. They +// search with the words of their problem, so these hold the other vocabulary: +// the phrasings a user would actually type, and the service names, ticket ids +// and error strings that appear in one context and nowhere else. Both are +// matched against and neither is ever displayed, which is what makes them cheap +// enough to keep in bulk. +// +// Optional, unlike the description. A bundle written before this existed, or by +// a host that does not generate them, is not broken — it just matches on less. +function normalizeRoutingList(value, limit) { + if (!Array.isArray(value)) { + return []; + } + const seen = new Set(); + const kept = []; + for (const entry of value) { + if (typeof entry !== "string") continue; + const clean = entry.trim().replace(/\s+/g, " ").slice(0, MAX_ROUTING_TERM); + const key = clean.toLowerCase(); + if (clean.length === 0 || seen.has(key)) continue; + seen.add(key); + kept.push(clean); + if (kept.length === limit) break; + } + return kept; +} + function normalizeRoutingDescription(value) { const description = (value ?? "").trim().replace(/\s+/g, " "); if (description.length === 0) { @@ -392,6 +431,8 @@ export async function createCapturedContext({ name, profile, routingDescription, + routingQuestions, + routingEntities, knowledge, extensions, capturedFrom = "conversation" @@ -399,6 +440,8 @@ export async function createCapturedContext({ const cleanName = normalizeName(name); const profileText = normalizeProfile(profile); const useWhen = normalizeRoutingDescription(routingDescription); + const questions = normalizeRoutingList(routingQuestions, MAX_ROUTING_QUESTIONS); + const entities = normalizeRoutingList(routingEntities, MAX_ROUTING_ENTITIES); const files = normalizeCaptureKnowledge(knowledge); const declarations = serializeExtensionDeclarations(extensions ?? []); await ensureUniqueName(cleanName); @@ -416,6 +459,10 @@ export async function createCapturedContext({ capturedFrom: isConversationCapture(capturedFrom) ? capturedFrom : "conversation", routingDescription: useWhen }; + // Absent rather than empty when there is nothing: a manifest should not carry + // a field that says only that a host did not fill it in. + if (questions.length > 0) record.routingQuestions = questions; + if (entities.length > 0) record.routingEntities = entities; if (declarations) record.extensions = declarations; record.updatedAt = record.createdAt; @@ -540,6 +587,8 @@ async function prepareCapturedContextUpdate({ name, profile, routingDescription, + routingQuestions, + routingEntities, knowledge, extensions }) { @@ -560,6 +609,17 @@ async function prepareCapturedContextUpdate({ requireCurrentBase(record, currentHash, baseHash); const profileText = normalizeProfile(profile); const useWhen = normalizeRoutingDescription(routingDescription); + // Same rule as extensions below: a save that says nothing about the matching + // material leaves what is there alone. A host that does not generate these + // must not silently strip what another host wrote. + const questions = + routingQuestions === undefined + ? record.routingQuestions + : normalizeRoutingList(routingQuestions, MAX_ROUTING_QUESTIONS); + const entities = + routingEntities === undefined + ? record.routingEntities + : normalizeRoutingList(routingEntities, MAX_ROUTING_ENTITIES); const files = normalizeCaptureKnowledge(knowledge); // A save that says nothing about extensions leaves them exactly as they are. // Declarations are usually added deliberately, by hand or by `extensions add`, @@ -569,13 +629,18 @@ async function prepareCapturedContextUpdate({ const currentKnowledge = await readGeneratedKnowledge(record); const changes = knowledgeChanges(currentKnowledge, files); const profileChanged = (await readProfileText(record)) !== profileText; - const routingChanged = record.routingDescription !== useWhen; + const routingChanged = + record.routingDescription !== useWhen || + JSON.stringify(record.routingQuestions) !== JSON.stringify(questions) || + JSON.stringify(record.routingEntities) !== JSON.stringify(entities); const extensionsChanged = JSON.stringify(record.extensions) !== JSON.stringify(declarations); return { record, profileText, useWhen, + questions, + entities, files, declarations, changes, @@ -681,6 +746,8 @@ export async function updateCapturedContext(capture) { revision: prepared.record.revision + 1, routingDescription: prepared.useWhen, extensions: serializeExtensionDeclarations(prepared.declarations), + routingQuestions: prepared.questions, + routingEntities: prepared.entities, updatedFrom: typeof capture.updatedFrom === "string" && capture.updatedFrom.trim().length > 0 ? capture.updatedFrom.trim() @@ -798,6 +865,10 @@ export async function importCapturedContext({ bundleFolder, name }) { name: typeof name === "string" && name.trim().length > 0 ? name : manifest.name, profile, routingDescription: manifest.routingDescription, + // The point of keeping these in the bundle: a teammate's copy is findable + // by the same words as the original, without them rediscovering any of it. + routingQuestions: manifest.routingQuestions, + routingEntities: manifest.routingEntities, knowledge, // What the bundle says it expects to reach, reduced to declarations. The // import creates no binding for any of them, so the imported context arrives diff --git a/plugins/kimi-code/neatcontext/src/core/routing-candidates.mjs b/plugins/kimi-code/neatcontext/src/core/routing-candidates.mjs index aab5808..ae3ad79 100644 --- a/plugins/kimi-code/neatcontext/src/core/routing-candidates.mjs +++ b/plugins/kimi-code/neatcontext/src/core/routing-candidates.mjs @@ -24,9 +24,10 @@ import { buildIndex, rank } from "./routing-search.mjs"; // string rather than kept as a list because the scorer counts words, and a // user who wrote two aliases meant both of them. // -// `entities` and `questions` are missing on purpose: nothing generates them -// yet. The scorer treats an absent field as absent rather than empty, so adding -// them later changes what matches without changing anything here. +// `questions` and `entities` come from the bundle rather than from this +// machine, which is what makes a context findable by the same words on every +// machine it reaches. They are matched against and never displayed, so their +// size costs nothing in the prompt. export function routingFields(context, card, files) { return { name: context.name ?? "", @@ -35,6 +36,8 @@ export function routingFields(context, card, files) { // that travelled with the bundle. description: card?.useWhen || context.routingDescription || "", aliases: (card?.aliases ?? []).join(" "), + questions: (context.routingQuestions ?? []).join(" "), + entities: (context.routingEntities ?? []).join(" "), files: files.join(" ") }; } diff --git a/plugins/pi/neatcontext/src/core/context-store.mjs b/plugins/pi/neatcontext/src/core/context-store.mjs index 665823b..9fb69f6 100644 --- a/plugins/pi/neatcontext/src/core/context-store.mjs +++ b/plugins/pi/neatcontext/src/core/context-store.mjs @@ -34,6 +34,11 @@ const MAX_CAPTURE_FILE_BYTES = 256 * 1024; const MAX_CAPTURE_TOTAL_BYTES = 1024 * 1024; const MAX_PROFILE_BYTES = 128 * 1024; const MAX_ROUTING_DESCRIPTION = 240; +// Matching material rather than reading material: none of this is ever shown to +// a session, so the limits are about keeping a bundle sane, not a prompt short. +const MAX_ROUTING_QUESTIONS = 20; +const MAX_ROUTING_ENTITIES = 40; +const MAX_ROUTING_TERM = 200; const UPDATE_LOCK_STALE_MS = 60_000; const SKIPPED_DIRECTORIES = new Set(["node_modules", ".git", ".svn", ".hg", "__pycache__"]); @@ -100,6 +105,11 @@ function recordFor(directory, parsed) { conversationKnowledgeFolder: knowledgeManaged ? null : path.join(directory, "knowledge"), routingDescription: typeof parsed.routingDescription === "string" ? parsed.routingDescription : "", + // Index-only matching material. It travels with the bundle so a teammate's + // copy is findable by the same words as yours, which is the whole reason it + // lives here rather than in this machine's routing cache. + routingQuestions: normalizeRoutingList(parsed.routingQuestions, MAX_ROUTING_QUESTIONS), + routingEntities: normalizeRoutingList(parsed.routingEntities, MAX_ROUTING_ENTITIES), // What this context expects to be able to reach. Read leniently: a // declaration this plugin cannot make sense of is dropped rather than // allowed to hide the profile and knowledge behind it. @@ -294,6 +304,35 @@ export async function createContext({ name, knowledgeFolder, profile, extensions }; } +// The questions a context should catch, and the names that appear in it. +// +// A description answers "what is this?", which is not how anyone searches. They +// search with the words of their problem, so these hold the other vocabulary: +// the phrasings a user would actually type, and the service names, ticket ids +// and error strings that appear in one context and nowhere else. Both are +// matched against and neither is ever displayed, which is what makes them cheap +// enough to keep in bulk. +// +// Optional, unlike the description. A bundle written before this existed, or by +// a host that does not generate them, is not broken — it just matches on less. +function normalizeRoutingList(value, limit) { + if (!Array.isArray(value)) { + return []; + } + const seen = new Set(); + const kept = []; + for (const entry of value) { + if (typeof entry !== "string") continue; + const clean = entry.trim().replace(/\s+/g, " ").slice(0, MAX_ROUTING_TERM); + const key = clean.toLowerCase(); + if (clean.length === 0 || seen.has(key)) continue; + seen.add(key); + kept.push(clean); + if (kept.length === limit) break; + } + return kept; +} + function normalizeRoutingDescription(value) { const description = (value ?? "").trim().replace(/\s+/g, " "); if (description.length === 0) { @@ -392,6 +431,8 @@ export async function createCapturedContext({ name, profile, routingDescription, + routingQuestions, + routingEntities, knowledge, extensions, capturedFrom = "conversation" @@ -399,6 +440,8 @@ export async function createCapturedContext({ const cleanName = normalizeName(name); const profileText = normalizeProfile(profile); const useWhen = normalizeRoutingDescription(routingDescription); + const questions = normalizeRoutingList(routingQuestions, MAX_ROUTING_QUESTIONS); + const entities = normalizeRoutingList(routingEntities, MAX_ROUTING_ENTITIES); const files = normalizeCaptureKnowledge(knowledge); const declarations = serializeExtensionDeclarations(extensions ?? []); await ensureUniqueName(cleanName); @@ -416,6 +459,10 @@ export async function createCapturedContext({ capturedFrom: isConversationCapture(capturedFrom) ? capturedFrom : "conversation", routingDescription: useWhen }; + // Absent rather than empty when there is nothing: a manifest should not carry + // a field that says only that a host did not fill it in. + if (questions.length > 0) record.routingQuestions = questions; + if (entities.length > 0) record.routingEntities = entities; if (declarations) record.extensions = declarations; record.updatedAt = record.createdAt; @@ -540,6 +587,8 @@ async function prepareCapturedContextUpdate({ name, profile, routingDescription, + routingQuestions, + routingEntities, knowledge, extensions }) { @@ -560,6 +609,17 @@ async function prepareCapturedContextUpdate({ requireCurrentBase(record, currentHash, baseHash); const profileText = normalizeProfile(profile); const useWhen = normalizeRoutingDescription(routingDescription); + // Same rule as extensions below: a save that says nothing about the matching + // material leaves what is there alone. A host that does not generate these + // must not silently strip what another host wrote. + const questions = + routingQuestions === undefined + ? record.routingQuestions + : normalizeRoutingList(routingQuestions, MAX_ROUTING_QUESTIONS); + const entities = + routingEntities === undefined + ? record.routingEntities + : normalizeRoutingList(routingEntities, MAX_ROUTING_ENTITIES); const files = normalizeCaptureKnowledge(knowledge); // A save that says nothing about extensions leaves them exactly as they are. // Declarations are usually added deliberately, by hand or by `extensions add`, @@ -569,13 +629,18 @@ async function prepareCapturedContextUpdate({ const currentKnowledge = await readGeneratedKnowledge(record); const changes = knowledgeChanges(currentKnowledge, files); const profileChanged = (await readProfileText(record)) !== profileText; - const routingChanged = record.routingDescription !== useWhen; + const routingChanged = + record.routingDescription !== useWhen || + JSON.stringify(record.routingQuestions) !== JSON.stringify(questions) || + JSON.stringify(record.routingEntities) !== JSON.stringify(entities); const extensionsChanged = JSON.stringify(record.extensions) !== JSON.stringify(declarations); return { record, profileText, useWhen, + questions, + entities, files, declarations, changes, @@ -681,6 +746,8 @@ export async function updateCapturedContext(capture) { revision: prepared.record.revision + 1, routingDescription: prepared.useWhen, extensions: serializeExtensionDeclarations(prepared.declarations), + routingQuestions: prepared.questions, + routingEntities: prepared.entities, updatedFrom: typeof capture.updatedFrom === "string" && capture.updatedFrom.trim().length > 0 ? capture.updatedFrom.trim() @@ -798,6 +865,10 @@ export async function importCapturedContext({ bundleFolder, name }) { name: typeof name === "string" && name.trim().length > 0 ? name : manifest.name, profile, routingDescription: manifest.routingDescription, + // The point of keeping these in the bundle: a teammate's copy is findable + // by the same words as the original, without them rediscovering any of it. + routingQuestions: manifest.routingQuestions, + routingEntities: manifest.routingEntities, knowledge, // What the bundle says it expects to reach, reduced to declarations. The // import creates no binding for any of them, so the imported context arrives diff --git a/plugins/pi/neatcontext/src/core/routing-candidates.mjs b/plugins/pi/neatcontext/src/core/routing-candidates.mjs index aab5808..ae3ad79 100644 --- a/plugins/pi/neatcontext/src/core/routing-candidates.mjs +++ b/plugins/pi/neatcontext/src/core/routing-candidates.mjs @@ -24,9 +24,10 @@ import { buildIndex, rank } from "./routing-search.mjs"; // string rather than kept as a list because the scorer counts words, and a // user who wrote two aliases meant both of them. // -// `entities` and `questions` are missing on purpose: nothing generates them -// yet. The scorer treats an absent field as absent rather than empty, so adding -// them later changes what matches without changing anything here. +// `questions` and `entities` come from the bundle rather than from this +// machine, which is what makes a context findable by the same words on every +// machine it reaches. They are matched against and never displayed, so their +// size costs nothing in the prompt. export function routingFields(context, card, files) { return { name: context.name ?? "", @@ -35,6 +36,8 @@ export function routingFields(context, card, files) { // that travelled with the bundle. description: card?.useWhen || context.routingDescription || "", aliases: (card?.aliases ?? []).join(" "), + questions: (context.routingQuestions ?? []).join(" "), + entities: (context.routingEntities ?? []).join(" "), files: files.join(" ") }; } diff --git a/shared/core/context-store.mjs b/shared/core/context-store.mjs index 665823b..9fb69f6 100644 --- a/shared/core/context-store.mjs +++ b/shared/core/context-store.mjs @@ -34,6 +34,11 @@ const MAX_CAPTURE_FILE_BYTES = 256 * 1024; const MAX_CAPTURE_TOTAL_BYTES = 1024 * 1024; const MAX_PROFILE_BYTES = 128 * 1024; const MAX_ROUTING_DESCRIPTION = 240; +// Matching material rather than reading material: none of this is ever shown to +// a session, so the limits are about keeping a bundle sane, not a prompt short. +const MAX_ROUTING_QUESTIONS = 20; +const MAX_ROUTING_ENTITIES = 40; +const MAX_ROUTING_TERM = 200; const UPDATE_LOCK_STALE_MS = 60_000; const SKIPPED_DIRECTORIES = new Set(["node_modules", ".git", ".svn", ".hg", "__pycache__"]); @@ -100,6 +105,11 @@ function recordFor(directory, parsed) { conversationKnowledgeFolder: knowledgeManaged ? null : path.join(directory, "knowledge"), routingDescription: typeof parsed.routingDescription === "string" ? parsed.routingDescription : "", + // Index-only matching material. It travels with the bundle so a teammate's + // copy is findable by the same words as yours, which is the whole reason it + // lives here rather than in this machine's routing cache. + routingQuestions: normalizeRoutingList(parsed.routingQuestions, MAX_ROUTING_QUESTIONS), + routingEntities: normalizeRoutingList(parsed.routingEntities, MAX_ROUTING_ENTITIES), // What this context expects to be able to reach. Read leniently: a // declaration this plugin cannot make sense of is dropped rather than // allowed to hide the profile and knowledge behind it. @@ -294,6 +304,35 @@ export async function createContext({ name, knowledgeFolder, profile, extensions }; } +// The questions a context should catch, and the names that appear in it. +// +// A description answers "what is this?", which is not how anyone searches. They +// search with the words of their problem, so these hold the other vocabulary: +// the phrasings a user would actually type, and the service names, ticket ids +// and error strings that appear in one context and nowhere else. Both are +// matched against and neither is ever displayed, which is what makes them cheap +// enough to keep in bulk. +// +// Optional, unlike the description. A bundle written before this existed, or by +// a host that does not generate them, is not broken — it just matches on less. +function normalizeRoutingList(value, limit) { + if (!Array.isArray(value)) { + return []; + } + const seen = new Set(); + const kept = []; + for (const entry of value) { + if (typeof entry !== "string") continue; + const clean = entry.trim().replace(/\s+/g, " ").slice(0, MAX_ROUTING_TERM); + const key = clean.toLowerCase(); + if (clean.length === 0 || seen.has(key)) continue; + seen.add(key); + kept.push(clean); + if (kept.length === limit) break; + } + return kept; +} + function normalizeRoutingDescription(value) { const description = (value ?? "").trim().replace(/\s+/g, " "); if (description.length === 0) { @@ -392,6 +431,8 @@ export async function createCapturedContext({ name, profile, routingDescription, + routingQuestions, + routingEntities, knowledge, extensions, capturedFrom = "conversation" @@ -399,6 +440,8 @@ export async function createCapturedContext({ const cleanName = normalizeName(name); const profileText = normalizeProfile(profile); const useWhen = normalizeRoutingDescription(routingDescription); + const questions = normalizeRoutingList(routingQuestions, MAX_ROUTING_QUESTIONS); + const entities = normalizeRoutingList(routingEntities, MAX_ROUTING_ENTITIES); const files = normalizeCaptureKnowledge(knowledge); const declarations = serializeExtensionDeclarations(extensions ?? []); await ensureUniqueName(cleanName); @@ -416,6 +459,10 @@ export async function createCapturedContext({ capturedFrom: isConversationCapture(capturedFrom) ? capturedFrom : "conversation", routingDescription: useWhen }; + // Absent rather than empty when there is nothing: a manifest should not carry + // a field that says only that a host did not fill it in. + if (questions.length > 0) record.routingQuestions = questions; + if (entities.length > 0) record.routingEntities = entities; if (declarations) record.extensions = declarations; record.updatedAt = record.createdAt; @@ -540,6 +587,8 @@ async function prepareCapturedContextUpdate({ name, profile, routingDescription, + routingQuestions, + routingEntities, knowledge, extensions }) { @@ -560,6 +609,17 @@ async function prepareCapturedContextUpdate({ requireCurrentBase(record, currentHash, baseHash); const profileText = normalizeProfile(profile); const useWhen = normalizeRoutingDescription(routingDescription); + // Same rule as extensions below: a save that says nothing about the matching + // material leaves what is there alone. A host that does not generate these + // must not silently strip what another host wrote. + const questions = + routingQuestions === undefined + ? record.routingQuestions + : normalizeRoutingList(routingQuestions, MAX_ROUTING_QUESTIONS); + const entities = + routingEntities === undefined + ? record.routingEntities + : normalizeRoutingList(routingEntities, MAX_ROUTING_ENTITIES); const files = normalizeCaptureKnowledge(knowledge); // A save that says nothing about extensions leaves them exactly as they are. // Declarations are usually added deliberately, by hand or by `extensions add`, @@ -569,13 +629,18 @@ async function prepareCapturedContextUpdate({ const currentKnowledge = await readGeneratedKnowledge(record); const changes = knowledgeChanges(currentKnowledge, files); const profileChanged = (await readProfileText(record)) !== profileText; - const routingChanged = record.routingDescription !== useWhen; + const routingChanged = + record.routingDescription !== useWhen || + JSON.stringify(record.routingQuestions) !== JSON.stringify(questions) || + JSON.stringify(record.routingEntities) !== JSON.stringify(entities); const extensionsChanged = JSON.stringify(record.extensions) !== JSON.stringify(declarations); return { record, profileText, useWhen, + questions, + entities, files, declarations, changes, @@ -681,6 +746,8 @@ export async function updateCapturedContext(capture) { revision: prepared.record.revision + 1, routingDescription: prepared.useWhen, extensions: serializeExtensionDeclarations(prepared.declarations), + routingQuestions: prepared.questions, + routingEntities: prepared.entities, updatedFrom: typeof capture.updatedFrom === "string" && capture.updatedFrom.trim().length > 0 ? capture.updatedFrom.trim() @@ -798,6 +865,10 @@ export async function importCapturedContext({ bundleFolder, name }) { name: typeof name === "string" && name.trim().length > 0 ? name : manifest.name, profile, routingDescription: manifest.routingDescription, + // The point of keeping these in the bundle: a teammate's copy is findable + // by the same words as the original, without them rediscovering any of it. + routingQuestions: manifest.routingQuestions, + routingEntities: manifest.routingEntities, knowledge, // What the bundle says it expects to reach, reduced to declarations. The // import creates no binding for any of them, so the imported context arrives diff --git a/shared/core/routing-candidates.mjs b/shared/core/routing-candidates.mjs index aab5808..ae3ad79 100644 --- a/shared/core/routing-candidates.mjs +++ b/shared/core/routing-candidates.mjs @@ -24,9 +24,10 @@ import { buildIndex, rank } from "./routing-search.mjs"; // string rather than kept as a list because the scorer counts words, and a // user who wrote two aliases meant both of them. // -// `entities` and `questions` are missing on purpose: nothing generates them -// yet. The scorer treats an absent field as absent rather than empty, so adding -// them later changes what matches without changing anything here. +// `questions` and `entities` come from the bundle rather than from this +// machine, which is what makes a context findable by the same words on every +// machine it reaches. They are matched against and never displayed, so their +// size costs nothing in the prompt. export function routingFields(context, card, files) { return { name: context.name ?? "", @@ -35,6 +36,8 @@ export function routingFields(context, card, files) { // that travelled with the bundle. description: card?.useWhen || context.routingDescription || "", aliases: (card?.aliases ?? []).join(" "), + questions: (context.routingQuestions ?? []).join(" "), + entities: (context.routingEntities ?? []).join(" "), files: files.join(" ") }; } diff --git a/tests/routing-candidates.test.mjs b/tests/routing-candidates.test.mjs index ab04cb1..c8f69a9 100644 --- a/tests/routing-candidates.test.mjs +++ b/tests/routing-candidates.test.mjs @@ -93,7 +93,14 @@ describe("routingFields", () => { it("survives a context with nothing but a name", () => { const fields = routingFields({ id: "bare" }, undefined, []); - assert.deepEqual(fields, { name: "", description: "", aliases: "", files: "" }); + assert.deepEqual(fields, { + name: "", + description: "", + aliases: "", + questions: "", + entities: "", + files: "" + }); }); }); diff --git a/tests/routing-expansion.test.mjs b/tests/routing-expansion.test.mjs new file mode 100644 index 0000000..97ef6df --- /dev/null +++ b/tests/routing-expansion.test.mjs @@ -0,0 +1,210 @@ +// The questions a context should catch, and the names inside it. +// +// This is the recall half of routing. A description answers "what is this?", +// which is not how anyone searches — they search with the words of their +// problem. These two lists hold that other vocabulary, are matched against, and +// are never shown to anyone. +// +// They live in the bundle rather than in this machine's routing cache on +// purpose: a context handed to a teammate has to be findable by the same words +// on their machine as on yours, without them rediscovering any of it. + +import assert from "node:assert/strict"; +import { mkdir, mkdtemp, readFile, rm } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { after, before, beforeEach, describe, it } from "node:test"; + +let home; + +before(async () => { + home = await mkdtemp(path.join(os.tmpdir(), "neatcontext-expansion-test-")); + await mkdir(path.join(home, "docs"), { recursive: true }); + process.env.NEATCONTEXT_HOME = home; +}); + +after(async () => { + await rm(home, { recursive: true, force: true }); +}); + +beforeEach(async () => { + await rm(path.join(home, "contexts"), { recursive: true, force: true }); + await rm(path.join(home, "plugin-routing.json"), { force: true }); +}); + +const store = await import("../plugins/claude-code/neatcontext/src/core/context-store.mjs"); +const { createRoutingIndex } = await import( + "../plugins/claude-code/neatcontext/src/core/routing-candidates.mjs" +); + +const INCIDENT = { + name: "Pool limits June", + profile: "# Pool limits June\n\n## Purpose\nThe incident.\n", + routingDescription: "billing-postgres default_pool_size regression after the June deploy", + routingQuestions: [ + "why was checkout throwing 5xx last week", + "did we ever fix that database timeout thing", + "what happened on the thirtieth" + ], + routingEntities: ["INC-1001", "checkout-api", "pgbouncer", "dep-9001"], + knowledge: [{ path: "session-summary.md", content: "# Summary\n\nPool exhaustion.\n" }] +}; + +async function manifestOf(record) { + return JSON.parse(await readFile(path.join(record.directory, "context.json"), "utf8")); +} + +describe("storing the matching material", () => { + it("keeps questions and entities in the bundle", async () => { + const { record } = await store.createCapturedContext(INCIDENT); + const manifest = await manifestOf(record); + assert.deepEqual(manifest.routingQuestions, INCIDENT.routingQuestions); + assert.deepEqual(manifest.routingEntities, INCIDENT.routingEntities); + assert.deepEqual(record.routingQuestions, INCIDENT.routingQuestions); + }); + + it("leaves the fields off entirely when there are none", async () => { + // A manifest should not carry a field that says only that a host did not + // fill it in. + const { record } = await store.createCapturedContext({ + ...INCIDENT, + routingQuestions: undefined, + routingEntities: undefined + }); + const manifest = await manifestOf(record); + assert.equal("routingQuestions" in manifest, false); + assert.equal("routingEntities" in manifest, false); + assert.deepEqual(record.routingQuestions, []); + }); + + it("tidies what it is given", async () => { + const { record } = await store.createCapturedContext({ + ...INCIDENT, + routingQuestions: [" spaced out ", "", "Spaced Out", 42, "kept"], + routingEntities: ["INC-1001", "inc-1001", " "] + }); + // Trimmed, de-duplicated case-insensitively, non-strings dropped. + assert.deepEqual(record.routingQuestions, ["spaced out", "kept"]); + assert.deepEqual(record.routingEntities, ["INC-1001"]); + }); + + it("ignores a field that is not a list at all", async () => { + const { record } = await store.createCapturedContext({ + ...INCIDENT, + routingQuestions: "not a list" + }); + assert.deepEqual(record.routingQuestions, []); + }); + + it("caps how much it will keep", async () => { + const many = Array.from({ length: 60 }, (_, index) => `question number ${index}`); + const { record } = await store.createCapturedContext({ ...INCIDENT, routingQuestions: many }); + assert.equal(record.routingQuestions.length, 20); + }); +}); + +describe("updating a context", () => { + it("replaces the material when the save supplies it", async () => { + const { record } = await store.createCapturedContext(INCIDENT); + const baseHash = await store.fingerprintContext(record); + const updated = await store.updateCapturedContext({ + targetId: record.id, + baseHash, + name: record.name, + profile: "# Pool limits June\n\n## Purpose\nThe incident, revisited.\n", + routingDescription: INCIDENT.routingDescription, + routingQuestions: ["one new question"], + routingEntities: ["INC-2002"], + knowledge: INCIDENT.knowledge + }); + assert.deepEqual(updated.record.routingQuestions, ["one new question"]); + assert.deepEqual(updated.record.routingEntities, ["INC-2002"]); + }); + + it("leaves the material alone when the save says nothing about it", async () => { + // A host that does not generate these must not silently strip what another + // host wrote — the same rule extensions already follow. + const { record } = await store.createCapturedContext(INCIDENT); + const baseHash = await store.fingerprintContext(record); + const updated = await store.updateCapturedContext({ + targetId: record.id, + baseHash, + name: record.name, + profile: "# Pool limits June\n\n## Purpose\nUntouched material.\n", + routingDescription: INCIDENT.routingDescription, + knowledge: INCIDENT.knowledge + }); + assert.deepEqual(updated.record.routingQuestions, INCIDENT.routingQuestions); + assert.deepEqual(updated.record.routingEntities, INCIDENT.routingEntities); + }); + + it("counts a change to the material as a routing change", async () => { + const { record } = await store.createCapturedContext(INCIDENT); + const baseHash = await store.fingerprintContext(record); + const preview = await store.previewCapturedContextUpdate({ + targetId: record.id, + baseHash, + name: record.name, + profile: await readFile(record.profilePath, "utf8"), + routingDescription: INCIDENT.routingDescription, + routingQuestions: ["a question nobody asked before"], + knowledge: INCIDENT.knowledge + }); + assert.equal(preview.routingChanged, true); + assert.equal(preview.changed, true); + }); +}); + +describe("what it buys", () => { + // The point of the whole feature: a question phrased in the user's words, + // sharing nothing with the description, still finds the context. + const OTHERS = [ + ["Queue lag", "order-events partition lag and consumer rebalancing"], + ["Codex design", "Codex CLI plugin design and marketplace packaging"], + ["Kimi plugin", "Kimi Code manifests, skills and commands"], + ["Evidence", "conversation evidence and transcript adapters"], + ["Refunds", "refunds and chargebacks"], + ["Docker container", "Ubuntu container with SSH"], + ["Marketplace config", "switching the marketplace source"], + ["Session drift", "bridge session and thread drift"] + ]; + + async function seed(incident) { + await store.createCapturedContext(incident); + for (const [name, routingDescription] of OTHERS) { + await store.createCapturedContext({ + name, + profile: `# ${name}\n\n## Purpose\n${routingDescription}\n`, + routingDescription, + knowledge: [{ path: "session-summary.md", content: `# ${name}\n` }] + }); + } + const contexts = await store.listContexts(); + const candidates = createRoutingIndex({ listFiles: () => Promise.resolve([]) }); + return (query) => candidates(contexts, { cards: {} }, query, { limit: 3 }); + } + + it("finds the context from words its description never uses", async () => { + const rank = await seed(INCIDENT); + const [top] = await rank("why was checkout throwing 5xx last week"); + assert.equal(top.name, "Pool limits June"); + }); + + it("finds it from a ticket id that appears nowhere else", async () => { + const rank = await seed(INCIDENT); + const [top] = await rank("anything on dep-9001?"); + assert.equal(top.name, "Pool limits June"); + }); + + it("does not find it without the material", async () => { + // The same corpus and the same question, minus the stored questions: this + // is the recall failure the feature exists to fix. + const rank = await seed({ + ...INCIDENT, + routingQuestions: undefined, + routingEntities: undefined + }); + const results = await rank("why was checkout throwing 5xx last week"); + assert.ok(!results.some((result) => result.name === "Pool limits June")); + }); +});