diff --git a/codex-marketplace/plugins/neatcontext/src/core/context-store.mjs b/codex-marketplace/plugins/neatcontext/src/core/context-store.mjs index 665823b..9fb69f6 100644 --- a/codex-marketplace/plugins/neatcontext/src/core/context-store.mjs +++ b/codex-marketplace/plugins/neatcontext/src/core/context-store.mjs @@ -34,6 +34,11 @@ const MAX_CAPTURE_FILE_BYTES = 256 * 1024; const MAX_CAPTURE_TOTAL_BYTES = 1024 * 1024; const MAX_PROFILE_BYTES = 128 * 1024; const MAX_ROUTING_DESCRIPTION = 240; +// Matching material rather than reading material: none of this is ever shown to +// a session, so the limits are about keeping a bundle sane, not a prompt short. +const MAX_ROUTING_QUESTIONS = 20; +const MAX_ROUTING_ENTITIES = 40; +const MAX_ROUTING_TERM = 200; const UPDATE_LOCK_STALE_MS = 60_000; const SKIPPED_DIRECTORIES = new Set(["node_modules", ".git", ".svn", ".hg", "__pycache__"]); @@ -100,6 +105,11 @@ function recordFor(directory, parsed) { conversationKnowledgeFolder: knowledgeManaged ? null : path.join(directory, "knowledge"), routingDescription: typeof parsed.routingDescription === "string" ? parsed.routingDescription : "", + // Index-only matching material. It travels with the bundle so a teammate's + // copy is findable by the same words as yours, which is the whole reason it + // lives here rather than in this machine's routing cache. + routingQuestions: normalizeRoutingList(parsed.routingQuestions, MAX_ROUTING_QUESTIONS), + routingEntities: normalizeRoutingList(parsed.routingEntities, MAX_ROUTING_ENTITIES), // What this context expects to be able to reach. Read leniently: a // declaration this plugin cannot make sense of is dropped rather than // allowed to hide the profile and knowledge behind it. @@ -294,6 +304,35 @@ export async function createContext({ name, knowledgeFolder, profile, extensions }; } +// The questions a context should catch, and the names that appear in it. +// +// A description answers "what is this?", which is not how anyone searches. They +// search with the words of their problem, so these hold the other vocabulary: +// the phrasings a user would actually type, and the service names, ticket ids +// and error strings that appear in one context and nowhere else. Both are +// matched against and neither is ever displayed, which is what makes them cheap +// enough to keep in bulk. +// +// Optional, unlike the description. A bundle written before this existed, or by +// a host that does not generate them, is not broken — it just matches on less. +function normalizeRoutingList(value, limit) { + if (!Array.isArray(value)) { + return []; + } + const seen = new Set(); + const kept = []; + for (const entry of value) { + if (typeof entry !== "string") continue; + const clean = entry.trim().replace(/\s+/g, " ").slice(0, MAX_ROUTING_TERM); + const key = clean.toLowerCase(); + if (clean.length === 0 || seen.has(key)) continue; + seen.add(key); + kept.push(clean); + if (kept.length === limit) break; + } + return kept; +} + function normalizeRoutingDescription(value) { const description = (value ?? "").trim().replace(/\s+/g, " "); if (description.length === 0) { @@ -392,6 +431,8 @@ export async function createCapturedContext({ name, profile, routingDescription, + routingQuestions, + routingEntities, knowledge, extensions, capturedFrom = "conversation" @@ -399,6 +440,8 @@ export async function createCapturedContext({ const cleanName = normalizeName(name); const profileText = normalizeProfile(profile); const useWhen = normalizeRoutingDescription(routingDescription); + const questions = normalizeRoutingList(routingQuestions, MAX_ROUTING_QUESTIONS); + const entities = normalizeRoutingList(routingEntities, MAX_ROUTING_ENTITIES); const files = normalizeCaptureKnowledge(knowledge); const declarations = serializeExtensionDeclarations(extensions ?? []); await ensureUniqueName(cleanName); @@ -416,6 +459,10 @@ export async function createCapturedContext({ capturedFrom: isConversationCapture(capturedFrom) ? capturedFrom : "conversation", routingDescription: useWhen }; + // Absent rather than empty when there is nothing: a manifest should not carry + // a field that says only that a host did not fill it in. + if (questions.length > 0) record.routingQuestions = questions; + if (entities.length > 0) record.routingEntities = entities; if (declarations) record.extensions = declarations; record.updatedAt = record.createdAt; @@ -540,6 +587,8 @@ async function prepareCapturedContextUpdate({ name, profile, routingDescription, + routingQuestions, + routingEntities, knowledge, extensions }) { @@ -560,6 +609,17 @@ async function prepareCapturedContextUpdate({ requireCurrentBase(record, currentHash, baseHash); const profileText = normalizeProfile(profile); const useWhen = normalizeRoutingDescription(routingDescription); + // Same rule as extensions below: a save that says nothing about the matching + // material leaves what is there alone. A host that does not generate these + // must not silently strip what another host wrote. + const questions = + routingQuestions === undefined + ? record.routingQuestions + : normalizeRoutingList(routingQuestions, MAX_ROUTING_QUESTIONS); + const entities = + routingEntities === undefined + ? record.routingEntities + : normalizeRoutingList(routingEntities, MAX_ROUTING_ENTITIES); const files = normalizeCaptureKnowledge(knowledge); // A save that says nothing about extensions leaves them exactly as they are. // Declarations are usually added deliberately, by hand or by `extensions add`, @@ -569,13 +629,18 @@ async function prepareCapturedContextUpdate({ const currentKnowledge = await readGeneratedKnowledge(record); const changes = knowledgeChanges(currentKnowledge, files); const profileChanged = (await readProfileText(record)) !== profileText; - const routingChanged = record.routingDescription !== useWhen; + const routingChanged = + record.routingDescription !== useWhen || + JSON.stringify(record.routingQuestions) !== JSON.stringify(questions) || + JSON.stringify(record.routingEntities) !== JSON.stringify(entities); const extensionsChanged = JSON.stringify(record.extensions) !== JSON.stringify(declarations); return { record, profileText, useWhen, + questions, + entities, files, declarations, changes, @@ -681,6 +746,8 @@ export async function updateCapturedContext(capture) { revision: prepared.record.revision + 1, routingDescription: prepared.useWhen, extensions: serializeExtensionDeclarations(prepared.declarations), + routingQuestions: prepared.questions, + routingEntities: prepared.entities, updatedFrom: typeof capture.updatedFrom === "string" && capture.updatedFrom.trim().length > 0 ? capture.updatedFrom.trim() @@ -798,6 +865,10 @@ export async function importCapturedContext({ bundleFolder, name }) { name: typeof name === "string" && name.trim().length > 0 ? name : manifest.name, profile, routingDescription: manifest.routingDescription, + // The point of keeping these in the bundle: a teammate's copy is findable + // by the same words as the original, without them rediscovering any of it. + routingQuestions: manifest.routingQuestions, + routingEntities: manifest.routingEntities, knowledge, // What the bundle says it expects to reach, reduced to declarations. The // import creates no binding for any of them, so the imported context arrives diff --git a/codex-marketplace/plugins/neatcontext/src/core/routing-candidates.mjs b/codex-marketplace/plugins/neatcontext/src/core/routing-candidates.mjs index aab5808..ae3ad79 100644 --- a/codex-marketplace/plugins/neatcontext/src/core/routing-candidates.mjs +++ b/codex-marketplace/plugins/neatcontext/src/core/routing-candidates.mjs @@ -24,9 +24,10 @@ import { buildIndex, rank } from "./routing-search.mjs"; // string rather than kept as a list because the scorer counts words, and a // user who wrote two aliases meant both of them. // -// `entities` and `questions` are missing on purpose: nothing generates them -// yet. The scorer treats an absent field as absent rather than empty, so adding -// them later changes what matches without changing anything here. +// `questions` and `entities` come from the bundle rather than from this +// machine, which is what makes a context findable by the same words on every +// machine it reaches. They are matched against and never displayed, so their +// size costs nothing in the prompt. export function routingFields(context, card, files) { return { name: context.name ?? "", @@ -35,6 +36,8 @@ export function routingFields(context, card, files) { // that travelled with the bundle. description: card?.useWhen || context.routingDescription || "", aliases: (card?.aliases ?? []).join(" "), + questions: (context.routingQuestions ?? []).join(" "), + entities: (context.routingEntities ?? []).join(" "), files: files.join(" ") }; } diff --git a/plugins/claude-code/neatcontext/src/core/context-store.mjs b/plugins/claude-code/neatcontext/src/core/context-store.mjs index 665823b..9fb69f6 100644 --- a/plugins/claude-code/neatcontext/src/core/context-store.mjs +++ b/plugins/claude-code/neatcontext/src/core/context-store.mjs @@ -34,6 +34,11 @@ const MAX_CAPTURE_FILE_BYTES = 256 * 1024; const MAX_CAPTURE_TOTAL_BYTES = 1024 * 1024; const MAX_PROFILE_BYTES = 128 * 1024; const MAX_ROUTING_DESCRIPTION = 240; +// Matching material rather than reading material: none of this is ever shown to +// a session, so the limits are about keeping a bundle sane, not a prompt short. +const MAX_ROUTING_QUESTIONS = 20; +const MAX_ROUTING_ENTITIES = 40; +const MAX_ROUTING_TERM = 200; const UPDATE_LOCK_STALE_MS = 60_000; const SKIPPED_DIRECTORIES = new Set(["node_modules", ".git", ".svn", ".hg", "__pycache__"]); @@ -100,6 +105,11 @@ function recordFor(directory, parsed) { conversationKnowledgeFolder: knowledgeManaged ? null : path.join(directory, "knowledge"), routingDescription: typeof parsed.routingDescription === "string" ? parsed.routingDescription : "", + // Index-only matching material. It travels with the bundle so a teammate's + // copy is findable by the same words as yours, which is the whole reason it + // lives here rather than in this machine's routing cache. + routingQuestions: normalizeRoutingList(parsed.routingQuestions, MAX_ROUTING_QUESTIONS), + routingEntities: normalizeRoutingList(parsed.routingEntities, MAX_ROUTING_ENTITIES), // What this context expects to be able to reach. Read leniently: a // declaration this plugin cannot make sense of is dropped rather than // allowed to hide the profile and knowledge behind it. @@ -294,6 +304,35 @@ export async function createContext({ name, knowledgeFolder, profile, extensions }; } +// The questions a context should catch, and the names that appear in it. +// +// A description answers "what is this?", which is not how anyone searches. They +// search with the words of their problem, so these hold the other vocabulary: +// the phrasings a user would actually type, and the service names, ticket ids +// and error strings that appear in one context and nowhere else. Both are +// matched against and neither is ever displayed, which is what makes them cheap +// enough to keep in bulk. +// +// Optional, unlike the description. A bundle written before this existed, or by +// a host that does not generate them, is not broken — it just matches on less. +function normalizeRoutingList(value, limit) { + if (!Array.isArray(value)) { + return []; + } + const seen = new Set(); + const kept = []; + for (const entry of value) { + if (typeof entry !== "string") continue; + const clean = entry.trim().replace(/\s+/g, " ").slice(0, MAX_ROUTING_TERM); + const key = clean.toLowerCase(); + if (clean.length === 0 || seen.has(key)) continue; + seen.add(key); + kept.push(clean); + if (kept.length === limit) break; + } + return kept; +} + function normalizeRoutingDescription(value) { const description = (value ?? "").trim().replace(/\s+/g, " "); if (description.length === 0) { @@ -392,6 +431,8 @@ export async function createCapturedContext({ name, profile, routingDescription, + routingQuestions, + routingEntities, knowledge, extensions, capturedFrom = "conversation" @@ -399,6 +440,8 @@ export async function createCapturedContext({ const cleanName = normalizeName(name); const profileText = normalizeProfile(profile); const useWhen = normalizeRoutingDescription(routingDescription); + const questions = normalizeRoutingList(routingQuestions, MAX_ROUTING_QUESTIONS); + const entities = normalizeRoutingList(routingEntities, MAX_ROUTING_ENTITIES); const files = normalizeCaptureKnowledge(knowledge); const declarations = serializeExtensionDeclarations(extensions ?? []); await ensureUniqueName(cleanName); @@ -416,6 +459,10 @@ export async function createCapturedContext({ capturedFrom: isConversationCapture(capturedFrom) ? capturedFrom : "conversation", routingDescription: useWhen }; + // Absent rather than empty when there is nothing: a manifest should not carry + // a field that says only that a host did not fill it in. + if (questions.length > 0) record.routingQuestions = questions; + if (entities.length > 0) record.routingEntities = entities; if (declarations) record.extensions = declarations; record.updatedAt = record.createdAt; @@ -540,6 +587,8 @@ async function prepareCapturedContextUpdate({ name, profile, routingDescription, + routingQuestions, + routingEntities, knowledge, extensions }) { @@ -560,6 +609,17 @@ async function prepareCapturedContextUpdate({ requireCurrentBase(record, currentHash, baseHash); const profileText = normalizeProfile(profile); const useWhen = normalizeRoutingDescription(routingDescription); + // Same rule as extensions below: a save that says nothing about the matching + // material leaves what is there alone. A host that does not generate these + // must not silently strip what another host wrote. + const questions = + routingQuestions === undefined + ? record.routingQuestions + : normalizeRoutingList(routingQuestions, MAX_ROUTING_QUESTIONS); + const entities = + routingEntities === undefined + ? record.routingEntities + : normalizeRoutingList(routingEntities, MAX_ROUTING_ENTITIES); const files = normalizeCaptureKnowledge(knowledge); // A save that says nothing about extensions leaves them exactly as they are. // Declarations are usually added deliberately, by hand or by `extensions add`, @@ -569,13 +629,18 @@ async function prepareCapturedContextUpdate({ const currentKnowledge = await readGeneratedKnowledge(record); const changes = knowledgeChanges(currentKnowledge, files); const profileChanged = (await readProfileText(record)) !== profileText; - const routingChanged = record.routingDescription !== useWhen; + const routingChanged = + record.routingDescription !== useWhen || + JSON.stringify(record.routingQuestions) !== JSON.stringify(questions) || + JSON.stringify(record.routingEntities) !== JSON.stringify(entities); const extensionsChanged = JSON.stringify(record.extensions) !== JSON.stringify(declarations); return { record, profileText, useWhen, + questions, + entities, files, declarations, changes, @@ -681,6 +746,8 @@ export async function updateCapturedContext(capture) { revision: prepared.record.revision + 1, routingDescription: prepared.useWhen, extensions: serializeExtensionDeclarations(prepared.declarations), + routingQuestions: prepared.questions, + routingEntities: prepared.entities, updatedFrom: typeof capture.updatedFrom === "string" && capture.updatedFrom.trim().length > 0 ? capture.updatedFrom.trim() @@ -798,6 +865,10 @@ export async function importCapturedContext({ bundleFolder, name }) { name: typeof name === "string" && name.trim().length > 0 ? name : manifest.name, profile, routingDescription: manifest.routingDescription, + // The point of keeping these in the bundle: a teammate's copy is findable + // by the same words as the original, without them rediscovering any of it. + routingQuestions: manifest.routingQuestions, + routingEntities: manifest.routingEntities, knowledge, // What the bundle says it expects to reach, reduced to declarations. The // import creates no binding for any of them, so the imported context arrives diff --git a/plugins/claude-code/neatcontext/src/core/routing-candidates.mjs b/plugins/claude-code/neatcontext/src/core/routing-candidates.mjs index aab5808..ae3ad79 100644 --- a/plugins/claude-code/neatcontext/src/core/routing-candidates.mjs +++ b/plugins/claude-code/neatcontext/src/core/routing-candidates.mjs @@ -24,9 +24,10 @@ import { buildIndex, rank } from "./routing-search.mjs"; // string rather than kept as a list because the scorer counts words, and a // user who wrote two aliases meant both of them. // -// `entities` and `questions` are missing on purpose: nothing generates them -// yet. The scorer treats an absent field as absent rather than empty, so adding -// them later changes what matches without changing anything here. +// `questions` and `entities` come from the bundle rather than from this +// machine, which is what makes a context findable by the same words on every +// machine it reaches. They are matched against and never displayed, so their +// size costs nothing in the prompt. export function routingFields(context, card, files) { return { name: context.name ?? "", @@ -35,6 +36,8 @@ export function routingFields(context, card, files) { // that travelled with the bundle. description: card?.useWhen || context.routingDescription || "", aliases: (card?.aliases ?? []).join(" "), + questions: (context.routingQuestions ?? []).join(" "), + entities: (context.routingEntities ?? []).join(" "), files: files.join(" ") }; } diff --git a/plugins/copilot/neatcontext/src/core/context-store.mjs b/plugins/copilot/neatcontext/src/core/context-store.mjs index 665823b..9fb69f6 100644 --- a/plugins/copilot/neatcontext/src/core/context-store.mjs +++ b/plugins/copilot/neatcontext/src/core/context-store.mjs @@ -34,6 +34,11 @@ const MAX_CAPTURE_FILE_BYTES = 256 * 1024; const MAX_CAPTURE_TOTAL_BYTES = 1024 * 1024; const MAX_PROFILE_BYTES = 128 * 1024; const MAX_ROUTING_DESCRIPTION = 240; +// Matching material rather than reading material: none of this is ever shown to +// a session, so the limits are about keeping a bundle sane, not a prompt short. +const MAX_ROUTING_QUESTIONS = 20; +const MAX_ROUTING_ENTITIES = 40; +const MAX_ROUTING_TERM = 200; const UPDATE_LOCK_STALE_MS = 60_000; const SKIPPED_DIRECTORIES = new Set(["node_modules", ".git", ".svn", ".hg", "__pycache__"]); @@ -100,6 +105,11 @@ function recordFor(directory, parsed) { conversationKnowledgeFolder: knowledgeManaged ? null : path.join(directory, "knowledge"), routingDescription: typeof parsed.routingDescription === "string" ? parsed.routingDescription : "", + // Index-only matching material. It travels with the bundle so a teammate's + // copy is findable by the same words as yours, which is the whole reason it + // lives here rather than in this machine's routing cache. + routingQuestions: normalizeRoutingList(parsed.routingQuestions, MAX_ROUTING_QUESTIONS), + routingEntities: normalizeRoutingList(parsed.routingEntities, MAX_ROUTING_ENTITIES), // What this context expects to be able to reach. Read leniently: a // declaration this plugin cannot make sense of is dropped rather than // allowed to hide the profile and knowledge behind it. @@ -294,6 +304,35 @@ export async function createContext({ name, knowledgeFolder, profile, extensions }; } +// The questions a context should catch, and the names that appear in it. +// +// A description answers "what is this?", which is not how anyone searches. They +// search with the words of their problem, so these hold the other vocabulary: +// the phrasings a user would actually type, and the service names, ticket ids +// and error strings that appear in one context and nowhere else. Both are +// matched against and neither is ever displayed, which is what makes them cheap +// enough to keep in bulk. +// +// Optional, unlike the description. A bundle written before this existed, or by +// a host that does not generate them, is not broken — it just matches on less. +function normalizeRoutingList(value, limit) { + if (!Array.isArray(value)) { + return []; + } + const seen = new Set(); + const kept = []; + for (const entry of value) { + if (typeof entry !== "string") continue; + const clean = entry.trim().replace(/\s+/g, " ").slice(0, MAX_ROUTING_TERM); + const key = clean.toLowerCase(); + if (clean.length === 0 || seen.has(key)) continue; + seen.add(key); + kept.push(clean); + if (kept.length === limit) break; + } + return kept; +} + function normalizeRoutingDescription(value) { const description = (value ?? "").trim().replace(/\s+/g, " "); if (description.length === 0) { @@ -392,6 +431,8 @@ export async function createCapturedContext({ name, profile, routingDescription, + routingQuestions, + routingEntities, knowledge, extensions, capturedFrom = "conversation" @@ -399,6 +440,8 @@ export async function createCapturedContext({ const cleanName = normalizeName(name); const profileText = normalizeProfile(profile); const useWhen = normalizeRoutingDescription(routingDescription); + const questions = normalizeRoutingList(routingQuestions, MAX_ROUTING_QUESTIONS); + const entities = normalizeRoutingList(routingEntities, MAX_ROUTING_ENTITIES); const files = normalizeCaptureKnowledge(knowledge); const declarations = serializeExtensionDeclarations(extensions ?? []); await ensureUniqueName(cleanName); @@ -416,6 +459,10 @@ export async function createCapturedContext({ capturedFrom: isConversationCapture(capturedFrom) ? capturedFrom : "conversation", routingDescription: useWhen }; + // Absent rather than empty when there is nothing: a manifest should not carry + // a field that says only that a host did not fill it in. + if (questions.length > 0) record.routingQuestions = questions; + if (entities.length > 0) record.routingEntities = entities; if (declarations) record.extensions = declarations; record.updatedAt = record.createdAt; @@ -540,6 +587,8 @@ async function prepareCapturedContextUpdate({ name, profile, routingDescription, + routingQuestions, + routingEntities, knowledge, extensions }) { @@ -560,6 +609,17 @@ async function prepareCapturedContextUpdate({ requireCurrentBase(record, currentHash, baseHash); const profileText = normalizeProfile(profile); const useWhen = normalizeRoutingDescription(routingDescription); + // Same rule as extensions below: a save that says nothing about the matching + // material leaves what is there alone. A host that does not generate these + // must not silently strip what another host wrote. + const questions = + routingQuestions === undefined + ? record.routingQuestions + : normalizeRoutingList(routingQuestions, MAX_ROUTING_QUESTIONS); + const entities = + routingEntities === undefined + ? record.routingEntities + : normalizeRoutingList(routingEntities, MAX_ROUTING_ENTITIES); const files = normalizeCaptureKnowledge(knowledge); // A save that says nothing about extensions leaves them exactly as they are. // Declarations are usually added deliberately, by hand or by `extensions add`, @@ -569,13 +629,18 @@ async function prepareCapturedContextUpdate({ const currentKnowledge = await readGeneratedKnowledge(record); const changes = knowledgeChanges(currentKnowledge, files); const profileChanged = (await readProfileText(record)) !== profileText; - const routingChanged = record.routingDescription !== useWhen; + const routingChanged = + record.routingDescription !== useWhen || + JSON.stringify(record.routingQuestions) !== JSON.stringify(questions) || + JSON.stringify(record.routingEntities) !== JSON.stringify(entities); const extensionsChanged = JSON.stringify(record.extensions) !== JSON.stringify(declarations); return { record, profileText, useWhen, + questions, + entities, files, declarations, changes, @@ -681,6 +746,8 @@ export async function updateCapturedContext(capture) { revision: prepared.record.revision + 1, routingDescription: prepared.useWhen, extensions: serializeExtensionDeclarations(prepared.declarations), + routingQuestions: prepared.questions, + routingEntities: prepared.entities, updatedFrom: typeof capture.updatedFrom === "string" && capture.updatedFrom.trim().length > 0 ? capture.updatedFrom.trim() @@ -798,6 +865,10 @@ export async function importCapturedContext({ bundleFolder, name }) { name: typeof name === "string" && name.trim().length > 0 ? name : manifest.name, profile, routingDescription: manifest.routingDescription, + // The point of keeping these in the bundle: a teammate's copy is findable + // by the same words as the original, without them rediscovering any of it. + routingQuestions: manifest.routingQuestions, + routingEntities: manifest.routingEntities, knowledge, // What the bundle says it expects to reach, reduced to declarations. The // import creates no binding for any of them, so the imported context arrives diff --git a/plugins/copilot/neatcontext/src/core/routing-candidates.mjs b/plugins/copilot/neatcontext/src/core/routing-candidates.mjs index aab5808..ae3ad79 100644 --- a/plugins/copilot/neatcontext/src/core/routing-candidates.mjs +++ b/plugins/copilot/neatcontext/src/core/routing-candidates.mjs @@ -24,9 +24,10 @@ import { buildIndex, rank } from "./routing-search.mjs"; // string rather than kept as a list because the scorer counts words, and a // user who wrote two aliases meant both of them. // -// `entities` and `questions` are missing on purpose: nothing generates them -// yet. The scorer treats an absent field as absent rather than empty, so adding -// them later changes what matches without changing anything here. +// `questions` and `entities` come from the bundle rather than from this +// machine, which is what makes a context findable by the same words on every +// machine it reaches. They are matched against and never displayed, so their +// size costs nothing in the prompt. export function routingFields(context, card, files) { return { name: context.name ?? "", @@ -35,6 +36,8 @@ export function routingFields(context, card, files) { // that travelled with the bundle. description: card?.useWhen || context.routingDescription || "", aliases: (card?.aliases ?? []).join(" "), + questions: (context.routingQuestions ?? []).join(" "), + entities: (context.routingEntities ?? []).join(" "), files: files.join(" ") }; } diff --git a/plugins/kimi-code/neatcontext/src/core/context-store.mjs b/plugins/kimi-code/neatcontext/src/core/context-store.mjs index 665823b..9fb69f6 100644 --- a/plugins/kimi-code/neatcontext/src/core/context-store.mjs +++ b/plugins/kimi-code/neatcontext/src/core/context-store.mjs @@ -34,6 +34,11 @@ const MAX_CAPTURE_FILE_BYTES = 256 * 1024; const MAX_CAPTURE_TOTAL_BYTES = 1024 * 1024; const MAX_PROFILE_BYTES = 128 * 1024; const MAX_ROUTING_DESCRIPTION = 240; +// Matching material rather than reading material: none of this is ever shown to +// a session, so the limits are about keeping a bundle sane, not a prompt short. +const MAX_ROUTING_QUESTIONS = 20; +const MAX_ROUTING_ENTITIES = 40; +const MAX_ROUTING_TERM = 200; const UPDATE_LOCK_STALE_MS = 60_000; const SKIPPED_DIRECTORIES = new Set(["node_modules", ".git", ".svn", ".hg", "__pycache__"]); @@ -100,6 +105,11 @@ function recordFor(directory, parsed) { conversationKnowledgeFolder: knowledgeManaged ? null : path.join(directory, "knowledge"), routingDescription: typeof parsed.routingDescription === "string" ? parsed.routingDescription : "", + // Index-only matching material. It travels with the bundle so a teammate's + // copy is findable by the same words as yours, which is the whole reason it + // lives here rather than in this machine's routing cache. + routingQuestions: normalizeRoutingList(parsed.routingQuestions, MAX_ROUTING_QUESTIONS), + routingEntities: normalizeRoutingList(parsed.routingEntities, MAX_ROUTING_ENTITIES), // What this context expects to be able to reach. Read leniently: a // declaration this plugin cannot make sense of is dropped rather than // allowed to hide the profile and knowledge behind it. @@ -294,6 +304,35 @@ export async function createContext({ name, knowledgeFolder, profile, extensions }; } +// The questions a context should catch, and the names that appear in it. +// +// A description answers "what is this?", which is not how anyone searches. They +// search with the words of their problem, so these hold the other vocabulary: +// the phrasings a user would actually type, and the service names, ticket ids +// and error strings that appear in one context and nowhere else. Both are +// matched against and neither is ever displayed, which is what makes them cheap +// enough to keep in bulk. +// +// Optional, unlike the description. A bundle written before this existed, or by +// a host that does not generate them, is not broken — it just matches on less. +function normalizeRoutingList(value, limit) { + if (!Array.isArray(value)) { + return []; + } + const seen = new Set(); + const kept = []; + for (const entry of value) { + if (typeof entry !== "string") continue; + const clean = entry.trim().replace(/\s+/g, " ").slice(0, MAX_ROUTING_TERM); + const key = clean.toLowerCase(); + if (clean.length === 0 || seen.has(key)) continue; + seen.add(key); + kept.push(clean); + if (kept.length === limit) break; + } + return kept; +} + function normalizeRoutingDescription(value) { const description = (value ?? "").trim().replace(/\s+/g, " "); if (description.length === 0) { @@ -392,6 +431,8 @@ export async function createCapturedContext({ name, profile, routingDescription, + routingQuestions, + routingEntities, knowledge, extensions, capturedFrom = "conversation" @@ -399,6 +440,8 @@ export async function createCapturedContext({ const cleanName = normalizeName(name); const profileText = normalizeProfile(profile); const useWhen = normalizeRoutingDescription(routingDescription); + const questions = normalizeRoutingList(routingQuestions, MAX_ROUTING_QUESTIONS); + const entities = normalizeRoutingList(routingEntities, MAX_ROUTING_ENTITIES); const files = normalizeCaptureKnowledge(knowledge); const declarations = serializeExtensionDeclarations(extensions ?? []); await ensureUniqueName(cleanName); @@ -416,6 +459,10 @@ export async function createCapturedContext({ capturedFrom: isConversationCapture(capturedFrom) ? capturedFrom : "conversation", routingDescription: useWhen }; + // Absent rather than empty when there is nothing: a manifest should not carry + // a field that says only that a host did not fill it in. + if (questions.length > 0) record.routingQuestions = questions; + if (entities.length > 0) record.routingEntities = entities; if (declarations) record.extensions = declarations; record.updatedAt = record.createdAt; @@ -540,6 +587,8 @@ async function prepareCapturedContextUpdate({ name, profile, routingDescription, + routingQuestions, + routingEntities, knowledge, extensions }) { @@ -560,6 +609,17 @@ async function prepareCapturedContextUpdate({ requireCurrentBase(record, currentHash, baseHash); const profileText = normalizeProfile(profile); const useWhen = normalizeRoutingDescription(routingDescription); + // Same rule as extensions below: a save that says nothing about the matching + // material leaves what is there alone. A host that does not generate these + // must not silently strip what another host wrote. + const questions = + routingQuestions === undefined + ? record.routingQuestions + : normalizeRoutingList(routingQuestions, MAX_ROUTING_QUESTIONS); + const entities = + routingEntities === undefined + ? record.routingEntities + : normalizeRoutingList(routingEntities, MAX_ROUTING_ENTITIES); const files = normalizeCaptureKnowledge(knowledge); // A save that says nothing about extensions leaves them exactly as they are. // Declarations are usually added deliberately, by hand or by `extensions add`, @@ -569,13 +629,18 @@ async function prepareCapturedContextUpdate({ const currentKnowledge = await readGeneratedKnowledge(record); const changes = knowledgeChanges(currentKnowledge, files); const profileChanged = (await readProfileText(record)) !== profileText; - const routingChanged = record.routingDescription !== useWhen; + const routingChanged = + record.routingDescription !== useWhen || + JSON.stringify(record.routingQuestions) !== JSON.stringify(questions) || + JSON.stringify(record.routingEntities) !== JSON.stringify(entities); const extensionsChanged = JSON.stringify(record.extensions) !== JSON.stringify(declarations); return { record, profileText, useWhen, + questions, + entities, files, declarations, changes, @@ -681,6 +746,8 @@ export async function updateCapturedContext(capture) { revision: prepared.record.revision + 1, routingDescription: prepared.useWhen, extensions: serializeExtensionDeclarations(prepared.declarations), + routingQuestions: prepared.questions, + routingEntities: prepared.entities, updatedFrom: typeof capture.updatedFrom === "string" && capture.updatedFrom.trim().length > 0 ? capture.updatedFrom.trim() @@ -798,6 +865,10 @@ export async function importCapturedContext({ bundleFolder, name }) { name: typeof name === "string" && name.trim().length > 0 ? name : manifest.name, profile, routingDescription: manifest.routingDescription, + // The point of keeping these in the bundle: a teammate's copy is findable + // by the same words as the original, without them rediscovering any of it. + routingQuestions: manifest.routingQuestions, + routingEntities: manifest.routingEntities, knowledge, // What the bundle says it expects to reach, reduced to declarations. The // import creates no binding for any of them, so the imported context arrives diff --git a/plugins/kimi-code/neatcontext/src/core/routing-candidates.mjs b/plugins/kimi-code/neatcontext/src/core/routing-candidates.mjs index aab5808..ae3ad79 100644 --- a/plugins/kimi-code/neatcontext/src/core/routing-candidates.mjs +++ b/plugins/kimi-code/neatcontext/src/core/routing-candidates.mjs @@ -24,9 +24,10 @@ import { buildIndex, rank } from "./routing-search.mjs"; // string rather than kept as a list because the scorer counts words, and a // user who wrote two aliases meant both of them. // -// `entities` and `questions` are missing on purpose: nothing generates them -// yet. The scorer treats an absent field as absent rather than empty, so adding -// them later changes what matches without changing anything here. +// `questions` and `entities` come from the bundle rather than from this +// machine, which is what makes a context findable by the same words on every +// machine it reaches. They are matched against and never displayed, so their +// size costs nothing in the prompt. export function routingFields(context, card, files) { return { name: context.name ?? "", @@ -35,6 +36,8 @@ export function routingFields(context, card, files) { // that travelled with the bundle. description: card?.useWhen || context.routingDescription || "", aliases: (card?.aliases ?? []).join(" "), + questions: (context.routingQuestions ?? []).join(" "), + entities: (context.routingEntities ?? []).join(" "), files: files.join(" ") }; } diff --git a/plugins/pi/neatcontext/src/core/context-store.mjs b/plugins/pi/neatcontext/src/core/context-store.mjs index 665823b..9fb69f6 100644 --- a/plugins/pi/neatcontext/src/core/context-store.mjs +++ b/plugins/pi/neatcontext/src/core/context-store.mjs @@ -34,6 +34,11 @@ const MAX_CAPTURE_FILE_BYTES = 256 * 1024; const MAX_CAPTURE_TOTAL_BYTES = 1024 * 1024; const MAX_PROFILE_BYTES = 128 * 1024; const MAX_ROUTING_DESCRIPTION = 240; +// Matching material rather than reading material: none of this is ever shown to +// a session, so the limits are about keeping a bundle sane, not a prompt short. +const MAX_ROUTING_QUESTIONS = 20; +const MAX_ROUTING_ENTITIES = 40; +const MAX_ROUTING_TERM = 200; const UPDATE_LOCK_STALE_MS = 60_000; const SKIPPED_DIRECTORIES = new Set(["node_modules", ".git", ".svn", ".hg", "__pycache__"]); @@ -100,6 +105,11 @@ function recordFor(directory, parsed) { conversationKnowledgeFolder: knowledgeManaged ? null : path.join(directory, "knowledge"), routingDescription: typeof parsed.routingDescription === "string" ? parsed.routingDescription : "", + // Index-only matching material. It travels with the bundle so a teammate's + // copy is findable by the same words as yours, which is the whole reason it + // lives here rather than in this machine's routing cache. + routingQuestions: normalizeRoutingList(parsed.routingQuestions, MAX_ROUTING_QUESTIONS), + routingEntities: normalizeRoutingList(parsed.routingEntities, MAX_ROUTING_ENTITIES), // What this context expects to be able to reach. Read leniently: a // declaration this plugin cannot make sense of is dropped rather than // allowed to hide the profile and knowledge behind it. @@ -294,6 +304,35 @@ export async function createContext({ name, knowledgeFolder, profile, extensions }; } +// The questions a context should catch, and the names that appear in it. +// +// A description answers "what is this?", which is not how anyone searches. They +// search with the words of their problem, so these hold the other vocabulary: +// the phrasings a user would actually type, and the service names, ticket ids +// and error strings that appear in one context and nowhere else. Both are +// matched against and neither is ever displayed, which is what makes them cheap +// enough to keep in bulk. +// +// Optional, unlike the description. A bundle written before this existed, or by +// a host that does not generate them, is not broken — it just matches on less. +function normalizeRoutingList(value, limit) { + if (!Array.isArray(value)) { + return []; + } + const seen = new Set(); + const kept = []; + for (const entry of value) { + if (typeof entry !== "string") continue; + const clean = entry.trim().replace(/\s+/g, " ").slice(0, MAX_ROUTING_TERM); + const key = clean.toLowerCase(); + if (clean.length === 0 || seen.has(key)) continue; + seen.add(key); + kept.push(clean); + if (kept.length === limit) break; + } + return kept; +} + function normalizeRoutingDescription(value) { const description = (value ?? "").trim().replace(/\s+/g, " "); if (description.length === 0) { @@ -392,6 +431,8 @@ export async function createCapturedContext({ name, profile, routingDescription, + routingQuestions, + routingEntities, knowledge, extensions, capturedFrom = "conversation" @@ -399,6 +440,8 @@ export async function createCapturedContext({ const cleanName = normalizeName(name); const profileText = normalizeProfile(profile); const useWhen = normalizeRoutingDescription(routingDescription); + const questions = normalizeRoutingList(routingQuestions, MAX_ROUTING_QUESTIONS); + const entities = normalizeRoutingList(routingEntities, MAX_ROUTING_ENTITIES); const files = normalizeCaptureKnowledge(knowledge); const declarations = serializeExtensionDeclarations(extensions ?? []); await ensureUniqueName(cleanName); @@ -416,6 +459,10 @@ export async function createCapturedContext({ capturedFrom: isConversationCapture(capturedFrom) ? capturedFrom : "conversation", routingDescription: useWhen }; + // Absent rather than empty when there is nothing: a manifest should not carry + // a field that says only that a host did not fill it in. + if (questions.length > 0) record.routingQuestions = questions; + if (entities.length > 0) record.routingEntities = entities; if (declarations) record.extensions = declarations; record.updatedAt = record.createdAt; @@ -540,6 +587,8 @@ async function prepareCapturedContextUpdate({ name, profile, routingDescription, + routingQuestions, + routingEntities, knowledge, extensions }) { @@ -560,6 +609,17 @@ async function prepareCapturedContextUpdate({ requireCurrentBase(record, currentHash, baseHash); const profileText = normalizeProfile(profile); const useWhen = normalizeRoutingDescription(routingDescription); + // Same rule as extensions below: a save that says nothing about the matching + // material leaves what is there alone. A host that does not generate these + // must not silently strip what another host wrote. + const questions = + routingQuestions === undefined + ? record.routingQuestions + : normalizeRoutingList(routingQuestions, MAX_ROUTING_QUESTIONS); + const entities = + routingEntities === undefined + ? record.routingEntities + : normalizeRoutingList(routingEntities, MAX_ROUTING_ENTITIES); const files = normalizeCaptureKnowledge(knowledge); // A save that says nothing about extensions leaves them exactly as they are. // Declarations are usually added deliberately, by hand or by `extensions add`, @@ -569,13 +629,18 @@ async function prepareCapturedContextUpdate({ const currentKnowledge = await readGeneratedKnowledge(record); const changes = knowledgeChanges(currentKnowledge, files); const profileChanged = (await readProfileText(record)) !== profileText; - const routingChanged = record.routingDescription !== useWhen; + const routingChanged = + record.routingDescription !== useWhen || + JSON.stringify(record.routingQuestions) !== JSON.stringify(questions) || + JSON.stringify(record.routingEntities) !== JSON.stringify(entities); const extensionsChanged = JSON.stringify(record.extensions) !== JSON.stringify(declarations); return { record, profileText, useWhen, + questions, + entities, files, declarations, changes, @@ -681,6 +746,8 @@ export async function updateCapturedContext(capture) { revision: prepared.record.revision + 1, routingDescription: prepared.useWhen, extensions: serializeExtensionDeclarations(prepared.declarations), + routingQuestions: prepared.questions, + routingEntities: prepared.entities, updatedFrom: typeof capture.updatedFrom === "string" && capture.updatedFrom.trim().length > 0 ? capture.updatedFrom.trim() @@ -798,6 +865,10 @@ export async function importCapturedContext({ bundleFolder, name }) { name: typeof name === "string" && name.trim().length > 0 ? name : manifest.name, profile, routingDescription: manifest.routingDescription, + // The point of keeping these in the bundle: a teammate's copy is findable + // by the same words as the original, without them rediscovering any of it. + routingQuestions: manifest.routingQuestions, + routingEntities: manifest.routingEntities, knowledge, // What the bundle says it expects to reach, reduced to declarations. The // import creates no binding for any of them, so the imported context arrives diff --git a/plugins/pi/neatcontext/src/core/routing-candidates.mjs b/plugins/pi/neatcontext/src/core/routing-candidates.mjs index aab5808..ae3ad79 100644 --- a/plugins/pi/neatcontext/src/core/routing-candidates.mjs +++ b/plugins/pi/neatcontext/src/core/routing-candidates.mjs @@ -24,9 +24,10 @@ import { buildIndex, rank } from "./routing-search.mjs"; // string rather than kept as a list because the scorer counts words, and a // user who wrote two aliases meant both of them. // -// `entities` and `questions` are missing on purpose: nothing generates them -// yet. The scorer treats an absent field as absent rather than empty, so adding -// them later changes what matches without changing anything here. +// `questions` and `entities` come from the bundle rather than from this +// machine, which is what makes a context findable by the same words on every +// machine it reaches. They are matched against and never displayed, so their +// size costs nothing in the prompt. export function routingFields(context, card, files) { return { name: context.name ?? "", @@ -35,6 +36,8 @@ export function routingFields(context, card, files) { // that travelled with the bundle. description: card?.useWhen || context.routingDescription || "", aliases: (card?.aliases ?? []).join(" "), + questions: (context.routingQuestions ?? []).join(" "), + entities: (context.routingEntities ?? []).join(" "), files: files.join(" ") }; } diff --git a/shared/core/context-store.mjs b/shared/core/context-store.mjs index 665823b..9fb69f6 100644 --- a/shared/core/context-store.mjs +++ b/shared/core/context-store.mjs @@ -34,6 +34,11 @@ const MAX_CAPTURE_FILE_BYTES = 256 * 1024; const MAX_CAPTURE_TOTAL_BYTES = 1024 * 1024; const MAX_PROFILE_BYTES = 128 * 1024; const MAX_ROUTING_DESCRIPTION = 240; +// Matching material rather than reading material: none of this is ever shown to +// a session, so the limits are about keeping a bundle sane, not a prompt short. +const MAX_ROUTING_QUESTIONS = 20; +const MAX_ROUTING_ENTITIES = 40; +const MAX_ROUTING_TERM = 200; const UPDATE_LOCK_STALE_MS = 60_000; const SKIPPED_DIRECTORIES = new Set(["node_modules", ".git", ".svn", ".hg", "__pycache__"]); @@ -100,6 +105,11 @@ function recordFor(directory, parsed) { conversationKnowledgeFolder: knowledgeManaged ? null : path.join(directory, "knowledge"), routingDescription: typeof parsed.routingDescription === "string" ? parsed.routingDescription : "", + // Index-only matching material. It travels with the bundle so a teammate's + // copy is findable by the same words as yours, which is the whole reason it + // lives here rather than in this machine's routing cache. + routingQuestions: normalizeRoutingList(parsed.routingQuestions, MAX_ROUTING_QUESTIONS), + routingEntities: normalizeRoutingList(parsed.routingEntities, MAX_ROUTING_ENTITIES), // What this context expects to be able to reach. Read leniently: a // declaration this plugin cannot make sense of is dropped rather than // allowed to hide the profile and knowledge behind it. @@ -294,6 +304,35 @@ export async function createContext({ name, knowledgeFolder, profile, extensions }; } +// The questions a context should catch, and the names that appear in it. +// +// A description answers "what is this?", which is not how anyone searches. They +// search with the words of their problem, so these hold the other vocabulary: +// the phrasings a user would actually type, and the service names, ticket ids +// and error strings that appear in one context and nowhere else. Both are +// matched against and neither is ever displayed, which is what makes them cheap +// enough to keep in bulk. +// +// Optional, unlike the description. A bundle written before this existed, or by +// a host that does not generate them, is not broken — it just matches on less. +function normalizeRoutingList(value, limit) { + if (!Array.isArray(value)) { + return []; + } + const seen = new Set(); + const kept = []; + for (const entry of value) { + if (typeof entry !== "string") continue; + const clean = entry.trim().replace(/\s+/g, " ").slice(0, MAX_ROUTING_TERM); + const key = clean.toLowerCase(); + if (clean.length === 0 || seen.has(key)) continue; + seen.add(key); + kept.push(clean); + if (kept.length === limit) break; + } + return kept; +} + function normalizeRoutingDescription(value) { const description = (value ?? "").trim().replace(/\s+/g, " "); if (description.length === 0) { @@ -392,6 +431,8 @@ export async function createCapturedContext({ name, profile, routingDescription, + routingQuestions, + routingEntities, knowledge, extensions, capturedFrom = "conversation" @@ -399,6 +440,8 @@ export async function createCapturedContext({ const cleanName = normalizeName(name); const profileText = normalizeProfile(profile); const useWhen = normalizeRoutingDescription(routingDescription); + const questions = normalizeRoutingList(routingQuestions, MAX_ROUTING_QUESTIONS); + const entities = normalizeRoutingList(routingEntities, MAX_ROUTING_ENTITIES); const files = normalizeCaptureKnowledge(knowledge); const declarations = serializeExtensionDeclarations(extensions ?? []); await ensureUniqueName(cleanName); @@ -416,6 +459,10 @@ export async function createCapturedContext({ capturedFrom: isConversationCapture(capturedFrom) ? capturedFrom : "conversation", routingDescription: useWhen }; + // Absent rather than empty when there is nothing: a manifest should not carry + // a field that says only that a host did not fill it in. + if (questions.length > 0) record.routingQuestions = questions; + if (entities.length > 0) record.routingEntities = entities; if (declarations) record.extensions = declarations; record.updatedAt = record.createdAt; @@ -540,6 +587,8 @@ async function prepareCapturedContextUpdate({ name, profile, routingDescription, + routingQuestions, + routingEntities, knowledge, extensions }) { @@ -560,6 +609,17 @@ async function prepareCapturedContextUpdate({ requireCurrentBase(record, currentHash, baseHash); const profileText = normalizeProfile(profile); const useWhen = normalizeRoutingDescription(routingDescription); + // Same rule as extensions below: a save that says nothing about the matching + // material leaves what is there alone. A host that does not generate these + // must not silently strip what another host wrote. + const questions = + routingQuestions === undefined + ? record.routingQuestions + : normalizeRoutingList(routingQuestions, MAX_ROUTING_QUESTIONS); + const entities = + routingEntities === undefined + ? record.routingEntities + : normalizeRoutingList(routingEntities, MAX_ROUTING_ENTITIES); const files = normalizeCaptureKnowledge(knowledge); // A save that says nothing about extensions leaves them exactly as they are. // Declarations are usually added deliberately, by hand or by `extensions add`, @@ -569,13 +629,18 @@ async function prepareCapturedContextUpdate({ const currentKnowledge = await readGeneratedKnowledge(record); const changes = knowledgeChanges(currentKnowledge, files); const profileChanged = (await readProfileText(record)) !== profileText; - const routingChanged = record.routingDescription !== useWhen; + const routingChanged = + record.routingDescription !== useWhen || + JSON.stringify(record.routingQuestions) !== JSON.stringify(questions) || + JSON.stringify(record.routingEntities) !== JSON.stringify(entities); const extensionsChanged = JSON.stringify(record.extensions) !== JSON.stringify(declarations); return { record, profileText, useWhen, + questions, + entities, files, declarations, changes, @@ -681,6 +746,8 @@ export async function updateCapturedContext(capture) { revision: prepared.record.revision + 1, routingDescription: prepared.useWhen, extensions: serializeExtensionDeclarations(prepared.declarations), + routingQuestions: prepared.questions, + routingEntities: prepared.entities, updatedFrom: typeof capture.updatedFrom === "string" && capture.updatedFrom.trim().length > 0 ? capture.updatedFrom.trim() @@ -798,6 +865,10 @@ export async function importCapturedContext({ bundleFolder, name }) { name: typeof name === "string" && name.trim().length > 0 ? name : manifest.name, profile, routingDescription: manifest.routingDescription, + // The point of keeping these in the bundle: a teammate's copy is findable + // by the same words as the original, without them rediscovering any of it. + routingQuestions: manifest.routingQuestions, + routingEntities: manifest.routingEntities, knowledge, // What the bundle says it expects to reach, reduced to declarations. The // import creates no binding for any of them, so the imported context arrives diff --git a/shared/core/routing-candidates.mjs b/shared/core/routing-candidates.mjs index aab5808..ae3ad79 100644 --- a/shared/core/routing-candidates.mjs +++ b/shared/core/routing-candidates.mjs @@ -24,9 +24,10 @@ import { buildIndex, rank } from "./routing-search.mjs"; // string rather than kept as a list because the scorer counts words, and a // user who wrote two aliases meant both of them. // -// `entities` and `questions` are missing on purpose: nothing generates them -// yet. The scorer treats an absent field as absent rather than empty, so adding -// them later changes what matches without changing anything here. +// `questions` and `entities` come from the bundle rather than from this +// machine, which is what makes a context findable by the same words on every +// machine it reaches. They are matched against and never displayed, so their +// size costs nothing in the prompt. export function routingFields(context, card, files) { return { name: context.name ?? "", @@ -35,6 +36,8 @@ export function routingFields(context, card, files) { // that travelled with the bundle. description: card?.useWhen || context.routingDescription || "", aliases: (card?.aliases ?? []).join(" "), + questions: (context.routingQuestions ?? []).join(" "), + entities: (context.routingEntities ?? []).join(" "), files: files.join(" ") }; } diff --git a/tests/routing-candidates.test.mjs b/tests/routing-candidates.test.mjs index ab04cb1..c8f69a9 100644 --- a/tests/routing-candidates.test.mjs +++ b/tests/routing-candidates.test.mjs @@ -93,7 +93,14 @@ describe("routingFields", () => { it("survives a context with nothing but a name", () => { const fields = routingFields({ id: "bare" }, undefined, []); - assert.deepEqual(fields, { name: "", description: "", aliases: "", files: "" }); + assert.deepEqual(fields, { + name: "", + description: "", + aliases: "", + questions: "", + entities: "", + files: "" + }); }); }); diff --git a/tests/routing-expansion.test.mjs b/tests/routing-expansion.test.mjs new file mode 100644 index 0000000..97ef6df --- /dev/null +++ b/tests/routing-expansion.test.mjs @@ -0,0 +1,210 @@ +// The questions a context should catch, and the names inside it. +// +// This is the recall half of routing. A description answers "what is this?", +// which is not how anyone searches — they search with the words of their +// problem. These two lists hold that other vocabulary, are matched against, and +// are never shown to anyone. +// +// They live in the bundle rather than in this machine's routing cache on +// purpose: a context handed to a teammate has to be findable by the same words +// on their machine as on yours, without them rediscovering any of it. + +import assert from "node:assert/strict"; +import { mkdir, mkdtemp, readFile, rm } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { after, before, beforeEach, describe, it } from "node:test"; + +let home; + +before(async () => { + home = await mkdtemp(path.join(os.tmpdir(), "neatcontext-expansion-test-")); + await mkdir(path.join(home, "docs"), { recursive: true }); + process.env.NEATCONTEXT_HOME = home; +}); + +after(async () => { + await rm(home, { recursive: true, force: true }); +}); + +beforeEach(async () => { + await rm(path.join(home, "contexts"), { recursive: true, force: true }); + await rm(path.join(home, "plugin-routing.json"), { force: true }); +}); + +const store = await import("../plugins/claude-code/neatcontext/src/core/context-store.mjs"); +const { createRoutingIndex } = await import( + "../plugins/claude-code/neatcontext/src/core/routing-candidates.mjs" +); + +const INCIDENT = { + name: "Pool limits June", + profile: "# Pool limits June\n\n## Purpose\nThe incident.\n", + routingDescription: "billing-postgres default_pool_size regression after the June deploy", + routingQuestions: [ + "why was checkout throwing 5xx last week", + "did we ever fix that database timeout thing", + "what happened on the thirtieth" + ], + routingEntities: ["INC-1001", "checkout-api", "pgbouncer", "dep-9001"], + knowledge: [{ path: "session-summary.md", content: "# Summary\n\nPool exhaustion.\n" }] +}; + +async function manifestOf(record) { + return JSON.parse(await readFile(path.join(record.directory, "context.json"), "utf8")); +} + +describe("storing the matching material", () => { + it("keeps questions and entities in the bundle", async () => { + const { record } = await store.createCapturedContext(INCIDENT); + const manifest = await manifestOf(record); + assert.deepEqual(manifest.routingQuestions, INCIDENT.routingQuestions); + assert.deepEqual(manifest.routingEntities, INCIDENT.routingEntities); + assert.deepEqual(record.routingQuestions, INCIDENT.routingQuestions); + }); + + it("leaves the fields off entirely when there are none", async () => { + // A manifest should not carry a field that says only that a host did not + // fill it in. + const { record } = await store.createCapturedContext({ + ...INCIDENT, + routingQuestions: undefined, + routingEntities: undefined + }); + const manifest = await manifestOf(record); + assert.equal("routingQuestions" in manifest, false); + assert.equal("routingEntities" in manifest, false); + assert.deepEqual(record.routingQuestions, []); + }); + + it("tidies what it is given", async () => { + const { record } = await store.createCapturedContext({ + ...INCIDENT, + routingQuestions: [" spaced out ", "", "Spaced Out", 42, "kept"], + routingEntities: ["INC-1001", "inc-1001", " "] + }); + // Trimmed, de-duplicated case-insensitively, non-strings dropped. + assert.deepEqual(record.routingQuestions, ["spaced out", "kept"]); + assert.deepEqual(record.routingEntities, ["INC-1001"]); + }); + + it("ignores a field that is not a list at all", async () => { + const { record } = await store.createCapturedContext({ + ...INCIDENT, + routingQuestions: "not a list" + }); + assert.deepEqual(record.routingQuestions, []); + }); + + it("caps how much it will keep", async () => { + const many = Array.from({ length: 60 }, (_, index) => `question number ${index}`); + const { record } = await store.createCapturedContext({ ...INCIDENT, routingQuestions: many }); + assert.equal(record.routingQuestions.length, 20); + }); +}); + +describe("updating a context", () => { + it("replaces the material when the save supplies it", async () => { + const { record } = await store.createCapturedContext(INCIDENT); + const baseHash = await store.fingerprintContext(record); + const updated = await store.updateCapturedContext({ + targetId: record.id, + baseHash, + name: record.name, + profile: "# Pool limits June\n\n## Purpose\nThe incident, revisited.\n", + routingDescription: INCIDENT.routingDescription, + routingQuestions: ["one new question"], + routingEntities: ["INC-2002"], + knowledge: INCIDENT.knowledge + }); + assert.deepEqual(updated.record.routingQuestions, ["one new question"]); + assert.deepEqual(updated.record.routingEntities, ["INC-2002"]); + }); + + it("leaves the material alone when the save says nothing about it", async () => { + // A host that does not generate these must not silently strip what another + // host wrote — the same rule extensions already follow. + const { record } = await store.createCapturedContext(INCIDENT); + const baseHash = await store.fingerprintContext(record); + const updated = await store.updateCapturedContext({ + targetId: record.id, + baseHash, + name: record.name, + profile: "# Pool limits June\n\n## Purpose\nUntouched material.\n", + routingDescription: INCIDENT.routingDescription, + knowledge: INCIDENT.knowledge + }); + assert.deepEqual(updated.record.routingQuestions, INCIDENT.routingQuestions); + assert.deepEqual(updated.record.routingEntities, INCIDENT.routingEntities); + }); + + it("counts a change to the material as a routing change", async () => { + const { record } = await store.createCapturedContext(INCIDENT); + const baseHash = await store.fingerprintContext(record); + const preview = await store.previewCapturedContextUpdate({ + targetId: record.id, + baseHash, + name: record.name, + profile: await readFile(record.profilePath, "utf8"), + routingDescription: INCIDENT.routingDescription, + routingQuestions: ["a question nobody asked before"], + knowledge: INCIDENT.knowledge + }); + assert.equal(preview.routingChanged, true); + assert.equal(preview.changed, true); + }); +}); + +describe("what it buys", () => { + // The point of the whole feature: a question phrased in the user's words, + // sharing nothing with the description, still finds the context. + const OTHERS = [ + ["Queue lag", "order-events partition lag and consumer rebalancing"], + ["Codex design", "Codex CLI plugin design and marketplace packaging"], + ["Kimi plugin", "Kimi Code manifests, skills and commands"], + ["Evidence", "conversation evidence and transcript adapters"], + ["Refunds", "refunds and chargebacks"], + ["Docker container", "Ubuntu container with SSH"], + ["Marketplace config", "switching the marketplace source"], + ["Session drift", "bridge session and thread drift"] + ]; + + async function seed(incident) { + await store.createCapturedContext(incident); + for (const [name, routingDescription] of OTHERS) { + await store.createCapturedContext({ + name, + profile: `# ${name}\n\n## Purpose\n${routingDescription}\n`, + routingDescription, + knowledge: [{ path: "session-summary.md", content: `# ${name}\n` }] + }); + } + const contexts = await store.listContexts(); + const candidates = createRoutingIndex({ listFiles: () => Promise.resolve([]) }); + return (query) => candidates(contexts, { cards: {} }, query, { limit: 3 }); + } + + it("finds the context from words its description never uses", async () => { + const rank = await seed(INCIDENT); + const [top] = await rank("why was checkout throwing 5xx last week"); + assert.equal(top.name, "Pool limits June"); + }); + + it("finds it from a ticket id that appears nowhere else", async () => { + const rank = await seed(INCIDENT); + const [top] = await rank("anything on dep-9001?"); + assert.equal(top.name, "Pool limits June"); + }); + + it("does not find it without the material", async () => { + // The same corpus and the same question, minus the stored questions: this + // is the recall failure the feature exists to fix. + const rank = await seed({ + ...INCIDENT, + routingQuestions: undefined, + routingEntities: undefined + }); + const results = await rank("why was checkout throwing 5xx last week"); + assert.ok(!results.some((result) => result.name === "Pool limits June")); + }); +});