diff --git a/src/main/ipc/chatHandlers.ts b/src/main/ipc/chatHandlers.ts index 855bf55..2a217a5 100644 --- a/src/main/ipc/chatHandlers.ts +++ b/src/main/ipc/chatHandlers.ts @@ -6,8 +6,9 @@ import { KnowledgeService } from '../services/KnowledgeService' import { buildRAGContext } from '../services/citations' import { validateAndCleanMessages } from '../utils/messageValidator' import Logger from '../../shared/utils/logger' +import { resolveCitations } from '../../shared/utils/citationResolution' import type { AnswerSource, ChatMessageMetadata, RetrievalStatus } from '../../shared/types/chat' -import type { Citation } from '../../shared/types/citation' +import type { Citation, CitationContext } from '../../shared/types/citation' import { ChatSchemas, validate } from './validation' // 管理活跃的流式请求 @@ -111,6 +112,7 @@ export function registerChatHandlers( let retrieval: RetrievalStatus = 'none' let answerSources: AnswerSource[] = [] let answerCitations: Citation[] = [] + let citationContexts: CitationContext[] = [] try { const embeddingClient = await connectionManager.getEmbeddingClient() @@ -127,6 +129,12 @@ export function registerChatHandlers( retrieval = 'used' answerSources = sources answerCitations = citations + // The span a quote is checked against lives only in the locator, so + // carry it alongside the citation for validation (#70). + citationContexts = citations.map((citation, index) => ({ + citation, + spanText: searchResults[index].locator.blocks.map((block) => block.text).join('\n') + })) Logger.debug( 'ChatHandlers', `RAG: Found ${searchResults.length} relevant chunks for query` @@ -155,6 +163,9 @@ export function registerChatHandlers( citations: answerCitations } queries.updateMessageMetadata(assistantMessage.id, answerMetadata) + // Rewritten at the end of the stream, once the answer text exists and its + // `[n]` markers can be resolved against the evidence. + let resolvedMetadata = answerMetadata // 4. 调用 Model Connection 流式生成 const client = await connectionManager.getChatClient() @@ -222,6 +233,19 @@ export function registerChatHandlers( usageMetadata = metadata } + // An answer that marked sources gets only the grounded ones: a + // fabricated `[9]` or a quote that is not in its span must not survive + // as a clickable source (#70). With no markers at all, keep the full + // evidence set — the model simply did not use the marker convention. + const resolution = resolveCitations(fullTextContent, citationContexts) + if (resolution.matches.length > 0) { + const grounded: Citation[] = [] + for (const match of resolution.resolved) { + if (match.citation) grounded.push(match.citation) + } + resolvedMetadata = { ...answerMetadata, citations: grounded } + } + // 发送完成事件 event.sender.send('message-chunk', { messageId: assistantMessage.id, @@ -230,7 +254,7 @@ export function registerChatHandlers( // The renderer's in-memory message never sees the DB row written // before streaming, so the persisted provenance rides along here or // the answer loses its citations until the session is reloaded. - messageMetadata: answerMetadata + messageMetadata: resolvedMetadata }) } }, @@ -249,6 +273,9 @@ export function registerChatHandlers( // 更新数据库中的完整内容(包含推理内容) queries.updateMessageContent(assistantMessage.id, fullTextContent, fullReasoningContent) + // Persist the marker-filtered citations now that the answer text is final. + queries.updateMessageMetadata(assistantMessage.id, resolvedMetadata) + // 计算 token 使用量 let tokensUsed = 0 if (usageMetadata?.usage?.totalTokens) { diff --git a/src/shared/types/citation.ts b/src/shared/types/citation.ts index 709a4c0..87a2852 100644 --- a/src/shared/types/citation.ts +++ b/src/shared/types/citation.ts @@ -29,3 +29,38 @@ export interface Citation { quote: string score: number } + +/** + * 一条 citation 连同它所覆盖的原文。 + * + * 校验「引文确实落在引用区间内」需要区间文本,而文本只在检索侧(`locator`) + * 存在,因此由调用方补齐。`spanText` 缺失时无法证伪引文 —— 解析结果保持为 + * resolved,而不是把一个证据不足的怀疑渲染成错误。 + */ +export interface CitationContext { + citation: Citation + spanText?: string +} + +/** 回答里一个 `[n]` 标记的归宿。 */ +export type CitationMatchStatus = 'resolved' | 'unresolved' | 'misattributed' + +export interface CitationMatch { + /** 回答里的 `[n]` 编号。 */ + marker: number + status: CitationMatchStatus + /** 标记在回答文本里的字符位置。 */ + position: number + /** 仅在 `resolved` / `misattributed` 时存在。 */ + citation?: Citation +} + +/** `resolveCitations` 的结果。三组是互斥且穷尽的,因此可从中直接算出 precision。 */ +export interface CitationResolution { + matches: CitationMatch[] + resolved: CitationMatch[] + unresolved: CitationMatch[] + misattributed: CitationMatch[] + /** `resolved / total`;回答里没有任何标记时为 0。 */ + precision: number +} diff --git a/src/shared/utils/citationResolution.ts b/src/shared/utils/citationResolution.ts new file mode 100644 index 0000000..d9764a9 --- /dev/null +++ b/src/shared/utils/citationResolution.ts @@ -0,0 +1,111 @@ +import type { + Citation, + CitationContext, + CitationMatch, + CitationResolution +} from '../types/citation' + +/** + * Citation resolution (#70). + * + * An answer that is asked to mark its sources with `[n]` can mark a source that + * was never retrieved, or attribute a passage to the wrong span. Both are + * first-class failure modes: a plausible-looking link to the wrong paragraph is + * worse than no link, because the reader cannot tell it is wrong. + * + * Resolution is **deterministic and pure** — no model call, no I/O. It maps + * markers to the citations produced during retrieval, so the same answer and the + * same evidence always produce the same result, and citation precision can be + * computed from the return value for the eval harness (#75). + */ + +/** + * Case- and whitespace-insensitive comparison, and nothing beyond that. + * + * Deliberately not fuzzy: a quote that "roughly" matches is exactly the failure + * this guards against. Collapsing whitespace is safe because the page text is + * reflowed; stemming or token overlap would let a fabricated quote through. + */ +export function normalizeForComparison(text: string): string { + return text.toLowerCase().replace(/\s+/g, ' ').trim() +} + +/** 标记里的编号去重后,按 prompt 位置查 citation;重复编号取第一个。 */ +function indexContexts( + contexts: readonly (Citation | CitationContext)[] +): Map { + const byMarker = new Map() + for (const entry of contexts) { + const context: CitationContext = 'citation' in entry ? entry : { citation: entry } + if (!byMarker.has(context.citation.index)) { + byMarker.set(context.citation.index, context) + } + } + return byMarker +} + +/** + * 校验引文是否落在区间内。 + * + * 区间文本缺失、或引文为空时返回 `null`,表示「无法证伪」。把未知当成错误会 + * 让一条正确的引用因为元数据不全而消失;把未知当成正确则是这里的默认立场, + * 因为 citation 本身来自检索,不是模型生成的。 + */ +function isQuoteInSpan(context: CitationContext): boolean | null { + const quote = normalizeForComparison(context.citation.quote) + const span = context.spanText ? normalizeForComparison(context.spanText) : '' + if (quote.length === 0 || span.length === 0) return null + return span.includes(quote) +} + +/** + * 解析回答里的 `[n]` 标记,映射回检索阶段产生的 citation。 + * + * 每个标记恰好落入三类之一: + * + * - `resolved` —— 有对应 context,且引文落在区间内(或无法证伪)。 + * - `unresolved` —— 没有任何 context(模型编造了 `[9]`)。 + * - `misattributed` —— 有 context,但引文不在其区间内。 + * + * 三类都被返回,而不是把坏标记静默丢掉:precision 要能算,界面也要能选择把 + * 它们按普通文本渲染。 + */ +export function resolveCitations( + answer: string, + contexts: readonly (Citation | CitationContext)[] +): CitationResolution { + const byMarker = indexContexts(contexts) + const matches: CitationMatch[] = [] + + // A fresh regex per call: `RegExp.lastIndex` is mutable state, and a pure + // resolver must not depend on how many times it has run before. + const pattern = /\[(\d+)\]/g + let found = pattern.exec(answer) + while (found !== null) { + const marker = Number(found[1]) + const position = found.index + const context = byMarker.get(marker) + + if (!context) { + matches.push({ marker, status: 'unresolved', position }) + } else if (isQuoteInSpan(context) === false) { + matches.push({ marker, status: 'misattributed', position, citation: context.citation }) + } else { + matches.push({ marker, status: 'resolved', position, citation: context.citation }) + } + + found = pattern.exec(answer) + } + + const resolved = matches.filter((match) => match.status === 'resolved') + const unresolved = matches.filter((match) => match.status === 'unresolved') + const misattributed = matches.filter((match) => match.status === 'misattributed') + + return { + matches, + resolved, + unresolved, + misattributed, + precision: matches.length === 0 ? 0 : resolved.length / matches.length + } +} diff --git a/test/citationResolution.test.ts b/test/citationResolution.test.ts new file mode 100644 index 0000000..a0d6e17 --- /dev/null +++ b/test/citationResolution.test.ts @@ -0,0 +1,113 @@ +import { test } from 'node:test' +import assert from 'node:assert/strict' +import { normalizeForComparison, resolveCitations } from '../src/shared/utils/citationResolution.ts' +import type { Citation, CitationContext } from '../src/shared/types/citation.ts' + +/** + * An answer asked to mark its sources with `[n]` can mark a source that was never + * retrieved, or attribute a passage to the wrong span (#70). Neither may render as + * a clickable source. Resolution is pure and deterministic so citation precision + * is computable for the eval harness (#75). + */ + +const citation = (over: Partial = {}): Citation => ({ + index: 1, + documentId: 'doc_1', + documentTitle: 'Attention Is All You Need', + chunkId: 'chunk_1', + quote: 'the attention mechanism', + score: 0.87, + ...over +}) + +const context = (over: Partial = {}): CitationContext => ({ + citation: citation(), + spanText: 'The attention mechanism is central to the transformer.', + ...over +}) + +test('a grounded marker resolves to its citation', () => { + const resolution = resolveCitations('The mechanism is central [1].', [context()]) + + assert.equal(resolution.resolved.length, 1) + assert.equal(resolution.unresolved.length, 0) + assert.equal(resolution.misattributed.length, 0) + assert.equal(resolution.resolved[0].marker, 1) + assert.equal(resolution.resolved[0].citation?.documentTitle, 'Attention Is All You Need') + assert.equal(resolution.precision, 1) +}) + +test('a fabricated marker with no context is unresolved, never clickable', () => { + const resolution = resolveCitations('A claim with a made-up source [9].', [context()]) + + assert.equal(resolution.resolved.length, 0) + assert.equal(resolution.unresolved.length, 1) + assert.equal(resolution.unresolved[0].marker, 9) + assert.equal(resolution.misattributed.length, 0) + assert.equal(resolution.precision, 0) +}) + +test('a marker whose span does not contain the claimed quote is flagged, not resolved', () => { + const wrongSpan: CitationContext = { + citation: citation({ index: 2, quote: 'a passage about attention' }), + spanText: 'an unrelated sentence about tokenisation' + } + const resolution = resolveCitations('Something [2].', [context(), wrongSpan]) + + assert.equal(resolution.resolved.length, 0) + assert.equal(resolution.misattributed.length, 1) + assert.equal(resolution.misattributed[0].marker, 2) + assert.equal(resolution.misattributed[0].citation?.chunkId, 'chunk_1') +}) + +test('validation is case- and whitespace-insensitive but nothing beyond that', () => { + const loose: CitationContext = { + citation: citation({ quote: 'THE Attention\n Mechanism' }), + spanText: 'the attention mechanism is central' + } + assert.equal(resolveCitations('x [1]', [loose]).resolved.length, 1) + + // "roughly matching" is exactly the failure this guards against: one changed + // word is a different claim, so it must not resolve. + const fuzzy: CitationContext = { + citation: citation({ quote: 'the attention mechanism' }), + spanText: 'the efficient attention mechanism differs' + } + assert.equal(resolveCitations('x [1]', [fuzzy]).misattributed.length, 1) +}) + +test('a missing span cannot refute the quote, so the citation resolves', () => { + const unknown = resolveCitations('x [1]', [{ citation: citation() }]) + assert.equal(unknown.resolved.length, 1) +}) + +test('precision counts resolved markers over all markers', () => { + const resolution = resolveCitations('a [1] b [2] c [9]', [ + context({ citation: citation({ index: 1 }) }), + { + citation: citation({ index: 2, quote: 'nowhere in the span' }), + spanText: 'span without the quote' + } + ]) + + assert.equal(resolution.matches.length, 3) + assert.equal(resolution.resolved.length, 1) + assert.equal(resolution.misattributed.length, 1) + assert.equal(resolution.unresolved.length, 1) + assert.equal(resolution.precision, 1 / 3) +}) + +test('an answer with no markers has no matches and zero precision', () => { + const resolution = resolveCitations('A plain answer.', [context()]) + assert.deepEqual(resolution.matches, []) + assert.equal(resolution.precision, 0) +}) + +test('every marker occurrence is reported, not just the first', () => { + const resolution = resolveCitations('first [1] and again [1]', [context()]) + assert.equal(resolution.resolved.length, 2) +}) + +test('normalisation collapses whitespace and case only', () => { + assert.equal(normalizeForComparison(' Hello\n\tWorld '), 'hello world') +})