feat(citations): resolve and validate [n] markers

src/main/ipc/chatHandlers.ts

@@ -6,8 +6,9 @@ import { KnowledgeService } from '../services/KnowledgeService'
 import { buildRAGContext } from '../services/citations'
 import { validateAndCleanMessages } from '../utils/messageValidator'
 import Logger from '../../shared/utils/logger'
+import { resolveCitations } from '../../shared/utils/citationResolution'
 import type { AnswerSource, ChatMessageMetadata, RetrievalStatus } from '../../shared/types/chat'
-import type { Citation } from '../../shared/types/citation'
+import type { Citation, CitationContext } from '../../shared/types/citation'
 import { ChatSchemas, validate } from './validation'
 
 // 管理活跃的流式请求
@@ -111,6 +112,7 @@ export function registerChatHandlers(
     let retrieval: RetrievalStatus = 'none'
     let answerSources: AnswerSource[] = []
     let answerCitations: Citation[] = []
+    let citationContexts: CitationContext[] = []
     try {
       const embeddingClient = await connectionManager.getEmbeddingClient()
 
@@ -127,6 +129,12 @@ export function registerChatHandlers(
             retrieval = 'used'
             answerSources = sources
             answerCitations = citations
+            // The span a quote is checked against lives only in the locator, so
+            // carry it alongside the citation for validation (#70).
+            citationContexts = citations.map((citation, index) => ({
+              citation,
+              spanText: searchResults[index].locator.blocks.map((block) => block.text).join('\n')
+            }))
             Logger.debug(
               'ChatHandlers',
               `RAG: Found ${searchResults.length} relevant chunks for query`
@@ -155,6 +163,9 @@ export function registerChatHandlers(
       citations: answerCitations
     }
     queries.updateMessageMetadata(assistantMessage.id, answerMetadata)
+    // Rewritten at the end of the stream, once the answer text exists and its
+    // `[n]` markers can be resolved against the evidence.
+    let resolvedMetadata = answerMetadata
 
     // 4. 调用 Model Connection 流式生成
     const client = await connectionManager.getChatClient()
@@ -222,6 +233,19 @@ export function registerChatHandlers(
             usageMetadata = metadata
           }
 
+          // An answer that marked sources gets only the grounded ones: a
+          // fabricated `[9]` or a quote that is not in its span must not survive
+          // as a clickable source (#70). With no markers at all, keep the full
+          // evidence set — the model simply did not use the marker convention.
+          const resolution = resolveCitations(fullTextContent, citationContexts)
+          if (resolution.matches.length > 0) {
+            const grounded: Citation[] = []
+            for (const match of resolution.resolved) {
+              if (match.citation) grounded.push(match.citation)
+            }
+            resolvedMetadata = { ...answerMetadata, citations: grounded }
+          }
+
           // 发送完成事件
           event.sender.send('message-chunk', {
             messageId: assistantMessage.id,
@@ -230,7 +254,7 @@ export function registerChatHandlers(
             // The renderer's in-memory message never sees the DB row written
             // before streaming, so the persisted provenance rides along here or
             // the answer loses its citations until the session is reloaded.
-            messageMetadata: answerMetadata
+            messageMetadata: resolvedMetadata
           })
         }
       },
@@ -249,6 +273,9 @@ export function registerChatHandlers(
           // 更新数据库中的完整内容（包含推理内容）
           queries.updateMessageContent(assistantMessage.id, fullTextContent, fullReasoningContent)
 
+          // Persist the marker-filtered citations now that the answer text is final.
+          queries.updateMessageMetadata(assistantMessage.id, resolvedMetadata)
+
           // 计算 token 使用量
           let tokensUsed = 0
           if (usageMetadata?.usage?.totalTokens) {

src/shared/types/citation.ts

@@ -29,3 +29,38 @@ export interface Citation {
   quote: string
   score: number
 }
+
+/**
+ * 一条 citation 连同它所覆盖的原文。
+ *
+ * 校验「引文确实落在引用区间内」需要区间文本，而文本只在检索侧（`locator`）
+ * 存在，因此由调用方补齐。`spanText` 缺失时无法证伪引文 —— 解析结果保持为
+ * resolved，而不是把一个证据不足的怀疑渲染成错误。
+ */
+export interface CitationContext {
+  citation: Citation
+  spanText?: string
+}
+
+/** 回答里一个 `[n]` 标记的归宿。 */
+export type CitationMatchStatus = 'resolved' | 'unresolved' | 'misattributed'
+
+export interface CitationMatch {
+  /** 回答里的 `[n]` 编号。 */
+  marker: number
+  status: CitationMatchStatus
+  /** 标记在回答文本里的字符位置。 */
+  position: number
+  /** 仅在 `resolved` / `misattributed` 时存在。 */
+  citation?: Citation
+}
+
+/** `resolveCitations` 的结果。三组是互斥且穷尽的，因此可从中直接算出 precision。 */
+export interface CitationResolution {
+  matches: CitationMatch[]
+  resolved: CitationMatch[]
+  unresolved: CitationMatch[]
+  misattributed: CitationMatch[]
+  /** `resolved / total`；回答里没有任何标记时为 0。 */
+  precision: number
+}

src/shared/utils/citationResolution.ts

@@ -0,0 +1,111 @@
+import type {
+  Citation,
+  CitationContext,
+  CitationMatch,
+  CitationResolution
+} from '../types/citation'
+
+/**
+ * Citation resolution (#70).
+ *
+ * An answer that is asked to mark its sources with `[n]` can mark a source that
+ * was never retrieved, or attribute a passage to the wrong span. Both are
+ * first-class failure modes: a plausible-looking link to the wrong paragraph is
+ * worse than no link, because the reader cannot tell it is wrong.
+ *
+ * Resolution is **deterministic and pure** — no model call, no I/O. It maps
+ * markers to the citations produced during retrieval, so the same answer and the
+ * same evidence always produce the same result, and citation precision can be
+ * computed from the return value for the eval harness (#75).
+ */
+
+/**
+ * Case- and whitespace-insensitive comparison, and nothing beyond that.
+ *
+ * Deliberately not fuzzy: a quote that "roughly" matches is exactly the failure
+ * this guards against. Collapsing whitespace is safe because the page text is
+ * reflowed; stemming or token overlap would let a fabricated quote through.
+ */
+export function normalizeForComparison(text: string): string {
+  return text.toLowerCase().replace(/\s+/g, ' ').trim()
+}
+
+/** 标记里的编号去重后，按 prompt 位置查 citation；重复编号取第一个。 */
+function indexContexts(
+  contexts: readonly (Citation | CitationContext)[]
+): Map<number, CitationContext> {
+  const byMarker = new Map<number, CitationContext>()
+  for (const entry of contexts) {
+    const context: CitationContext = 'citation' in entry ? entry : { citation: entry }
+    if (!byMarker.has(context.citation.index)) {
+      byMarker.set(context.citation.index, context)
+    }
+  }
+  return byMarker
+}
+
+/**
+ * 校验引文是否落在区间内。
+ *
+ * 区间文本缺失、或引文为空时返回 `null`，表示「无法证伪」。把未知当成错误会
+ * 让一条正确的引用因为元数据不全而消失；把未知当成正确则是这里的默认立场，
+ * 因为 citation 本身来自检索，不是模型生成的。
+ */
+function isQuoteInSpan(context: CitationContext): boolean | null {
+  const quote = normalizeForComparison(context.citation.quote)
+  const span = context.spanText ? normalizeForComparison(context.spanText) : ''
+  if (quote.length === 0 || span.length === 0) return null
+  return span.includes(quote)
+}
+
+/**
+ * 解析回答里的 `[n]` 标记，映射回检索阶段产生的 citation。
+ *
+ * 每个标记恰好落入三类之一：
+ *
+ * - `resolved`       —— 有对应 context，且引文落在区间内（或无法证伪）。
+ * - `unresolved`     —— 没有任何 context（模型编造了 `[9]`）。
+ * - `misattributed`  —— 有 context，但引文不在其区间内。
+ *
+ * 三类都被返回，而不是把坏标记静默丢掉：precision 要能算，界面也要能选择把
+ * 它们按普通文本渲染。
+ */
+export function resolveCitations(
+  answer: string,
+  contexts: readonly (Citation | CitationContext)[]
+): CitationResolution {
+  const byMarker = indexContexts(contexts)
+  const matches: CitationMatch[] = []
+
+  // A fresh regex per call: `RegExp.lastIndex` is mutable state, and a pure
+  // resolver must not depend on how many times it has run before.
+  const pattern = /\[(\d+)\]/g
+  let found = pattern.exec(answer)
+  while (found !== null) {
+    const marker = Number(found[1])
+    const position = found.index
+    const context = byMarker.get(marker)
+
+    if (!context) {
+      matches.push({ marker, status: 'unresolved', position })
+    } else if (isQuoteInSpan(context) === false) {
+      matches.push({ marker, status: 'misattributed', position, citation: context.citation })
+    } else {
+      matches.push({ marker, status: 'resolved', position, citation: context.citation })
+    }
+
+    found = pattern.exec(answer)
+  }
+
+  const resolved = matches.filter((match) => match.status === 'resolved')
+  const unresolved = matches.filter((match) => match.status === 'unresolved')
+  const misattributed = matches.filter((match) => match.status === 'misattributed')
+
+  return {
+    matches,
+    resolved,
+    unresolved,
+    misattributed,
+    precision: matches.length === 0 ? 0 : resolved.length / matches.length
+  }
+}

test/citationResolution.test.ts

@@ -0,0 +1,113 @@
+import { test } from 'node:test'
+import assert from 'node:assert/strict'
+import { normalizeForComparison, resolveCitations } from '../src/shared/utils/citationResolution.ts'
+import type { Citation, CitationContext } from '../src/shared/types/citation.ts'
+
+/**
+ * An answer asked to mark its sources with `[n]` can mark a source that was never
+ * retrieved, or attribute a passage to the wrong span (#70). Neither may render as
+ * a clickable source. Resolution is pure and deterministic so citation precision
+ * is computable for the eval harness (#75).
+ */
+
+const citation = (over: Partial<Citation> = {}): Citation => ({
+  index: 1,
+  documentId: 'doc_1',
+  documentTitle: 'Attention Is All You Need',
+  chunkId: 'chunk_1',
+  quote: 'the attention mechanism',
+  score: 0.87,
+  ...over
+})
+
+const context = (over: Partial<CitationContext> = {}): CitationContext => ({
+  citation: citation(),
+  spanText: 'The attention mechanism is central to the transformer.',
+  ...over
+})
+
+test('a grounded marker resolves to its citation', () => {
+  const resolution = resolveCitations('The mechanism is central [1].', [context()])
+
+  assert.equal(resolution.resolved.length, 1)
+  assert.equal(resolution.unresolved.length, 0)
+  assert.equal(resolution.misattributed.length, 0)
+  assert.equal(resolution.resolved[0].marker, 1)
+  assert.equal(resolution.resolved[0].citation?.documentTitle, 'Attention Is All You Need')
+  assert.equal(resolution.precision, 1)
+})
+
+test('a fabricated marker with no context is unresolved, never clickable', () => {
+  const resolution = resolveCitations('A claim with a made-up source [9].', [context()])
+
+  assert.equal(resolution.resolved.length, 0)
+  assert.equal(resolution.unresolved.length, 1)
+  assert.equal(resolution.unresolved[0].marker, 9)
+  assert.equal(resolution.misattributed.length, 0)
+  assert.equal(resolution.precision, 0)
+})
+
+test('a marker whose span does not contain the claimed quote is flagged, not resolved', () => {
+  const wrongSpan: CitationContext = {
+    citation: citation({ index: 2, quote: 'a passage about attention' }),
+    spanText: 'an unrelated sentence about tokenisation'
+  }
+  const resolution = resolveCitations('Something [2].', [context(), wrongSpan])
+
+  assert.equal(resolution.resolved.length, 0)
+  assert.equal(resolution.misattributed.length, 1)
+  assert.equal(resolution.misattributed[0].marker, 2)
+  assert.equal(resolution.misattributed[0].citation?.chunkId, 'chunk_1')
+})
+
+test('validation is case- and whitespace-insensitive but nothing beyond that', () => {
+  const loose: CitationContext = {
+    citation: citation({ quote: 'THE   Attention\n Mechanism' }),
+    spanText: 'the attention mechanism is central'
+  }
+  assert.equal(resolveCitations('x [1]', [loose]).resolved.length, 1)
+
+  // "roughly matching" is exactly the failure this guards against: one changed
+  // word is a different claim, so it must not resolve.
+  const fuzzy: CitationContext = {
+    citation: citation({ quote: 'the attention mechanism' }),
+    spanText: 'the efficient attention mechanism differs'
+  }
+  assert.equal(resolveCitations('x [1]', [fuzzy]).misattributed.length, 1)
+})
+
+test('a missing span cannot refute the quote, so the citation resolves', () => {
+  const unknown = resolveCitations('x [1]', [{ citation: citation() }])
+  assert.equal(unknown.resolved.length, 1)
+})
+
+test('precision counts resolved markers over all markers', () => {
+  const resolution = resolveCitations('a [1] b [2] c [9]', [
+    context({ citation: citation({ index: 1 }) }),
+    {
+      citation: citation({ index: 2, quote: 'nowhere in the span' }),
+      spanText: 'span without the quote'
+    }
+  ])
+
+  assert.equal(resolution.matches.length, 3)
+  assert.equal(resolution.resolved.length, 1)
+  assert.equal(resolution.misattributed.length, 1)
+  assert.equal(resolution.unresolved.length, 1)
+  assert.equal(resolution.precision, 1 / 3)
+})
+
+test('an answer with no markers has no matches and zero precision', () => {
+  const resolution = resolveCitations('A plain answer.', [context()])
+  assert.deepEqual(resolution.matches, [])
+  assert.equal(resolution.precision, 0)
+})
+
+test('every marker occurrence is reported, not just the first', () => {
+  const resolution = resolveCitations('first [1] and again [1]', [context()])
+  assert.equal(resolution.resolved.length, 2)
+})
+
+test('normalisation collapses whitespace and case only', () => {
+  assert.equal(normalizeForComparison('  Hello\n\tWorld  '), 'hello world')
+})