diff --git a/src/main/services/ChunkingService.ts b/src/main/services/ChunkingService.ts index b5a3fc9..5c459fe 100644 --- a/src/main/services/ChunkingService.ts +++ b/src/main/services/ChunkingService.ts @@ -1,9 +1,16 @@ /** * ChunkingService - * 智能文档分块服务,保持语义完整性 + * 分块服务:按块序列切分,保留规范文本偏移。 + * + * 关键约束:chunk 的 `content` 一律是 `content.slice(startOffset, endOffset)`。 + * 分块器不再预处理文本、不再改写空白,所以 `documents.content`(每个文档唯一的 + * 规范字符串)与 `chunks.start_offset/end_offset` 永远指向同一份字节。旧的 + * `preprocessText()` 会把偏移算到清洗后的字符串上,这正是"能看到来源标题却无法 + * 高亮来源"的根因。 */ import Logger from '../../shared/utils/logger' +import type { BlockKind } from './blocks/documentBlocks' /** * 分块选项 @@ -11,30 +18,67 @@ import Logger from '../../shared/utils/logger' export interface ChunkOptions { chunkSize?: number // 每块的目标字符数,默认 500 chunkOverlap?: number // 块之间的重叠字符数,默认 50 - separators?: string[] // 分隔符优先级列表 - minChunkSize?: number // 最小块大小,默认 100 + separators?: string[] // 回退窗口在何处断开的优先级列表 + minChunkSize?: number // 整篇文本不超过它时直接作为一块 + allowSpanPages?: boolean // 允许一个 chunk 跨页,默认 false(分页文档在页边界断开) +} + +/** + * 分块器接受的块。字段是 `document_blocks` 的子集。 + */ +export interface ChunkBlock { + id: string + kind: BlockKind + page: number | null + level: number | null + text: string + startOffset: number + endOffset: number +} + +/** 一个 chunk 覆盖的某个块内的字符区间。 */ +export interface ChunkBlockSpan { + blockId: string + startInBlock: number + endInBlock: number } /** * 分块结果 */ export interface ChunkResult { - content: string // 块内容 + content: string // 块内容(规范文本的精确切片) index: number // 块索引 - startOffset: number // 原文起始位置 - endOffset: number // 原文结束位置 + startOffset: number // 规范文本起始位置 + endOffset: number // 规范文本结束位置 tokenCount: number // 估算 token 数 + blockSpans: ChunkBlockSpan[] // 覆盖的块及其区间(按文档顺序) + pageStart: number | null // 覆盖块的最小页码;无分页块时为 null + pageEnd: number | null // 覆盖块的最大页码;无分页块时为 null +} + +interface Unit { + blockIndex: number + start: number // 规范文本偏移 + end: number + page: number | null +} + +interface Range { + start: number + end: number } /** * 文档分块服务 - * 支持智能分块,保持语义完整性 + * 支持块感知分块(保留来源结构),以及无块时的字符窗口回退。 */ export class ChunkingService { private defaultOptions: Required = { chunkSize: 500, chunkOverlap: 50, minChunkSize: 100, + allowSpanPages: false, separators: [ '\n\n\n', // 多个空行(章节分隔) '\n\n', // 段落分隔 @@ -54,81 +98,80 @@ export class ChunkingService { } /** - * 对文本进行分块 + * 块感知分块。偏移与内容都锚定在 `content` 这份规范字符串上。 + * + * 分块边界只落在语义单元之间(标题 → 段落 → 行 → 句子 → 空白),标题块是原子 + * 单元,永远不会被切开;分页文档默认不跨页。`blockSpans` 记录每个 chunk 覆盖 + * 的块及其区间,供调用方持久化。 + */ + chunkBlocks(content: string, blocks: ChunkBlock[], options?: ChunkOptions): ChunkResult[] { + const opts = { ...this.defaultOptions, ...options } + const ordered = [...blocks] + .filter((block) => block.endOffset > block.startOffset) + .sort((a, b) => a.startOffset - b.startOffset) + + if (ordered.length === 0) { + return this.chunk(content, options) + } + + const units = this.buildUnits(ordered, opts.chunkSize) + const packed = this.packUnits(content, ordered, units, opts) + + const chunks = packed.map((chunk, index) => ({ + ...chunk, + index, + tokenCount: this.estimateTokens(chunk.content) + })) + + Logger.debug('ChunkingService', `Split ${ordered.length} blocks into ${chunks.length} chunks`) + return chunks + } + + /** + * 无块回退:按字符窗口切分规范文本。不预处理、不改写长度,偏移可精确切回。 */ chunk(text: string, options?: ChunkOptions): ChunkResult[] { const opts = { ...this.defaultOptions, ...options } const { chunkSize, chunkOverlap, separators, minChunkSize } = opts - // 预处理:移除多余空白 - const cleanedText = this.preprocessText(text) + if (!text || text.trim().length === 0) return [] - if (!cleanedText || cleanedText.length === 0) { - return [] - } + const chunks: ChunkResult[] = [] + const push = (start: number, end: number): void => { + while (start < end && isWhitespace(text[start])) start++ + while (end > start && isWhitespace(text[end - 1])) end-- + if (end <= start) return - // 如果文本小于最小块大小,直接返回整个文本 - if (cleanedText.length <= minChunkSize) { - return [ - { - content: cleanedText, - index: 0, - startOffset: 0, - endOffset: cleanedText.length, - tokenCount: this.estimateTokens(cleanedText) - } - ] + const content = text.slice(start, end) + chunks.push({ + content, + index: chunks.length, + startOffset: start, + endOffset: end, + tokenCount: this.estimateTokens(content), + blockSpans: [], + pageStart: null, + pageEnd: null + }) } - const chunks: ChunkResult[] = [] - let currentStart = 0 - - while (currentStart < cleanedText.length) { - let currentEnd = Math.min(currentStart + chunkSize, cleanedText.length) - - // 如果不是文本末尾,尝试在分隔符处断开 - if (currentEnd < cleanedText.length) { - const searchEnd = currentEnd - const searchStart = Math.max(currentStart + Math.floor(chunkSize * 0.5), currentStart) - - let bestSplitPos = -1 - let bestSeparatorPriority = separators.length - - // 在范围内查找最佳分割点 - for (let i = searchEnd; i >= searchStart; i--) { - for (let j = 0; j < separators.length; j++) { - const sep = separators[j] - if (cleanedText.slice(i, i + sep.length) === sep) { - if (j < bestSeparatorPriority) { - bestSplitPos = i + sep.length - bestSeparatorPriority = j - } - break - } - } - // 找到高优先级分隔符就停止 - if (bestSeparatorPriority <= 2) break - } + if (text.length <= minChunkSize) { + push(0, text.length) + return chunks + } - if (bestSplitPos > currentStart) { - currentEnd = bestSplitPos - } + let start = 0 + while (start < text.length) { + let end = Math.min(start + chunkSize, text.length) + if (end < text.length) { + const split = this.findSeparatorSplit(text, start, end, separators) + if (split > start) end = split } - const content = cleanedText.slice(currentStart, currentEnd).trim() - - if (content.length >= minChunkSize) { - chunks.push({ - content, - index: chunks.length, - startOffset: currentStart, - endOffset: currentEnd, - tokenCount: this.estimateTokens(content) - }) - } + push(start, end) + if (end >= text.length) break - // 计算下一块的起始位置(考虑重叠) - currentStart = Math.max(currentEnd - chunkOverlap, currentStart + 1) + start = Math.max(end - chunkOverlap, start + 1) } Logger.debug('ChunkingService', `Split text into ${chunks.length} chunks`) @@ -136,90 +179,183 @@ export class ChunkingService { } /** - * 按句子分块(更保守的分块策略) + * 把块序列拆成语义单元。标题块整体作为一个原子单元;段落按行、句子、空白逐步 + * 细分,但绝不跨越块边界(块边界本身就是段落级语义边界)。 */ - chunkBySentence(text: string, options?: ChunkOptions): ChunkResult[] { - const opts = { ...this.defaultOptions, ...options } + private buildUnits(blocks: ChunkBlock[], chunkSize: number): Unit[] { + const units: Unit[] = [] + + blocks.forEach((block, blockIndex) => { + const page = block.page + const push = (start: number, end: number, trim: boolean): void => { + if (trim) { + while (start < end && isWhitespace(block.text[start])) start++ + while (end > start && isWhitespace(block.text[end - 1])) end-- + } + if (end > start) { + units.push({ + blockIndex, + start: block.startOffset + start, + end: block.startOffset + end, + page + }) + } + } - // 分句 - const sentences = this.splitIntoSentences(text) - const chunks: ChunkResult[] = [] - let currentChunk: string[] = [] - let currentLength = 0 - let currentStartOffset = 0 - - for (const sentence of sentences) { - const sentenceLength = sentence.length - - if (currentLength + sentenceLength > opts.chunkSize && currentChunk.length > 0) { - // 保存当前块 - const content = currentChunk.join('') - chunks.push({ - content, - index: chunks.length, - startOffset: currentStartOffset, - endOffset: currentStartOffset + content.length, - tokenCount: this.estimateTokens(content) - }) - - // 开始新块(可以考虑重叠) - currentStartOffset += content.length - currentChunk = [] - currentLength = 0 + if (block.kind === 'heading') { + push(0, block.text.length, true) + return + } + + // 代码/表格保留行首缩进;其余块的行首行尾空白不属于内容 + const preserveWhitespace = block.kind === 'code' || block.kind === 'table' + + for (const line of lineRanges(block.text)) { + if (line.end - line.start <= chunkSize) { + push(line.start, line.end, !preserveWhitespace) + continue + } + + for (const sentence of sentenceRanges(block.text, line.start, line.end)) { + if (sentence.end - sentence.start <= chunkSize) { + push(sentence.start, sentence.end, !preserveWhitespace) + continue + } + for (const piece of hardSplit(block.text, sentence.start, sentence.end, chunkSize)) { + push(piece.start, piece.end, !preserveWhitespace) + } + } } + }) + + return units + } - currentChunk.push(sentence) - currentLength += sentenceLength + /** + * 贪心装箱:装满 chunkSize 前尽量合并单元;页边界处默认断开;重叠按规范偏移对齐 + * 到单元起点,所以重叠区域两块的字节完全一致。 + */ + private packUnits( + content: string, + blocks: ChunkBlock[], + units: Unit[], + opts: Required + ): Array> { + const chunks: Array> = [] + let current: Unit[] = [] + + const flush = (): void => { + if (current.length === 0) return + chunks.push(this.materialize(content, blocks, current)) + current = [] } - // 处理最后一个块 - if (currentChunk.length > 0) { - const content = currentChunk.join('') - chunks.push({ - content, - index: chunks.length, - startOffset: currentStartOffset, - endOffset: currentStartOffset + content.length, - tokenCount: this.estimateTokens(content) - }) + for (const unit of units) { + if (current.length === 0) { + current.push(unit) + continue + } + + const pageBreak = + !opts.allowSpanPages && + unit.page !== null && + current[current.length - 1].page !== null && + unit.page !== current[current.length - 1].page + const wouldOverflow = unit.end - current[0].start > opts.chunkSize + + if (pageBreak || wouldOverflow) { + const overlap = pageBreak ? [] : this.overlapUnits(current, opts.chunkOverlap) + flush() + current = overlap + } + + current.push(unit) } + flush() return chunks } - /** - * 预处理文本 - */ - private preprocessText(text: string): string { - return ( - text - // 统一换行符 - .replace(/\r\n/g, '\n') - .replace(/\r/g, '\n') - // 移除连续的多余空白行(保留最多两个换行) - .replace(/\n{4,}/g, '\n\n\n') - // 移除行首行尾空白 - .trim() - ) + /** 从已满 chunk 的尾部取不超过 chunkOverlap 的单元作为下一块的开头。 */ + private overlapUnits(current: Unit[], chunkOverlap: number): Unit[] { + if (chunkOverlap <= 0) return [] + + const end = current[current.length - 1].end + const picked: Unit[] = [] + for (let i = current.length - 1; i >= 0; i--) { + if (end - current[i].start > chunkOverlap) break + picked.unshift(current[i]) + } + + // 不要把整个上一块当作重叠,否则新块只会不停变长 + if (picked.length === current.length && picked.length > 1) picked.shift() + return picked } - /** - * 分句 - */ - private splitIntoSentences(text: string): string[] { - // 使用正则匹配句子结束符 - const sentenceEndings = /([。!?.!?]+)/g - const parts = text.split(sentenceEndings) - const sentences: string[] = [] - - for (let i = 0; i < parts.length; i += 2) { - const sentence = parts[i] + (parts[i + 1] || '') - if (sentence.trim()) { - sentences.push(sentence) + /** 依据单元范围切出 chunk 内容与块区间。内容永远是规范文本的精确切片。 */ + private materialize( + content: string, + blocks: ChunkBlock[], + units: Unit[] + ): Omit { + const startOffset = units[0].start + const endOffset = units[units.length - 1].end + const pieces = content.slice(startOffset, endOffset) + + const blockSpans: ChunkBlockSpan[] = [] + const pages: number[] = [] + + for (const unit of units) { + const block = blocks[unit.blockIndex] + if (block.page !== null) pages.push(block.page) + + const startInBlock = unit.start - block.startOffset + const endInBlock = unit.end - block.startOffset + const last = blockSpans[blockSpans.length - 1] + + if (last && last.blockId === block.id) { + last.startInBlock = Math.min(last.startInBlock, startInBlock) + last.endInBlock = Math.max(last.endInBlock, endInBlock) + } else { + blockSpans.push({ blockId: block.id, startInBlock, endInBlock }) } } - return sentences + return { + content: pieces, + startOffset, + endOffset, + blockSpans, + pageStart: pages.length > 0 ? Math.min(...pages) : null, + pageEnd: pages.length > 0 ? Math.max(...pages) : null + } + } + + /** 在 [start, end) 内从右向左找优先级最高的分隔符,返回断开位置。 */ + private findSeparatorSplit( + text: string, + start: number, + end: number, + separators: string[] + ): number { + const searchStart = Math.max(start + Math.floor((end - start) / 2), start) + let bestPos = -1 + let bestPriority = separators.length + + for (let i = end; i >= searchStart; i--) { + for (let j = 0; j < separators.length; j++) { + if (text.startsWith(separators[j], i)) { + if (j < bestPriority) { + bestPos = i + separators[j].length + bestPriority = j + } + break + } + } + if (bestPriority <= 2) break + } + + return bestPos > start ? bestPos : end } /** @@ -260,3 +396,73 @@ export class ChunkingService { this.defaultOptions = { ...this.defaultOptions, ...options } } } + +/** 文本的每一行(不含换行符)的相对区间。 */ +function lineRanges(text: string): Range[] { + const ranges: Range[] = [] + let start = 0 + + for (let i = 0; i <= text.length; i++) { + if (i === text.length || text[i] === '\n') { + ranges.push({ start, end: i }) + start = i + 1 + } + } + + return ranges +} + +/** 在 [from, to) 内按句末标点切分。 */ +function sentenceRanges(text: string, from: number, to: number): Range[] { + const ranges: Range[] = [] + let start = from + + for (let i = from; i < to; i++) { + if (!isSentenceEnd(text[i])) continue + + let end = i + 1 + while (end < to && isSentenceEnd(text[end])) end++ + ranges.push({ start, end }) + start = end + i = end - 1 + } + + if (start < to) ranges.push({ start, end: to }) + return ranges +} + +/** 超长片段的最后手段:按空白装箱,单词本身仍超长时硬切。 */ +function hardSplit(text: string, from: number, to: number, maxLength: number): Range[] { + const ranges: Range[] = [] + let start = from + let i = from + let lastWhitespace = -1 + + while (i < to) { + if (i - start >= maxLength) { + const cut = lastWhitespace > start ? lastWhitespace : i + ranges.push({ start, end: cut }) + start = cut + while (start < to && isWhitespace(text[start])) start++ + i = start + lastWhitespace = -1 + continue + } + + if (isWhitespace(text[i])) lastWhitespace = i + i++ + } + + if (start < to) ranges.push({ start, end: to }) + return ranges +} + +function isSentenceEnd(char: string): boolean { + return ( + char === '。' || char === '!' || char === '?' || char === '.' || char === '!' || char === '?' + ) +} + +function isWhitespace(char: string): boolean { + return char === ' ' || char === '\t' || char === '\n' || char === '\r' +} diff --git a/src/main/services/KnowledgeService.ts b/src/main/services/KnowledgeService.ts index c89c181..89d90d6 100644 --- a/src/main/services/KnowledgeService.ts +++ b/src/main/services/KnowledgeService.ts @@ -35,7 +35,11 @@ import { EmbeddingService } from './EmbeddingService' import type { EmbeddingSpace } from '../../shared/types' import { ChunkingService, type ChunkOptions } from './ChunkingService' import { FileParserService } from './FileParserService' -import { buildDocumentBlocks, type DocumentBlockDraft } from './blocks/documentBlocks' +import { + buildDocumentBlocks, + assignBlockIds, + type IdentifiedBlockDraft +} from './blocks/documentBlocks' import { WebFetchService } from './WebFetchService' import { vectorStoreManager } from '../vectorstore' import Logger from '../../shared/utils/logger' @@ -196,12 +200,17 @@ export class KnowledgeService { db.insert(documents).values(newDoc).run() - // 1b. 持久化文档块(文本/URL/笔记没有结构,按段落平铺) - this.persistDocumentBlocks(documentId, buildDocumentBlocks({ content: options.content })) + // 1b. 持久化文档块(文本/URL/笔记没有结构,按段落平铺),并把同一批块交给分块器 + const blocks = assignBlockIds(documentId, buildDocumentBlocks({ content: options.content })) + this.persistDocumentBlocks(blocks) - // 2. 分块 + // 2. 分块(偏移锚定 options.content,即 documents.content) onProgress?.('chunking', 10) - const chunkResults = this.chunkingService.chunk(options.content, options.chunkOptions) + const chunkResults = this.chunkingService.chunkBlocks( + options.content, + blocks, + options.chunkOptions + ) if (chunkResults.length === 0) { throw new Error('No chunks generated from document') @@ -383,15 +392,16 @@ export class KnowledgeService { db.insert(documents).values(newDoc).run() - // 1b. 持久化文档块,保留解析器给出的页/标题结构 - this.persistDocumentBlocks( + // 1b. 持久化文档块,保留解析器给出的页/标题结构,并把同一批块交给分块器 + const blocks = assignBlockIds( documentId, buildDocumentBlocks({ content: parseResult.content, structure: parseResult.structure }) ) + this.persistDocumentBlocks(blocks) - // 2. 分块 + // 2. 分块(偏移锚定 parseResult.content,即 documents.content) onProgress?.('chunking', 10) - const chunkResults = this.chunkingService.chunk(parseResult.content) + const chunkResults = this.chunkingService.chunkBlocks(parseResult.content, blocks) if (chunkResults.length === 0) { throw new Error('No chunks generated from document') @@ -525,28 +535,27 @@ export class KnowledgeService { } /** - * 持久化文档块。块 id 由 documentId 与顺序确定,天然唯一且可读; - * 重新索引会换新 documentId,不会与旧块冲突。 + * 持久化文档块。id 已由 `assignBlockIds()` 生成,与分块器看到的是同一批块。 */ - private persistDocumentBlocks(documentId: string, drafts: DocumentBlockDraft[]): void { - if (drafts.length === 0) return - - const rows: NewDocumentBlock[] = drafts.map((draft) => ({ - id: `blk_${documentId}_${draft.order}`, - documentId, - kind: draft.kind, - order: draft.order, - page: draft.page, - level: draft.level, - text: draft.text, - startOffset: draft.startOffset, - endOffset: draft.endOffset, - bbox: draft.bbox, - metadata: draft.metadata + private persistDocumentBlocks(blocks: IdentifiedBlockDraft[]): void { + if (blocks.length === 0) return + + const rows: NewDocumentBlock[] = blocks.map((block) => ({ + id: block.id, + documentId: block.documentId, + kind: block.kind, + order: block.order, + page: block.page, + level: block.level, + text: block.text, + startOffset: block.startOffset, + endOffset: block.endOffset, + bbox: block.bbox, + metadata: block.metadata })) getDatabase().insert(documentBlocks).values(rows).run() - Logger.debug('KnowledgeService', `Document ${documentId}: ${rows.length} blocks`) + Logger.debug('KnowledgeService', `Document ${blocks[0].documentId}: ${rows.length} blocks`) } /** diff --git a/src/main/services/blocks/documentBlocks.ts b/src/main/services/blocks/documentBlocks.ts index 6b5bcb5..743266f 100644 --- a/src/main/services/blocks/documentBlocks.ts +++ b/src/main/services/blocks/documentBlocks.ts @@ -29,6 +29,27 @@ export interface DocumentBlockDraft { metadata: Record | null } +/** 已分配 id 的块;持久化与分块共用同一批对象,id 只生成一次。 */ +export interface IdentifiedBlockDraft extends DocumentBlockDraft { + id: string + documentId: string +} + +/** + * 给块分配 id。id 由 documentId 与顺序确定,天然唯一且可读;重新索引会换新 + * documentId,不会与旧块冲突。 + */ +export function assignBlockIds( + documentId: string, + drafts: DocumentBlockDraft[] +): IdentifiedBlockDraft[] { + return drafts.map((draft) => ({ + ...draft, + id: `blk_${documentId}_${draft.order}`, + documentId + })) +} + export interface BlockSource { content: string structure?: DocumentStructure diff --git a/test/chunkBlocks.test.ts b/test/chunkBlocks.test.ts new file mode 100644 index 0000000..b5b5cbb --- /dev/null +++ b/test/chunkBlocks.test.ts @@ -0,0 +1,258 @@ +import { test } from 'node:test' +import assert from 'node:assert/strict' +import { join } from 'node:path' +import { fileURLToPath } from 'node:url' +import { PdfLoader } from '../src/main/services/loaders/PdfLoader.ts' +import { + assignBlockIds, + buildDocumentBlocks, + type BlockKind +} from '../src/main/services/blocks/documentBlocks.ts' +import { + ChunkingService, + type ChunkBlock, + type ChunkResult +} from '../src/main/services/ChunkingService.ts' + +/** + * The chunker is the safety net for the whole provenance epic: if + * `chunks.start_offset/end_offset` stop addressing `documents.content`, no + * citation can ever highlight the right text again. Every test here asserts the + * contract + * + * content.slice(startOffset, endOffset) === chunk.content + * + * plus the properties `#67`/`#68` must preserve: dense indexing, block spans that + * point back into real blocks, page ranges, overlap that shares exact bytes, and + * headings that are never split. + */ + +const FIXTURES = fileURLToPath(new URL('./fixtures/', import.meta.url)) +const fixture = (name: string): string => join(FIXTURES, name) + +interface Synthetic { + content: string + blocks: ChunkBlock[] +} + +/** A small heading + paragraph document with hand-computed canonical offsets. */ +function synthetic(): Synthetic { + const heading1 = 'Overview' + const paragraph1 = 'First paragraph sentence one. Sentence two.' + const heading2 = 'Details' + const paragraph2 = 'Second paragraph with more text.' + const content = [heading1, paragraph1, heading2, paragraph2].join('\n\n') + + const make = ( + id: string, + kind: BlockKind, + level: number | null, + text: string, + page: number | null = null + ): ChunkBlock => { + const startOffset = content.indexOf(text) + return { id, kind, level, page, text, startOffset, endOffset: startOffset + text.length } + } + + return { + content, + blocks: [ + make('b1', 'heading', 1, heading1), + make('b2', 'paragraph', null, paragraph1), + make('b3', 'heading', 2, heading2), + make('b4', 'paragraph', null, paragraph2) + ] + } +} + +function assertCanonicalOffsets(content: string, chunks: ChunkResult[]): void { + assert.deepEqual( + chunks.map((chunk) => chunk.index), + chunks.map((_, index) => index), + 'chunk index must be dense and ascending' + ) + + for (const chunk of chunks) { + assert.equal( + content.slice(chunk.startOffset, chunk.endOffset), + chunk.content, + `chunk ${chunk.index} does not slice back to documents.content` + ) + assert.ok(chunk.tokenCount > 0, `chunk ${chunk.index} has no token estimate`) + } +} + +function assertSpansResolve(content: string, blocks: ChunkBlock[], chunks: ChunkResult[]): void { + const order = new Map(blocks.map((block, index) => [block.id, index])) + + for (const chunk of chunks) { + let previousOrder = -1 + + for (const span of chunk.blockSpans) { + const block = blocks.find((candidate) => candidate.id === span.blockId) + assert.ok(block, `chunk ${chunk.index} references unknown block ${span.blockId}`) + assert.ok( + span.startInBlock >= 0 && span.endInBlock <= block.text.length, + `chunk ${chunk.index} span for ${span.blockId} leaves the block` + ) + assert.ok(span.endInBlock > span.startInBlock, 'span must not be empty') + + const spanText = content.slice( + block.startOffset + span.startInBlock, + block.startOffset + span.endInBlock + ) + assert.ok( + chunk.content.includes(spanText), + `chunk ${chunk.index} content does not contain the text of ${span.blockId}` + ) + + const blockOrder = order.get(span.blockId)! + assert.ok(blockOrder > previousOrder, 'block spans must follow document order') + previousOrder = blockOrder + } + } +} + +test('chunk offsets and content address the canonical document exactly', () => { + const { content, blocks } = synthetic() + const chunks = new ChunkingService().chunkBlocks(content, blocks, { chunkSize: 40 }) + + assert.ok(chunks.length > 1, 'a small chunk size must produce several chunks') + assertCanonicalOffsets(content, chunks) + assertSpansResolve(content, blocks, chunks) +}) + +test('a heading block is never split, even when it forces a chunk boundary', () => { + const { content, blocks } = synthetic() + const chunks = new ChunkingService().chunkBlocks(content, blocks, { + chunkSize: 20, + chunkOverlap: 5 + }) + + assertCanonicalOffsets(content, chunks) + + const headings = new Map(blocks.filter((b) => b.kind === 'heading').map((b) => [b.id, b])) + let seen = 0 + + for (const chunk of chunks) { + for (const span of chunk.blockSpans) { + const heading = headings.get(span.blockId) + if (!heading) continue + seen++ + assert.equal(span.startInBlock, 0, `chunk ${chunk.index} starts inside a heading`) + assert.equal( + span.endInBlock, + heading.text.length, + `chunk ${chunk.index} cut heading ${heading.id} short` + ) + } + } + + assert.ok(seen >= headings.size, 'every heading should be covered at least once') +}) + +test('overlapping chunks share identical bytes', () => { + const { content, blocks } = synthetic() + const chunks = new ChunkingService().chunkBlocks(content, blocks, { + chunkSize: 30, + chunkOverlap: 15 + }) + + let overlaps = 0 + for (let i = 0; i < chunks.length - 1; i++) { + const current = chunks[i] + const next = chunks[i + 1] + if (next.startOffset >= current.endOffset) continue + + overlaps++ + const fromCurrent = current.content.slice(next.startOffset - current.startOffset) + const fromNext = next.content.slice(0, current.endOffset - next.startOffset) + assert.equal( + fromCurrent, + fromNext, + `chunk ${i} and ${i + 1} do not share the same overlap text` + ) + } + + assert.ok(overlaps > 0, 'expected at least one overlapping chunk pair') +}) + +test('PDF chunks stay on one page and report the page range', async () => { + const result = await new PdfLoader().loadFromPath(fixture('multipage.pdf')) + const blocks = assignBlockIds( + 'doc_pdf', + buildDocumentBlocks({ content: result.content, structure: result.structure }) + ) + const chunks = new ChunkingService().chunkBlocks(result.content, blocks) + + assertCanonicalOffsets(result.content, chunks) + assertSpansResolve(result.content, blocks, chunks) + + assert.deepEqual( + chunks.map((chunk) => [chunk.pageStart, chunk.pageEnd]), + [ + [1, 1], + [2, 2], + [3, 3] + ] + ) +}) + +test('cross-page chunks only appear when explicitly allowed', async () => { + const result = await new PdfLoader().loadFromPath(fixture('multipage.pdf')) + const blocks = assignBlockIds( + 'doc_pdf', + buildDocumentBlocks({ content: result.content, structure: result.structure }) + ) + const service = new ChunkingService() + + const merged = service.chunkBlocks(result.content, blocks, { allowSpanPages: true }) + assert.equal(merged.length, 1, 'a large chunk size with allowSpanPages must merge the pages') + assert.equal(merged[0].pageStart, 1) + assert.equal(merged[0].pageEnd, 3) + assertCanonicalOffsets(result.content, merged) +}) + +test('indexing test/fixtures/sample.pdf yields chunks that resolve to its blocks', async () => { + const result = await new PdfLoader().loadFromPath(fixture('sample.pdf')) + const blocks = assignBlockIds( + 'doc_sample', + buildDocumentBlocks({ content: result.content, structure: result.structure }) + ) + const chunks = new ChunkingService().chunkBlocks(result.content, blocks) + + assert.ok(chunks.length >= 1) + assertCanonicalOffsets(result.content, chunks) + assertSpansResolve(result.content, blocks, chunks) + + // The single page is one block, so the chunk must resolve to exactly that page. + for (const chunk of chunks) { + assert.equal(chunk.pageStart, 1) + assert.equal(chunk.pageEnd, 1) + } +}) + +test('the char-window fallback also produces canonical offsets and overlap', () => { + const content = Array.from({ length: 40 }, (_, i) => `Sentence number ${i}.`).join(' ') + const chunks = new ChunkingService().chunk(content, { chunkSize: 120, chunkOverlap: 40 }) + + assert.ok(chunks.length > 1) + assertCanonicalOffsets(content, chunks) + assert.ok(chunks.every((chunk) => chunk.blockSpans.length === 0)) + + for (let i = 0; i < chunks.length - 1; i++) { + const current = chunks[i] + const next = chunks[i + 1] + assert.ok(next.startOffset < current.endOffset, 'windows should overlap') + assert.equal( + current.content.slice(next.startOffset - current.startOffset), + next.content.slice(0, current.endOffset - next.startOffset) + ) + } +}) + +test('no blocks and no text produces no chunks', () => { + const service = new ChunkingService() + assert.deepEqual(service.chunkBlocks('', []), []) + assert.deepEqual(service.chunk(' \n\n '), []) +})