diff --git a/package.json b/package.json index 6fbd153..4e3826e 100644 --- a/package.json +++ b/package.json @@ -21,7 +21,7 @@ "typecheck:web": "tsc --noEmit -p tsconfig.web.json --composite false", "typecheck:test": "tsc -p tsconfig.test.json", "typecheck": "npm run typecheck:node && npm run typecheck:web && npm run typecheck:test", - "test": "node --import ./test/ts-resolve.mjs --disable-warning=MODULE_TYPELESS_PACKAGE_JSON --test \"test/**/*.test.ts\"", + "test": "node --experimental-transform-types --disable-warning=ExperimentalWarning --import ./test/ts-resolve.mjs --disable-warning=MODULE_TYPELESS_PACKAGE_JSON --test \"test/**/*.test.ts\"", "start": "electron-vite preview", "dev": "electron-vite dev", "build": "npm run typecheck && electron-vite build", @@ -36,6 +36,13 @@ "db:push": "drizzle-kit push", "db:studio": "drizzle-kit studio" }, + "//test": [ + "`node --test` strips TypeScript types rather than compiling them, and strip-only", + "mode rejects the enums the production sources use (src/shared/utils/logger.ts is", + "on every loader's import path). `--experimental-transform-types` lets those", + "modules load unmodified, so parser tests can import the real loaders instead of", + "a test-only copy." + ], "//dependencies": [ "ONLY packages that cannot be bundled belong here.", "electron-vite defaults build.externalizeDeps to true, so every `dependencies`", diff --git a/test/fixtures/headings.docx b/test/fixtures/headings.docx new file mode 100644 index 0000000..0d8bd68 Binary files /dev/null and b/test/fixtures/headings.docx differ diff --git a/test/fixtures/multipage.pdf b/test/fixtures/multipage.pdf new file mode 100644 index 0000000..f3d6431 Binary files /dev/null and b/test/fixtures/multipage.pdf differ diff --git a/test/fixtures/sample.md b/test/fixtures/sample.md new file mode 100644 index 0000000..d99f4e4 --- /dev/null +++ b/test/fixtures/sample.md @@ -0,0 +1,11 @@ +# Sample Markdown + +Intro paragraph for the fixture. + +## Section One + +Body of section one. + +## Section Two + +Body of section two. diff --git a/test/parsers.test.ts b/test/parsers.test.ts new file mode 100644 index 0000000..627c902 --- /dev/null +++ b/test/parsers.test.ts @@ -0,0 +1,258 @@ +import { test } from 'node:test' +import assert from 'node:assert/strict' +import { readFile } from 'node:fs/promises' +import { join } from 'node:path' +import { fileURLToPath } from 'node:url' +import { FileParserService } from '../src/main/services/FileParserService.ts' +import { PdfLoader } from '../src/main/services/loaders/PdfLoader.ts' +import { DocxLoader } from '../src/main/services/loaders/DocxLoader.ts' +import { MarkdownLoader } from '../src/main/services/loaders/MarkdownLoader.ts' +import { WebLoader } from '../src/main/services/loaders/WebLoader.ts' +import type { DocumentLoadResult, SectionInfo } from '../src/main/services/loaders/types.ts' + +/** + * Golden-file tests for the document loaders. + * + * `DocumentLoadResult.structure` is about to become load-bearing for provenance + * (#66/#67) and nothing under `test/` exercised a loader before this file. These + * tests parse the committed fixtures in `test/fixtures/` and assert the *shape* + * of the result — page count, page boundaries, heading outlines, offset + * arithmetic — rather than the extracted prose. A slightly different extraction + * still passes; a loader that stops emitting `structure`, miscounts a page + * boundary, or loses a heading level fails. + * + * Fixtures: + * multipage.pdf three pages, one text run each (generated for this issue) + * sample.pdf the existing single-page smoke fixture + * headings.docx minimal OOXML with Heading1/Heading2 styles (generated) + * sample.docx the existing fixture, deliberately heading-less + * sample.md ATX headings: one h1 and two h2 + * sample.html the existing Readability smoke fixture + */ + +const FIXTURES = fileURLToPath(new URL('./fixtures/', import.meta.url)) +const fixture = (name: string): string => join(FIXTURES, name) + +/** The committed outline each fixture is expected to produce. */ +const DOCX_OUTLINE = [ + { level: 1, title: 'Overview' }, + { level: 2, title: 'Details' }, + { level: 1, title: 'Summary' } +] +const MARKDOWN_OUTLINE = [ + { level: 1, title: 'Sample Markdown' }, + { level: 2, title: 'Section One' }, + { level: 2, title: 'Section Two' } +] + +/** `structure` is optional on the result, so every structural test has to ask for it. */ +function requirePages(result: DocumentLoadResult) { + assert.ok(result.structure, 'the loader must populate `structure`') + assert.equal(result.structure.type, 'pages') + assert.ok(result.structure.pages, 'a `pages` structure must carry `pages`') + return result.structure.pages +} + +function requireSections(result: DocumentLoadResult) { + assert.ok(result.structure, 'the loader must populate `structure`') + assert.equal(result.structure.type, 'sections') + assert.ok(result.structure.sections, 'a `sections` structure must carry `sections`') + return result.structure.sections +} + +function flattenSections(sections: SectionInfo[]): SectionInfo[] { + const flat: SectionInfo[] = [] + const walk = (nodes: SectionInfo[]): void => { + for (const node of nodes) { + flat.push(node) + if (node.children?.length) walk(node.children) + } + } + walk(sections) + return flat +} + +function outline(sections: SectionInfo[]): { level: number; title: string }[] { + return flattenSections(sections).map(({ level, title }) => ({ level, title })) +} + +/** + * Siblings must not overlap, children must stay inside their parent, and no + * range may be inverted. This is the invariant #66/#67 are most likely to break + * while they rework how offsets are computed. + */ +function assertSectionOffsets(nodes: SectionInfo[], label: string, parent?: SectionInfo): void { + nodes.forEach((node, index) => { + assert.ok(node.endOffset >= node.startOffset, `${label}: ${node.title} has an inverted range`) + if (parent) { + assert.ok( + node.startOffset >= parent.startOffset && node.endOffset <= parent.endOffset, + `${label}: ${node.title} escapes its parent ${parent.title}` + ) + } + if (index > 0) { + assert.ok( + node.startOffset >= nodes[index - 1].endOffset, + `${label}: ${node.title} overlaps ${nodes[index - 1].title}` + ) + } + if (node.children?.length) assertSectionOffsets(node.children, label, node) + }) +} + +// --- PDF --------------------------------------------------------------------- + +test('PDF: pages are numbered in order and their offsets slice back to the page text', async () => { + const result = await new PdfLoader().loadFromPath(fixture('multipage.pdf')) + assert.equal(result.mimeType, 'application/pdf') + assert.equal(result.metadata?.pageCount, 3) + + const pages = requirePages(result) + assert.equal(pages.length, 3) + assert.deepEqual( + pages.map((page) => page.pageNumber), + [1, 2, 3] + ) + + for (const page of pages) { + assert.equal( + result.content.slice(page.startOffset, page.endOffset), + page.content, + `page ${page.pageNumber} offsets do not slice back to its text` + ) + } + + for (let i = 1; i < pages.length; i++) { + assert.ok(pages[i].startOffset >= pages[i - 1].endOffset, 'page boundaries overlap') + assert.ok(pages[i].startOffset >= pages[i - 1].startOffset, 'page offsets are not monotonic') + } + assert.ok( + pages[pages.length - 1].endOffset <= result.content.length, + 'the last page boundary runs past the document' + ) +}) + +test('PDF: the existing single-page fixture keeps its text and page box', async () => { + const result = await new PdfLoader().loadFromPath(fixture('sample.pdf')) + assert.equal(result.metadata?.pageCount, 1) + + const pages = requirePages(result) + assert.equal(pages.length, 1) + assert.match(pages[0].content, /KnowNote Import Test/) + assert.equal(result.content.slice(pages[0].startOffset, pages[0].endOffset), pages[0].content) + assert.equal(pages[0].metadata?.width, 612) + assert.equal(pages[0].metadata?.height, 792) +}) + +// --- DOCX -------------------------------------------------------------------- + +test('DOCX: heading levels and their nesting survive the round trip', async () => { + const result = await new DocxLoader().loadFromPath(fixture('headings.docx')) + assert.equal(result.title, 'Overview') + assert.equal( + result.mimeType, + 'application/vnd.openxmlformats-officedocument.wordprocessingml.document' + ) + + const sections = requireSections(result) + assert.deepEqual(outline(sections), DOCX_OUTLINE) + assert.equal(sections.length, 2, 'Overview and Summary are the roots') + assert.equal(sections[0].children?.length, 1, 'Details nests under Overview') + assertSectionOffsets(sections, 'DOCX') + + for (const section of flattenSections(sections)) { + assert.ok( + result.content.slice(section.startOffset).startsWith(section.title), + `DOCX: ${section.title} startOffset does not point at its title` + ) + } +}) + +test('DOCX: a heading-less document still reports an (empty) section structure', async () => { + const result = await new DocxLoader().loadFromPath(fixture('sample.docx')) + assert.match(result.content, /KnowNote Import Test/) + assert.deepEqual(requireSections(result), []) +}) + +// --- Markdown ---------------------------------------------------------------- + +test('Markdown: ATX headings become a nested outline with consistent offsets', async () => { + const result = await new MarkdownLoader().loadFromPath(fixture('sample.md')) + assert.equal(result.title, 'Sample Markdown') + assert.equal(result.mimeType, 'text/markdown') + + const sections = requireSections(result) + assert.deepEqual(outline(sections), MARKDOWN_OUTLINE) + assertSectionOffsets(sections, 'Markdown') + + // Heading offsets are positions in the raw fixture text while `content` is + // trimmed, so the root section can end a newline past `result.content`; the + // assertions stay relative to the headings themselves. + for (const section of flattenSections(sections)) { + const marker = `${'#'.repeat(section.level)} ${section.title}` + assert.ok( + result.content.slice(section.startOffset).startsWith(marker), + `Markdown: ${section.title} startOffset does not point at its ATX marker` + ) + } +}) + +// --- HTML -------------------------------------------------------------------- + +test('HTML: Readability extracts the title and body and still carries a structure', async () => { + const html = await readFile(fixture('sample.html')) + const result = await new WebLoader().loadFromBuffer(html) + assert.equal(result.title, 'Sample Document') + assert.equal(result.mimeType, 'text/html') + assert.equal(result.metadata?.excerpt, 'Fixture for the packaged-app smoke test') + assert.match(result.content, /KnowNote Import Test/) + assert.match(result.content, /Hello World/) + + // Readability hoists the document's lone

into `title` and drops it from + // the body, so this fixture has no heading left to become a section. The test + // still guards the presence of the structure object itself. + assert.deepEqual(requireSections(result), []) +}) + +// --- structure opt-out ------------------------------------------------------- + +test('preserveStructure: false drops the structure without dropping the text', async () => { + const buffer = await readFile(fixture('multipage.pdf')) + const result = await new PdfLoader().loadFromBuffer(buffer, { preserveStructure: false }) + assert.equal(result.structure, undefined) + assert.match(result.content, /Page One Text/) +}) + +// --- dispatch ---------------------------------------------------------------- + +test('FileParserService routes each extension and MIME type to the matching loader', async () => { + const parser = new FileParserService() + + const pdf = await parser.parseFile(fixture('sample.pdf')) + assert.equal(pdf.mimeType, 'application/pdf') + assert.equal(pdf.structure?.type, 'pages') + + const docx = await parser.parseFile(fixture('sample.docx')) + assert.equal( + docx.mimeType, + 'application/vnd.openxmlformats-officedocument.wordprocessingml.document' + ) + assert.equal(docx.structure?.type, 'sections') + + const markdown = await parser.parseBuffer(await readFile(fixture('sample.md')), 'md') + assert.equal(markdown.mimeType, 'text/markdown') + assert.equal(markdown.structure?.type, 'sections') + + for (const supported of ['pdf', 'docx', 'md', 'html', 'htm', 'pptx']) { + assert.ok(parser.isSupported(supported), `${supported} should be routable`) + } + assert.equal(parser.isSupported('exe'), false) + + assert.equal(parser.getFileTypeFromMime('application/pdf'), 'pdf') + assert.equal(parser.getFileTypeFromMime('text/markdown'), 'md') + assert.equal(parser.getMimeType('md'), 'text/markdown') + assert.equal( + parser.getMimeType('docx'), + 'application/vnd.openxmlformats-officedocument.wordprocessingml.document' + ) +})