diff --git a/electron-builder.yml b/electron-builder.yml index 37151f4..5f9cacc 100644 --- a/electron-builder.yml +++ b/electron-builder.yml @@ -11,6 +11,20 @@ files: - '!{tsconfig.json,tsconfig.node.json,tsconfig.web.json}' # Dev-only: test fixtures and build scripts are not part of the shipped app. - '!{test,scripts}/*' +# PDF.js treats CMaps, standard fonts and the wasm image decoders as *external* +# assets fetched by filename. They ship as real files under +# `resources/pdfjs/`, outside app.asar, because both consumers need them there: +# the main-process loader reads them with `fs` (pdfjs resolves its own optional +# native peer that way), and the renderer reads them through the +# `knownote-asset://` protocol. Chromium cannot read inside app.asar, so these +# cannot ride along in the bundle. See `src/main/pdfjsAssetPaths.ts`. +extraResources: + - from: node_modules/pdfjs-dist/cmaps + to: pdfjs/cmaps + - from: node_modules/pdfjs-dist/standard_fonts + to: pdfjs/standard_fonts + - from: node_modules/pdfjs-dist/wasm + to: pdfjs/wasm # Native modules must live outside the asar so they can be loaded by dlopen() # AND so electron-builder signs them individually. Keep these patterns explicit: # if a native module stays inside app.asar it cannot be signed or loaded. diff --git a/src/main/index.ts b/src/main/index.ts index 9ccdba3..213204e 100644 --- a/src/main/index.ts +++ b/src/main/index.ts @@ -25,11 +25,16 @@ import { registerDocumentProtocolHandler, registerDocumentScheme } from './protocol/documentProtocol' +import { + registerPdfjsAssetProtocolHandler, + registerPdfjsAssetScheme +} from './protocol/pdfjsAssetProtocol' import Logger from '../shared/utils/logger' // Scheme privileges must be registered before the app is ready, so this is a // module-scope call rather than part of the whenReady sequence. registerDocumentScheme() +registerPdfjsAssetScheme() let connectionManager: ConnectionManager | null = null let embeddingService: EmbeddingService | null = null @@ -84,6 +89,9 @@ app.whenReady().then(async () => { registerDocumentProtocolHandler( (documentId) => knowledgeService?.getDocumentLocalFilePath(documentId) ?? null ) + // PDF.js 的 CMap / 标准字体 / wasm 资源。它不依赖任何服务初始化顺序，只是把 + // resources/pdfjs 暴露给渲染进程。 + registerPdfjsAssetProtocolHandler() Logger.info('Main', 'Database initialized') // Initialize electron-store diff --git a/src/main/pdfjsAssetPaths.ts b/src/main/pdfjsAssetPaths.ts new file mode 100644 index 0000000..a7409aa --- /dev/null +++ b/src/main/pdfjsAssetPaths.ts @@ -0,0 +1,59 @@ +/** + * 主进程侧的 PDF.js 资源目录解析。 + * + * 打包后资源由 electron-builder 的 `extraResources` 放到 `resources/pdfjs/`，是 asar + * 之外的**真实文件**：主进程要 `fs.readFile` 它们，渲染进程的自定义协议也要能从 + * 磁盘读到。放进 asar 会让 Chromium 读不到，`process.getBuiltinModule('fs')` 的路子 + * 也不保证走 Electron 的 asar 补丁 —— 所以这里刻意不用 asar 内路径。 + * + * 开发时直接读 `node_modules/pdfjs-dist`，省掉"先打包再调试"。 + */ + +import * as electron from 'electron' +import { join } from 'path' +import type { PdfjsAssetDir } from '../shared/utils/pdfjsAssets' + +type ElectronApp = { isPackaged?: boolean; getAppPath?: () => string } + +/** + * `electron` 在纯 Node 下解析成一个可执行文件路径字符串，`app` 因此是 `undefined`； + * `test/parsers.test.ts`、`test/blocks.test.ts` 等会在不启动 Electron 的情况下跑 + * `PdfLoader`，所以这里不能假设自己运行在 Electron 里。 + */ +// SAFETY: `electron` outside a running Electron process resolves to a path string +// with no `app` property, so the runtime value is genuinely `undefined` there. The +// cast narrows that one case away instead of pretending `app` is always present. +const electronApp = (electron as unknown as { app?: ElectronApp }).app + +/** 三类资源的共同根目录。 */ +export function pdfjsAssetRoot(): string { + if (electronApp?.isPackaged === true) { + return join(process.resourcesPath, 'pdfjs') + } + // electron-vite dev 下 `app.getAppPath()` 是项目根；测试里退回 cwd（同样是项目根）。 + const projectRoot = electronApp?.getAppPath?.() ?? process.cwd() + return join(projectRoot, 'node_modules', 'pdfjs-dist') +} + +/** + * 供 PDF.js 的 Node 侧使用的目录路径。 + * + * `NodeBinaryDataFactory` 直接 `fs.readFile(`${baseUrl}${filename}`)`，所以这里给的是 + * 目录路径，并保留 PDF.js 要求的结尾斜杠（`getFactoryUrlProp` 会校验）。 + */ +export function pdfjsNodeAssetDir(dir: PdfjsAssetDir): string { + return `${join(pdfjsAssetRoot(), dir)}/` +} + +/** `PdfLoader.getDocument()` 要的三个资源参数。 */ +export function pdfjsNodeAssetUrls(): { + cMapUrl: string + standardFontDataUrl: string + wasmUrl: string +} { + return { + cMapUrl: pdfjsNodeAssetDir('cmaps'), + standardFontDataUrl: pdfjsNodeAssetDir('standard_fonts'), + wasmUrl: pdfjsNodeAssetDir('wasm') + } +} diff --git a/src/main/protocol/pdfjsAssetProtocol.ts b/src/main/protocol/pdfjsAssetProtocol.ts new file mode 100644 index 0000000..60a26b9 --- /dev/null +++ b/src/main/protocol/pdfjsAssetProtocol.ts @@ -0,0 +1,69 @@ +/** + * `knownote-asset://` 协议（PDF.js 外部资源） + * + * 渲染进程不接触文件路径：它只知道资源目录名，主进程按白名单解析并只服务 + * `resources/pdfjs/{cmaps,standard_fonts,wasm}` 下的文件。路径来自这里的 + * `pdfjsAssetRoot()`，不来自请求。 + * + * 和 `documentProtocol` 一样，scheme 必须在 app ready 之前注册为 privileged； + * handler 在 ready 之后安装。 + */ + +import { net, protocol } from 'electron' +import { join, normalize, sep } from 'path' +import { pathToFileURL } from 'url' +import { PDFJS_ASSET_SCHEME, parsePdfjsAssetUrl } from '../../shared/utils/pdfjsAssets' +import { pdfjsAssetRoot } from '../pdfjsAssetPaths' +import Logger from '../../shared/utils/logger' + +/** + * Origins allowed to read an asset. Same set as `documentProtocol`: the renderer + * is `file://` in production (serialised as `null`) or the electron-vite dev + * server in development. + */ +const ALLOWED_ORIGINS = new Set(['null', 'http://localhost:5173', 'http://127.0.0.1:5173']) + +/** 必须在 app ready 之前调用一次。 */ +export function registerPdfjsAssetScheme(): void { + protocol.registerSchemesAsPrivileged([ + { + scheme: PDFJS_ASSET_SCHEME, + privileges: { + standard: true, + secure: true, + supportFetchAPI: true, + stream: true, + // The renderer fetches this scheme from a different origin (`file://` in + // production, the dev server in development), so it is a cross-origin + // request and must be CORS-enabled to succeed. + corsEnabled: true + } + } + ]) +} + +/** 在 app ready 之后调用，安装只读的资源 handler。 */ +export function registerPdfjsAssetProtocolHandler(): void { + protocol.handle(PDFJS_ASSET_SCHEME, async (request) => { + const parsed = parsePdfjsAssetUrl(request.url) + if (!parsed) return new Response('Not found', { status: 404 }) + + const root = pdfjsAssetRoot() + const filePath = normalize(join(root, parsed.dir, parsed.file)) + // 解析层已经限制了目录白名单和文件名，这里再确认一次最终路径没跑出根目录。 + if (!filePath.startsWith(root + sep)) return new Response('Not found', { status: 404 }) + + try { + const response = await net.fetch(pathToFileURL(filePath).toString()) + const headers = new Headers(response.headers) + const origin = request.headers.get('Origin') ?? 'null' + if (ALLOWED_ORIGINS.has(origin)) headers.set('Access-Control-Allow-Origin', origin) + // Chromium 只接受这个 MIME 做 `WebAssembly.instantiateStreaming`。 + if (parsed.file.endsWith('.wasm')) headers.set('Content-Type', 'application/wasm') + return new Response(response.body, { status: response.status, headers }) + } catch (error) { + Logger.warn('PdfjsAssetProtocol', `Failed to serve ${parsed.dir}/${parsed.file}:`, error) + return new Response('Not found', { status: 404 }) + } + }) +} diff --git a/src/main/services/loaders/PdfLoader.ts b/src/main/services/loaders/PdfLoader.ts index f7dcec3..a16b232 100644 --- a/src/main/services/loaders/PdfLoader.ts +++ b/src/main/services/loaders/PdfLoader.ts @@ -8,6 +8,7 @@ import { extname } from 'path' // 使用 legacy build for Node.js 环境 import * as pdfjsLib from 'pdfjs-dist/legacy/build/pdf.mjs' import type { PDFDocumentProxy, PDFPageProxy, TextItem } from 'pdfjs-dist/types/src/display/api' +import { pdfjsNodeAssetUrls } from '../../pdfjsAssetPaths' import Logger from '../../../shared/utils/logger' import { layoutPageText, type PositionedTextItem } from './pdfTextLayout' import type { @@ -74,7 +75,12 @@ export class PdfLoader implements IDocumentLoader { // 拉进来，那个测试会失败。 const loadingTask = pdfjsLib.getDocument({ data: new Uint8Array(buffer), - password: opts.password + password: opts.password, + // CMap / 标准字体 / wasm 是外部资源：缺了 `cMapUrl`，CJK 文档的 + // `translateFont` 会抛 `Ensure that the cMapUrl API parameter is provided.`， + // 文本层随之取不到中文。Node 侧 pdfjs 直接 `fs.readFile` 这些目录。 + ...pdfjsNodeAssetUrls(), + cMapPacked: true }) const pdfDoc: PDFDocumentProxy = await loadingTask.promise diff --git a/src/main/smokeTest.ts b/src/main/smokeTest.ts index 2f24cc4..085f20b 100644 --- a/src/main/smokeTest.ts +++ b/src/main/smokeTest.ts @@ -23,7 +23,7 @@ */ import Logger from '../shared/utils/logger' -import { readFile } from 'fs/promises' +import { readFile, readdir } from 'fs/promises' import { mkdtempSync, rmSync, writeFileSync } from 'fs' import { tmpdir } from 'os' import { join } from 'path' @@ -52,7 +52,9 @@ import { insertChunkBlocks, resolveChunkProvenance } from './services/chunkProve import { KnowledgeService } from './services/KnowledgeService' import { DenseRetriever } from './services/retrieval' import { documentUrl } from '../shared/utils/documentUrl' +import { PDFJS_ASSET_DIRS } from '../shared/utils/pdfjsAssets' import { registerDocumentProtocolHandler } from './protocol/documentProtocol' +import { pdfjsAssetRoot } from './pdfjsAssetPaths' import type { EmbeddingService } from './services/EmbeddingService' export const SMOKE_TEST_FLAG = '--smoke-test' @@ -440,6 +442,22 @@ async function runChecks(): Promise { ) pass('PDF import extracts text (pdfjs-dist)') + // PDF.js fetches CMaps, standard fonts and wasm decoders by filename, and the + // packaged app serves them from `resources/pdfjs/` (electron-builder + // `extraResources`). Getting that wrong does not fail `npm run build`; it + // makes every CJK document die at runtime with "Ensure that the `cMapUrl` API + // parameter is provided." - the failure #125 found. Assert the files actually + // shipped, in the real packaged layout. + const assetRoot = pdfjsAssetRoot() + for (const dir of PDFJS_ASSET_DIRS) { + const entries = await readdir(join(assetRoot, dir)).catch(() => []) + assert( + entries.length > 0, + `PDF.js assets missing or empty in the packaged app: ${join(assetRoot, dir)}` + ) + } + pass('PDF.js CMap / standard font / wasm assets are packaged') + const docx = await parser.parseFile(join(fixtures, 'sample.docx')) assert( docx.content.includes(IMPORT_MARKER), diff --git a/src/renderer/index.html b/src/renderer/index.html index eb454c8..99134e7 100644 --- a/src/renderer/index.html +++ b/src/renderer/index.html @@ -6,7 +6,7 @@ diff --git a/src/renderer/src/components/notebook/source/reader/PdfSourceReader.tsx b/src/renderer/src/components/notebook/source/reader/PdfSourceReader.tsx index f56fc07..1ac30db 100644 --- a/src/renderer/src/components/notebook/source/reader/PdfSourceReader.tsx +++ b/src/renderer/src/components/notebook/source/reader/PdfSourceReader.tsx @@ -22,6 +22,7 @@ import workerUrl from 'pdfjs-dist/legacy/build/pdf.worker.min.mjs?url' import { useTranslation } from 'react-i18next' import { Minus, Plus, Loader2 } from 'lucide-react' import { documentUrl } from '../../../../../../shared/utils/documentUrl' +import { pdfjsAssetUrl } from '../../../../../../shared/utils/pdfjsAssets' import type { ReaderAnchor, ReaderHandle, @@ -223,7 +224,17 @@ const PdfSourceReader = forwardRef(function // 上，不在 `getDocument` 参数里，所以这里没有可以传的开关（#121 / // GHSA-hq66-cqwq-w95j）。也就是说**在当前检查过的调用点下漏洞路径不可达**， // 前提由 `test/pdfjsScriptingBoundary.test.ts` 守住。 - const task = pdfjs.getDocument({ data }) + // `cMapUrl` / `standardFontDataUrl` / `wasmUrl` 是 PDF.js 的外部资源入口。 + // 缺了 `cMapUrl`，CJK 文档的 `translateFont` 会抛 + // `Ensure that the cMapUrl API parameter is provided.`，中文整页画不出来； + // 两个 URL 都走 `knownote-asset://`，由主进程从 resources/pdfjs 只读服务。 + const task = pdfjs.getDocument({ + data, + cMapUrl: pdfjsAssetUrl('cmaps'), + cMapPacked: true, + standardFontDataUrl: pdfjsAssetUrl('standard_fonts'), + wasmUrl: pdfjsAssetUrl('wasm') + }) loaded = task const doc = await task.promise if (cancelled) { diff --git a/src/shared/utils/pdfjsAssets.ts b/src/shared/utils/pdfjsAssets.ts new file mode 100644 index 0000000..f4c5a35 --- /dev/null +++ b/src/shared/utils/pdfjsAssets.ts @@ -0,0 +1,73 @@ +/** + * PDF.js 外部资源（CMap / 标准字体 / wasm 解码器）的共享契约。 + * + * PDF.js 把这三类资源当作**外部资源按文件名拉取**：`getDocument` 收到 + * `cMapUrl` 之后，会去请求 `${cMapUrl}${name}.bcmap`。缺了 `cMapUrl`，CJK + * 文档的 `translateFont` 直接抛 + * `Ensure that the cMapUrl API parameter is provided.` —— 中文既画不出来，文本层 + * 也取不到。`standardFontDataUrl` / `wasmUrl` 同理，只是触发条件更窄 + * （非嵌入标准字体；JBIG2 / JPEG2000 / qcms 图像）。 + * + * 主进程用文件系统路径（pdfjs 在 Node 侧直接 `fs.readFile`），渲染进程用这里的 + * scheme URL（走自定义协议，避免渲染进程接触任意文件路径）。两边共享"需要哪些 + * 目录"和 URL 形状的校验，但**不共享拼 URL 的实现**。 + */ + +/** 渲染进程访问 PDF.js 资源的自定义 scheme。见 `protocol/pdfjsAssetProtocol.ts`。 */ +export const PDFJS_ASSET_SCHEME = 'knownote-asset' + +/** URL 里的 host，和 `knownote-doc://docs` 一样只是形状的一部分，不表示主机。 */ +const PDFJS_ASSET_HOST = 'pdfjs' + +/** 要随包携带的目录名，与 `node_modules/pdfjs-dist/` 下一致。 */ +export const PDFJS_ASSET_DIRS = ['cmaps', 'standard_fonts', 'wasm'] as const + +export type PdfjsAssetDir = (typeof PDFJS_ASSET_DIRS)[number] + +/** + * 渲染进程侧的资源目录 URL。 + * + * 必须以 `/` 结尾：PDF.js 直接做字符串拼接，`getFactoryUrlProp` 对没有结尾斜杠的 + * 值会抛 `Invalid factory url: "..." must include trailing slash.`。 + */ +export function pdfjsAssetUrl(dir: PdfjsAssetDir): string { + return `${PDFJS_ASSET_SCHEME}://${PDFJS_ASSET_HOST}/${dir}/` +} + +/** pdfjs 的资源文件名都是单层、无分隔符的名字（`H.bcmap`、`qcms_bg.wasm` 等）。 */ +const ASSET_FILE_NAME = /^[A-Za-z0-9._-]+$/ + +/** + * 只接受 `knownote-asset://pdfjs//`，其余形状一律返回 null。 + * + * 目录必须在白名单里、文件名不允许路径分隔符也不允许 `.` / `..`，所以路径穿越在 + * 解析这一层就被挡掉，协议 handler 不需要再相信调用方。 + */ +export function parsePdfjsAssetUrl(url: string): { dir: PdfjsAssetDir; file: string } | null { + let parsed: URL + try { + parsed = new URL(url) + } catch { + return null + } + if (parsed.protocol !== `${PDFJS_ASSET_SCHEME}:`) return null + if (parsed.hostname !== PDFJS_ASSET_HOST) return null + + const match = /^\/([^/]+)\/([^/]+)$/.exec(parsed.pathname) + if (!match) return null + + let dir: string + let file: string + try { + dir = decodeURIComponent(match[1]) + file = decodeURIComponent(match[2]) + } catch { + return null + } + + if (!(PDFJS_ASSET_DIRS as readonly string[]).includes(dir)) return null + if (file === '.' || file === '..') return null + if (!ASSET_FILE_NAME.test(file)) return null + + return { dir: dir as PdfjsAssetDir, file } +} diff --git a/test/pdfjsAssets.test.ts b/test/pdfjsAssets.test.ts new file mode 100644 index 0000000..385fb39 --- /dev/null +++ b/test/pdfjsAssets.test.ts @@ -0,0 +1,124 @@ +import { test } from 'node:test' +import assert from 'node:assert/strict' +import { readFileSync } from 'node:fs' +import { join } from 'node:path' +import { + PDFJS_ASSET_SCHEME, + PDFJS_ASSET_DIRS, + pdfjsAssetUrl, + parsePdfjsAssetUrl +} from '../src/shared/utils/pdfjsAssets.ts' + +/** + * PDF.js 把 CMap / 标准字体 / wasm 当外部资源按文件名拉取。缺了 `cMapUrl`，CJK + * 文档的 `translateFont` 直接抛 `Ensure that the cMapUrl API parameter is + * provided.`，中文既画不出来、文本层也取不到（#125）。这些测试守住两件容易静默 + * 回归的事：URL 形状（PDF.js 对结尾斜杠有硬校验）和两个 `getDocument` 调用点确实 + * 配置了资源。 + */ + +const read = (path: string): string => readFileSync(join(process.cwd(), path), 'utf8') + +test('every asset URL ends with the slash PDF.js requires', () => { + for (const dir of PDFJS_ASSET_DIRS) { + const url = pdfjsAssetUrl(dir) + assert.ok(url.endsWith('/'), `pdfjs rejects "${url}" without a trailing slash`) + assert.equal(url, `${PDFJS_ASSET_SCHEME}://pdfjs/${dir}/`) + } +}) + +test('parse accepts the allowlisted dirs and flat filenames', () => { + assert.deepEqual(parsePdfjsAssetUrl('knownote-asset://pdfjs/cmaps/H.bcmap'), { + dir: 'cmaps', + file: 'H.bcmap' + }) + assert.deepEqual(parsePdfjsAssetUrl('knownote-asset://pdfjs/wasm/qcms_bg.wasm'), { + dir: 'wasm', + file: 'qcms_bg.wasm' + }) + assert.deepEqual(parsePdfjsAssetUrl('knownote-asset://pdfjs/standard_fonts/FoxitSans.pfb'), { + dir: 'standard_fonts', + file: 'FoxitSans.pfb' + }) +}) + +test('parse rejects path traversal, unknown dirs and foreign origins', () => { + const rejected = [ + 'knownote-asset://pdfjs/cmaps/../../secret', + 'knownote-asset://pdfjs/cmaps/%2e%2e', + 'knownote-asset://pdfjs/cmaps/..', + 'knownote-asset://pdfjs/../secret', + 'knownote-asset://pdfjs/other/H.bcmap', + 'knownote-asset://pdfjs/cmaps/sub/H.bcmap', + 'knownote-asset://evil/cmaps/H.bcmap', + 'https://pdfjs/cmaps/H.bcmap', + 'knownote-doc://docs/abc', + 'not a url' + ] + + for (const url of rejected) { + assert.equal(parsePdfjsAssetUrl(url), null, `must reject ${url}`) + } +}) + +test('both getDocument call sites configure the external assets', () => { + const reader = read('src/renderer/src/components/notebook/source/reader/PdfSourceReader.tsx') + const loader = read('src/main/services/loaders/PdfLoader.ts') + + // Match object properties (`cMapUrl:`), not bare mentions: the explanatory + // comments name these parameters, and a guard that accepts a comment would + // pass with the call site reverted to `getDocument({ data })`. + for (const property of ['cMapUrl', 'standardFontDataUrl', 'wasmUrl', 'cMapPacked']) { + assert.match( + reader, + new RegExp(`\\b${property}\\s*:`), + `PdfSourceReader no longer passes \`${property}\` to getDocument; CJK PDFs will break` + ) + } + assert.ok( + reader.includes('pdfjsAssetUrl'), + 'PdfSourceReader must resolve assets through pdfjsAssetUrl()' + ) + + // PdfLoader folds the three URLs in from the helper, so assert the call and the + // helper's own contract rather than literal keys at the call site. + assert.ok( + loader.includes('pdfjsNodeAssetUrls'), + 'PdfLoader must resolve assets through pdfjsNodeAssetUrls()' + ) + assert.match( + loader, + /\bcMapPacked\s*:/, + 'PdfLoader must pass cMapPacked, matching the packed .bcmap files on disk' + ) + + const loaderAssets = read('src/main/pdfjsAssetPaths.ts') + for (const key of ['cMapUrl', 'standardFontDataUrl', 'wasmUrl']) { + assert.match( + loaderAssets, + new RegExp(`\\b${key}\\s*:`), + `pdfjsNodeAssetUrls() stopped returning ${key}; CJK PDFs will break` + ) + } +}) + +test('the renderer CSP permits the asset scheme and wasm compilation', () => { + const html = read('src/renderer/index.html') + const csp = /http-equiv="Content-Security-Policy"\s+content="([^"]+)"/.exec(html)?.[1] + assert.ok(csp, 'index.html must carry a Content-Security-Policy meta tag') + + const directive = (name: string): string | undefined => + csp + .split(';') + .map((part) => part.trim()) + .find((part) => part.startsWith(name)) + + assert.ok( + directive('connect-src')?.includes(`${PDFJS_ASSET_SCHEME}:`), + 'connect-src must allow the asset scheme, or PDF.js cannot fetch CMaps' + ) + assert.ok( + directive('script-src')?.includes("'wasm-unsafe-eval'"), + "script-src needs 'wasm-unsafe-eval' or wasm image decoders cannot compile" + ) +})