fix(pdf): bundle PDF.js CMap/font assets so CJK documents render and index

electron-builder.yml

@@ -11,6 +11,20 @@ files:
   - '!{tsconfig.json,tsconfig.node.json,tsconfig.web.json}'
   # Dev-only: test fixtures and build scripts are not part of the shipped app.
   - '!{test,scripts}/*'
+# PDF.js treats CMaps, standard fonts and the wasm image decoders as *external*
+# assets fetched by filename. They ship as real files under
+# `resources/pdfjs/`, outside app.asar, because both consumers need them there:
+# the main-process loader reads them with `fs` (pdfjs resolves its own optional
+# native peer that way), and the renderer reads them through the
+# `knownote-asset://` protocol. Chromium cannot read inside app.asar, so these
+# cannot ride along in the bundle. See `src/main/pdfjsAssetPaths.ts`.
+extraResources:
+  - from: node_modules/pdfjs-dist/cmaps
+    to: pdfjs/cmaps
+  - from: node_modules/pdfjs-dist/standard_fonts
+    to: pdfjs/standard_fonts
+  - from: node_modules/pdfjs-dist/wasm
+    to: pdfjs/wasm
 # Native modules must live outside the asar so they can be loaded by dlopen()
 # AND so electron-builder signs them individually. Keep these patterns explicit:
 # if a native module stays inside app.asar it cannot be signed or loaded.

src/main/index.ts

@@ -25,11 +25,16 @@ import {
   registerDocumentProtocolHandler,
   registerDocumentScheme
 } from './protocol/documentProtocol'
+import {
+  registerPdfjsAssetProtocolHandler,
+  registerPdfjsAssetScheme
+} from './protocol/pdfjsAssetProtocol'
 import Logger from '../shared/utils/logger'
 
 // Scheme privileges must be registered before the app is ready, so this is a
 // module-scope call rather than part of the whenReady sequence.
 registerDocumentScheme()
+registerPdfjsAssetScheme()
 
 let connectionManager: ConnectionManager | null = null
 let embeddingService: EmbeddingService | null = null
@@ -84,6 +89,9 @@ app.whenReady().then(async () => {
   registerDocumentProtocolHandler(
     (documentId) => knowledgeService?.getDocumentLocalFilePath(documentId) ?? null
   )
+  // PDF.js 的 CMap / 标准字体 / wasm 资源。它不依赖任何服务初始化顺序，只是把
+  // resources/pdfjs 暴露给渲染进程。
+  registerPdfjsAssetProtocolHandler()
   Logger.info('Main', 'Database initialized')
 
   // Initialize electron-store

src/main/pdfjsAssetPaths.ts

@@ -0,0 +1,59 @@
+/**
+ * 主进程侧的 PDF.js 资源目录解析。
+ *
+ * 打包后资源由 electron-builder 的 `extraResources` 放到 `resources/pdfjs/`，是 asar
+ * 之外的**真实文件**：主进程要 `fs.readFile` 它们，渲染进程的自定义协议也要能从
+ * 磁盘读到。放进 asar 会让 Chromium 读不到，`process.getBuiltinModule('fs')` 的路子
+ * 也不保证走 Electron 的 asar 补丁 —— 所以这里刻意不用 asar 内路径。
+ *
+ * 开发时直接读 `node_modules/pdfjs-dist`，省掉"先打包再调试"。
+ */
+
+import * as electron from 'electron'
+import { join } from 'path'
+import type { PdfjsAssetDir } from '../shared/utils/pdfjsAssets'
+
+type ElectronApp = { isPackaged?: boolean; getAppPath?: () => string }
+
+/**
+ * `electron` 在纯 Node 下解析成一个可执行文件路径字符串，`app` 因此是 `undefined`；
+ * `test/parsers.test.ts`、`test/blocks.test.ts` 等会在不启动 Electron 的情况下跑
+ * `PdfLoader`，所以这里不能假设自己运行在 Electron 里。
+ */
+// SAFETY: `electron` outside a running Electron process resolves to a path string
+// with no `app` property, so the runtime value is genuinely `undefined` there. The
+// cast narrows that one case away instead of pretending `app` is always present.
+const electronApp = (electron as unknown as { app?: ElectronApp }).app
+
+/** 三类资源的共同根目录。 */
+export function pdfjsAssetRoot(): string {
+  if (electronApp?.isPackaged === true) {
+    return join(process.resourcesPath, 'pdfjs')
+  }
+  // electron-vite dev 下 `app.getAppPath()` 是项目根；测试里退回 cwd（同样是项目根）。
+  const projectRoot = electronApp?.getAppPath?.() ?? process.cwd()
+  return join(projectRoot, 'node_modules', 'pdfjs-dist')
+}
+
+/**
+ * 供 PDF.js 的 Node 侧使用的目录路径。
+ *
+ * `NodeBinaryDataFactory` 直接 `fs.readFile(`${baseUrl}${filename}`)`，所以这里给的是
+ * 目录路径，并保留 PDF.js 要求的结尾斜杠（`getFactoryUrlProp` 会校验）。
+ */
+export function pdfjsNodeAssetDir(dir: PdfjsAssetDir): string {
+  return `${join(pdfjsAssetRoot(), dir)}/`
+}
+
+/** `PdfLoader.getDocument()` 要的三个资源参数。 */
+export function pdfjsNodeAssetUrls(): {
+  cMapUrl: string
+  standardFontDataUrl: string
+  wasmUrl: string
+} {
+  return {
+    cMapUrl: pdfjsNodeAssetDir('cmaps'),
+    standardFontDataUrl: pdfjsNodeAssetDir('standard_fonts'),
+    wasmUrl: pdfjsNodeAssetDir('wasm')
+  }
+}

src/main/protocol/pdfjsAssetProtocol.ts

@@ -0,0 +1,69 @@
+/**
+ * `knownote-asset://` 协议（PDF.js 外部资源）
+ *
+ * 渲染进程不接触文件路径：它只知道资源目录名，主进程按白名单解析并只服务
+ * `resources/pdfjs/{cmaps,standard_fonts,wasm}` 下的文件。路径来自这里的
+ * `pdfjsAssetRoot()`，不来自请求。
+ *
+ * 和 `documentProtocol` 一样，scheme 必须在 app ready 之前注册为 privileged；
+ * handler 在 ready 之后安装。
+ */
+
+import { net, protocol } from 'electron'
+import { join, normalize, sep } from 'path'
+import { pathToFileURL } from 'url'
+import { PDFJS_ASSET_SCHEME, parsePdfjsAssetUrl } from '../../shared/utils/pdfjsAssets'
+import { pdfjsAssetRoot } from '../pdfjsAssetPaths'
+import Logger from '../../shared/utils/logger'
+
+/**
+ * Origins allowed to read an asset. Same set as `documentProtocol`: the renderer
+ * is `file://` in production (serialised as `null`) or the electron-vite dev
+ * server in development.
+ */
+const ALLOWED_ORIGINS = new Set(['null', 'http://localhost:5173', 'http://127.0.0.1:5173'])
+
+/** 必须在 app ready 之前调用一次。 */
+export function registerPdfjsAssetScheme(): void {
+  protocol.registerSchemesAsPrivileged([
+    {
+      scheme: PDFJS_ASSET_SCHEME,
+      privileges: {
+        standard: true,
+        secure: true,
+        supportFetchAPI: true,
+        stream: true,
+        // The renderer fetches this scheme from a different origin (`file://` in
+        // production, the dev server in development), so it is a cross-origin
+        // request and must be CORS-enabled to succeed.
+        corsEnabled: true
+      }
+    }
+  ])
+}
+
+/** 在 app ready 之后调用，安装只读的资源 handler。 */
+export function registerPdfjsAssetProtocolHandler(): void {
+  protocol.handle(PDFJS_ASSET_SCHEME, async (request) => {
+    const parsed = parsePdfjsAssetUrl(request.url)
+    if (!parsed) return new Response('Not found', { status: 404 })
+
+    const root = pdfjsAssetRoot()
+    const filePath = normalize(join(root, parsed.dir, parsed.file))
+    // 解析层已经限制了目录白名单和文件名，这里再确认一次最终路径没跑出根目录。
+    if (!filePath.startsWith(root + sep)) return new Response('Not found', { status: 404 })
+
+    try {
+      const response = await net.fetch(pathToFileURL(filePath).toString())
+      const headers = new Headers(response.headers)
+      const origin = request.headers.get('Origin') ?? 'null'
+      if (ALLOWED_ORIGINS.has(origin)) headers.set('Access-Control-Allow-Origin', origin)
+      // Chromium 只接受这个 MIME 做 `WebAssembly.instantiateStreaming`。
+      if (parsed.file.endsWith('.wasm')) headers.set('Content-Type', 'application/wasm')
+      return new Response(response.body, { status: response.status, headers })
+    } catch (error) {
+      Logger.warn('PdfjsAssetProtocol', `Failed to serve ${parsed.dir}/${parsed.file}:`, error)
+      return new Response('Not found', { status: 404 })
+    }
+  })
+}

src/main/services/loaders/PdfLoader.ts

@@ -8,6 +8,7 @@ import { extname } from 'path'
 // 使用 legacy build for Node.js 环境
 import * as pdfjsLib from 'pdfjs-dist/legacy/build/pdf.mjs'
 import type { PDFDocumentProxy, PDFPageProxy, TextItem } from 'pdfjs-dist/types/src/display/api'
+import { pdfjsNodeAssetUrls } from '../../pdfjsAssetPaths'
 import Logger from '../../../shared/utils/logger'
 import { layoutPageText, type PositionedTextItem } from './pdfTextLayout'
 import type {
@@ -74,7 +75,12 @@ export class PdfLoader implements IDocumentLoader {
       // 拉进来，那个测试会失败。
       const loadingTask = pdfjsLib.getDocument({
         data: new Uint8Array(buffer),
-        password: opts.password
+        password: opts.password,
+        // CMap / 标准字体 / wasm 是外部资源：缺了 `cMapUrl`，CJK 文档的
+        // `translateFont` 会抛 `Ensure that the cMapUrl API parameter is provided.`，
+        // 文本层随之取不到中文。Node 侧 pdfjs 直接 `fs.readFile` 这些目录。
+        ...pdfjsNodeAssetUrls(),
+        cMapPacked: true
       })
       const pdfDoc: PDFDocumentProxy = await loadingTask.promise
 

src/main/smokeTest.ts

@@ -23,7 +23,7 @@
  */
 
 import Logger from '../shared/utils/logger'
-import { readFile } from 'fs/promises'
+import { readFile, readdir } from 'fs/promises'
 import { mkdtempSync, rmSync, writeFileSync } from 'fs'
 import { tmpdir } from 'os'
 import { join } from 'path'
@@ -52,7 +52,9 @@ import { insertChunkBlocks, resolveChunkProvenance } from './services/chunkProve
 import { KnowledgeService } from './services/KnowledgeService'
 import { DenseRetriever } from './services/retrieval'
 import { documentUrl } from '../shared/utils/documentUrl'
+import { PDFJS_ASSET_DIRS } from '../shared/utils/pdfjsAssets'
 import { registerDocumentProtocolHandler } from './protocol/documentProtocol'
+import { pdfjsAssetRoot } from './pdfjsAssetPaths'
 import type { EmbeddingService } from './services/EmbeddingService'
 
 export const SMOKE_TEST_FLAG = '--smoke-test'
@@ -440,6 +442,22 @@ async function runChecks(): Promise<string[]> {
   )
   pass('PDF import extracts text (pdfjs-dist)')
 
+  // PDF.js fetches CMaps, standard fonts and wasm decoders by filename, and the
+  // packaged app serves them from `resources/pdfjs/` (electron-builder
+  // `extraResources`). Getting that wrong does not fail `npm run build`; it
+  // makes every CJK document die at runtime with "Ensure that the `cMapUrl` API
+  // parameter is provided." - the failure #125 found. Assert the files actually
+  // shipped, in the real packaged layout.
+  const assetRoot = pdfjsAssetRoot()
+  for (const dir of PDFJS_ASSET_DIRS) {
+    const entries = await readdir(join(assetRoot, dir)).catch(() => [])
+    assert(
+      entries.length > 0,
+      `PDF.js assets missing or empty in the packaged app: ${join(assetRoot, dir)}`
+    )
+  }
+  pass('PDF.js CMap / standard font / wasm assets are packaged')
+
   const docx = await parser.parseFile(join(fixtures, 'sample.docx'))
   assert(
     docx.content.includes(IMPORT_MARKER),

src/renderer/index.html

@@ -6,7 +6,7 @@
     <!-- https://developer.mozilla.org/en-US/docs/Web/HTTP/CSP -->
     <meta
       http-equiv="Content-Security-Policy"
-      content="default-src 'self'; script-src 'self'; style-src 'self' 'unsafe-inline'; img-src 'self' data: blob: https://fonts.googleapis.com https://fonts.gstatic.com; font-src https://fonts.gstatic.com; connect-src 'self' knownote-doc:; worker-src 'self' blob:;"
+      content="default-src 'self'; script-src 'self' 'wasm-unsafe-eval'; style-src 'self' 'unsafe-inline'; img-src 'self' data: blob: https://fonts.googleapis.com https://fonts.gstatic.com; font-src https://fonts.gstatic.com; connect-src 'self' knownote-doc: knownote-asset:; worker-src 'self' blob:;"
     />
     <link rel="preconnect" href="https://fonts.googleapis.com" />
     <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin />

src/renderer/src/components/notebook/source/reader/PdfSourceReader.tsx

@@ -22,6 +22,7 @@ import workerUrl from 'pdfjs-dist/legacy/build/pdf.worker.min.mjs?url'
 import { useTranslation } from 'react-i18next'
 import { Minus, Plus, Loader2 } from 'lucide-react'
 import { documentUrl } from '../../../../../../shared/utils/documentUrl'
+import { pdfjsAssetUrl } from '../../../../../../shared/utils/pdfjsAssets'
 import type {
   ReaderAnchor,
   ReaderHandle,
@@ -223,7 +224,17 @@ const PdfSourceReader = forwardRef<ReaderHandle, PdfSourceReaderProps>(function
       // 上，不在 `getDocument` 参数里，所以这里没有可以传的开关（#121 /
       // GHSA-hq66-cqwq-w95j）。也就是说**在当前检查过的调用点下漏洞路径不可达**，
       // 前提由 `test/pdfjsScriptingBoundary.test.ts` 守住。
-      const task = pdfjs.getDocument({ data })
+      // `cMapUrl` / `standardFontDataUrl` / `wasmUrl` 是 PDF.js 的外部资源入口。
+      // 缺了 `cMapUrl`，CJK 文档的 `translateFont` 会抛
+      // `Ensure that the cMapUrl API parameter is provided.`，中文整页画不出来；
+      // 两个 URL 都走 `knownote-asset://`，由主进程从 resources/pdfjs 只读服务。
+      const task = pdfjs.getDocument({
+        data,
+        cMapUrl: pdfjsAssetUrl('cmaps'),
+        cMapPacked: true,
+        standardFontDataUrl: pdfjsAssetUrl('standard_fonts'),
+        wasmUrl: pdfjsAssetUrl('wasm')
+      })
       loaded = task
       const doc = await task.promise
       if (cancelled) {

src/shared/utils/pdfjsAssets.ts

@@ -0,0 +1,73 @@
+/**
+ * PDF.js 外部资源（CMap / 标准字体 / wasm 解码器）的共享契约。
+ *
+ * PDF.js 把这三类资源当作**外部资源按文件名拉取**：`getDocument` 收到
+ * `cMapUrl` 之后，会去请求 `${cMapUrl}${name}.bcmap`。缺了 `cMapUrl`，CJK
+ * 文档的 `translateFont` 直接抛
+ * `Ensure that the cMapUrl API parameter is provided.` —— 中文既画不出来，文本层
+ * 也取不到。`standardFontDataUrl` / `wasmUrl` 同理，只是触发条件更窄
+ * （非嵌入标准字体；JBIG2 / JPEG2000 / qcms 图像）。
+ *
+ * 主进程用文件系统路径（pdfjs 在 Node 侧直接 `fs.readFile`），渲染进程用这里的
+ * scheme URL（走自定义协议，避免渲染进程接触任意文件路径）。两边共享"需要哪些
+ * 目录"和 URL 形状的校验，但**不共享拼 URL 的实现**。
+ */
+
+/** 渲染进程访问 PDF.js 资源的自定义 scheme。见 `protocol/pdfjsAssetProtocol.ts`。 */
+export const PDFJS_ASSET_SCHEME = 'knownote-asset'
+
+/** URL 里的 host，和 `knownote-doc://docs` 一样只是形状的一部分，不表示主机。 */
+const PDFJS_ASSET_HOST = 'pdfjs'
+
+/** 要随包携带的目录名，与 `node_modules/pdfjs-dist/` 下一致。 */
+export const PDFJS_ASSET_DIRS = ['cmaps', 'standard_fonts', 'wasm'] as const
+
+export type PdfjsAssetDir = (typeof PDFJS_ASSET_DIRS)[number]
+
+/**
+ * 渲染进程侧的资源目录 URL。
+ *
+ * 必须以 `/` 结尾：PDF.js 直接做字符串拼接，`getFactoryUrlProp` 对没有结尾斜杠的
+ * 值会抛 `Invalid factory url: "..." must include trailing slash.`。
+ */
+export function pdfjsAssetUrl(dir: PdfjsAssetDir): string {
+  return `${PDFJS_ASSET_SCHEME}://${PDFJS_ASSET_HOST}/${dir}/`
+}
+
+/** pdfjs 的资源文件名都是单层、无分隔符的名字（`H.bcmap`、`qcms_bg.wasm` 等）。 */
+const ASSET_FILE_NAME = /^[A-Za-z0-9._-]+$/
+
+/**
+ * 只接受 `knownote-asset://pdfjs/<dir>/<file>`，其余形状一律返回 null。
+ *
+ * 目录必须在白名单里、文件名不允许路径分隔符也不允许 `.` / `..`，所以路径穿越在
+ * 解析这一层就被挡掉，协议 handler 不需要再相信调用方。
+ */
+export function parsePdfjsAssetUrl(url: string): { dir: PdfjsAssetDir; file: string } | null {
+  let parsed: URL
+  try {
+    parsed = new URL(url)
+  } catch {
+    return null
+  }
+  if (parsed.protocol !== `${PDFJS_ASSET_SCHEME}:`) return null
+  if (parsed.hostname !== PDFJS_ASSET_HOST) return null
+
+  const match = /^\/([^/]+)\/([^/]+)$/.exec(parsed.pathname)
+  if (!match) return null
+
+  let dir: string
+  let file: string
+  try {
+    dir = decodeURIComponent(match[1])
+    file = decodeURIComponent(match[2])
+  } catch {
+    return null
+  }
+
+  if (!(PDFJS_ASSET_DIRS as readonly string[]).includes(dir)) return null
+  if (file === '.' || file === '..') return null
+  if (!ASSET_FILE_NAME.test(file)) return null
+
+  return { dir: dir as PdfjsAssetDir, file }
+}