From 55fd1718d8b3745b9ac4455bdd364f2e8256b843 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Orell=20B=C3=BChler?= Date: Sun, 16 Aug 2026 00:50:27 +0200 Subject: [PATCH] add local pdf page extraction and text search tools extract_document_pages pulls pages out of a document's pdf into a new file on disk (pdf-lib), and find_document_pages locates the pages containing a text snippet via the pdf text layer (unpdf). both run locally and never modify the document in paperless. --- README.md | 15 ++-- package-lock.json | 61 +++++++++++++ package.json | 2 + src/__tests__/pdf-tools.test.ts | 146 ++++++++++++++++++++++++++++++++ src/tools/documents.ts | 79 ++++++++++++++++- 5 files changed, 296 insertions(+), 7 deletions(-) create mode 100644 src/__tests__/pdf-tools.test.ts diff --git a/README.md b/README.md index b8643db..13cb66e 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,8 @@ Things you can ask once it's connected: > > "Which documents in my inbox are missing a correspondent or document type?" > +> "Give me just the page of my insurance policy that mentions the deductible, as a PDF" +> > "Find the document about the espresso machine warranty" _(semantic search — no keyword match needed)_ ## Quick start @@ -99,12 +101,13 @@ Semantic search is off by default. To enable it, add `"EMBEDDINGS_ENABLED": "tru ### Extended Tools -| Category | Tools | Description | -| --------------- | ---------------------------------------------------------------------- | -------------------------------------------------------- | -| Semantic search | `semantic_search`, `sync_embeddings`, `embedding_status` | Vector similarity search using local sqlite-vec database | -| Content | `get_document_content` | Extract OCR'd text content from documents | -| Workflows | `auto_classify_document`, `process_inbox`, `bulk_tag_by_content` | AI-assisted classification and bulk operations | -| Helpers | `get_documents_by_correspondent`, `monthly_summary`, `upload_from_url` | Convenience tools for common workflows | +| Category | Tools | Description | +| --------------- | ---------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------- | +| Semantic search | `semantic_search`, `sync_embeddings`, `embedding_status` | Vector similarity search using local sqlite-vec database | +| Content | `get_document_content` | Extract OCR'd text content from documents | +| Workflows | `auto_classify_document`, `process_inbox`, `bulk_tag_by_content` | AI-assisted classification and bulk operations | +| Helpers | `get_documents_by_correspondent`, `monthly_summary`, `upload_from_url` | Convenience tools for common workflows | +| PDF pages | `extract_document_pages`, `find_document_pages` | Extract pages into a new PDF file or find the pages containing a text snippet — processed locally, without modifying the document in Paperless | ### Paperless-ngx 3.0+ Tools diff --git a/package-lock.json b/package-lock.json index 96d74f6..8b16dab 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,6 +10,8 @@ "license": "MIT", "dependencies": { "@modelcontextprotocol/sdk": "^1.30.0", + "pdf-lib": "^1.17.1", + "unpdf": "^1.8.1", "zod": "^3.24.4" }, "bin": { @@ -297,6 +299,24 @@ "url": "https://github.com/sponsors/Boshen" } }, + "node_modules/@pdf-lib/standard-fonts": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@pdf-lib/standard-fonts/-/standard-fonts-1.0.0.tgz", + "integrity": "sha512-hU30BK9IUN/su0Mn9VdlVKsWBS6GyhVfqjwl1FjZN4TxP6cCw0jP2w7V3Hf5uX7M0AZJ16vey9yE0ny7Sa59ZA==", + "license": "MIT", + "dependencies": { + "pako": "^1.0.6" + } + }, + "node_modules/@pdf-lib/upng": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@pdf-lib/upng/-/upng-1.0.1.tgz", + "integrity": "sha512-dQK2FUMQtowVP00mtIksrlZhdFXQZPC+taih1q4CvPZ5vqdxR/LKBaFg0oAfzd1GlHZXXSPdQfzQnt+ViGvEIQ==", + "license": "MIT", + "dependencies": { + "pako": "^1.0.10" + } + }, "node_modules/@rolldown/binding-android-arm64": { "version": "1.2.4", "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm64/-/binding-android-arm64-1.2.4.tgz", @@ -2824,6 +2844,12 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/pako": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/pako/-/pako-1.0.11.tgz", + "integrity": "sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==", + "license": "(MIT AND Zlib)" + }, "node_modules/parseurl": { "version": "1.3.3", "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", @@ -2869,6 +2895,18 @@ "dev": true, "license": "MIT" }, + "node_modules/pdf-lib": { + "version": "1.17.1", + "resolved": "https://registry.npmjs.org/pdf-lib/-/pdf-lib-1.17.1.tgz", + "integrity": "sha512-V/mpyJAoTsN4cnP31vc0wfNA1+p20evqqnap0KLoRUN0Yk/p3wN52DOEsL4oBFcLdb76hlpKPtzJIgo67j/XLw==", + "license": "MIT", + "dependencies": { + "@pdf-lib/standard-fonts": "^1.0.0", + "@pdf-lib/upng": "^1.0.1", + "pako": "^1.0.11", + "tslib": "^1.11.1" + } + }, "node_modules/picocolors": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", @@ -3609,6 +3647,12 @@ "typescript": ">=4.8.4" } }, + "node_modules/tslib": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", + "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==", + "license": "0BSD" + }, "node_modules/tunnel-agent": { "version": "0.6.0", "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", @@ -3711,6 +3755,23 @@ "dev": true, "license": "MIT" }, + "node_modules/unpdf": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/unpdf/-/unpdf-1.8.1.tgz", + "integrity": "sha512-xkURhy2SoGpOIH0a1gLHNkASPIQYonadDJs2AQwPEfUakafeD9EA1WTWWsaR++gfTCXJpV27W7tU1nXuk82UKQ==", + "license": "MIT", + "engines": { + "node": ">=22" + }, + "peerDependencies": { + "@napi-rs/canvas": "^0.1.69 || ^1.0.0" + }, + "peerDependenciesMeta": { + "@napi-rs/canvas": { + "optional": true + } + } + }, "node_modules/unpipe": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", diff --git a/package.json b/package.json index 1b03fff..8c6c36f 100644 --- a/package.json +++ b/package.json @@ -53,6 +53,8 @@ }, "dependencies": { "@modelcontextprotocol/sdk": "^1.30.0", + "pdf-lib": "^1.17.1", + "unpdf": "^1.8.1", "zod": "^3.24.4" }, "optionalDependencies": { diff --git a/src/__tests__/pdf-tools.test.ts b/src/__tests__/pdf-tools.test.ts new file mode 100644 index 0000000..99c3cda --- /dev/null +++ b/src/__tests__/pdf-tools.test.ts @@ -0,0 +1,146 @@ +import { describe, it, expect, vi, afterEach } from "vitest"; +import { mkdtemp, readFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { PDFDocument, StandardFonts } from "pdf-lib"; + +vi.stubEnv("PAPERLESS_URL", "http://localhost:8000"); +vi.stubEnv("PAPERLESS_TOKEN", "test-token-123"); + +const { registerDocumentTools } = await import("../tools/documents.js"); +const { PaperlessClient } = await import("../paperless/client.js"); + +type ToolHandler = (args: any) => Promise<{ content: { text: string }[]; isError?: boolean }>; + +const client = new PaperlessClient("http://localhost:8000", "test-token-123"); +const tools = new Map(); +const server = { + tool: (name: string, _desc: string, _schema: unknown, handler: ToolHandler) => { + tools.set(name, handler); + }, +}; +registerDocumentTools(server as any, client); + +async function makePdf(pageTexts: string[]) { + const doc = await PDFDocument.create(); + const font = await doc.embedFont(StandardFonts.Helvetica); + for (const text of pageTexts) { + const page = doc.addPage([300, 300]); + page.drawText(text, { x: 20, y: 150, size: 12, font }); + } + return doc.save(); +} + +function pdfResponse(bytes: Uint8Array) { + return new Response(Buffer.from(bytes), { + headers: { "content-type": "application/pdf" }, + }); +} + +function parseResult(result: { content: { text: string }[] }) { + return JSON.parse(result.content[0].text); +} + +describe("pdf page tools", () => { + afterEach(() => { + vi.restoreAllMocks(); + }); + + it("registers both tools", () => { + expect(tools.has("extract_document_pages")).toBe(true); + expect(tools.has("find_document_pages")).toBe(true); + }); + + it("extract_document_pages writes the requested pages in order", async () => { + const bytes = await makePdf(["page one", "page two", "page three"]); + const dl = vi.spyOn(client, "download").mockResolvedValue(pdfResponse(bytes)); + const dir = await mkdtemp(join(tmpdir(), "pdf-tools-")); + const output_path = join(dir, "out.pdf"); + + const result = await tools.get("extract_document_pages")!({ + id: 5, + pages: [3, 1], + output_path, + }); + + expect(dl).toHaveBeenCalledWith("/api/documents/5/download/"); + expect(result.isError).toBeUndefined(); + expect(parseResult(result)).toMatchObject({ path: output_path, pages: [3, 1], total_pages: 3 }); + const written = await PDFDocument.load(await readFile(output_path)); + expect(written.getPageCount()).toBe(2); + }); + + it("extract_document_pages passes the original flag", async () => { + const bytes = await makePdf(["only page"]); + const dl = vi.spyOn(client, "download").mockResolvedValue(pdfResponse(bytes)); + const dir = await mkdtemp(join(tmpdir(), "pdf-tools-")); + + await tools.get("extract_document_pages")!({ + id: 9, + pages: [1], + output_path: join(dir, "orig.pdf"), + original: true, + }); + + expect(dl).toHaveBeenCalledWith("/api/documents/9/download/?original=true"); + }); + + it("extract_document_pages errors on out-of-range pages", async () => { + const bytes = await makePdf(["page one", "page two"]); + vi.spyOn(client, "download").mockResolvedValue(pdfResponse(bytes)); + const dir = await mkdtemp(join(tmpdir(), "pdf-tools-")); + + const result = await tools.get("extract_document_pages")!({ + id: 5, + pages: [1, 4], + output_path: join(dir, "bad.pdf"), + }); + + expect(result.isError).toBe(true); + expect(result.content[0].text).toContain("out of range: 4"); + }); + + it("extract_document_pages errors on non-PDF content", async () => { + vi.spyOn(client, "download").mockResolvedValue( + new Response("hello", { headers: { "content-type": "text/plain" } }), + ); + const dir = await mkdtemp(join(tmpdir(), "pdf-tools-")); + + const result = await tools.get("extract_document_pages")!({ + id: 5, + pages: [1], + output_path: join(dir, "nope.pdf"), + }); + + expect(result.isError).toBe(true); + expect(result.content[0].text).toContain("not a PDF"); + }); + + it("find_document_pages returns matching pages with snippets", async () => { + const bytes = await makePdf([ + "Invoice from ACME Corp", + "Total amount 42.50 CHF", + "Terms and conditions apply", + ]); + const dl = vi.spyOn(client, "download").mockResolvedValue(pdfResponse(bytes)); + + const result = await tools.get("find_document_pages")!({ id: 7, query: "Total Amount" }); + + expect(dl).toHaveBeenCalledWith("/api/documents/7/download/"); + expect(result.isError).toBeUndefined(); + const data = parseResult(result); + expect(data.total_pages).toBe(3); + expect(data.matches).toHaveLength(1); + expect(data.matches[0].page).toBe(2); + expect(data.matches[0].snippet).toContain("total amount 42.50"); + }); + + it("find_document_pages returns no matches for absent text", async () => { + const bytes = await makePdf(["page one", "page two"]); + vi.spyOn(client, "download").mockResolvedValue(pdfResponse(bytes)); + + const result = await tools.get("find_document_pages")!({ id: 7, query: "nonexistent" }); + + expect(parseResult(result).matches).toEqual([]); + }); +}); diff --git a/src/tools/documents.ts b/src/tools/documents.ts index cd44985..694d83a 100644 --- a/src/tools/documents.ts +++ b/src/tools/documents.ts @@ -1,7 +1,9 @@ import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; import { z } from "zod"; import { readFile, writeFile } from "node:fs/promises"; -import { ok, err } from "../paperless/format.js"; +import { PDFDocument } from "pdf-lib"; +import { extractText } from "unpdf"; +import { ok, err, buildQS } from "../paperless/format.js"; import type { PaperlessClient } from "../paperless/client.js"; const docSelection = { @@ -170,6 +172,81 @@ export function registerDocumentTools(server: McpServer, client: PaperlessClient }, ); + server.tool( + "extract_document_pages", + "Extract pages from a document's PDF into a new PDF written to disk. Processes the file locally; the document in Paperless is not modified.", + { + id: z.number().describe("Document ID"), + pages: z.array(z.number()).min(1).describe("1-based page numbers, kept in the given order"), + output_path: z.string().describe("Absolute path to write the extracted .pdf to"), + original: z + .boolean() + .optional() + .describe("Use the original file instead of the archived version"), + }, + async ({ id, pages, output_path, original }) => { + try { + const res = await client.download(`/api/documents/${id}/download/${buildQS({ original })}`); + if (!res.ok) throw new Error(`${res.status} ${res.statusText}`); + const ct = res.headers.get("content-type") || ""; + if (!ct.includes("pdf")) throw new Error(`document is not a PDF (content-type: ${ct})`); + const src = await PDFDocument.load(await res.arrayBuffer()); + const total = src.getPageCount(); + const outOfRange = pages.filter((p) => p < 1 || p > total); + if (outOfRange.length) + throw new Error( + `pages out of range: ${outOfRange.join(", ")} (document has ${total} pages)`, + ); + const out = await PDFDocument.create(); + const copied = await out.copyPages( + src, + pages.map((p) => p - 1), + ); + for (const page of copied) out.addPage(page); + const bytes = await out.save(); + await writeFile(output_path, bytes); + return ok({ path: output_path, pages, total_pages: total, bytes: bytes.byteLength }); + } catch (e) { + return err(e); + } + }, + ); + + server.tool( + "find_document_pages", + "Find which pages of a document's PDF contain a text snippet. Searches the PDF text layer locally (scanned documents need an OCR'd archive version). Pair with extract_document_pages to pull out the matching pages.", + { + id: z.number().describe("Document ID"), + query: z.string().describe("Text to search for (case-insensitive)"), + original: z + .boolean() + .optional() + .describe("Search the original file instead of the archived version"), + }, + async ({ id, query, original }) => { + try { + const res = await client.download(`/api/documents/${id}/download/${buildQS({ original })}`); + if (!res.ok) throw new Error(`${res.status} ${res.statusText}`); + const ct = res.headers.get("content-type") || ""; + if (!ct.includes("pdf")) throw new Error(`document is not a PDF (content-type: ${ct})`); + const { totalPages, text } = await extractText(new Uint8Array(await res.arrayBuffer())); + const normalize = (s: string) => s.replace(/\s+/g, " ").toLowerCase(); + const needle = normalize(query); + const matches = text.flatMap((pageText, i) => { + const hay = normalize(pageText); + const idx = hay.indexOf(needle); + if (idx === -1) return []; + const start = Math.max(0, idx - 80); + const snippet = hay.slice(start, idx + needle.length + 80).trim(); + return [{ page: i + 1, snippet }]; + }); + return ok({ query, total_pages: totalPages, matches }); + } catch (e) { + return err(e); + } + }, + ); + server.tool( "test_storage_path", "Preview the filename a storage path template produces for a document, without saving anything",