From 27d814945df08fc0fe7ba8921db2eb6b7fe8d881 Mon Sep 17 00:00:00 2001 From: Aleksandr Slokov Date: Sat, 5 Sep 2026 20:40:08 +0700 Subject: [PATCH 1/2] feat(pdf): read PDF files, including two-column papers (OXD-060) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `oxidiris paper.pdf` now works the same way `oxidiris paper.md` does. Detection is by file header rather than extension, so `cat paper.pdf | oxidiris -` works and a mis-named file still reads. The spike overturned its own premise. OXD-060 assumed shelling out to Poppler's `pdftotext -layout` would be the pragmatic answer; measured on five real two-column papers, the pure-Rust `pdf-extract` beat it on both reading order and speed, so there is no external dependency and no subprocess. Where the two extractors disagreed most (Omega, 0.699 ordered overlap) it was Poppler that was wrong, dropping figure labels into the middle of the abstract. Measurements and the rejected alternatives are in ADR 003. Everything PDF lives in a new `oxidiris-pdf` crate. `oxidiris-core` is untouched and still compiles for wasm32, which is the reason the split exists at all. Extracted text is tidied before the reader sees it: ligatures expanded, hyphenated line breaks welded back into words, isolated short numbers dropped. That last rule was written for running page numbers; measuring the corpus showed it mostly catches chart axis labels instead, and the code now says so. Failures explain themselves — encrypted, damaged, or a scan with no text layer. `pdf-extract` panics rather than returning `Err` on some malformed files, so the panic is caught and converted (spec §4.4). Known gaps, tracked on OXD-061: figure and table text still lands in the body stream (Poppler has the same defect), a PDF yields no heading structure, and a compound breaking at its own hyphen comes back welded shut. `deny.toml` now ignores RUSTSEC-2026-0192 (`ttf-parser` unmaintained), reached through lopdf -> pdf-extract. The advisory says no upgrade exists; the entry names it and says to delete it when lopdf moves to skrifa. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_0189gxRgTxfvzjNYezCfS2iR --- BACKLOG.md | 47 +++- CHANGELOG.md | 25 ++ Cargo.lock | 320 +++++++++++++++++++++++++- Cargo.toml | 3 + README.md | 5 + crates/oxidiris-pdf/Cargo.toml | 16 ++ crates/oxidiris-pdf/src/cleanup.rs | 250 ++++++++++++++++++++ crates/oxidiris-pdf/src/lib.rs | 225 ++++++++++++++++++ crates/oxidiris-pdf/tests/fixtures.rs | 70 ++++++ crates/oxidiris/Cargo.toml | 1 + crates/oxidiris/src/cli.rs | 1 + crates/oxidiris/src/main.rs | 55 ++++- deny.toml | 9 + docs/decisions/pdf-extraction.md | 153 ++++++++++++ testdata/README.md | 18 ++ testdata/make_pdf_fixtures.py | 149 ++++++++++++ testdata/pdf_no_text_layer.pdf | Bin 0 -> 632 bytes testdata/pdf_two_column.pdf | Bin 0 -> 942 bytes testdata/pdf_typography.pdf | Bin 0 -> 912 bytes 19 files changed, 1331 insertions(+), 16 deletions(-) create mode 100644 crates/oxidiris-pdf/Cargo.toml create mode 100644 crates/oxidiris-pdf/src/cleanup.rs create mode 100644 crates/oxidiris-pdf/src/lib.rs create mode 100644 crates/oxidiris-pdf/tests/fixtures.rs create mode 100644 docs/decisions/pdf-extraction.md create mode 100644 testdata/make_pdf_fixtures.py create mode 100644 testdata/pdf_no_text_layer.pdf create mode 100644 testdata/pdf_two_column.pdf create mode 100644 testdata/pdf_typography.pdf diff --git a/BACKLOG.md b/BACKLOG.md index e224aec..c74c5ae 100644 --- a/BACKLOG.md +++ b/BACKLOG.md @@ -157,7 +157,7 @@ Known gaps carried forward, all deliberate and tracked: | OXD-050 | LaTeX Parser | 5 | L | OXD-017 | ⬜ Todo | | OXD-051 | Formula display strategy | 5 | M | OXD-050 | ⬜ Todo | | OXD-052 | Typst Parser | 5 | M | OXD-051 | ⬜ Todo | -| OXD-060 | Spike: PDF extraction | 6 | M | OXD-017 | ⬜ Todo | +| OXD-060 | Spike: PDF extraction | 6 | M | OXD-017 | ✅ Done | | OXD-061 | PDF De-columnizing | 6 | XL | OXD-060 | ⬜ Todo | | OXD-062 | Lazy parsing & cache | 6 | L | OXD-018 | ⬜ Todo | | OXD-070 | EPUB Parser | 7 | L | OXD-062 | ⬜ Todo | @@ -999,25 +999,50 @@ Scope — compare on same 5 real two-column papers (IEEE, ACM, arXiv): Criteria: reading order accuracy · header/footer/footnote handling · performance · maintenance cost · cost of external dependency. Acceptance -- [ ] `docs/decisions/pdf-extraction.md` created with clear recommendation and measured benchmarks -- [ ] 5 sample PDFs added to `testdata/` -- [ ] Human approval obtained before starting OXD-061 +- [x] `docs/decisions/pdf-extraction.md` created with clear recommendation and measured benchmarks +- [x] 5 sample PDFs benchmarked — Borg, Omega, MapReduce, Bigtable, Spanner. **Not committed to + `testdata/`**: they are under publisher copyright, and a fixture has to be redistributable. + Three generated PDFs (`testdata/make_pdf_fixtures.py`) pin the behaviours instead +- [x] Human approval obtained before starting OXD-061 + +Outcome — the spike overturned its own premise. `pdf-extract` (pure Rust) beat Poppler on both +reading order and speed, so there is no external dependency and no shelling out. Shipped as the +`oxidiris-pdf` crate; `oxidiris-core` is untouched and still builds for wasm. Reading order comes +from content-stream order, which is correct on all five papers. --- ### OXD-061 · PDF De-Columnizing -Phase 6 · Depends OXD-060 · Crate core · Spec §8.2 · Size XL +Phase 6 · Depends OXD-060 · Crate `oxidiris-pdf` · Spec §8.2 · Size XL → M -> XL — MUST be broken into sub-tasks before work begins. Do not pick as single block. +> Re-sized after OXD-060. Column clustering turned out **not** to be needed for the papers +> measured: LaTeX emits columns in reading order and the content stream preserves it. What is +> left is the geometry work for figures, which is a smaller and separable job. -Scope — column detection via x-coordinate clustering · reorder reading order · filter header/footer/page numbers · group footnotes · handle figures and captions. +Scope — what OXD-060 knowingly left open: +- Filter figure and table text out of the body stream. Reading Borg, the Figure 1 labels + (`BorgMaster`, `link shard`, `UI shard`) land in the text after the first-page footer. Poppler + has the same defect, so there is no tool to defer to. Needs the figure's bounding box. +- Group footnotes rather than leaving them mid-paragraph. +- Column detection via x-coordinate clustering, as a fallback for files whose content stream is + *not* in reading order. No such file is in the corpus yet; find one before building this. Acceptance -- [ ] 5 sample PDFs read in correct column order (manually verified) -- [ ] No column interleaving (left column text jumping into right column) -- [ ] Single-column PDFs remain functional -- [ ] Encrypted/scanned image PDFs → clean error message, no panic (§4.4) +- [x] 5 sample PDFs read in correct column order (verified by cross-checking two independent + extractors; the one disagreement, Omega, was Poppler's fault — see ADR 003) +- [x] No column interleaving (left column text jumping into right column) +- [x] Single-column PDFs remain functional +- [x] Scanned image PDFs → clean error message, no panic (§4.4) +- [ ] Encrypted PDFs → clean error message, no panic. Implemented and refused with a message, but + **untested**: there is no encrypted fixture. Writing an RC4-encrypted PDF by hand was judged + not worth the cost +- [ ] Figure and table text no longer appears in the body stream + +Carried debt from OXD-060: `deny.toml` now ignores **RUSTSEC-2026-0192** (`ttf-parser` +unmaintained), reached through `lopdf` → `pdf-extract`. The advisory says no upgrade exists; the +replacement, `skrifa`, has to be adopted upstream in lopdf. Re-check on every dependency bump and +delete the ignore entry as soon as it lands. --- diff --git a/CHANGELOG.md b/CHANGELOG.md index f2599b0..fd236ea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,31 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). While the Phase 3 is not finished: `OXD-033` (`%` jump) and `OXD-035` (ramp-up on resume) remain. Both are small; see [`BACKLOG.md`](BACKLOG.md). +### Added + +- **PDF support.** `oxidiris paper.pdf` opens a PDF the same way it opens Markdown, including + two-column conference papers. Detection is by file header rather than extension, so + `cat paper.pdf | oxidiris -` works and a mis-named file still reads. Closes `OXD-060`. +- New crate `oxidiris-pdf`, holding the whole PDF dependency tree. `oxidiris-core` gains nothing + and still compiles for `wasm32-unknown-unknown`. +- Extracted text is tidied before it reaches the reader: ligatures expanded (`efficient` → + `efficient`), hyphenated line breaks welded back into words (`admission con-` / `trol` → + `admission control`), and isolated short numbers — chart axis labels and page numbers — dropped + so they do not each get their own pause in the RSVP frame. +- A PDF that cannot be read says why: encrypted, damaged, or a scan with no text layer, the last + naming the page count and pointing at OCR. `pdf-extract` panics rather than erroring on some + malformed files, so the panic is caught and turned into a message (spec §4.4). +- `testdata/make_pdf_fixtures.py` generates three small PDF fixtures with no third-party library. + +### Known limitations + +- Figure and table text still lands in the body stream — reading Borg, the Figure 1 labels appear + after the first-page footer. Poppler has the same defect. Tracked as `OXD-061`. +- A PDF produces no heading structure, so the outline panel is empty for one. +- A compound that breaks across lines at its own hyphen (`over-` / `commitment`) comes back + welded shut. Telling that apart from a typesetter's hyphen needs a dictionary. +- The encrypted-PDF path is implemented but has no test fixture. + ## [0.2.1] - 2026-09-02 Packaging only — the reader itself is byte-for-byte the same as v0.2.0. Installing no longer diff --git a/Cargo.lock b/Cargo.lock index 3850c36..5c9384b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,32 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "adobe-cmap-parser" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae8abfa9a4688de8fc9f42b3f013b6fffec18ed8a554f5f113577e0b9b3212a3" +dependencies = [ + "pom", +] + +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures 0.2.17", +] + [[package]] name = "aho-corasick" version = "1.1.5" @@ -154,6 +180,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block-padding" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93" +dependencies = [ + "generic-array", +] + [[package]] name = "bumpalo" version = "3.20.3" @@ -181,6 +216,21 @@ dependencies = [ "rustversion", ] +[[package]] +name = "cbc" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26b52a9543ae338f279b96b0b9fed9c8093744685043739079ce85cd58f289a6" +dependencies = [ + "cipher", +] + +[[package]] +name = "cff-parser" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5810ca1a2b5870df2aab1c03e11c40c361ba51d6e3e361e56310f1cb3b4e087" + [[package]] name = "cfg-if" version = "1.0.4" @@ -193,6 +243,27 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" +[[package]] +name = "chacha20" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.1", + "rand_core 0.10.1", +] + +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common", + "inout", +] + [[package]] name = "clap" version = "4.6.6" @@ -282,6 +353,24 @@ dependencies = [ "libc", ] +[[package]] +name = "cpufeatures" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32fast" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8498c871161e1742aaa9d52551b2d6ebdd4c3d45a3be423e3728f33b955be550" +dependencies = [ + "cfg-if", +] + [[package]] name = "critical-section" version = "1.2.0" @@ -422,6 +511,15 @@ dependencies = [ "litrs", ] +[[package]] +name = "ecb" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a8bfa975b1aec2145850fcaa1c6fe269a16578c44705a532ae3edc92b8881c7" +dependencies = [ + "cipher", +] + [[package]] name = "either" version = "1.18.0" @@ -459,6 +557,15 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "euclid" +version = "0.20.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bb7ef65b3777a325d1eeefefab5b6d4959da54747e33bd6258e789640f307ad" +dependencies = [ + "num-traits", +] + [[package]] name = "euclid" version = "0.22.14" @@ -507,6 +614,17 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" +[[package]] +name = "flate2" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb" +dependencies = [ + "crc32fast", + "miniz_oxide", + "zlib-rs", +] + [[package]] name = "fnv" version = "1.0.7" @@ -572,8 +690,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" dependencies = [ "cfg-if", + "js-sys", "libc", "r-efi 6.0.0", + "rand_core 0.10.1", + "wasm-bindgen", ] [[package]] @@ -616,6 +737,16 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" +[[package]] +name = "indexmap" +version = "2.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" +dependencies = [ + "equivalent", + "hashbrown 0.17.1", +] + [[package]] name = "indoc" version = "2.0.7" @@ -625,6 +756,16 @@ dependencies = [ "rustversion", ] +[[package]] +name = "inout" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +dependencies = [ + "block-padding", + "generic-array", +] + [[package]] name = "insta" version = "1.48.0" @@ -753,6 +894,33 @@ version = "0.4.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" +[[package]] +name = "lopdf" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25aab26d99567469098e64a02f42679f8965c6401263eefa31d8f2dcc37a221c" +dependencies = [ + "aes", + "bitflags 2.13.1", + "cbc", + "ecb", + "encoding_rs", + "flate2", + "getrandom 0.4.3", + "indexmap", + "itoa", + "log", + "md-5", + "nom 8.0.0", + "rand 0.10.2", + "rangemap", + "sha2", + "stringprep", + "thiserror 2.0.20", + "ttf-parser", + "weezl", +] + [[package]] name = "lru" version = "0.18.3" @@ -772,6 +940,16 @@ dependencies = [ "winapi", ] +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest", +] + [[package]] name = "memchr" version = "2.8.3" @@ -799,6 +977,16 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" +[[package]] +name = "miniz_oxide" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c" +dependencies = [ + "adler2", + "simd-adler32", +] + [[package]] name = "mio" version = "1.2.2" @@ -834,6 +1022,15 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "nom" +version = "8.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df9761775871bdef83bee530e60050f7e54b1105350d6884eb0fb4f46c2f9405" +dependencies = [ + "memchr", +] + [[package]] name = "num-conv" version = "0.2.2" @@ -899,6 +1096,7 @@ dependencies = [ "crossterm", "insta", "oxidiris-core", + "oxidiris-pdf", "ratatui", ] @@ -915,6 +1113,14 @@ dependencies = [ "unicode-width", ] +[[package]] +name = "oxidiris-pdf" +version = "0.2.1" +dependencies = [ + "pdf-extract", + "thiserror 2.0.20", +] + [[package]] name = "palette" version = "0.7.7" @@ -971,6 +1177,23 @@ dependencies = [ "windows-link", ] +[[package]] +name = "pdf-extract" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "417e8fdc940f1d5bc62c5f89864c3a2255f74f69aa353c98509213d67df61e73" +dependencies = [ + "adobe-cmap-parser", + "cff-parser", + "encoding_rs", + "euclid 0.20.14", + "log", + "lopdf", + "postscript", + "type1-encoding-parser", + "unicode-normalization", +] + [[package]] name = "pest" version = "2.9.0" @@ -1071,12 +1294,24 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" +[[package]] +name = "pom" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60f6ce597ecdcc9a098e7fddacb1065093a3d66446fa16c675e7e71d1b5c28e6" + [[package]] name = "portable-atomic" version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" +[[package]] +name = "postscript" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78451badbdaebaf17f053fd9152b3ffb33b516104eacb45e7864aaa9c712f306" + [[package]] name = "powerfmt" version = "0.2.0" @@ -1177,6 +1412,17 @@ dependencies = [ "rand_core 0.9.5", ] +[[package]] +name = "rand" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" +dependencies = [ + "chacha20", + "getrandom 0.4.3", + "rand_core 0.10.1", +] + [[package]] name = "rand_chacha" version = "0.9.0" @@ -1202,6 +1448,12 @@ dependencies = [ "getrandom 0.3.4", ] +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + [[package]] name = "rand_xorshift" version = "0.4.0" @@ -1211,6 +1463,12 @@ dependencies = [ "rand_core 0.9.5", ] +[[package]] +name = "rangemap" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a611d15b50743feb4c76b7d03edcb0e64f399c26961e4efe6975bc398be6aa3d" + [[package]] name = "ratatui" version = "0.30.2" @@ -1445,7 +1703,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "digest", ] @@ -1480,6 +1738,12 @@ dependencies = [ "libc", ] +[[package]] +name = "simd-adler32" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" + [[package]] name = "similar" version = "2.7.0" @@ -1510,6 +1774,17 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +[[package]] +name = "stringprep" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1" +dependencies = [ + "unicode-bidi", + "unicode-normalization", + "unicode-properties", +] + [[package]] name = "strsim" version = "0.11.1" @@ -1603,7 +1878,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d4ea810f0692f9f51b382fff5893887bb4580f5fa246fde546e0b13e7fcee662" dependencies = [ "fnv", - "nom", + "nom 7.1.3", "phf", "phf_codegen", ] @@ -1735,6 +2010,21 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" +[[package]] +name = "ttf-parser" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2df906b07856748fa3f6e0ad0cbaa047052d4a7dd609e231c4f72cee8c36f31" + +[[package]] +name = "type1-encoding-parser" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa10c302f5a53b7ad27fd42a3996e23d096ba39b5b8dd6d9e683a05b01bee749" +dependencies = [ + "pom", +] + [[package]] name = "typenum" version = "1.20.1" @@ -1759,6 +2049,12 @@ version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142" +[[package]] +name = "unicode-bidi" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" + [[package]] name = "unicode-ident" version = "1.0.24" @@ -1774,6 +2070,12 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "unicode-properties" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d" + [[package]] name = "unicode-segmentation" version = "1.13.3" @@ -1899,6 +2201,12 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "weezl" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a28ac98ddc8b9274cb41bb4d9d4d5c425b6020c50c46f25559911905610b4a88" + [[package]] name = "wezterm-bidi" version = "0.2.3" @@ -1965,7 +2273,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7012add459f951456ec9d6c7e6fc340b1ce15d6fc9629f8c42853412c029e57e" dependencies = [ "bitflags 1.3.2", - "euclid", + "euclid 0.22.14", "lazy_static", "serde", "wezterm-dynamic", @@ -2033,3 +2341,9 @@ dependencies = [ "quote", "syn 2.0.119", ] + +[[package]] +name = "zlib-rs" +version = "0.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34b31d188d9d685a4f9c7b46d6e36631b07058d2cfe190267adce54dc230bf12" diff --git a/Cargo.toml b/Cargo.toml index 4a2c8a7..df8b30d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,6 +22,9 @@ unicode-width = "0.2" pulldown-cmark = { version = "0.13", default-features = false } thiserror = "2" +# --- pdf extraction (native only: pulls in font and codec crates, never linked into core) --- +pdf-extract = "0.12" + # --- binary only --- ratatui = "0.30.2" crossterm = "0.29" diff --git a/README.md b/README.md index ffd689e..ac2c2ee 100644 --- a/README.md +++ b/README.md @@ -43,6 +43,10 @@ reader frame is showing. It appears whenever the window is at least 80 columns w [`CHANGELOG.md`](CHANGELOG.md) for what this release contains and [`BACKLOG.md`](BACKLOG.md) for what comes next. +**Unreleased** — PDFs read too, including two-column conference papers. Ligatures, hyphenated +line breaks and stray page numbers are cleaned up on the way in; figure labels are not, yet. How +that was decided, and measured, is in [ADR 003](docs/decisions/pdf-extraction.md). + ## What it is not RSVP removes *regressions*, the small backward eye movements that make up 10-15% of natural reading @@ -141,6 +145,7 @@ oxidiris README.md # read at the default 300 WPM oxidiris paper.md -w 450 # start faster oxidiris paper.md -m focus # hide the text panel, reader frame only oxidiris notes.txt --pacing linear # even timing, no punctuation pauses +oxidiris borg.pdf # PDFs too, two columns included oxidiris BACKLOG.md --dump # clean plain text on stdout oxidiris BACKLOG.md | less # piping produces text, not a broken TUI ``` diff --git a/crates/oxidiris-pdf/Cargo.toml b/crates/oxidiris-pdf/Cargo.toml new file mode 100644 index 0000000..33262c1 --- /dev/null +++ b/crates/oxidiris-pdf/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "oxidiris-pdf" +description = "PDF text extraction for Oxidiris: turns a PDF into the plain text the core engine reads." +license = "GPL-3.0-or-later" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +authors.workspace = true +repository.workspace = true +readme.workspace = true +keywords.workspace = true +categories.workspace = true + +[dependencies] +pdf-extract.workspace = true +thiserror.workspace = true \ No newline at end of file diff --git a/crates/oxidiris-pdf/src/cleanup.rs b/crates/oxidiris-pdf/src/cleanup.rs new file mode 100644 index 0000000..cc89de3 --- /dev/null +++ b/crates/oxidiris-pdf/src/cleanup.rs @@ -0,0 +1,250 @@ +//! Turning raw extracted text into something worth reading. +//! +//! A PDF stores glyphs at coordinates, not sentences. What comes back out therefore carries three +//! artefacts that would each be visible in the RSVP frame, one word at a time, at 400 WPM: +//! +//! 1. **Typographic ligatures.** `efficient` is stored as `e`, `f`, U+FB01, `c`… The frame would +//! show a glyph most terminal fonts cannot draw, and a search for "efficient" would miss it. +//! 2. **Hyphenated line breaks.** Justified two-column text breaks `hun-` / `dreds` across lines. +//! The plain-text parser rejoins wrapped lines with a space, which would yield `hun- dreds` — +//! two nonsense display units instead of one word. +//! 3. **Stray numbers from page furniture and charts.** A bare `7` standing between two +//! paragraphs becomes a paragraph of its own, so the reader gets a full pause on something +//! that is not part of the text. Measured on five real papers, most of these turn out to be +//! axis tick labels lifted out of figures rather than running page numbers; the rule catches +//! both, because to a reader they are the same interruption. +//! +//! Every step here is a pure `&str -> String` function, so the whole pipeline is testable without +//! a PDF anywhere in sight. + +/// Ligatures that must be expanded before the text reaches a terminal. +/// +/// Deliberately a short explicit table rather than NFKC normalisation: NFKC would also rewrite +/// superscripts, fractions, Roman numeral characters and CJK compatibility forms, all of which +/// carry meaning in a technical paper and must survive untouched. +const LIGATURES: [(char, &str); 7] = [ + ('\u{FB00}', "ff"), + ('\u{FB01}', "fi"), + ('\u{FB02}', "fl"), + ('\u{FB03}', "ffi"), + ('\u{FB04}', "ffl"), + ('\u{FB05}', "st"), // long s + t + ('\u{FB06}', "st"), +]; + +/// Hyphen characters that a typesetter may use to break a word across two lines. +const BREAKING_HYPHENS: [char; 2] = ['-', '\u{2010}']; + +/// Longest numeric line still treated as page furniture rather than content. +const STRAY_NUMBER_MAX_LEN: usize = 4; + +/// Clean raw extracted text into readable prose. +/// +/// ``` +/// let raw = "hun-\ndreds of e\u{FB03}cient jobs\n\n7\n\nNext paragraph.\n"; +/// let text = oxidiris_pdf::tidy(raw); +/// assert_eq!(text, "hundreds of efficient jobs\n\nNext paragraph.\n"); +/// ``` +pub fn tidy(raw: &str) -> String { + let folded = expand_ligatures(raw); + let welded = weld_hyphenated_breaks(&folded); + let stripped = drop_isolated_numbers(&welded); + collapse_blank_runs(&stripped) +} + +/// Replace typographic ligature code points with the letters they stand for. +fn expand_ligatures(text: &str) -> String { + let mut out = String::with_capacity(text.len()); + for c in text.chars() { + match LIGATURES.iter().find(|(ligature, _)| *ligature == c) { + Some((_, letters)) => out.push_str(letters), + None => out.push(c), + } + } + out +} + +/// Join a line ending in a hyphen onto the next one, dropping the hyphen. +/// +/// Every line is trimmed on both sides on the way through: a PDF's leading whitespace is an +/// artefact of glyph positions, never indentation the author typed. +fn weld_hyphenated_breaks(text: &str) -> String { + let mut out = String::with_capacity(text.len()); + let mut lines = text.lines().peekable(); + while let Some(line) = lines.next() { + let current = line.trim(); + let next = lines.peek().map(|l| l.trim()); + if let Some(stem) = word_split_across_lines(current, next) { + out.push_str(stem); + continue; + } + out.push_str(current); + out.push('\n'); + } + out +} + +/// The line without its trailing hyphen, when that hyphen breaks a word across two lines. +/// +/// Requires a letter on both sides of the break, so a dash used as punctuation (`text - `) and a +/// list bullet are left alone. A compound that genuinely contains a hyphen and happens to break at +/// it (`Google-` / `wide`) is welded into one word; distinguishing those needs a dictionary, and +/// every extractor in the field makes the same trade. +fn word_split_across_lines<'a>(current: &'a str, next: Option<&str>) -> Option<&'a str> { + let next = next?; + if !next.chars().next()?.is_lowercase() { + return None; + } + let hyphen = *BREAKING_HYPHENS.iter().find(|h| current.ends_with(**h))?; + let stem = ¤t[..current.len() - hyphen.len_utf8()]; + stem.chars().next_back()?.is_alphanumeric().then_some(stem) +} + +/// Remove short numeric lines that stand alone between blank lines. +/// +/// Isolation is what makes this safe: a `7` or an `I` inside a paragraph keeps its neighbours and +/// is never touched, so only a number that already reads as its own paragraph can match. +fn drop_isolated_numbers(text: &str) -> String { + let lines: Vec<&str> = text.lines().collect(); + let mut out = String::with_capacity(text.len()); + for (index, line) in lines.iter().enumerate() { + let alone_above = index == 0 || lines[index - 1].trim().is_empty(); + let alone_below = index + 1 >= lines.len() || lines[index + 1].trim().is_empty(); + if alone_above && alone_below && is_stray_number(line) { + continue; + } + out.push_str(line); + out.push('\n'); + } + out +} + +/// Whether a line is nothing but a short number, in arabic or roman form. +fn is_stray_number(line: &str) -> bool { + let text = line.trim(); + if text.is_empty() || text.len() > STRAY_NUMBER_MAX_LEN { + return false; + } + text.chars().all(|c| c.is_ascii_digit()) + || text.chars().all(|c| matches!(c, 'i' | 'v' | 'x' | 'l' | 'I' | 'V' | 'X' | 'L')) +} + +/// Reduce runs of blank lines to a single one and guarantee a trailing newline. +/// +/// The plain-text parser reads one blank line as a paragraph break; more than one adds nothing, +/// and removing a stray number leaves a doubled gap behind every time. +fn collapse_blank_runs(text: &str) -> String { + let mut out = String::with_capacity(text.len()); + let mut blank_pending = false; + for line in text.lines() { + if line.trim().is_empty() { + blank_pending = !out.is_empty(); + continue; + } + if blank_pending { + out.push('\n'); + } + out.push_str(line); + out.push('\n'); + blank_pending = false; + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn ligatures_become_the_letters_they_stand_for() { + assert_eq!(expand_ligatures("e\u{FB03}cient"), "efficient"); + assert_eq!(expand_ligatures("bene\u{FB01}ts and \u{FB02}ags"), "benefits and flags"); + assert_eq!(expand_ligatures("o\u{FB00}er"), "offer"); + } + + /// NFKC would fold these too; the explicit table must not. + #[test] + fn superscripts_and_fractions_survive_untouched() { + assert_eq!(expand_ligatures("x\u{00B2} and \u{00BD}"), "x\u{00B2} and \u{00BD}"); + } + + #[test] + fn a_word_broken_across_lines_is_welded_back_together() { + assert_eq!(weld_hyphenated_breaks("hun-\ndreds\n"), "hundreds\n"); + assert_eq!(weld_hyphenated_breaks("e\u{2010}\nmail\n"), "email\n"); + } + + #[test] + fn a_hyphen_before_a_capital_stays_a_hyphen() { + assert_eq!(weld_hyphenated_breaks("Sub-\nSaharan\n"), "Sub-\nSaharan\n"); + } + + #[test] + fn a_dash_used_as_punctuation_is_not_a_word_break() { + assert_eq!(weld_hyphenated_breaks("the answer -\nmaybe\n"), "the answer -\nmaybe\n"); + } + + #[test] + fn a_trailing_hyphen_on_the_last_line_is_kept() { + assert_eq!(weld_hyphenated_breaks("dangling-\n"), "dangling-\n"); + } + + #[test] + fn an_isolated_number_is_dropped() { + assert_eq!(drop_isolated_numbers("End.\n\n7\n\nStart.\n"), "End.\n\n\nStart.\n"); + assert_eq!(drop_isolated_numbers("End.\n\nxiv\n\nStart.\n"), "End.\n\n\nStart.\n"); + } + + /// The isolation rule is the whole safety net: a number inside prose must survive. + #[test] + fn a_number_inside_a_paragraph_is_kept() { + let text = "we ran\n7\nmachines\n"; + assert_eq!(drop_isolated_numbers(text), text); + } + + #[test] + fn a_long_number_is_content_not_furniture() { + let text = "End.\n\n20155\n\nStart.\n"; + assert_eq!(drop_isolated_numbers(text), text); + } + + #[test] + fn blank_runs_collapse_to_one_paragraph_break() { + assert_eq!(collapse_blank_runs("a\n\n\n\nb\n"), "a\n\nb\n"); + assert_eq!(collapse_blank_runs("\n\n\na\n"), "a\n"); + assert_eq!(collapse_blank_runs(""), ""); + } + + /// A hyphen the author typed must survive; only the one the typesetter added disappears. + /// + /// Both appear in this excerpt from the Borg paper: `task-packing` sits mid-line and is real, + /// `con-` / `trol` spans a line break and is not. + #[test] + fn the_whole_pipeline_produces_readable_paragraphs() { + let raw = "It achieves high utilization by combining admission con-\n\ + trol, ef\u{FB01}cient task-packing, over-commitment, and machine\n\ + sharing with isolation.\n\ + \n\ + \n\ + 12\n\ + \n\ + The next section explains how.\n"; + let text = tidy(raw); + assert!(text.contains("admission control,"), "got {text:?}"); + assert!(text.contains("efficient task-packing,"), "got {text:?}"); + assert!(!text.contains("\n12\n"), "page number survived: {text:?}"); + assert_eq!(text.matches("\n\n").count(), 1, "one paragraph break expected: {text:?}"); + } + + /// Known limitation, recorded so a future change to the heuristic is a deliberate one: a real + /// compound that happens to break at its own hyphen comes back welded shut. + #[test] + fn a_compound_broken_at_its_own_hyphen_loses_it() { + assert_eq!(weld_hyphenated_breaks("over-\ncommitment\n"), "overcommitment\n"); + } + + #[test] + fn empty_input_stays_empty() { + assert_eq!(tidy(""), ""); + } +} diff --git a/crates/oxidiris-pdf/src/lib.rs b/crates/oxidiris-pdf/src/lib.rs new file mode 100644 index 0000000..30ed93a --- /dev/null +++ b/crates/oxidiris-pdf/src/lib.rs @@ -0,0 +1,225 @@ +//! PDF text extraction for Oxidiris. +//! +//! Implements OXD-060. See spec §8.2 and `docs/decisions/pdf-extraction.md`. +//! +//! # Why this is a crate of its own +//! +//! `oxidiris-core` must keep building for `wasm32-unknown-unknown`, and the PDF backend drags in +//! font parsers, cryptography and image codecs that have no business in that build. Keeping the +//! dependency here means the core stays small and the wasm job keeps passing. +//! +//! # Where this sits in the pipeline +//! +//! ```text +//! pdf bytes -> oxidiris_pdf::extract -> plain text -> oxidiris_core::load -> Document +//! ``` +//! +//! The output is deliberately plain text rather than a `Document`: a PDF carries no reliable +//! block structure, so the existing plain-text parser — which already rejoins hard-wrapped lines +//! — is exactly the right consumer. +//! +//! # Example +//! +//! ```no_run +//! let bytes = std::fs::read("paper.pdf").unwrap(); +//! let text = oxidiris_pdf::extract(&bytes).unwrap(); +//! assert!(text.contains(' ')); +//! ``` + +#![forbid(unsafe_code)] +#![warn(missing_docs)] + +mod cleanup; + +use std::panic::{self, AssertUnwindSafe}; + +use pdf_extract::{Document, PlainTextOutput, output_doc}; +use thiserror::Error; + +pub use cleanup::tidy; + +/// Why a PDF could not be turned into readable text. +#[derive(Debug, Error, PartialEq, Eq)] +pub enum PdfError { + /// The bytes do not begin with a PDF header. + #[error("not a PDF: expected the file to start with `%PDF-`, found {head:?}")] + NotPdf { + /// The first few bytes, rendered printably, so the message names the offending value. + head: String, + }, + + /// The file is password protected. + #[error("the PDF is encrypted; oxidiris cannot open password-protected files")] + Encrypted, + + /// The backend could not make sense of the file structure. + #[error("the PDF is damaged or uses an unsupported feature: {detail}")] + Malformed { + /// What the backend reported. + detail: String, + }, + + /// The pages carry no text, only images. + #[error( + "the PDF has no text layer ({pages} {} scanned); it is probably a scan and needs OCR", + if *pages == 1 { "page" } else { "pages" } + )] + NoTextLayer { + /// How many pages were inspected. + pages: usize, + }, +} + +/// Number of leading bytes quoted back in a [`PdfError::NotPdf`] message. +const HEAD_LEN: usize = 8; + +/// Whether `bytes` start with a PDF header. +/// +/// Used to route a file to this crate before the core's text decoder rejects it as binary, and to +/// recognise a PDF arriving on standard input, where there is no extension to go by. +/// +/// ``` +/// assert!(oxidiris_pdf::looks_like_pdf(b"%PDF-1.7\n...")); +/// assert!(!oxidiris_pdf::looks_like_pdf(b"# Just markdown\n")); +/// ``` +pub fn looks_like_pdf(bytes: &[u8]) -> bool { + bytes.starts_with(b"%PDF-") +} + +/// Extract readable plain text from the bytes of a PDF. +/// +/// The returned text is already tidied for reading: ligatures expanded, hyphenated line breaks +/// welded back together, isolated short numbers removed. Feed it straight to +/// `oxidiris_core::load` as plain text. +/// +/// ```no_run +/// let bytes = std::fs::read("paper.pdf").unwrap(); +/// match oxidiris_pdf::extract(&bytes) { +/// Ok(text) => println!("{} characters", text.len()), +/// Err(err) => eprintln!("{err}"), +/// } +/// ``` +pub fn extract(bytes: &[u8]) -> Result { + if !looks_like_pdf(bytes) { + return Err(PdfError::NotPdf { head: printable_head(bytes) }); + } + + let document = load(bytes)?; + if document.is_encrypted() { + return Err(PdfError::Encrypted); + } + + let pages = document.get_pages().len(); + let text = tidy(&render(&document)?); + if !text.chars().any(char::is_alphanumeric) { + return Err(PdfError::NoTextLayer { pages }); + } + Ok(text) +} + +/// Parse the file structure. +fn load(bytes: &[u8]) -> Result { + let outcome = guard(|| Document::load_mem(bytes))?; + outcome.map_err(|e| PdfError::Malformed { detail: e.to_string() }) +} + +/// Walk every page and collect the text in content-stream order. +fn render(document: &Document) -> Result { + let mut text = String::new(); + let outcome = guard(|| { + let mut sink = PlainTextOutput::new(&mut text); + output_doc(document, &mut sink) + })?; + outcome.map_err(|e| PdfError::Malformed { detail: e.to_string() })?; + Ok(text) +} + +/// Run `job`, turning a panic inside the PDF backend into a [`PdfError::Malformed`]. +/// +/// `pdf-extract` panics instead of returning `Err` on several classes of broken file — unknown +/// font encodings, truncated cross-reference tables, unbalanced content streams. Spec §4.4 says a +/// reader must answer those with a clean message, not a crash, so the panic is caught here. +/// +/// The default panic printer is silenced for the duration of the call, otherwise the backtrace +/// would still reach the terminal. That hook is process-wide: a panic on another thread during +/// this window would be swallowed too. Extraction happens once, before any threads or the +/// terminal exist, so the window is closed again long before that could matter. +fn guard(job: impl FnOnce() -> T) -> Result { + let previous = panic::take_hook(); + panic::set_hook(Box::new(|_| {})); + let outcome = panic::catch_unwind(AssertUnwindSafe(job)); + panic::set_hook(previous); + outcome.map_err(|payload| PdfError::Malformed { detail: panic_detail(&payload) }) +} + +/// Recover the message from a caught panic payload. +fn panic_detail(payload: &Box) -> String { + if let Some(s) = payload.downcast_ref::<&str>() { + return (*s).to_string(); + } + if let Some(s) = payload.downcast_ref::() { + return s.clone(); + } + "the PDF backend panicked".to_string() +} + +/// Render the first few bytes printably, for the "this is not a PDF" message. +fn printable_head(bytes: &[u8]) -> String { + bytes + .iter() + .take(HEAD_LEN) + .map(|b| { + if b.is_ascii_graphic() || *b == b' ' { + (*b as char).to_string() + } else { + format!("\\x{b:02x}") + } + }) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_pdf_header_is_what_identifies_a_pdf() { + assert!(looks_like_pdf(b"%PDF-1.4\n")); + assert!(!looks_like_pdf(b"%PDF"), "a truncated header is not a PDF"); + assert!(!looks_like_pdf(b"")); + assert!(!looks_like_pdf(b"# Markdown\n")); + } + + #[test] + fn non_pdf_input_is_rejected_with_the_offending_bytes_in_the_message() { + let err = extract(b"\x00\x01not a pdf").unwrap_err(); + let message = err.to_string(); + assert!(message.contains("\\x00"), "message should quote the head: {message}"); + assert!(message.contains("%PDF-"), "message should name the expected shape: {message}"); + } + + #[test] + fn empty_input_is_rejected_rather_than_panicking() { + assert!(matches!(extract(b""), Err(PdfError::NotPdf { .. }))); + } + + /// Spec §4.4: a damaged file must produce a message, never a crash. + #[test] + fn a_truncated_pdf_is_an_error_not_a_panic() { + let err = extract(b"%PDF-1.7\nthis file stops right here").unwrap_err(); + assert!(matches!(err, PdfError::Malformed { .. }), "got {err:?}"); + } + + /// A one-page scan is the common case; "1 pages scanned" would look like a bug in the tool. + #[test] + fn the_no_text_message_counts_pages_in_the_right_number() { + assert!(PdfError::NoTextLayer { pages: 1 }.to_string().contains("1 page scanned")); + assert!(PdfError::NoTextLayer { pages: 14 }.to_string().contains("14 pages scanned")); + } + + #[test] + fn printable_head_escapes_non_printable_bytes() { + assert_eq!(printable_head(b"%PDF-1.7extra"), "%PDF-1.7"); + assert_eq!(printable_head(&[0x00, 0xff]), "\\x00\\xff"); + } +} diff --git a/crates/oxidiris-pdf/tests/fixtures.rs b/crates/oxidiris-pdf/tests/fixtures.rs new file mode 100644 index 0000000..3426bf2 --- /dev/null +++ b/crates/oxidiris-pdf/tests/fixtures.rs @@ -0,0 +1,70 @@ +//! Corpus-driven tests against the generated PDF fixtures. +//! +//! Implements the verification half of OXD-060. The fixtures are produced by +//! `testdata/make_pdf_fixtures.py`; see `testdata/README.md` for what each one pins down. + +use std::path::{Path, PathBuf}; + +use oxidiris_pdf::PdfError; + +fn testdata(name: &str) -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).join("../../testdata").join(name) +} + +fn extract(name: &str) -> Result { + let bytes = std::fs::read(testdata(name)).unwrap_or_else(|e| panic!("cannot read {name}: {e}")); + oxidiris_pdf::extract(&bytes) +} + +#[test] +fn every_fixture_is_recognised_as_a_pdf() { + for name in ["pdf_typography.pdf", "pdf_two_column.pdf", "pdf_no_text_layer.pdf"] { + let bytes = std::fs::read(testdata(name)).unwrap(); + assert!(oxidiris_pdf::looks_like_pdf(&bytes), "{name} lost its header"); + } +} + +/// The three artefacts that would otherwise reach the RSVP frame, on a real PDF rather than a +/// hand-written string. +#[test] +fn typography_artefacts_do_not_survive_extraction() { + let text = extract("pdf_typography.pdf").expect("fixture should extract"); + + assert!(text.contains("efficient"), "ligature not expanded: {text:?}"); + assert!(!text.contains('\u{FB01}'), "raw fi ligature left in: {text:?}"); + assert!(text.contains("admission control"), "hyphenated break not welded: {text:?}"); + assert!(!text.contains("con-"), "hyphen left behind: {text:?}"); + assert!(!text.contains("12"), "page number survived: {text:?}"); +} + +/// A hyphen the author typed is not a line-break hyphen and must survive. +#[test] +fn a_real_compound_keeps_its_hyphen() { + let text = extract("pdf_typography.pdf").unwrap(); + assert!(text.contains("process-level"), "got {text:?}"); +} + +/// Spec §8.2: no column interleaving. The left column must be finished before the right starts. +#[test] +fn two_columns_are_read_one_after_the_other() { + let text = extract("pdf_two_column.pdf").expect("fixture should extract"); + + let left_end = text.find("bottom of the").expect("left column missing"); + let right_start = text.find("right column is read second").expect("right column missing"); + assert!(left_end < right_start, "columns interleaved: {text:?}"); + + // Line breaks are still present here; rejoining them into paragraphs is the plain-text + // parser's job, one layer further on. + assert!( + text.contains("The left column starts here and\ncontinues to the bottom of the\npage."), + "left column lost its own lines: {text:?}" + ); +} + +/// Spec §4.4: a scan must produce an explanation, not an empty reader or a crash. +#[test] +fn a_page_without_a_text_layer_is_reported_as_such() { + let err = extract("pdf_no_text_layer.pdf").unwrap_err(); + assert_eq!(err, PdfError::NoTextLayer { pages: 1 }); + assert!(err.to_string().contains("OCR"), "message should say why: {err}"); +} diff --git a/crates/oxidiris/Cargo.toml b/crates/oxidiris/Cargo.toml index 6375566..511ab5a 100644 --- a/crates/oxidiris/Cargo.toml +++ b/crates/oxidiris/Cargo.toml @@ -13,6 +13,7 @@ categories.workspace = true [dependencies] oxidiris-core = { path = "../oxidiris-core", version = "0.2.1" } +oxidiris-pdf = { path = "../oxidiris-pdf", version = "0.2.1" } ratatui.workspace = true crossterm.workspace = true clap.workspace = true diff --git a/crates/oxidiris/src/cli.rs b/crates/oxidiris/src/cli.rs index 5fbb8d0..0913e4c 100644 --- a/crates/oxidiris/src/cli.rs +++ b/crates/oxidiris/src/cli.rs @@ -54,6 +54,7 @@ EXAMPLES: oxidiris README.md Read a file at the default 300 WPM oxidiris paper.md -w 450 Start at 450 words per minute oxidiris notes.txt --pacing linear Disable punctuation-aware pauses + oxidiris borg.pdf Read a PDF, two-column papers included oxidiris paper.md -m focus Hide the text panel, RSVP frame only oxidiris BACKLOG.md --dump Print clean plain text and exit oxidiris BACKLOG.md | less Piping also produces plain text diff --git a/crates/oxidiris/src/main.rs b/crates/oxidiris/src/main.rs index 69256a4..0cc8625 100644 --- a/crates/oxidiris/src/main.rs +++ b/crates/oxidiris/src/main.rs @@ -74,7 +74,7 @@ fn read_input(cli: &Cli) -> Result<(Vec, String, Option)> { if buf.is_empty() { bail!("standard input was empty"); } - return Ok((buf, "stdin".to_string(), None)); + return unpack_pdf(buf, "stdin".to_string(), None); } let path = &cli.file; @@ -84,7 +84,24 @@ fn read_input(cli: &Cli) -> Result<(Vec, String, Option)> { .map(|n| n.to_string_lossy().into_owned()) .unwrap_or_else(|| path.display().to_string()); let format = Some(Format::from_extension(path.extension().and_then(|e| e.to_str()))); - Ok((bytes, title, format)) + unpack_pdf(bytes, title, format) +} + +/// Replace PDF bytes with the plain text extracted from them, leaving anything else alone. +/// +/// Detection is by header rather than by extension so that `cat paper.pdf | oxidiris -` works, +/// and so that a PDF saved under the wrong name still reads. Without this the core's decoder +/// would reject the file as binary, which is true but unhelpful (OXD-060). +fn unpack_pdf( + bytes: Vec, + title: String, + format: Option, +) -> Result<(Vec, String, Option)> { + if !oxidiris_pdf::looks_like_pdf(&bytes) { + return Ok((bytes, title, format)); + } + let text = oxidiris_pdf::extract(&bytes).with_context(|| format!("cannot read {title}"))?; + Ok((text.into_bytes(), title, Some(Format::PlainText))) } /// Compose the note shown in the status bar on the first frame. @@ -138,6 +155,40 @@ mod tests { assert!(notice.contains("--chunk"), "got {notice:?}"); } + fn testdata(name: &str) -> std::path::PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).join("../../testdata").join(name) + } + + /// A PDF must arrive at the engine as plain text, whatever the extension claimed (OXD-060). + #[test] + fn pdf_bytes_are_replaced_by_their_text_and_marked_plain() { + let bytes = std::fs::read(testdata("pdf_typography.pdf")).unwrap(); + let (out, title, format) = unpack_pdf(bytes, "paper.pdf".into(), None).unwrap(); + + assert_eq!(format, Some(Format::PlainText)); + assert_eq!(title, "paper.pdf"); + let text = String::from_utf8(out).unwrap(); + assert!(text.contains("efficient"), "got {text:?}"); + } + + #[test] + fn non_pdf_bytes_pass_through_untouched() { + let markdown = b"# Title\n\nbody\n".to_vec(); + let (out, _, format) = + unpack_pdf(markdown.clone(), "a.md".into(), Some(Format::Markdown)).unwrap(); + + assert_eq!(out, markdown); + assert_eq!(format, Some(Format::Markdown), "the format hint must survive"); + } + + /// The failure has to name the file, since the reader may have been handed a directory of them. + #[test] + fn an_unreadable_pdf_reports_which_file_failed() { + let err = + unpack_pdf(b"%PDF-1.7 truncated".to_vec(), "broken.pdf".into(), None).unwrap_err(); + assert!(format!("{err:#}").contains("broken.pdf"), "got {err:#}"); + } + #[test] fn the_notice_falls_back_to_a_hint() { let cli = Cli::try_parse_from(["oxidiris", "a.md"]).unwrap(); diff --git a/deny.toml b/deny.toml index e8686c5..64cbb0d 100644 --- a/deny.toml +++ b/deny.toml @@ -8,6 +8,14 @@ targets = [ [advisories] yanked = "deny" +ignore = [ + # RUSTSEC-2026-0192 — `ttf-parser` is unmaintained. It arrives transitively through + # lopdf -> pdf-extract, the PDF backend chosen in ADR 003, and the advisory itself says there + # is no upgrade to move to: the suggested replacement, `skrifa`, is a change lopdf has to make + # upstream. This is a maintenance notice, not a vulnerability — no unsoundness or exploitable + # behaviour is claimed. Delete this entry the moment lopdf switches. + "RUSTSEC-2026-0192", +] [licenses] # Permissive licences only for dependencies. Oxidiris itself is GPL-3.0-or-later, which can @@ -30,6 +38,7 @@ confidence-threshold = 0.9 exceptions = [ { name = "oxidiris", allow = ["GPL-3.0-or-later"] }, { name = "oxidiris-core", allow = ["GPL-3.0-or-later"] }, + { name = "oxidiris-pdf", allow = ["GPL-3.0-or-later"] }, ] [bans] diff --git a/docs/decisions/pdf-extraction.md b/docs/decisions/pdf-extraction.md new file mode 100644 index 0000000..cd7c516 --- /dev/null +++ b/docs/decisions/pdf-extraction.md @@ -0,0 +1,153 @@ +# ADR 003 — PDF text comes from `pdf-extract`, in content-stream order + +**Status:** Accepted +**Date:** 2026-09-05 +**Task:** OXD-060 +**Amends:** `docs/informations/proposals.md` §8.2 + +## Context + +Spec §8.2 warns that de-columnizing PDFs is hard enough to consume the whole project, and OXD-060 +was written on the assumption that shelling out to Poppler's `pdftotext -layout` would probably be +the pragmatic answer. This is the measurement that assumption asked for. + +**Corpus.** Five real two-column papers, all publicly downloadable: Borg (EuroSys'15, 18 pages), +Omega (EuroSys'13, 14 pages, 4.3 MB), MapReduce (OSDI'04), Bigtable (OSDI'06), Spanner (OSDI'12). +They are deliberately *not* committed to `testdata/` — they are under publisher copyright and a +fixture has to be redistributable. See "Fixtures" below for what was committed instead. + +**Candidates.** `pdf-extract` 0.12 (pure Rust, wraps `lopdf`), `lopdf` used directly, and Poppler's +`pdftotext` in both its plain and `-layout` modes. + +`lopdf` on its own was dropped before benchmarking: it hands back content streams and font +dictionaries, so using it directly means writing the glyph-to-Unicode mapping, the font encoding +tables and the text-positioning maths ourselves. That is precisely the work `pdf-extract` has +already done on top of it, and it is not work this project wants to own. + +## What the measurements said + +**Reading order.** There is no ground truth to diff against, so the two extractors were compared +to each other: word sequences normalised (NFKC, de-hyphenated, letters only) and matched in order +with a longest-common-subsequence ratio. Agreement between two independent implementations is +weak evidence that both are right; disagreement says where to look by hand. + +| Paper | Ordered overlap | +|---|---| +| Borg | 0.953 | +| Bigtable | 0.954 | +| Spanner | 0.959 | +| MapReduce | 0.925 | +| Omega | 0.699 | + +Omega is the interesting one, and it is **Poppler that is wrong there**: it lifts the figure +labels out of Figure 1 (`Monolithic`, `Two-level`, `scheduling logic`, `cluster state +information`) and drops them into the middle of the abstract. `pdf-extract` keeps the body text +running clean. The remaining 5% on the other four papers is the same class of difference — +figure and table furniture, ordered differently — not columns interleaving. + +`pdftotext -layout` was rejected outright: preserving the visual layout puts the two columns +side by side on every line, which is the one output shape a reader must never be handed. + +**Speed**, best of three, whole pipeline (`oxidiris --dump`: extract, tidy, parse, +tokenize, print) against `pdftotext /dev/null` which only extracts: + +| Paper | Pages | Size | oxidiris | pdftotext | +|---|---|---|---|---| +| Borg | 18 | 0.9 MB | 176 ms | 144 ms | +| Omega | 14 | 4.3 MB | 288 ms | 319 ms | +| MapReduce | 13 | 0.2 MB | 24 ms | 60 ms | +| Bigtable | 14 | 0.2 MB | 36 ms | 71 ms | +| Spanner | 14 | 0.4 MB | 73 ms | 89 ms | + +Doing more work in-process still beats spawning a subprocess on four of five papers. Note these +are release numbers: the debug build is 8-20x slower, so `make run` on a large PDF feels sluggish +in a way the shipped binary does not. + +**Cost of the external dependency.** `pdftotext` would have to be found, version-checked and +error-mapped at runtime, on three platforms, and its absence explained to a reader who just +wanted to open a file. The prebuilt binaries shipped since v0.2.1 (OXD-077) exist precisely so +that installing oxidiris needs nothing else; a hard runtime dependency on a Poppler install would +undo that for the one format most likely to bring new users in. + +## Decision + +**Use `pdf-extract` 0.12, and take its content-stream order as the reading order.** + +**In a crate of its own, `oxidiris-pdf`.** The backend pulls in font parsers, AES and image +codecs. `oxidiris-core` must keep compiling for `wasm32-unknown-unknown` — CI enforces it — so +none of that can live there. The crate exposes two functions, `looks_like_pdf` and `extract`, and +owns the whole third-party surface behind them. + +**Output plain text, not a `Document`.** A PDF carries no reliable block structure, and the +plain-text parser already does the one thing that matters: rejoining hard-wrapped lines into +paragraphs. The pipeline is therefore: + +```text +pdf bytes -> oxidiris_pdf::extract -> plain text -> oxidiris_core::load -> Document +``` + +**Detect by header, not by extension**, so `cat paper.pdf | oxidiris -` works and a mis-named file +still reads. + +**Do not implement x-coordinate column clustering yet.** OXD-061 sized it XL on the assumption it +would be needed for correct reading order. On this corpus it is not: LaTeX emits its columns in +order, and so does the content stream. The clustering work is worth doing when a PDF that +genuinely interleaves shows up — a Word-produced or OCR-reconstructed file is the likely first +one — and not before. + +## What the extracted text still needs + +Three artefacts reach the RSVP frame unless something removes them, all handled in +`oxidiris-pdf`'s `cleanup` module as pure `&str -> String` steps: + +- **Ligatures.** `efficient` is stored with U+FB01. Expanded through a seven-entry table rather + than NFKC, which would also rewrite superscripts, fractions and CJK compatibility forms that + carry meaning in a paper. +- **Hyphenated line breaks.** `admission con-` / `trol` would otherwise become two display units, + `con-` and `trol`. Welded when there is a letter on both sides and the next line starts + lowercase. A compound that breaks at its own hyphen (`over-` / `commitment`) is welded shut by + the same rule; separating those needs a dictionary, and every extractor in the field makes the + same trade. +- **Isolated short numbers.** Measuring this on the corpus corrected the original guess: the bare + numeric lines that show up are mostly **chart axis tick labels** (`20`, `40`, `60`) pulled out + of figures, not running page numbers, which usually get glued onto the preceding line instead. + Bigtable produces none at all. The rule drops a short numeric line only when blank lines already + isolate it, which is what keeps a `7` inside a sentence safe. + +## What is knowingly not handled + +- **Figure and table text still lands in the stream.** Reading Borg, the labels from Figure 1 + (`BorgMaster`, `link shard`, `UI shard`) appear after the first-page footer. Poppler does the + same thing. Filtering it needs the figure's bounding box, which is the same geometry work as + OXD-061. +- **No heading structure.** `Document::headings` is empty for a PDF, so the outline panel has + nothing to show. Section numbers survive as ordinary paragraphs. +- **Encrypted PDFs are refused, not decrypted**, with a message saying so. There is no fixture for + this path: writing an RC4-encrypted PDF by hand to test it was judged not worth the cost, so the + branch is exercised by inspection only. +- **Scanned pages** produce `PdfError::NoTextLayer` naming the page count and pointing at OCR. + +## Fixtures + +`testdata/make_pdf_fixtures.py` generates three small PDFs with no third-party library, so a +contributor can change one without installing anything: + +| Fixture | Pins down | +|---|---| +| `pdf_typography.pdf` | `fi` ligature via an `/Encoding /Differences` array, a hyphenated line break, a real compound hyphen that must survive, an isolated page number | +| `pdf_two_column.pdf` | Two columns must not interleave | +| `pdf_no_text_layer.pdf` | A drawn rectangle and no text: must produce an error, not an empty reader | + +## Consequences + +- `oxidiris` gains a dependency tree of about 70 crates. All are permissive-licensed and the + highest MSRV in it is 1.85, below the project's declared 1.88. +- **One advisory is now ignored in `deny.toml`.** `cargo deny` flags `ttf-parser` as unmaintained + (RUSTSEC-2026-0192); it arrives through `lopdf` → `pdf-extract`, and the advisory states no + upgrade exists — moving to `skrifa` is a change lopdf has to make upstream. It is a maintenance + notice, not a vulnerability. The ignore entry names the advisory and says to delete it as soon + as lopdf switches, so this does not quietly become permanent. +- `oxidiris-core` is untouched, and the wasm build stays clean. +- `pdf-extract` panics rather than returning `Err` on some malformed files. Spec §4.4 requires a + message instead of a crash, so `extract` catches the panic and converts it. This is the only + place in the project that catches a panic, and it exists because of the backend's error style. \ No newline at end of file diff --git a/testdata/README.md b/testdata/README.md index c1f3eb5..3ed464a 100644 --- a/testdata/README.md +++ b/testdata/README.md @@ -18,6 +18,24 @@ behaviour, so please do not "tidy" the contents — the odd characters are the p | `utf16_bom.txt` | UTF-16LE with a byte order mark | | `latin1.txt` | Windows-1252 bytes that are invalid UTF-8 | | `empty.txt` | Zero bytes: must not panic anywhere | +| `pdf_typography.pdf` | A `fi` ligature, a hyphenated line break, a real compound hyphen that must survive, and a page number standing alone | +| `pdf_two_column.pdf` | Two columns that must be read one after the other, never interleaved | +| `pdf_no_text_layer.pdf` | A drawn rectangle and no text, the way a scan looks: must produce an error, not an empty reader | + +## Regenerating the PDF fixtures + +The three PDFs are generated, not collected: + +```sh +python3 testdata/make_pdf_fixtures.py +``` + +No third-party library is involved — the script writes the PDF structure itself, so changing a +fixture does not mean installing a toolchain. The real papers the extractor was benchmarked +against (Borg, Omega, MapReduce, Bigtable, Spanner) are deliberately absent: they are under +publisher copyright, and a fixture has to be redistributable. See +[`docs/decisions/pdf-extraction.md`](../docs/decisions/pdf-extraction.md) for the measurements +they produced. ## Regenerating the Unicode fixtures diff --git a/testdata/make_pdf_fixtures.py b/testdata/make_pdf_fixtures.py new file mode 100644 index 0000000..f2fadbe --- /dev/null +++ b/testdata/make_pdf_fixtures.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 +"""Generate the PDF fixtures used by the oxidiris-pdf tests (OXD-060). + +Run from the repository root: + + python3 testdata/make_pdf_fixtures.py + +Real papers are not committed: the ones worth testing against (EuroSys, OSDI, IEEE) are under +publisher copyright, and a fixture has to be redistributable. These files are generated instead, +each one pinning down a single extraction behaviour, and each small enough to read by hand with +`less`. + +No third-party library on purpose. A PDF writer that only has to emit uncompressed text is about +sixty lines, and that is cheaper than making every contributor install reportlab to touch a +fixture. +""" + +import pathlib + +PAGE_WIDTH, PAGE_HEIGHT = 612, 792 +FONT_SIZE = 11 +LINE_HEIGHT = 14 + +# Code 1 is remapped to the `fi` ligature glyph through an /Encoding /Differences array, which is +# exactly how a TeX-produced paper stores it. Extraction has to turn it back into "fi". +FI = b"\x01" + + +def escape(text: bytes) -> bytes: + """Escape the three characters that terminate or nest a PDF literal string.""" + for special in (b"\\", b"(", b")"): + text = text.replace(special, b"\\" + special) + return text + + +def text_block(lines: list[bytes], x: int, top: int) -> bytes: + """Draw one line per entry, top down, at a fixed left edge. + + Each line is positioned absolutely rather than with T*, so the vertical gaps are explicit and + a reader of this file can see where the extractor's line breaks come from. + """ + out = [b"BT", b"/F1 %d Tf" % FONT_SIZE] + for index, line in enumerate(lines): + out.append(b"1 0 0 1 %d %d Tm" % (x, top - index * LINE_HEIGHT)) + out.append(b"(%s) Tj" % escape(line)) + out.append(b"ET") + return b"\n".join(out) + + +def build(objects: list[bytes]) -> bytes: + """Assemble numbered objects into a PDF file with a correct cross-reference table.""" + out = bytearray(b"%PDF-1.4\n") + offsets = [] + for number, body in enumerate(objects, start=1): + offsets.append(len(out)) + out += b"%d 0 obj\n" % number + body + b"\nendobj\n" + + xref_at = len(out) + out += b"xref\n0 %d\n" % (len(objects) + 1) + out += b"0000000000 65535 f \n" + for offset in offsets: + out += b"%010d 00000 n \n" % offset + out += b"trailer\n<< /Size %d /Root 1 0 R >>\n" % (len(objects) + 1) + out += b"startxref\n%d\n%%%%EOF\n" % xref_at + return bytes(out) + + +def document(streams: list[bytes], with_font: bool = True) -> bytes: + """One page per content stream, all sharing a single Helvetica font resource.""" + page_count = len(streams) + kids = b" ".join(b"%d 0 R" % (4 + index) for index in range(page_count)) + resources = b"<< /Font << /F1 2 0 R >> >>" if with_font else b"<< >>" + + objects = [ + b"<< /Type /Catalog /Pages 3 0 R >>", + b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding " + b"<< /Type /Encoding /BaseEncoding /WinAnsiEncoding /Differences [1 /fi] >> >>", + b"<< /Type /Pages /Kids [%s] /Count %d >>" % (kids, page_count), + ] + content_first = 4 + page_count + for index in range(page_count): + objects.append( + b"<< /Type /Page /Parent 3 0 R /MediaBox [0 0 %d %d] /Resources %s /Contents %d 0 R >>" + % (PAGE_WIDTH, PAGE_HEIGHT, resources, content_first + index) + ) + for stream in streams: + objects.append(b"<< /Length %d >>\nstream\n" % len(stream) + stream + b"\nendstream") + return build(objects) + + +def typography_pdf() -> bytes: + """Ligature, hyphenated line break, and a running page number standing on its own. + + The page number sits at the same left edge as the body, not centred. That is what makes the + extractor treat it as a paragraph of its own: it emits a blank line when the next glyph is + both far below the previous one and no further right than where that line ended. A centred + number lands to the right of the short last line and gets glued onto it instead — the + behaviour measured on real papers, and the reason this fixture is positioned deliberately. + """ + body = text_block( + [ + b"Borg achieves high utilization by combining ef" + FI + b"cient", + b"admission con-", + b"trol and machine sharing with process-level", + b"performance isolation.", + ], + x=72, + top=700, + ) + page_number = text_block([b"12"], x=72, top=60) + return document([body + b"\n" + page_number]) + + +def two_column_pdf() -> bytes: + """Two columns whose sentences must not interleave. + + The content stream emits the left column completely before the right one, which is what a + LaTeX-produced paper does. Reading order therefore falls out of stream order; a PDF that + emits its columns interleaved is the OXD-061 case and is not handled yet. + """ + left = text_block( + [b"The left column starts here and", b"continues to the bottom of the", b"page."], + x=72, + top=700, + ) + right = text_block( + [b"The right column is read second,", b"after the left one ends, never", b"before."], + x=330, + top=700, + ) + return document([left + b"\n" + right]) + + +def no_text_layer_pdf() -> bytes: + """A page carrying a drawn rectangle and no text at all, the way a scan does.""" + return document([b"0.5 g\n100 100 400 500 re\nf"], with_font=False) + + +FIXTURES = { + "pdf_typography.pdf": typography_pdf, + "pdf_two_column.pdf": two_column_pdf, + "pdf_no_text_layer.pdf": no_text_layer_pdf, +} + +if __name__ == "__main__": + here = pathlib.Path(__file__).parent + for name, make in FIXTURES.items(): + (here / name).write_bytes(make()) + print(f"wrote {here / name}") \ No newline at end of file diff --git a/testdata/pdf_no_text_layer.pdf b/testdata/pdf_no_text_layer.pdf new file mode 100644 index 0000000000000000000000000000000000000000..77a4b9c884bf5b1edc45cf277fcfc1dd83b537dc GIT binary patch literal 632 zcmZWnO;5r=5WVlOmCUAZ(Y{`^xiOV)3o~65=+I;&YC{Bt-jb{f zWvukCm6HQ`uXBRLY-CIR?1A)#7eMZGfgAwQ`6dDRh`Cl%dxT9u^Jy5ta1_zL6;}4( zSQVKfK}29Y;VMX&@q&lnzdR3UO8*K`%KgeaRPQ3_Cs2wo2!L-)H=*lL6dZmt)O?1c z!$Ob@5u6XCNis;FfVMV*x(lYQCD)x=L|*>SMQIm17*fQ1c%OGl@6pvi&RTyXq!fj< VoNVK|^2+&IXB>}2uQz{Mh#zC}s?`7h literal 0 HcmV?d00001 diff --git a/testdata/pdf_two_column.pdf b/testdata/pdf_two_column.pdf new file mode 100644 index 0000000000000000000000000000000000000000..a78fbb5a68a350938db385b964c0112432e445d6 GIT binary patch literal 942 zcmZvbQES^U5Xay3Defh(L04OL?KDsd%bGCQ#!740L+OKJpBxoODw15fefpggCr;)W zKUgRI@Bh2gd3Jxd{=_e@7>5+h=7rtdKvMmFMM%B~FFMmgaxYqRkkf$&xV>eln{Y6j z2-Ze>NFMu*kNS{>L$VYO_d8#)dqb~k0m(|&rct_u(e4Cf7!F>t4(M2os6 zs3$Uz>Y1=k$O7qkDfk~5RVd_{>N7p@@)Kd$nF>P#$j%QPNPb|W#L~ROQ%dJc zp22*PQN9OsrniAA6ctO39~?rfLhHgyw%SuR2gMLQG&HOa?%z?jehWFz1D^91#f~j2 zmaI8&4wa1el@k4&?9&vg9s5vi5jvC}YSZ;Q4bBToK3i%Q2;F>y>g7~cUYAi;6Oht9 zaeD)va%_zEW(P)w)kH;x!R(cW;xcZos{fPad3uIsRl7a1DF;+j0}jc)xj1tP!ULR; z9&MzGl^Kmd^XD!=;~UzkY@pkqGxl}*aG4%=rLAY zPhuHOz`y6Ra;nW2bi?5Z`MedXLmPj$NA(AxqyaJ}%;WHz#oLRV^J2kfv(?Wv`wL0- B{<{DG literal 0 HcmV?d00001 diff --git a/testdata/pdf_typography.pdf b/testdata/pdf_typography.pdf new file mode 100644 index 0000000000000000000000000000000000000000..0284754a66c1ab538b019bce3eddc9100d4aaedc GIT binary patch literal 912 zcmZ`&OK;mS492ed6}$x44yhlhodhTfENg-R+pwZFutU*xHX5{SWUpTyM_M8)e)LG(p>*_j5STiGZFGfKFF>uaHO#fj+% zurb<0bid#FfM=EjqLp;&aPn1k&&p#dAzJIwRHz#mfe=ad<*Gf#`c`IO7>z^ zh-i~Ql0Z?1dBx;@PV#XKMK36|NHdB7R?xXW1-+8iv>s z!YSw>M!x=yr8MJFEFWWMOPV1s=JQr!r)+po_xPqDrz>I%-?-yLa$Z^=e9qIQm`v6` GH{uVg9Qq3Y literal 0 HcmV?d00001 From 6823822c3db82c1886a1681b068294e1e242cf39 Mon Sep 17 00:00:00 2001 From: Aleksandr Slokov Date: Sat, 5 Sep 2026 20:51:07 +0700 Subject: [PATCH 2/2] fix(pdf): keep the PDF fixtures binary so Windows checkouts stay valid MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The windows-latest job failed on this branch while ubuntu and macos passed. The fixtures are almost all ASCII, so git's own heuristic reads them as text and rewrote their line endings on checkout. One added byte per line shifts every offset in the cross-reference table, and the file stops parsing — reported as `InvalidTrailer`, on a file nobody edited. Reproduced on Linux by replacing LF with CRLF in `pdf_typography.pdf`: same error, exactly. `*.pdf binary` in `.gitattributes` is the fix, and it is the same trap `*.snap text eol=lf` was already there to avoid (OXD-036). The regression test is the other half. Without it the next fixture to be mangled reports a trailer problem, which points nowhere near the cause; now it says which file was rewritten and which `.gitattributes` line prevents it. Verified by CRLF-ifying a fixture and watching the guard fire before restoring it. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_0189gxRgTxfvzjNYezCfS2iR --- .gitattributes | 6 ++++++ crates/oxidiris-pdf/tests/fixtures.rs | 24 +++++++++++++++++++++++- 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/.gitattributes b/.gitattributes index 1907931..0f96157 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,3 +1,9 @@ # Golden frames are compared byte for byte. A CRLF checkout on Windows would fail every # snapshot test in CI for reasons that have nothing to do with the code (OXD-036). *.snap text eol=lf + +# The PDF fixtures are mostly ASCII, so git's heuristic reads them as text and rewrites their +# line endings on a Windows checkout. Every added byte shifts the cross-reference table, and the +# file stops parsing: `InvalidTrailer`, from a file nobody edited. That is exactly how the +# windows-latest job failed on the OXD-060 pull request (#13). +*.pdf binary diff --git a/crates/oxidiris-pdf/tests/fixtures.rs b/crates/oxidiris-pdf/tests/fixtures.rs index 3426bf2..256fd1d 100644 --- a/crates/oxidiris-pdf/tests/fixtures.rs +++ b/crates/oxidiris-pdf/tests/fixtures.rs @@ -7,6 +7,9 @@ use std::path::{Path, PathBuf}; use oxidiris_pdf::PdfError; +/// Every generated fixture, in the order `testdata/README.md` lists them. +const FIXTURES: [&str; 3] = ["pdf_typography.pdf", "pdf_two_column.pdf", "pdf_no_text_layer.pdf"]; + fn testdata(name: &str) -> PathBuf { Path::new(env!("CARGO_MANIFEST_DIR")).join("../../testdata").join(name) } @@ -18,12 +21,31 @@ fn extract(name: &str) -> Result { #[test] fn every_fixture_is_recognised_as_a_pdf() { - for name in ["pdf_typography.pdf", "pdf_two_column.pdf", "pdf_no_text_layer.pdf"] { + for name in FIXTURES { let bytes = std::fs::read(testdata(name)).unwrap(); assert!(oxidiris_pdf::looks_like_pdf(&bytes), "{name} lost its header"); } } +/// Regression guard for the `windows-latest` failure on PR #13. +/// +/// These fixtures are almost all ASCII, so git's own heuristic calls them text and rewrites their +/// line endings on a Windows checkout. One added byte per line shifts every offset in the +/// cross-reference table, and the file stops parsing — reported as `InvalidTrailer`, which points +/// nowhere near the cause, on a file nobody touched. `*.pdf binary` in `.gitattributes` is what +/// prevents it; failing here says so out loud instead. +#[test] +fn fixtures_are_checked_out_byte_for_byte() { + for name in FIXTURES { + let bytes = std::fs::read(testdata(name)).unwrap(); + assert!( + !bytes.windows(2).any(|pair| pair == b"\r\n"), + "{name} has CRLF line endings: the checkout rewrote it. \ + `.gitattributes` must keep `*.pdf binary`." + ); + } +} + /// The three artefacts that would otherwise reach the RSVP frame, on a real PDF rather than a /// hand-written string. #[test]