From 41f3bcf4458be6aa33e59671a7e038988011be2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=9C=E7=9A=87=C2=B7Beelzebub?= Date: Mon, 17 Aug 2026 17:45:59 +0800 Subject: [PATCH] eval: benchmark controlled open-source parsers --- ...controlled-open-source-parser-benchmark.md | 241 ++++ ...14B-controlled-parser-benchmark-matrix.csv | 34 + ...B-controlled-parser-benchmark-results.json | 1070 ++++++++++++++++ .../14B-parser-selection-manifest.json | 46 + .../candidate_probe_14b.py | 245 ++++ .../controlled_benchmark_14b.py | 1107 +++++++++++++++++ .../run_controlled_parser_benchmark_14b.py | 364 ++++++ tests/test_controlled_parser_benchmark_14b.py | 318 +++++ 8 files changed, 3425 insertions(+) create mode 100644 docs/evaluations/14B-controlled-open-source-parser-benchmark.md create mode 100644 docs/evaluations/artifacts/14B-controlled-parser-benchmark-matrix.csv create mode 100644 docs/evaluations/artifacts/14B-controlled-parser-benchmark-results.json create mode 100644 docs/evaluations/artifacts/14B-parser-selection-manifest.json create mode 100644 scripts/evaluations/open_source_parser_eval/candidate_probe_14b.py create mode 100644 scripts/evaluations/open_source_parser_eval/controlled_benchmark_14b.py create mode 100644 scripts/evaluations/run_controlled_parser_benchmark_14b.py create mode 100644 tests/test_controlled_parser_benchmark_14b.py diff --git a/docs/evaluations/14B-controlled-open-source-parser-benchmark.md b/docs/evaluations/14B-controlled-open-source-parser-benchmark.md new file mode 100644 index 0000000..f6c57d3 --- /dev/null +++ b/docs/evaluations/14B-controlled-open-source-parser-benchmark.md @@ -0,0 +1,241 @@ +# Task 14B Controlled Open-source Parser Benchmark + +## Status + +- Technical status: `CONTROLLED_OPEN_SOURCE_PARSER_BENCHMARK_CLOSED` +- Quality status: `CONDITIONAL_PARSER_INTEGRATION_CANDIDATE_ESTABLISHED` +- Baseline: `6cca8cb0881226c6031f3adafff0d89c1146d584` +- Selected integration candidate: `docling@2.117.0` +- Selected role: `conditional_complex_document_parser_candidate` +- Production parser changed: false +- Production adapter authorized: false + +## Decision + +Docling is the best eligible candidate in this controlled comparison, but it +did not pass every document-class gate and must not replace the current parser +globally. It is recommended for the next adapter task only for: + +- scanned English and Chinese PDFs; +- simple table PDFs with a preserved row/column shape. + +The following classes remain excluded from Docling routing until their +individual gates are repaired and rerun: + +- multi-column PDFs: reading-order accuracy was `0.8214`; +- mixed-layout table PDFs: the table was not retained; +- formula PDFs without composition with the existing `FormulaRegion` path: + formula structure retention was `0.0`. + +The current native parser remains the simple-document fallback. The next task +must introduce per-document quality routing through the existing parser and +KnowledgeChunk contracts; it must not create a second ingestion or retrieval +system. + +MinerU is not eligible. Its installed `3.4.4` metadata reports the non-standard +`LicenseRef-MinerU-Open-Source-License`, its multi-column score was lower, and +its Chinese headings were commonly merged into paragraph bodies. That reduced +the retrieval Chinese corpus to two chunks and prevented a downstream gain. + +## Why This Is Not Merely A Strict Threshold Problem + +The rejected cases have observable structural failures: + +- Docling and MinerU interleaved content across columns in the synthetic + two-column PDF. +- Both candidates detected the simple table, but neither retained the table in + the mixed-layout blocker. +- Neither candidate emitted a governed `formula` block for the raster formula + fixture. +- MinerU merged nine Chinese term headings with their definitions as ordinary + paragraphs, so the existing heading-aware KnowledgeChunk builder produced + two broad chunks rather than eleven concept-preserving chunks. + +These are content and provenance defects, not arbitrary policy failures. A +global replacement would therefore make some real documents worse despite the +higher aggregate score. + +## Controlled Corpus + +The corpus contains eleven repository-independent, runtime-generated synthetic +PDFs (fifteen pages total): + +- single-column born-digital English; +- two-column born-digital English; +- scanned English; +- scanned Chinese; +- mixed-layout blocker; +- simple table; +- raster formula; +- negative/no-term document; +- three-page repeated header/footer document; +- two-page English retrieval concepts; +- two-page independent Chinese retrieval evidence. + +The retrieval pair contains ten concepts in physics, mechanics, control +engineering, and applied mathematics, including three intentionally confusable +groups. The Chinese PDF contains no complete English gold term. Fixtures do +not contain private course material, inline bilingual answers, aliases, or +Provider output. + +## Execution Contract + +Each parser ran in an isolated process and normalized to a common evaluation +block contract. Candidate model artifacts and caches remained under +repository-external temporary storage and are not tracked by Git. + +| Parser | Runtime | Local model policy | License gate | +|---|---|---|---| +| current native/Tesseract/FormulaRegion | project runtime | existing project configuration | pass | +| Docling 2.117.0 | isolated Conda environment | explicit pre-provisioned artifacts, offline mode | pass (MIT metadata) | +| MinerU 3.4.4 | isolated Conda environment | local pipeline, CPU, offline mode | blocked (non-standard license) | + +No parser was allowed to call an external API. MinerU's local CLI used only a +loopback endpoint; the network audit observed zero external hosts and zero +external parser requests. + +## Parser Metrics + +| Metric | Current | Docling | MinerU | +|---|---:|---:|---:| +| Parse success | 0.8182 | 1.0000 | 1.0000 | +| Reading order (mean where defined) | 0.7778 | 0.9802 | 0.9762 | +| Heading-definition integrity | 1.0000 | 1.0000 | 1.0000 | +| Page provenance completeness | 0.8182 | 1.0000 | 1.0000 | +| Block provenance completeness | 0.8182 | 1.0000 | 1.0000 | +| BBox provenance completeness | 0.0000 | 1.0000 | 1.0000 | +| Table retention (simple + mixed) | 0.0000 | 0.5000 | 0.5000 | +| Formula structure retention | 0.0000 | 0.0000 | 0.0000 | +| Repeated header/footer filter | 0.0000 | 1.0000 | 1.0000 | +| Duplicate body blocks | 0 | 0 | 0 | +| Median runtime | 32.42 ms | 5,049.68 ms | 10,366.44 ms | +| Peak RSS | 76.83 MB | 2,446.58 MB | 1,830.62 MB | + +`Parse success` requires non-empty governed content, not merely a zero process +exit code. The current parser's scanned English and Chinese runs exited cleanly +but produced no eligible content, so both correctly fail that metric. + +Aggregate values do not authorize routing. The selection manifest separately +retains critical per-fixture gates: + +| Critical fixture | Current | Docling | MinerU | +|---|---:|---:|---:| +| Two-column reading order | 1.0000 | 0.8214 | 0.7857 | +| Scanned English content success | false | true | true | +| Scanned Chinese content success | false | true | true | +| Simple table retention | 0.0000 | 1.0000 | 1.0000 | +| Mixed-layout table retention | 0.0000 | 0.0000 | 0.0000 | +| Formula structure retention | 0.0000 | 0.0000 | 0.0000 | +| Repeated header/footer filter | 0.0000 | 1.0000 | 1.0000 | + +## Existing-pipeline Retrieval Impact + +Normalized blocks were converted through +`knowledge_governance.build_knowledge_chunks_from_parse_blocks` and ranked by +the existing fixed local backend: + +- model: `intfloat/multilingual-e5-small`; +- revision: `614241f622f53c4eeff9890bdc4f31cfecc418b3`; +- backend: `local_multilingual_e5_pytorch_cpu_v1`. + +No new embedding abstraction, model, vector database, translation step, or +gold mapping was introduced. Gold concept IDs were used only after ranking to +calculate metrics. + +| Parser | EN chunks | ZH chunks | hit@1 | hit@3 | MRR | No result | +|---|---:|---:|---:|---:|---:|---:| +| Current | 2 | 2 | 0.1000 | 0.1000 | 0.1000 | 0 | +| Docling | 11 | 11 | 0.5000 | 0.6000 | 0.5500 | 0 | +| MinerU | 11 | 2 | 0.1000 | 0.1000 | 0.1000 | 0 | + +Docling materially improves concept-preserving chunk construction and the +controlled downstream retrieval baseline, but `hit@3 = 0.6000` is not a claim +of broad product quality. It establishes a conditional adapter candidate only. + +## Architecture Boundary + +Before this task: + +```text +PDF -> current parser/OCR/layout diagnostics -> parse blocks + -> existing heading-aware KnowledgeChunk builder + -> existing multilingual retrieval +``` + +This task adds only an offline comparison path: + +```text +synthetic PDF + -> isolated current / Docling / MinerU probe + -> neutral evaluation blocks + -> existing KnowledgeChunk builder + -> existing multilingual retrieval + -> metric and selection artifacts +``` + +Production remains unchanged. The ordered next task may add: + +```text +production PDF + -> existing parser adapter and document-quality router + -> simple digital: current native parser + -> scanned or validated simple-table: Docling candidate + -> formula: compose existing FormulaRegion + -> failed class gate: current fallback or fail closed + -> existing parse blocks -> existing KnowledgeChunk -> existing retrieval +``` + +The PDF selection interface and concept learning card are deliberately not +changed here. Direct PDF text-layer selection remains an input mechanism; the +student-facing concept learning card remains the intended output and will be +implemented only after the parser adapter is closed. + +## Safety + +- Application external API used: false. +- External parser request count: 0. +- Real Provider requests: 0. +- Real credentials read: false. +- Private fixture used: false. +- Incident database accessed: false. +- Production parser changed: false. +- Production adapter changed: false. +- Model/cache tracked: false. +- Request/response/complete source bodies stored in artifacts: false. + +## Artifacts + +- `14B-controlled-parser-benchmark-results.json` + - SHA-256: `293abe451b713f9f68d354f5dc8f6472760558a06104d7aba248e426429cad5d` +- `14B-controlled-parser-benchmark-matrix.csv` + - SHA-256: `62908931e193141407e832323a5048d98e860cc4d9ed3426e6674324386ac6d0` +- `14B-parser-selection-manifest.json` + - SHA-256: `7eddca05ac7c0aa72e6b5d4c28a567960768de5a339c17404cc22e65d6ec6232` + +## Verification + +- New and related parser/layout/chunk tests: `66 passed, 1 skipped`. +- Full pytest in the task runtime: `1722 passed, 5 skipped`. +- `dev_check` in the canonical runtime: `1720 passed, 7 skipped`; migration + and backend API smoke passed. The two additional skips reflect optional + layout-runtime availability in that separately pinned runtime, not failures. +- Release safety: pass. +- `git diff --check`: pass. +- Cross-Corpus V2 frozen hashes remain: + - manifest: `a53a5152aa66189c954b703c42fb815a6a8c7f83933051f80f1538e78bdc3f88`; + - gold: `3379bc1e6c589256dcec384a9f7435c96277f6754b7cb44cff69f86244a9b3f0`; + - English bundle: `e84fc99a993c099628fe7d740c1e248fefe1e03cbece7b4df43d02cd7dfaddc5`; + - Chinese bundle: `cebd3274cb0802fa645948d9b641b1776c60ab99700bbf3cd4a80999bc45b5f7`. +- Accident database before/final: + - SHA-256: `9e6fb68ab13dff2763e2fff18d2aee531486360c557f058eb8a471d9209a9eaa`; + - size: `1015808`; + - mtime: `1785496597`; + - WAL/SHM: absent/absent. + +## Next Ordered Step + +Add a bounded Docling adapter through the existing production parser contract, +with explicit document-class routing, existing FormulaRegion composition, +fallback/fail-closed behavior, offline model provisioning, resource bounds, +and no change to KnowledgeChunk or retrieval semantics. Do not begin the PDF.js +reader or concept-card redesign until that adapter task is merged and verified. diff --git a/docs/evaluations/artifacts/14B-controlled-parser-benchmark-matrix.csv b/docs/evaluations/artifacts/14B-controlled-parser-benchmark-matrix.csv new file mode 100644 index 0000000..d6a33a1 --- /dev/null +++ b/docs/evaluations/artifacts/14B-controlled-parser-benchmark-matrix.csv @@ -0,0 +1,34 @@ +parser_id,fixture_id,parse_success,reading_order_accuracy,heading_definition_integrity,page_provenance_completeness,block_provenance_completeness,bbox_provenance_completeness,table_retention,formula_retention,header_footer_filter_rate,duplicate_block_count,runtime_ms,peak_rss_mb +baseline_native_tesseract_formula_region,single_column_born_digital,True,1.0,,1.0,1.0,0.0,,,,0,29.63,46.7 +baseline_native_tesseract_formula_region,two_column_born_digital,True,1.0,,1.0,1.0,0.0,,,,0,54.58,47.38 +baseline_native_tesseract_formula_region,scanned_english,False,0.0,,0.0,0.0,0.0,,,,0,85.7,76.59 +baseline_native_tesseract_formula_region,scanned_chinese,False,0.0,,0.0,0.0,0.0,,,,0,85.79,76.83 +baseline_native_tesseract_formula_region,mixed_layout_blocker,True,1.0,,1.0,1.0,0.0,0.0,,,0,31.0,47.06 +baseline_native_tesseract_formula_region,simple_table,True,1.0,,1.0,1.0,0.0,0.0,,,0,30.43,46.83 +baseline_native_tesseract_formula_region,raster_formula,True,,,1.0,1.0,0.0,,0.0,,0,80.31,52.31 +baseline_native_tesseract_formula_region,negative_no_terms,True,,,1.0,1.0,0.0,,,,0,30.73,47.08 +baseline_native_tesseract_formula_region,repeated_header_footer,True,1.0,1.0,1.0,1.0,0.0,,,0.0,0,31.97,47.0 +baseline_native_tesseract_formula_region,retrieval_english,True,1.0,1.0,1.0,1.0,0.0,,,,0,35.97,46.97 +baseline_native_tesseract_formula_region,retrieval_chinese,True,1.0,1.0,1.0,1.0,0.0,,,,0,32.42,47.28 +docling,single_column_born_digital,True,1.0,,1.0,1.0,1.0,,,,0,7105.47,1469.59 +docling,two_column_born_digital,True,0.8214,,1.0,1.0,1.0,,,,0,5488.17,1549.94 +docling,scanned_english,True,1.0,,1.0,1.0,1.0,,,,0,5989.8,1998.62 +docling,scanned_chinese,True,1.0,,1.0,1.0,1.0,,,,0,7488.16,2446.58 +docling,mixed_layout_blocker,True,1.0,,1.0,1.0,1.0,0.0,,,0,4509.57,1651.75 +docling,simple_table,True,1.0,,1.0,1.0,1.0,1.0,,,0,4573.45,1693.83 +docling,raster_formula,True,,,1.0,1.0,1.0,,0.0,,0,6621.94,2022.55 +docling,negative_no_terms,True,,,1.0,1.0,1.0,,,,0,4540.28,1640.38 +docling,repeated_header_footer,True,1.0,1.0,1.0,1.0,1.0,,,1.0,0,5049.68,1505.3 +docling,retrieval_english,True,1.0,1.0,1.0,1.0,1.0,,,,0,4942.29,1645.81 +docling,retrieval_chinese,True,1.0,1.0,1.0,1.0,1.0,,,,0,4670.07,1644.03 +mineru,single_column_born_digital,True,1.0,,1.0,1.0,1.0,,,,0,10366.44,1416.88 +mineru,two_column_born_digital,True,0.7857,,1.0,1.0,1.0,,,,0,8639.2,1414.5 +mineru,scanned_english,True,1.0,,1.0,1.0,1.0,,,,0,7730.35,1421.36 +mineru,scanned_chinese,True,1.0,,1.0,1.0,1.0,,,,0,8838.18,1425.73 +mineru,mixed_layout_blocker,True,1.0,,1.0,1.0,1.0,0.0,,,0,9145.91,1424.64 +mineru,simple_table,True,1.0,,1.0,1.0,1.0,1.0,,,0,10429.65,1415.84 +mineru,raster_formula,True,,,1.0,1.0,1.0,,0.0,,0,13224.48,1830.62 +mineru,negative_no_terms,True,,,1.0,1.0,1.0,,,,0,9541.72,1403.34 +mineru,repeated_header_footer,True,1.0,1.0,1.0,1.0,1.0,,,1.0,0,11807.61,1527.08 +mineru,retrieval_english,True,1.0,1.0,1.0,1.0,1.0,,,,0,12114.65,1483.81 +mineru,retrieval_chinese,True,1.0,1.0,1.0,1.0,1.0,,,,0,12112.85,1443.22 diff --git a/docs/evaluations/artifacts/14B-controlled-parser-benchmark-results.json b/docs/evaluations/artifacts/14B-controlled-parser-benchmark-results.json new file mode 100644 index 0000000..f5a0b55 --- /dev/null +++ b/docs/evaluations/artifacts/14B-controlled-parser-benchmark-results.json @@ -0,0 +1,1070 @@ +{ + "aggregates": [ + { + "bbox_provenance_completeness": 0.0, + "block_provenance_completeness": 0.8182, + "critical_fixture_gates": { + "formula_retention": 0.0, + "mixed_layout_table_retention": 0.0, + "repeated_header_footer_filter_rate": 0.0, + "scanned_chinese_parse_success": false, + "scanned_english_parse_success": false, + "simple_table_retention": 0.0, + "two_column_reading_order_accuracy": 1.0 + }, + "document_count": 11, + "duplicate_block_count": 0, + "formula_retention": 0.0, + "header_footer_filter_rate": 0.0, + "heading_definition_integrity": 1.0, + "license_gate": "pass", + "median_runtime_ms": 32.42, + "page_provenance_completeness": 0.8182, + "parse_success_rate": 0.8182, + "parser_id": "baseline_native_tesseract_formula_region", + "peak_rss_mb": 76.83, + "reading_order_accuracy": 0.7778, + "retrieval_chinese_chunk_count": 2, + "retrieval_english_chunk_count": 2, + "retrieval_hit_at_1": 0.1, + "retrieval_hit_at_3": 0.1, + "retrieval_mrr": 0.1, + "table_retention": 0.0 + }, + { + "bbox_provenance_completeness": 1.0, + "block_provenance_completeness": 1.0, + "critical_fixture_gates": { + "formula_retention": 0.0, + "mixed_layout_table_retention": 0.0, + "repeated_header_footer_filter_rate": 1.0, + "scanned_chinese_parse_success": true, + "scanned_english_parse_success": true, + "simple_table_retention": 1.0, + "two_column_reading_order_accuracy": 0.8214 + }, + "document_count": 11, + "duplicate_block_count": 0, + "formula_retention": 0.0, + "header_footer_filter_rate": 1.0, + "heading_definition_integrity": 1.0, + "license_gate": "pass", + "median_runtime_ms": 5049.68, + "page_provenance_completeness": 1.0, + "parse_success_rate": 1.0, + "parser_id": "docling", + "peak_rss_mb": 2446.58, + "reading_order_accuracy": 0.9802, + "retrieval_chinese_chunk_count": 11, + "retrieval_english_chunk_count": 11, + "retrieval_hit_at_1": 0.5, + "retrieval_hit_at_3": 0.6, + "retrieval_mrr": 0.55, + "table_retention": 0.5 + }, + { + "bbox_provenance_completeness": 1.0, + "block_provenance_completeness": 1.0, + "critical_fixture_gates": { + "formula_retention": 0.0, + "mixed_layout_table_retention": 0.0, + "repeated_header_footer_filter_rate": 1.0, + "scanned_chinese_parse_success": true, + "scanned_english_parse_success": true, + "simple_table_retention": 1.0, + "two_column_reading_order_accuracy": 0.7857 + }, + "document_count": 11, + "duplicate_block_count": 0, + "formula_retention": 0.0, + "header_footer_filter_rate": 1.0, + "heading_definition_integrity": 1.0, + "license_gate": "blocked_nonstandard_license", + "median_runtime_ms": 10366.44, + "page_provenance_completeness": 1.0, + "parse_success_rate": 1.0, + "parser_id": "mineru", + "peak_rss_mb": 1830.62, + "reading_order_accuracy": 0.9762, + "retrieval_chinese_chunk_count": 2, + "retrieval_english_chunk_count": 11, + "retrieval_hit_at_1": 0.1, + "retrieval_hit_at_3": 0.1, + "retrieval_mrr": 0.1, + "table_retention": 0.5 + } + ], + "baseline_commit": "6cca8cb0881226c6031f3adafff0d89c1146d584", + "branch": "codex/open-source-parser-benchmark-14b", + "created_at": "2026-08-17T09:26:36.701204Z", + "document_scores": [ + { + "bbox_provenance_completeness": 0.0, + "block_count": 1, + "block_provenance_completeness": 1.0, + "duplicate_block_count": 0, + "errors": [], + "fixture_id": "single_column_born_digital", + "formula_retention": null, + "header_footer_filter_rate": null, + "heading_definition_integrity": null, + "page_provenance_completeness": 1.0, + "parse_success": true, + "parser_id": "baseline_native_tesseract_formula_region", + "peak_rss_mb": 46.7, + "reading_order_accuracy": 1.0, + "runtime_ms": 29.63, + "table_retention": null, + "warnings": [] + }, + { + "bbox_provenance_completeness": 0.0, + "block_count": 1, + "block_provenance_completeness": 1.0, + "duplicate_block_count": 0, + "errors": [], + "fixture_id": "two_column_born_digital", + "formula_retention": null, + "header_footer_filter_rate": null, + "heading_definition_integrity": null, + "page_provenance_completeness": 1.0, + "parse_success": true, + "parser_id": "baseline_native_tesseract_formula_region", + "peak_rss_mb": 47.38, + "reading_order_accuracy": 1.0, + "runtime_ms": 54.58, + "table_retention": null, + "warnings": [] + }, + { + "bbox_provenance_completeness": 0.0, + "block_count": 0, + "block_provenance_completeness": 0.0, + "duplicate_block_count": 0, + "errors": [], + "fixture_id": "scanned_english", + "formula_retention": null, + "header_footer_filter_rate": null, + "heading_definition_integrity": null, + "page_provenance_completeness": 0.0, + "parse_success": false, + "parser_id": "baseline_native_tesseract_formula_region", + "peak_rss_mb": 76.59, + "reading_order_accuracy": 0.0, + "runtime_ms": 85.7, + "table_retention": null, + "warnings": [ + "OCR is required but no OCR provider is available." + ] + }, + { + "bbox_provenance_completeness": 0.0, + "block_count": 0, + "block_provenance_completeness": 0.0, + "duplicate_block_count": 0, + "errors": [], + "fixture_id": "scanned_chinese", + "formula_retention": null, + "header_footer_filter_rate": null, + "heading_definition_integrity": null, + "page_provenance_completeness": 0.0, + "parse_success": false, + "parser_id": "baseline_native_tesseract_formula_region", + "peak_rss_mb": 76.83, + "reading_order_accuracy": 0.0, + "runtime_ms": 85.79, + "table_retention": null, + "warnings": [ + "OCR is required but no OCR provider is available." + ] + }, + { + "bbox_provenance_completeness": 0.0, + "block_count": 1, + "block_provenance_completeness": 1.0, + "duplicate_block_count": 0, + "errors": [], + "fixture_id": "mixed_layout_blocker", + "formula_retention": null, + "header_footer_filter_rate": null, + "heading_definition_integrity": null, + "page_provenance_completeness": 1.0, + "parse_success": true, + "parser_id": "baseline_native_tesseract_formula_region", + "peak_rss_mb": 47.06, + "reading_order_accuracy": 1.0, + "runtime_ms": 31.0, + "table_retention": 0.0, + "warnings": [] + }, + { + "bbox_provenance_completeness": 0.0, + "block_count": 1, + "block_provenance_completeness": 1.0, + "duplicate_block_count": 0, + "errors": [], + "fixture_id": "simple_table", + "formula_retention": null, + "header_footer_filter_rate": null, + "heading_definition_integrity": null, + "page_provenance_completeness": 1.0, + "parse_success": true, + "parser_id": "baseline_native_tesseract_formula_region", + "peak_rss_mb": 46.83, + "reading_order_accuracy": 1.0, + "runtime_ms": 30.43, + "table_retention": 0.0, + "warnings": [] + }, + { + "bbox_provenance_completeness": 0.0, + "block_count": 1, + "block_provenance_completeness": 1.0, + "duplicate_block_count": 0, + "errors": [], + "fixture_id": "raster_formula", + "formula_retention": 0.0, + "header_footer_filter_rate": null, + "heading_definition_integrity": null, + "page_provenance_completeness": 1.0, + "parse_success": true, + "parser_id": "baseline_native_tesseract_formula_region", + "peak_rss_mb": 52.31, + "reading_order_accuracy": null, + "runtime_ms": 80.31, + "table_retention": null, + "warnings": [] + }, + { + "bbox_provenance_completeness": 0.0, + "block_count": 1, + "block_provenance_completeness": 1.0, + "duplicate_block_count": 0, + "errors": [], + "fixture_id": "negative_no_terms", + "formula_retention": null, + "header_footer_filter_rate": null, + "heading_definition_integrity": null, + "page_provenance_completeness": 1.0, + "parse_success": true, + "parser_id": "baseline_native_tesseract_formula_region", + "peak_rss_mb": 47.08, + "reading_order_accuracy": null, + "runtime_ms": 30.73, + "table_retention": null, + "warnings": [] + }, + { + "bbox_provenance_completeness": 0.0, + "block_count": 3, + "block_provenance_completeness": 1.0, + "duplicate_block_count": 0, + "errors": [], + "fixture_id": "repeated_header_footer", + "formula_retention": null, + "header_footer_filter_rate": 0.0, + "heading_definition_integrity": 1.0, + "page_provenance_completeness": 1.0, + "parse_success": true, + "parser_id": "baseline_native_tesseract_formula_region", + "peak_rss_mb": 47.0, + "reading_order_accuracy": 1.0, + "runtime_ms": 31.97, + "table_retention": null, + "warnings": [] + }, + { + "bbox_provenance_completeness": 0.0, + "block_count": 2, + "block_provenance_completeness": 1.0, + "duplicate_block_count": 0, + "errors": [], + "fixture_id": "retrieval_english", + "formula_retention": null, + "header_footer_filter_rate": null, + "heading_definition_integrity": 1.0, + "page_provenance_completeness": 1.0, + "parse_success": true, + "parser_id": "baseline_native_tesseract_formula_region", + "peak_rss_mb": 46.97, + "reading_order_accuracy": 1.0, + "runtime_ms": 35.97, + "table_retention": null, + "warnings": [] + }, + { + "bbox_provenance_completeness": 0.0, + "block_count": 2, + "block_provenance_completeness": 1.0, + "duplicate_block_count": 0, + "errors": [], + "fixture_id": "retrieval_chinese", + "formula_retention": null, + "header_footer_filter_rate": null, + "heading_definition_integrity": 1.0, + "page_provenance_completeness": 1.0, + "parse_success": true, + "parser_id": "baseline_native_tesseract_formula_region", + "peak_rss_mb": 47.28, + "reading_order_accuracy": 1.0, + "runtime_ms": 32.42, + "table_retention": null, + "warnings": [] + }, + { + "bbox_provenance_completeness": 1.0, + "block_count": 2, + "block_provenance_completeness": 1.0, + "duplicate_block_count": 0, + "errors": [], + "fixture_id": "single_column_born_digital", + "formula_retention": null, + "header_footer_filter_rate": null, + "heading_definition_integrity": null, + "page_provenance_completeness": 1.0, + "parse_success": true, + "parser_id": "docling", + "peak_rss_mb": 1469.59, + "reading_order_accuracy": 1.0, + "runtime_ms": 7105.47, + "table_retention": null, + "warnings": [] + }, + { + "bbox_provenance_completeness": 1.0, + "block_count": 7, + "block_provenance_completeness": 1.0, + "duplicate_block_count": 0, + "errors": [], + "fixture_id": "two_column_born_digital", + "formula_retention": null, + "header_footer_filter_rate": null, + "heading_definition_integrity": null, + "page_provenance_completeness": 1.0, + "parse_success": true, + "parser_id": "docling", + "peak_rss_mb": 1549.94, + "reading_order_accuracy": 0.8214, + "runtime_ms": 5488.17, + "table_retention": null, + "warnings": [] + }, + { + "bbox_provenance_completeness": 1.0, + "block_count": 5, + "block_provenance_completeness": 1.0, + "duplicate_block_count": 0, + "errors": [], + "fixture_id": "scanned_english", + "formula_retention": null, + "header_footer_filter_rate": null, + "heading_definition_integrity": null, + "page_provenance_completeness": 1.0, + "parse_success": true, + "parser_id": "docling", + "peak_rss_mb": 1998.62, + "reading_order_accuracy": 1.0, + "runtime_ms": 5989.8, + "table_retention": null, + "warnings": [] + }, + { + "bbox_provenance_completeness": 1.0, + "block_count": 5, + "block_provenance_completeness": 1.0, + "duplicate_block_count": 0, + "errors": [], + "fixture_id": "scanned_chinese", + "formula_retention": null, + "header_footer_filter_rate": null, + "heading_definition_integrity": null, + "page_provenance_completeness": 1.0, + "parse_success": true, + "parser_id": "docling", + "peak_rss_mb": 2446.58, + "reading_order_accuracy": 1.0, + "runtime_ms": 7488.16, + "table_retention": null, + "warnings": [] + }, + { + "bbox_provenance_completeness": 1.0, + "block_count": 6, + "block_provenance_completeness": 1.0, + "duplicate_block_count": 0, + "errors": [], + "fixture_id": "mixed_layout_blocker", + "formula_retention": null, + "header_footer_filter_rate": null, + "heading_definition_integrity": null, + "page_provenance_completeness": 1.0, + "parse_success": true, + "parser_id": "docling", + "peak_rss_mb": 1651.75, + "reading_order_accuracy": 1.0, + "runtime_ms": 4509.57, + "table_retention": 0.0, + "warnings": [] + }, + { + "bbox_provenance_completeness": 1.0, + "block_count": 1, + "block_provenance_completeness": 1.0, + "duplicate_block_count": 0, + "errors": [], + "fixture_id": "simple_table", + "formula_retention": null, + "header_footer_filter_rate": null, + "heading_definition_integrity": null, + "page_provenance_completeness": 1.0, + "parse_success": true, + "parser_id": "docling", + "peak_rss_mb": 1693.83, + "reading_order_accuracy": 1.0, + "runtime_ms": 4573.45, + "table_retention": 1.0, + "warnings": [] + }, + { + "bbox_provenance_completeness": 1.0, + "block_count": 3, + "block_provenance_completeness": 1.0, + "duplicate_block_count": 0, + "errors": [], + "fixture_id": "raster_formula", + "formula_retention": 0.0, + "header_footer_filter_rate": null, + "heading_definition_integrity": null, + "page_provenance_completeness": 1.0, + "parse_success": true, + "parser_id": "docling", + "peak_rss_mb": 2022.55, + "reading_order_accuracy": null, + "runtime_ms": 6621.94, + "table_retention": null, + "warnings": [] + }, + { + "bbox_provenance_completeness": 1.0, + "block_count": 2, + "block_provenance_completeness": 1.0, + "duplicate_block_count": 0, + "errors": [], + "fixture_id": "negative_no_terms", + "formula_retention": null, + "header_footer_filter_rate": null, + "heading_definition_integrity": null, + "page_provenance_completeness": 1.0, + "parse_success": true, + "parser_id": "docling", + "peak_rss_mb": 1640.38, + "reading_order_accuracy": null, + "runtime_ms": 4540.28, + "table_retention": null, + "warnings": [] + }, + { + "bbox_provenance_completeness": 1.0, + "block_count": 6, + "block_provenance_completeness": 1.0, + "duplicate_block_count": 0, + "errors": [], + "fixture_id": "repeated_header_footer", + "formula_retention": null, + "header_footer_filter_rate": 1.0, + "heading_definition_integrity": 1.0, + "page_provenance_completeness": 1.0, + "parse_success": true, + "parser_id": "docling", + "peak_rss_mb": 1505.3, + "reading_order_accuracy": 1.0, + "runtime_ms": 5049.68, + "table_retention": null, + "warnings": [] + }, + { + "bbox_provenance_completeness": 1.0, + "block_count": 21, + "block_provenance_completeness": 1.0, + "duplicate_block_count": 0, + "errors": [], + "fixture_id": "retrieval_english", + "formula_retention": null, + "header_footer_filter_rate": null, + "heading_definition_integrity": 1.0, + "page_provenance_completeness": 1.0, + "parse_success": true, + "parser_id": "docling", + "peak_rss_mb": 1645.81, + "reading_order_accuracy": 1.0, + "runtime_ms": 4942.29, + "table_retention": null, + "warnings": [] + }, + { + "bbox_provenance_completeness": 1.0, + "block_count": 21, + "block_provenance_completeness": 1.0, + "duplicate_block_count": 0, + "errors": [], + "fixture_id": "retrieval_chinese", + "formula_retention": null, + "header_footer_filter_rate": null, + "heading_definition_integrity": 1.0, + "page_provenance_completeness": 1.0, + "parse_success": true, + "parser_id": "docling", + "peak_rss_mb": 1644.03, + "reading_order_accuracy": 1.0, + "runtime_ms": 4670.07, + "table_retention": null, + "warnings": [] + }, + { + "bbox_provenance_completeness": 1.0, + "block_count": 5, + "block_provenance_completeness": 1.0, + "duplicate_block_count": 0, + "errors": [], + "fixture_id": "single_column_born_digital", + "formula_retention": null, + "header_footer_filter_rate": null, + "heading_definition_integrity": null, + "page_provenance_completeness": 1.0, + "parse_success": true, + "parser_id": "mineru", + "peak_rss_mb": 1416.88, + "reading_order_accuracy": 1.0, + "runtime_ms": 10366.44, + "table_retention": null, + "warnings": [] + }, + { + "bbox_provenance_completeness": 1.0, + "block_count": 7, + "block_provenance_completeness": 1.0, + "duplicate_block_count": 0, + "errors": [], + "fixture_id": "two_column_born_digital", + "formula_retention": null, + "header_footer_filter_rate": null, + "heading_definition_integrity": null, + "page_provenance_completeness": 1.0, + "parse_success": true, + "parser_id": "mineru", + "peak_rss_mb": 1414.5, + "reading_order_accuracy": 0.7857, + "runtime_ms": 8639.2, + "table_retention": null, + "warnings": [] + }, + { + "bbox_provenance_completeness": 1.0, + "block_count": 5, + "block_provenance_completeness": 1.0, + "duplicate_block_count": 0, + "errors": [], + "fixture_id": "scanned_english", + "formula_retention": null, + "header_footer_filter_rate": null, + "heading_definition_integrity": null, + "page_provenance_completeness": 1.0, + "parse_success": true, + "parser_id": "mineru", + "peak_rss_mb": 1421.36, + "reading_order_accuracy": 1.0, + "runtime_ms": 7730.35, + "table_retention": null, + "warnings": [] + }, + { + "bbox_provenance_completeness": 1.0, + "block_count": 5, + "block_provenance_completeness": 1.0, + "duplicate_block_count": 0, + "errors": [], + "fixture_id": "scanned_chinese", + "formula_retention": null, + "header_footer_filter_rate": null, + "heading_definition_integrity": null, + "page_provenance_completeness": 1.0, + "parse_success": true, + "parser_id": "mineru", + "peak_rss_mb": 1425.73, + "reading_order_accuracy": 1.0, + "runtime_ms": 8838.18, + "table_retention": null, + "warnings": [] + }, + { + "bbox_provenance_completeness": 1.0, + "block_count": 6, + "block_provenance_completeness": 1.0, + "duplicate_block_count": 0, + "errors": [], + "fixture_id": "mixed_layout_blocker", + "formula_retention": null, + "header_footer_filter_rate": null, + "heading_definition_integrity": null, + "page_provenance_completeness": 1.0, + "parse_success": true, + "parser_id": "mineru", + "peak_rss_mb": 1424.64, + "reading_order_accuracy": 1.0, + "runtime_ms": 9145.91, + "table_retention": 0.0, + "warnings": [] + }, + { + "bbox_provenance_completeness": 1.0, + "block_count": 2, + "block_provenance_completeness": 1.0, + "duplicate_block_count": 0, + "errors": [], + "fixture_id": "simple_table", + "formula_retention": null, + "header_footer_filter_rate": null, + "heading_definition_integrity": null, + "page_provenance_completeness": 1.0, + "parse_success": true, + "parser_id": "mineru", + "peak_rss_mb": 1415.84, + "reading_order_accuracy": 1.0, + "runtime_ms": 10429.65, + "table_retention": 1.0, + "warnings": [] + }, + { + "bbox_provenance_completeness": 1.0, + "block_count": 1, + "block_provenance_completeness": 1.0, + "duplicate_block_count": 0, + "errors": [], + "fixture_id": "raster_formula", + "formula_retention": 0.0, + "header_footer_filter_rate": null, + "heading_definition_integrity": null, + "page_provenance_completeness": 1.0, + "parse_success": true, + "parser_id": "mineru", + "peak_rss_mb": 1830.62, + "reading_order_accuracy": null, + "runtime_ms": 13224.48, + "table_retention": null, + "warnings": [] + }, + { + "bbox_provenance_completeness": 1.0, + "block_count": 2, + "block_provenance_completeness": 1.0, + "duplicate_block_count": 0, + "errors": [], + "fixture_id": "negative_no_terms", + "formula_retention": null, + "header_footer_filter_rate": null, + "heading_definition_integrity": null, + "page_provenance_completeness": 1.0, + "parse_success": true, + "parser_id": "mineru", + "peak_rss_mb": 1403.34, + "reading_order_accuracy": null, + "runtime_ms": 9541.72, + "table_retention": null, + "warnings": [] + }, + { + "bbox_provenance_completeness": 1.0, + "block_count": 6, + "block_provenance_completeness": 1.0, + "duplicate_block_count": 0, + "errors": [], + "fixture_id": "repeated_header_footer", + "formula_retention": null, + "header_footer_filter_rate": 1.0, + "heading_definition_integrity": 1.0, + "page_provenance_completeness": 1.0, + "parse_success": true, + "parser_id": "mineru", + "peak_rss_mb": 1527.08, + "reading_order_accuracy": 1.0, + "runtime_ms": 11807.61, + "table_retention": null, + "warnings": [] + }, + { + "bbox_provenance_completeness": 1.0, + "block_count": 21, + "block_provenance_completeness": 1.0, + "duplicate_block_count": 0, + "errors": [], + "fixture_id": "retrieval_english", + "formula_retention": null, + "header_footer_filter_rate": null, + "heading_definition_integrity": 1.0, + "page_provenance_completeness": 1.0, + "parse_success": true, + "parser_id": "mineru", + "peak_rss_mb": 1483.81, + "reading_order_accuracy": 1.0, + "runtime_ms": 12114.65, + "table_retention": null, + "warnings": [] + }, + { + "bbox_provenance_completeness": 1.0, + "block_count": 12, + "block_provenance_completeness": 1.0, + "duplicate_block_count": 0, + "errors": [], + "fixture_id": "retrieval_chinese", + "formula_retention": null, + "header_footer_filter_rate": null, + "heading_definition_integrity": 1.0, + "page_provenance_completeness": 1.0, + "parse_success": true, + "parser_id": "mineru", + "peak_rss_mb": 1443.22, + "reading_order_accuracy": 1.0, + "runtime_ms": 12112.85, + "table_retention": null, + "warnings": [] + } + ], + "fixture_count": 11, + "fixtures": [ + { + "filename": "single-column-born-digital.pdf", + "fixture_id": "single_column_born_digital", + "language": "en", + "page_count": 1, + "privacy_classification": "SYNTHETIC", + "purpose": "parser_quality", + "source_hash": "ece61f3a09db3ab9546a4e556111a72e9d73cbaab411ac9eb8b78d6c75b1faab" + }, + { + "filename": "two-column-born-digital.pdf", + "fixture_id": "two_column_born_digital", + "language": "en", + "page_count": 1, + "privacy_classification": "SYNTHETIC", + "purpose": "parser_quality", + "source_hash": "cc4f48b78728215a2a7b40dab11cb5958808cabb1af9f59ec174a6f4a1d3f36d" + }, + { + "filename": "scanned-english.pdf", + "fixture_id": "scanned_english", + "language": "en", + "page_count": 1, + "privacy_classification": "SYNTHETIC", + "purpose": "parser_quality", + "source_hash": "7c3516664577c9b6bd3d19d40a57f6299a72da1d17b097aa1003b5760d365b6d" + }, + { + "filename": "scanned-chinese.pdf", + "fixture_id": "scanned_chinese", + "language": "zh", + "page_count": 1, + "privacy_classification": "SYNTHETIC", + "purpose": "parser_quality", + "source_hash": "9ce59c1174feecd9b24ff7dd3f616c687f52c9b9cc646cde4bbf745ce5ea9e45" + }, + { + "filename": "mixed-layout-blocker.pdf", + "fixture_id": "mixed_layout_blocker", + "language": "en", + "page_count": 1, + "privacy_classification": "SYNTHETIC", + "purpose": "parser_quality", + "source_hash": "7b1f5633b72d72729b1aa77c633ea892bddb12185ce1a9ece43d938c989d0450" + }, + { + "filename": "simple-table.pdf", + "fixture_id": "simple_table", + "language": "en", + "page_count": 1, + "privacy_classification": "SYNTHETIC", + "purpose": "parser_quality", + "source_hash": "23a18ae4d0721f025aff86b8c7f77275950d9853f6e570d8283274b34b058898" + }, + { + "filename": "raster-formula.pdf", + "fixture_id": "raster_formula", + "language": "en", + "page_count": 1, + "privacy_classification": "SYNTHETIC", + "purpose": "parser_quality", + "source_hash": "090cc807bf1ef9846be72d48a037bdd1906ec092efda038f726f1e9c682ada28" + }, + { + "filename": "negative-no-terms.pdf", + "fixture_id": "negative_no_terms", + "language": "en", + "page_count": 1, + "privacy_classification": "SYNTHETIC", + "purpose": "parser_quality", + "source_hash": "6f02fd4adda9a7530a67f76ca5ff3dfb6f411a2da62ba3e48919c622f038096d" + }, + { + "filename": "repeated-header-footer.pdf", + "fixture_id": "repeated_header_footer", + "language": "en", + "page_count": 3, + "privacy_classification": "SYNTHETIC", + "purpose": "parser_quality", + "source_hash": "35910dcbcd8844ad25b3e20cd4bdb18e1cfc8fa1eafaf013fb6c71de91fecb93" + }, + { + "filename": "retrieval-english-concepts.pdf", + "fixture_id": "retrieval_english", + "language": "en", + "page_count": 2, + "privacy_classification": "SYNTHETIC", + "purpose": "downstream_retrieval", + "source_hash": "b66ff1a039e7967f96de7269b69e24d89d4c40dd7e8b7ddba3c2066913f6c8fa" + }, + { + "filename": "retrieval-chinese-evidence.pdf", + "fixture_id": "retrieval_chinese", + "language": "zh", + "page_count": 2, + "privacy_classification": "SYNTHETIC", + "purpose": "downstream_retrieval", + "source_hash": "5a88e5f3536eff2808adc5a61e55281918b4e6f69ae2bf286a5df603b6007991" + } + ], + "platform": { + "machine": "arm64", + "python": "3.9.6", + "system": "Darwin" + }, + "retrieval": { + "baseline_native_tesseract_formula_region": { + "average_correct_rank": 1.0, + "backend_id": "local_multilingual_e5_pytorch_cpu_v1", + "chinese_chunk_count": 2, + "denominator": 10, + "english_chunk_count": 2, + "external_api_used": false, + "gold_used_in_parser": false, + "gold_used_in_retrieval": false, + "gold_used_post_ranking": true, + "hit_at_1": 0.1, + "hit_at_3": 0.1, + "model_id": "intfloat/multilingual-e5-small", + "model_revision": "614241f622f53c4eeff9890bdc4f31cfecc418b3", + "mrr": 0.1, + "no_result_count": 0, + "parser_id": "baseline_native_tesseract_formula_region", + "parser_version": "current_commit", + "rankings": { + "boundary_condition": [ + "ambiguous:9bd2bdac-ccfe-544b-87fe-da0b8d816f71", + "transfer_function" + ], + "damping_ratio": [ + "ambiguous:9bd2bdac-ccfe-544b-87fe-da0b8d816f71", + "transfer_function" + ], + "electric_charge": [ + "ambiguous:9bd2bdac-ccfe-544b-87fe-da0b8d816f71", + "transfer_function" + ], + "electric_field": [ + "ambiguous:9bd2bdac-ccfe-544b-87fe-da0b8d816f71", + "transfer_function" + ], + "electric_potential": [ + "ambiguous:9bd2bdac-ccfe-544b-87fe-da0b8d816f71", + "transfer_function" + ], + "impulse": [ + "ambiguous:9bd2bdac-ccfe-544b-87fe-da0b8d816f71", + "transfer_function" + ], + "moment_of_inertia": [ + "ambiguous:9bd2bdac-ccfe-544b-87fe-da0b8d816f71", + "transfer_function" + ], + "momentum": [ + "ambiguous:9bd2bdac-ccfe-544b-87fe-da0b8d816f71", + "transfer_function" + ], + "torque": [ + "ambiguous:9bd2bdac-ccfe-544b-87fe-da0b8d816f71", + "transfer_function" + ], + "transfer_function": [ + "transfer_function", + "ambiguous:9bd2bdac-ccfe-544b-87fe-da0b8d816f71" + ] + } + }, + "docling": { + "average_correct_rank": 1.1667, + "backend_id": "local_multilingual_e5_pytorch_cpu_v1", + "chinese_chunk_count": 11, + "denominator": 10, + "english_chunk_count": 11, + "external_api_used": false, + "gold_used_in_parser": false, + "gold_used_in_retrieval": false, + "gold_used_post_ranking": true, + "hit_at_1": 0.5, + "hit_at_3": 0.6, + "model_id": "intfloat/multilingual-e5-small", + "model_revision": "614241f622f53c4eeff9890bdc4f31cfecc418b3", + "mrr": 0.55, + "no_result_count": 0, + "parser_id": "docling", + "parser_version": "2.117.0", + "rankings": { + "boundary_condition": [ + "boundary_condition", + "ambiguous:86d5e620-3c70-5918-ac60-6aa2b088761f", + "electric_charge" + ], + "damping_ratio": [ + "ambiguous:86d5e620-3c70-5918-ac60-6aa2b088761f", + "ambiguous:4bbd99cf-4fbb-5566-a910-7dcada36d6b8", + "torque" + ], + "electric_charge": [ + "electric_charge", + "momentum", + "ambiguous:bb55ebcc-18d1-5b4c-b882-add7aa024621" + ], + "electric_field": [ + "ambiguous:24dc7ef7-d08b-5b5c-9c11-2d0012cc7801", + "momentum", + "ambiguous:bb55ebcc-18d1-5b4c-b882-add7aa024621" + ], + "electric_potential": [ + "ambiguous:bb55ebcc-18d1-5b4c-b882-add7aa024621", + "momentum", + "ambiguous:24dc7ef7-d08b-5b5c-9c11-2d0012cc7801" + ], + "impulse": [ + "ambiguous:4bbd99cf-4fbb-5566-a910-7dcada36d6b8", + "torque", + "momentum" + ], + "moment_of_inertia": [ + "ambiguous:4bbd99cf-4fbb-5566-a910-7dcada36d6b8", + "moment_of_inertia", + "torque" + ], + "momentum": [ + "momentum", + "torque", + "ambiguous:4bbd99cf-4fbb-5566-a910-7dcada36d6b8" + ], + "torque": [ + "torque", + "ambiguous:4bbd99cf-4fbb-5566-a910-7dcada36d6b8", + "momentum" + ], + "transfer_function": [ + "transfer_function", + "torque", + "ambiguous:4bbd99cf-4fbb-5566-a910-7dcada36d6b8" + ] + } + }, + "mineru": { + "average_correct_rank": 1.0, + "backend_id": "local_multilingual_e5_pytorch_cpu_v1", + "chinese_chunk_count": 2, + "denominator": 10, + "english_chunk_count": 11, + "external_api_used": false, + "gold_used_in_parser": false, + "gold_used_in_retrieval": false, + "gold_used_post_ranking": true, + "hit_at_1": 0.1, + "hit_at_3": 0.1, + "model_id": "intfloat/multilingual-e5-small", + "model_revision": "614241f622f53c4eeff9890bdc4f31cfecc418b3", + "mrr": 0.1, + "no_result_count": 0, + "parser_id": "mineru", + "parser_version": "3.4.4", + "rankings": { + "boundary_condition": [ + "ambiguous:b42c7ebe-0cd1-577d-a188-12d7440964ed", + "transfer_function" + ], + "damping_ratio": [ + "ambiguous:b42c7ebe-0cd1-577d-a188-12d7440964ed", + "transfer_function" + ], + "electric_charge": [ + "ambiguous:b42c7ebe-0cd1-577d-a188-12d7440964ed", + "transfer_function" + ], + "electric_field": [ + "ambiguous:b42c7ebe-0cd1-577d-a188-12d7440964ed", + "transfer_function" + ], + "electric_potential": [ + "ambiguous:b42c7ebe-0cd1-577d-a188-12d7440964ed", + "transfer_function" + ], + "impulse": [ + "ambiguous:b42c7ebe-0cd1-577d-a188-12d7440964ed", + "transfer_function" + ], + "moment_of_inertia": [ + "ambiguous:b42c7ebe-0cd1-577d-a188-12d7440964ed", + "transfer_function" + ], + "momentum": [ + "ambiguous:b42c7ebe-0cd1-577d-a188-12d7440964ed", + "transfer_function" + ], + "torque": [ + "ambiguous:b42c7ebe-0cd1-577d-a188-12d7440964ed", + "transfer_function" + ], + "transfer_function": [ + "transfer_function", + "ambiguous:b42c7ebe-0cd1-577d-a188-12d7440964ed" + ] + } + } + }, + "safety": { + "external_api_used": false, + "external_parser_request_count": 0, + "fixtures_private": false, + "incident_database_accessed": false, + "production_adapter_changed": false, + "production_parser_changed": false, + "real_credentials_read": false, + "real_provider_requests": 0 + }, + "schema_version": "controlled-parser-benchmark-14b@1.0.0", + "selection": { + "composition_requirements": [ + "per_document_quality_routing", + "current_native_parser_fallback", + "existing_formula_region" + ], + "excluded_scope": [ + "multi_column_pdf", + "mixed_layout_table_pdf", + "formula_pdf_without_existing_formula_region" + ], + "fallback_parser_id": "baseline_native_tesseract_formula_region", + "mineru_eligible": false, + "production_adapter_authorized": false, + "reason": "Selected a conditional integration candidate; failed document-class gates remain excluded and production routing is unchanged.", + "recommended_scope": [ + "scanned_pdf", + "simple_table_pdf" + ], + "selected_parser_id": "docling", + "selected_role": "conditional_complex_document_parser_candidate", + "simple_document_policy": "retain_current_native_parser" + }, + "task": "14B" +} diff --git a/docs/evaluations/artifacts/14B-parser-selection-manifest.json b/docs/evaluations/artifacts/14B-parser-selection-manifest.json new file mode 100644 index 0000000..bbf3131 --- /dev/null +++ b/docs/evaluations/artifacts/14B-parser-selection-manifest.json @@ -0,0 +1,46 @@ +{ + "baseline_commit": "6cca8cb0881226c6031f3adafff0d89c1146d584", + "parsers": [ + { + "license_gate": "pass", + "parser_id": "baseline_native_tesseract_formula_region", + "parser_version": "current_commit" + }, + { + "license_gate": "pass", + "parser_id": "docling", + "parser_version": "2.117.0" + }, + { + "license_gate": "blocked_nonstandard_license", + "parser_id": "mineru", + "parser_version": "3.4.4" + } + ], + "production_adapter_authorized": false, + "schema_version": "parser-selection-manifest-14b@1.0.0", + "selection": { + "composition_requirements": [ + "per_document_quality_routing", + "current_native_parser_fallback", + "existing_formula_region" + ], + "excluded_scope": [ + "multi_column_pdf", + "mixed_layout_table_pdf", + "formula_pdf_without_existing_formula_region" + ], + "fallback_parser_id": "baseline_native_tesseract_formula_region", + "mineru_eligible": false, + "production_adapter_authorized": false, + "reason": "Selected a conditional integration candidate; failed document-class gates remain excluded and production routing is unchanged.", + "recommended_scope": [ + "scanned_pdf", + "simple_table_pdf" + ], + "selected_parser_id": "docling", + "selected_role": "conditional_complex_document_parser_candidate", + "simple_document_policy": "retain_current_native_parser" + }, + "task": "14B" +} diff --git a/scripts/evaluations/open_source_parser_eval/candidate_probe_14b.py b/scripts/evaluations/open_source_parser_eval/candidate_probe_14b.py new file mode 100644 index 0000000..748f3c5 --- /dev/null +++ b/scripts/evaluations/open_source_parser_eval/candidate_probe_14b.py @@ -0,0 +1,245 @@ +#!/usr/bin/env python3 +"""Process-isolated parser probe for the Task 14B controlled benchmark.""" + +from __future__ import annotations + +import argparse +import importlib.metadata +import json +import os +import re +import resource +import shutil +import subprocess +import sys +import tempfile +import time +from pathlib import Path +from typing import Any + + +ROOT = Path(__file__).resolve().parents[3] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from scripts.evaluations.open_source_parser_eval import controlled_benchmark_14b as benchmark # noqa: E402 + + +URL_RE = re.compile(r"https?://([^\s/:]+)(?::\d+)?", re.IGNORECASE) + + +def _version(package: str) -> str: + try: + return importlib.metadata.version(package) + except importlib.metadata.PackageNotFoundError: + return "" + + +def _peak_rss_mb(usage: resource.struct_rusage) -> float: + value = float(usage.ru_maxrss or 0) + # macOS reports bytes; Linux reports KiB. + divisor = 1024 * 1024 if sys.platform == "darwin" else 1024 + return round(value / divisor, 2) + + +def _network_metadata(log_text: str) -> dict[str, Any]: + hosts = sorted(set(match.casefold() for match in URL_RE.findall(log_text or ""))) + external = [host for host in hosts if host not in {"127.0.0.1", "localhost", "::1"}] + return { + "observed_hosts": hosts, + "external_hosts": external, + "external_request_count": len(external), + } + + +def _safe_error(value: Any) -> str: + sanitized = benchmark.sanitize_artifact(str(value or "")) + return str(sanitized)[-1200:] + + +def run_baseline(path: Path, fixture_id: str) -> dict[str, Any]: + from scripts.evaluations.open_source_parser_eval import evaluate + + fixture = evaluate.ParserFixture( + fixture_id=fixture_id, + filename=path.name, + path=path, + privacy_classification="SYNTHETIC", + domains=("controlled_benchmark",), + expected_anchors=(), + ) + result = evaluate.run_baseline_fixture(fixture) + # The legacy evaluator stores a compact locator marker as bbox. Recover the + # actual numeric coordinates for the common page:bbox:x0,y0,x1,y1 shape. + for block in result.get("blocks") or []: + locator = str((block.get("provenance") or {}).get("source_locator") or "") + match = re.search(r"bbox:([-\d.]+),([-\d.]+),([-\d.]+),([-\d.]+)", locator) + if match: + block["bbox"] = { + "x0": float(match.group(1)), + "y0": float(match.group(2)), + "x1": float(match.group(3)), + "y1": float(match.group(4)), + } + block.setdefault("provenance", {})["source_parser"] = "baseline_native_tesseract_formula_region" + block["block_type"] = _baseline_block_type(block) + result["peak_rss_mb"] = _peak_rss_mb(resource.getrusage(resource.RUSAGE_SELF)) + result["network"] = {"observed_hosts": [], "external_hosts": [], "external_request_count": 0} + return result + + +def _baseline_block_type(block: dict[str, Any]) -> str: + if str(block.get("block_type") or "") == "formula": + return "formula" + parser_type = str((block.get("provenance") or {}).get("parser_type") or "") + # Production layout parse emits the original type in quality/locator only + # in older records, so retain paragraph when no governed type exists. + return "paragraph" if parser_type else str(block.get("block_type") or "paragraph") + + +def run_docling(path: Path, fixture_id: str, artifacts_path: Path) -> dict[str, Any]: + from docling.datamodel.base_models import InputFormat + from docling.datamodel.pipeline_options import PdfPipelineOptions + from docling.document_converter import DocumentConverter, PdfFormatOption + from scripts.evaluations.open_source_parser_eval import docling_failure_attribution + + options = PdfPipelineOptions( + artifacts_path=artifacts_path, + enable_remote_services=False, + do_ocr=True, + do_table_structure=True, + do_formula_enrichment=True, + do_picture_classification=False, + ) + converter = DocumentConverter( + format_options={InputFormat.PDF: PdfFormatOption(pipeline_options=options)} + ) + started = time.perf_counter() + converted = converter.convert(str(path)) + duration_ms = round((time.perf_counter() - started) * 1000, 2) + exported = converted.document.export_to_dict() + canonical = docling_failure_attribution.canonicalize_docling_export(exported) + blocks = docling_failure_attribution.canonical_blocks_to_evaluation_blocks( + canonical, fixture_id, parser_id="docling" + ) + return { + "parser_id": "docling", + "parser_version": _version("docling"), + "fixture_id": fixture_id, + "page_count": len(exported.get("pages") or {}), + "blocks": blocks, + "parse_duration_ms": duration_ms, + "peak_rss_mb": _peak_rss_mb(resource.getrusage(resource.RUSAGE_SELF)), + "warnings": [], + "errors": [], + "network": {"observed_hosts": [], "external_hosts": [], "external_request_count": 0}, + } + + +def _find_mineru_v2(output_root: Path) -> Path | None: + candidates = sorted(output_root.rglob("*_content_list_v2.json")) + return candidates[0] if candidates else None + + +def run_mineru(path: Path, fixture_id: str, runtime_root: Path) -> dict[str, Any]: + mineru = shutil.which("mineru") + if not mineru: + raise RuntimeError("MINERU_CLI_NOT_FOUND") + runtime_root.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(prefix="mineru-14b-", dir=str(runtime_root)) as directory: + output_root = Path(directory) / "output" + command = [ + mineru, + "--path", + str(path), + "--output", + str(output_root), + "--backend", + "pipeline", + "--method", + "auto", + "--formula", + "true", + "--table", + "true", + ] + started = time.perf_counter() + completed = subprocess.run( + command, + check=False, + shell=False, + capture_output=True, + text=True, + timeout=240, + env=os.environ.copy(), + ) + duration_ms = round((time.perf_counter() - started) * 1000, 2) + log_text = f"{completed.stdout}\n{completed.stderr}" + network = _network_metadata(log_text) + if completed.returncode != 0: + raise RuntimeError(f"MINERU_EXECUTION_FAILED: {_safe_error(log_text)}") + content_path = _find_mineru_v2(output_root) + if content_path is None: + raise RuntimeError("MINERU_CONTENT_LIST_V2_MISSING") + pages = json.loads(content_path.read_text(encoding="utf-8")) + blocks = benchmark.normalize_mineru_content_list_v2(pages, fixture_id=fixture_id) + return { + "parser_id": "mineru", + "parser_version": _version("mineru"), + "fixture_id": fixture_id, + "page_count": len(pages) if isinstance(pages, list) else 0, + "blocks": blocks, + "parse_duration_ms": duration_ms, + "peak_rss_mb": max( + _peak_rss_mb(resource.getrusage(resource.RUSAGE_SELF)), + _peak_rss_mb(resource.getrusage(resource.RUSAGE_CHILDREN)), + ), + "warnings": [], + "errors": [], + "network": network, + } + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--parser", required=True, choices=benchmark.PARSER_IDS) + parser.add_argument("--input", required=True) + parser.add_argument("--fixture-id", required=True) + parser.add_argument("--output", required=True) + parser.add_argument("--model-root", default="") + parser.add_argument("--runtime-root", required=True) + args = parser.parse_args(argv) + path = Path(args.input) + try: + if args.parser == "baseline_native_tesseract_formula_region": + payload = run_baseline(path, args.fixture_id) + elif args.parser == "docling": + if not args.model_root: + raise RuntimeError("DOCLING_MODEL_ROOT_REQUIRED") + payload = run_docling(path, args.fixture_id, Path(args.model_root)) + else: + payload = run_mineru(path, args.fixture_id, Path(args.runtime_root)) + except Exception as exc: # fail closed but keep the batch running + payload = { + "parser_id": args.parser, + "parser_version": _version("mineru" if args.parser == "mineru" else args.parser), + "fixture_id": args.fixture_id, + "page_count": 0, + "blocks": [], + "parse_duration_ms": 0, + "peak_rss_mb": max( + _peak_rss_mb(resource.getrusage(resource.RUSAGE_SELF)), + _peak_rss_mb(resource.getrusage(resource.RUSAGE_CHILDREN)), + ), + "warnings": [], + "errors": [{"code": "PARSER_PROBE_FAILED", "message": _safe_error(f"{type(exc).__name__}: {exc}")}], + "network": {"observed_hosts": [], "external_hosts": [], "external_request_count": 0}, + } + output = Path(args.output) + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(json.dumps(benchmark.sanitize_artifact(payload), ensure_ascii=False, indent=2, sort_keys=True), encoding="utf-8") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/evaluations/open_source_parser_eval/controlled_benchmark_14b.py b/scripts/evaluations/open_source_parser_eval/controlled_benchmark_14b.py new file mode 100644 index 0000000..67c1043 --- /dev/null +++ b/scripts/evaluations/open_source_parser_eval/controlled_benchmark_14b.py @@ -0,0 +1,1107 @@ +#!/usr/bin/env python3 +"""Controlled current/Docling/MinerU parser benchmark for LexiBridge. + +The module deliberately normalizes parser output into the existing parse-block +and KnowledgeChunk contracts. It is evaluation-only: importing it does not +load a third-party parser, open a database, download a model, or change +production parser routing. +""" + +from __future__ import annotations + +import argparse +import csv +import hashlib +import json +import math +import os +import re +import statistics +import subprocess +import sys +import time +import uuid +from collections import Counter +from dataclasses import asdict, dataclass, field +from datetime import datetime, timezone +from pathlib import Path +from types import SimpleNamespace +from typing import Any, Iterable, Mapping, Sequence + + +ROOT = Path(__file__).resolve().parents[3] +BACKEND = ROOT / "backend" +PARSER_IDS = ( + "baseline_native_tesseract_formula_region", + "docling", + "mineru", +) +SCHEMA_VERSION = "controlled-parser-benchmark-14b@1.0.0" +LOCAL_PATH_RE = re.compile( + r"(?:/Users/|/private/(?:tmp|var)/|file://)[^\s\"']+", re.IGNORECASE +) +SECRET_RE = re.compile( + r"(?:authorization|cookie)\s*[:=][^\s,}]+|Bearer\s+[^\s,}]+|sk-[A-Za-z0-9_-]+", + re.IGNORECASE, +) +BODY_KEYS = frozenset( + { + "request_body", + "response_body", + "source_body", + "full_source", + "raw_output", + "prompt_body", + } +) +NOISE_TYPES = frozenset({"header", "footer", "page_number"}) +CONTENT_TYPES = frozenset( + {"heading", "paragraph", "list_item", "table", "formula", "image", "caption"} +) + + +@dataclass(frozen=True) +class BenchmarkFixture: + fixture_id: str + filename: str + path: Path + privacy_classification: str + expected_anchors: tuple[Any, ...] = () + expected_heading_definition_pairs: tuple[tuple[str, str], ...] = () + expected_noise: tuple[str, ...] = () + expected_table_rows: int = 0 + expected_table_cols: int = 0 + expected_formula_count: int = 0 + language: str = "unknown" + purpose: str = "parser_quality" + + +@dataclass(frozen=True) +class RetrievalConcept: + concept_id: str + english_term: str + english_definition: str + chinese_term: str + chinese_definition: str + discipline: str + + +RETRIEVAL_CONCEPTS: tuple[RetrievalConcept, ...] = ( + RetrievalConcept( + "electric_charge", + "Electric Charge", + "A conserved property of matter that determines electrical interaction and can be positive or negative.", + "电荷", + "物质的一种守恒属性，决定物体参与电相互作用的方式，并可表现为正或负。", + "physics", + ), + RetrievalConcept( + "electric_field", + "Electric Field", + "A vector field assigning the force per unit positive test charge at each point in space.", + "电场", + "空间中每一点对单位正试探电荷作用力的矢量场。", + "physics", + ), + RetrievalConcept( + "electric_potential", + "Electric Potential", + "The electric potential energy per unit charge at a point, measured relative to a reference.", + "电势", + "某点单位电荷具有的电势能，以选定参考点为基准进行度量。", + "physics", + ), + RetrievalConcept( + "momentum", + "Momentum", + "A vector quantity equal to mass times velocity that describes translational motion.", + "动量", + "质量与速度乘积构成的矢量物理量，用于描述物体的平动状态。", + "mechanics", + ), + RetrievalConcept( + "impulse", + "Impulse", + "The time integral of force, equal to the change in momentum over an interval.", + "冲量", + "力对时间的积分，数值上等于物体在该时间间隔内动量的变化。", + "mechanics", + ), + RetrievalConcept( + "torque", + "Torque", + "The moment of a force about an axis that measures its tendency to cause rotation.", + "力矩", + "力对某一转轴的矩，用于度量该力使物体产生转动的趋势。", + "mechanics", + ), + RetrievalConcept( + "moment_of_inertia", + "Moment of Inertia", + "A mass-distribution property describing resistance to changes in rotational motion.", + "转动惯量", + "由质量分布决定、描述物体抵抗转动状态改变能力的物理量。", + "mechanics", + ), + RetrievalConcept( + "damping_ratio", + "Damping Ratio", + "A dimensionless ratio comparing actual damping with critical damping in a second-order system.", + "阻尼比", + "二阶系统实际阻尼与临界阻尼之比，是描述振动衰减程度的无量纲参数。", + "control engineering", + ), + RetrievalConcept( + "boundary_condition", + "Boundary Condition", + "A constraint imposed on a differential-equation solution at the boundary of its domain.", + "边界条件", + "在定义域边界处对微分方程解施加的约束条件。", + "applied mathematics", + ), + RetrievalConcept( + "transfer_function", + "Transfer Function", + "The ratio of output to input in the transform domain for a linear time-invariant system under zero initial conditions.", + "传递函数", + "在线性时不变系统零初始条件下，变换域中输出与输入之比。", + "control engineering", + ), +) + + +def _anchor(text: str, order: int) -> Any: + return SimpleNamespace(text=text, block_type="paragraph", order=order) + + +def _write_concept_pdf( + path: Path, + *, + document_title: str, + concepts: Sequence[tuple[str, str]], + language: str, +) -> None: + from reportlab.lib.pagesizes import letter + from reportlab.pdfgen import canvas + from scripts.evaluations.open_source_parser_eval import evaluate + + path.parent.mkdir(parents=True, exist_ok=True) + font = evaluate._register_font() + document = canvas.Canvas(str(path), pagesize=letter) + _, height = letter + document.setTitle(document_title) + y = height - 56 + document.setFont(font, 18) + document.drawString(54, y, document_title) + y -= 42 + for term, definition in concepts: + if y < 120: + document.showPage() + y = height - 56 + document.setFont(font, 14) + document.drawString(54, y, term) + y -= 24 + document.setFont(font, 10.5) + # Keep the synthetic definition on one line so every parser receives + # the same explicit heading-definition relation. + document.drawString(68, y, definition[:165]) + y -= 42 + document.save() + + +def _write_repeated_header_footer_pdf(path: Path) -> None: + from reportlab.lib.pagesizes import letter + from reportlab.pdfgen import canvas + from scripts.evaluations.open_source_parser_eval import evaluate + + path.parent.mkdir(parents=True, exist_ok=True) + font = evaluate._register_font() + document = canvas.Canvas(str(path), pagesize=letter) + width, height = letter + for page_number, body in enumerate( + ( + "Conservation law body paragraph for the first page.", + "Rotational dynamics body paragraph for the second page.", + "Control-system response body paragraph for the third page.", + ), + start=1, + ): + document.setFont(font, 8) + document.drawString(54, height - 24, "LexiBridge Evaluation Header") + document.drawString(54, 22, "LexiBridge Evaluation Footer") + document.drawRightString(width - 54, 22, f"Page {page_number}") + document.setFont(font, 15) + document.drawString(54, height - 90, f"Section {page_number}") + document.setFont(font, 11) + document.drawString(54, height - 122, body) + document.showPage() + document.save() + + +def build_controlled_fixture_set(root: Path) -> list[BenchmarkFixture]: + """Build the bounded safe corpus used by the 14B comparison.""" + + from scripts.evaluations.open_source_parser_eval import evaluate + + root.mkdir(parents=True, exist_ok=True) + baseline_fixtures = { + fixture.fixture_id: fixture for fixture in evaluate.build_fixture_set(root / "legacy") + } + selected_ids = ( + "single_column_born_digital", + "two_column_born_digital", + "scanned_english", + "scanned_chinese", + "mixed_layout_blocker", + "simple_table", + "raster_formula", + "negative_no_terms", + ) + fixtures = [ + BenchmarkFixture( + fixture_id=fixture.fixture_id, + filename=fixture.filename, + path=fixture.path, + privacy_classification="SYNTHETIC", + expected_anchors=fixture.expected_anchors, + expected_table_rows=fixture.expected_table_rows, + expected_table_cols=fixture.expected_table_cols, + expected_formula_count=fixture.expected_formula_count, + language=("zh" if fixture.fixture_id == "scanned_chinese" else "en"), + ) + for fixture in (baseline_fixtures[fixture_id] for fixture_id in selected_ids) + ] + + repeated = root / "repeated-header-footer.pdf" + _write_repeated_header_footer_pdf(repeated) + fixtures.append( + BenchmarkFixture( + fixture_id="repeated_header_footer", + filename=repeated.name, + path=repeated, + privacy_classification="SYNTHETIC", + expected_anchors=( + _anchor("Conservation law body paragraph", 1), + _anchor("Rotational dynamics body paragraph", 2), + _anchor("Control-system response body paragraph", 3), + ), + expected_heading_definition_pairs=( + ("Section 1", "Conservation law body paragraph"), + ("Section 2", "Rotational dynamics body paragraph"), + ("Section 3", "Control-system response body paragraph"), + ), + expected_noise=( + "LexiBridge Evaluation Header", + "LexiBridge Evaluation Footer", + "Page 1", + "Page 2", + "Page 3", + ), + language="en", + ) + ) + + english_path = root / "retrieval-english-concepts.pdf" + chinese_path = root / "retrieval-chinese-evidence.pdf" + _write_concept_pdf( + english_path, + document_title="Engineering Concept Notes", + concepts=[(item.english_term, item.english_definition) for item in RETRIEVAL_CONCEPTS], + language="en", + ) + _write_concept_pdf( + chinese_path, + document_title="工程概念参考资料", + concepts=[(item.chinese_term, item.chinese_definition) for item in RETRIEVAL_CONCEPTS], + language="zh", + ) + fixtures.extend( + ( + BenchmarkFixture( + fixture_id="retrieval_english", + filename=english_path.name, + path=english_path, + privacy_classification="SYNTHETIC", + expected_anchors=tuple( + _anchor(item.english_term, index) + for index, item in enumerate(RETRIEVAL_CONCEPTS, start=1) + ), + expected_heading_definition_pairs=tuple( + (item.english_term, item.english_definition[:40]) + for item in RETRIEVAL_CONCEPTS + ), + language="en", + purpose="downstream_retrieval", + ), + BenchmarkFixture( + fixture_id="retrieval_chinese", + filename=chinese_path.name, + path=chinese_path, + privacy_classification="SYNTHETIC", + expected_anchors=tuple( + _anchor(item.chinese_term, index) + for index, item in enumerate(RETRIEVAL_CONCEPTS, start=1) + ), + expected_heading_definition_pairs=tuple( + (item.chinese_term, item.chinese_definition[:20]) + for item in RETRIEVAL_CONCEPTS + ), + language="zh", + purpose="downstream_retrieval", + ), + ) + ) + return fixtures + + +def utc_now() -> str: + return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") + + +def _text(value: Any) -> str: + return " ".join(str(value or "").replace("\x00", " ").split()).strip() + + +def _normalized(value: Any) -> str: + return _text(value).casefold() + + +def _flatten_content(value: Any) -> str: + parts: list[str] = [] + + def walk(node: Any) -> None: + if isinstance(node, str): + if node.strip(): + parts.append(node.strip()) + return + if isinstance(node, list): + for child in node: + walk(child) + return + if not isinstance(node, dict): + return + for key, child in node.items(): + if key in {"content", "text", "title", "latex", "html"}: + walk(child) + elif key.endswith("_content"): + walk(child) + + walk(value) + ordered: list[str] = [] + seen: set[str] = set() + for part in parts: + normalized = _normalized(part) + if normalized and normalized not in seen: + ordered.append(_text(part)) + seen.add(normalized) + return "\n".join(ordered) + + +def _mineru_type(value: Any) -> str: + kind = str(value or "").strip().casefold() + if kind in {"title", "heading", "section_header"}: + return "heading" + if kind in {"paragraph", "text"}: + return "paragraph" + if kind in {"list", "list_item"}: + return "list_item" + if kind in {"table"}: + return "table" + if kind in {"equation", "formula", "interline_equation", "inline_equation"}: + return "formula" + if kind in {"image", "figure", "picture"}: + return "image" + if "caption" in kind: + return "caption" + if "header" in kind: + return "header" + if "footer" in kind: + return "footer" + if kind in {"page_number", "page-number"}: + return "page_number" + return "unknown" + + +def _bbox(value: Any) -> dict[str, float]: + if isinstance(value, (list, tuple)) and len(value) >= 4: + try: + x0, y0, x1, y1 = (float(value[index]) for index in range(4)) + return {"x0": x0, "y0": y0, "x1": x1, "y1": y1} + except (TypeError, ValueError): + return {} + if not isinstance(value, dict): + return {} + aliases = { + "x0": value.get("x0", value.get("l", value.get("left"))), + "y0": value.get("y0", value.get("t", value.get("top"))), + "x1": value.get("x1", value.get("r", value.get("right"))), + "y1": value.get("y1", value.get("b", value.get("bottom"))), + } + try: + return {key: float(child) for key, child in aliases.items() if child is not None} + except (TypeError, ValueError): + return {} + + +def normalize_mineru_content_list_v2( + pages: Sequence[Any], *, fixture_id: str +) -> list[dict[str, Any]]: + """Normalize MinerU's page-grouped content-list-v2 without raw bodies.""" + + blocks: list[dict[str, Any]] = [] + for page_index, page in enumerate(pages or (), start=1): + if isinstance(page, dict): + page_items = page.get("blocks") or page.get("content") or [] + else: + page_items = page + if not isinstance(page_items, list): + continue + for item in page_items: + if not isinstance(item, dict): + continue + block_type = _mineru_type(item.get("type")) + content = item.get("content") + text = _flatten_content(content) + if not text: + text = _text(item.get("text") or item.get("img_caption")) + order = len(blocks) + 1 + block_id = f"mineru-{fixture_id}-p{page_index}-b{order}" + table_structure = None + if block_type == "table": + table_structure = _mineru_table_shape(content) + blocks.append( + { + "parser_id": "mineru", + "fixture_id": fixture_id, + "block_id": block_id, + "parent_block_id": "", + "block_type": block_type, + "text": text, + "page_number": page_index, + "bbox": _bbox(item.get("bbox")), + "reading_order": order, + "confidence": item.get("score"), + "language": "", + "is_ocr": False, + "table_structure": table_structure, + "formula_text": text if block_type == "formula" else "", + "formula_format": "latex" if block_type == "formula" and text else "", + "image_ref": "", + "provenance": { + "source_parser": "mineru", + "source_item_ref": block_id, + "page_index_zero_based": page_index - 1, + }, + } + ) + return blocks + + +def _mineru_table_shape(content: Any) -> dict[str, int] | None: + text = _flatten_content(content) + html = text if "]*>(.*?)", html, flags=re.IGNORECASE | re.DOTALL) + ] + return {"rows": rows, "cols": max(cells, default=0)} + return None + + +def _ordered_blocks(blocks: Iterable[Mapping[str, Any]]) -> list[Mapping[str, Any]]: + return sorted( + list(blocks or ()), + key=lambda block: ( + int(block.get("page_number") or 0), + int(block.get("reading_order") or 0), + str(block.get("block_id") or ""), + ), + ) + + +def _anchor_text(anchor: Any) -> str: + return str(getattr(anchor, "text", anchor if isinstance(anchor, str) else "") or "") + + +def reading_order_accuracy( + blocks: Iterable[Mapping[str, Any]], expected_anchors: Sequence[Any] +) -> float | None: + expected = [_anchor_text(anchor) for anchor in expected_anchors if _anchor_text(anchor)] + if len(expected) < 2: + return None + body = "\n".join(str(block.get("text") or "") for block in _ordered_blocks(blocks)).casefold() + positions = {anchor: body.find(anchor.casefold()) for anchor in expected} + present = [anchor for anchor in expected if positions[anchor] >= 0] + if len(present) < 2: + return 0.0 + correct = 0 + total = 0 + for left_index, left in enumerate(present): + for right in present[left_index + 1 :]: + total += 1 + correct += int(positions[left] < positions[right]) + return round(correct / total, 4) if total else 0.0 + + +def heading_definition_integrity( + blocks: Iterable[Mapping[str, Any]], pairs: Sequence[tuple[str, str]] +) -> float | None: + if not pairs: + return None + ordered = _ordered_blocks(blocks) + matched = 0 + for heading, definition in pairs: + heading_norm = heading.casefold() + definition_norm = definition.casefold() + heading_indexes = [ + index + for index, block in enumerate(ordered) + if heading_norm in str(block.get("text") or "").casefold() + ] + definition_indexes = [ + index + for index, block in enumerate(ordered) + if definition_norm in str(block.get("text") or "").casefold() + ] + if any(abs(left - right) <= 1 for left in heading_indexes for right in definition_indexes): + matched += 1 + return round(matched / len(pairs), 4) + + +def duplicate_block_count(blocks: Iterable[Mapping[str, Any]]) -> int: + counts = Counter( + _normalized(block.get("text")) + for block in blocks or () + if _normalized(block.get("text")) and str(block.get("block_type") or "") not in NOISE_TYPES + ) + return sum(max(0, count - 1) for count in counts.values()) + + +def _bbox_complete(block: Mapping[str, Any]) -> bool: + bbox = _bbox(block.get("bbox")) + return ( + len(bbox) == 4 + and bbox["x1"] > bbox["x0"] + and bbox["y1"] > bbox["y0"] + ) + + +def _rate(values: Sequence[bool]) -> float: + return round(sum(bool(value) for value in values) / len(values), 4) if values else 0.0 + + +def _noise_filter_rate(blocks: Sequence[Mapping[str, Any]], expected: Sequence[str]) -> float | None: + if not expected: + return None + body_text = "\n".join( + str(block.get("text") or "") + for block in blocks + if str(block.get("block_type") or "") not in NOISE_TYPES + ).casefold() + filtered = sum(1 for noise in expected if noise.casefold() not in body_text) + return round(filtered / len(expected), 4) + + +def _table_retention(blocks: Sequence[Mapping[str, Any]], fixture: Any) -> float | None: + rows = int(getattr(fixture, "expected_table_rows", 0) or 0) + cols = int(getattr(fixture, "expected_table_cols", 0) or 0) + if not rows and not cols: + return None + tables = [block for block in blocks if block.get("block_type") == "table"] + if not tables: + return 0.0 + shapes = [block.get("table_structure") for block in tables if isinstance(block.get("table_structure"), dict)] + if not shapes: + return 0.5 + best_rows = max(int(shape.get("rows") or 0) for shape in shapes) + best_cols = max(int(shape.get("cols") or 0) for shape in shapes) + return round(min(1.0, min(best_rows / max(rows, 1), best_cols / max(cols, 1))), 4) + + +def _formula_retention(blocks: Sequence[Mapping[str, Any]], fixture: Any) -> float | None: + expected = int(getattr(fixture, "expected_formula_count", 0) or 0) + if not expected: + return None + found = sum(1 for block in blocks if block.get("block_type") == "formula") + return round(min(1.0, found / expected), 4) + + +def score_document(fixture: Any, result: Mapping[str, Any]) -> dict[str, Any]: + blocks = list(result.get("blocks") or []) + content = [ + block + for block in blocks + if str(block.get("block_type") or "") in CONTENT_TYPES + and str(block.get("text") or "").strip() + ] + return { + "fixture_id": getattr(fixture, "fixture_id", ""), + "parser_id": result.get("parser_id"), + "parse_success": bool(content) and not bool(result.get("errors")), + "block_count": len(content), + "reading_order_accuracy": reading_order_accuracy( + content, getattr(fixture, "expected_anchors", ()) + ), + "heading_definition_integrity": heading_definition_integrity( + content, getattr(fixture, "expected_heading_definition_pairs", ()) + ), + "page_provenance_completeness": _rate( + [block.get("page_number") not in (None, "") for block in content] + ), + "block_provenance_completeness": _rate( + [ + bool(block.get("block_id")) + and isinstance(block.get("provenance"), dict) + and bool(block.get("provenance")) + for block in content + ] + ), + "bbox_provenance_completeness": _rate([_bbox_complete(block) for block in content]), + "table_retention": _table_retention(content, fixture), + "formula_retention": _formula_retention(content, fixture), + "header_footer_filter_rate": _noise_filter_rate( + blocks, getattr(fixture, "expected_noise", ()) + ), + "duplicate_block_count": duplicate_block_count(content), + "runtime_ms": float(result.get("parse_duration_ms") or 0), + "peak_rss_mb": float(result.get("peak_rss_mb") or 0), + "errors": list(result.get("errors") or []), + "warnings": list(result.get("warnings") or []), + } + + +def parser_result_to_ingestion_contract( + *, + parser_id: str, + parser_version: str, + fixture_id: str, + blocks: Sequence[Mapping[str, Any]], + language: str, +) -> tuple[Any, list[Any]]: + parse_uid = str(uuid.uuid5(uuid.NAMESPACE_URL, f"14b:{parser_id}:{fixture_id}")) + parse_record = SimpleNamespace( + parse_uid=parse_uid, + source_filename=f"{fixture_id}.pdf", + file_type="pdf", + parser_name=parser_id, + parser_version=parser_version, + quality_status="native_text_ok", + quality_flags=["layout_applied", f"parser_backend_{parser_id}"], + warnings=[], + ) + parse_blocks = [] + for index, block in enumerate(_ordered_blocks(blocks), start=1): + block_type = str(block.get("block_type") or "text") + if block_type in NOISE_TYPES or not str(block.get("text") or "").strip(): + continue + bbox = _bbox(block.get("bbox")) + locator = f"page:{block.get('page_number') or 'unknown'}" + if len(bbox) == 4: + locator += ";bbox:{x0},{y0},{x1},{y1}".format(**bbox) + parse_blocks.append( + SimpleNamespace( + block_uid=str(block.get("block_id") or f"{parser_id}-{fixture_id}-{index}"), + page_number=block.get("page_number"), + block_index=index, + block_type=("title" if block_type == "heading" else block_type), + text=str(block.get("text") or ""), + confidence=block.get("confidence"), + parser_type=parser_id, + source_locator=locator[:160], + quality_flags=["layout", f"layout_type_{block_type}"], + language=language, + ) + ) + return parse_record, parse_blocks + + +def build_existing_pipeline_chunks( + *, parse_record: Any, parse_blocks: list[Any], source_uid: str, language: str +) -> list[dict[str, Any]]: + if str(BACKEND) not in sys.path: + sys.path.insert(0, str(BACKEND)) + from services import knowledge_governance + + return knowledge_governance.build_knowledge_chunks_from_parse_blocks( + parse_record, + parse_blocks, + source_uid, + { + "language": language, + "course": "14B Synthetic Parser Benchmark", + "source_type": "reference" if language == "zh" else "course_material", + "trust_level": "reference_material", + "scope_type": "personal", + "knowledge_base_type": "synthetic_evaluation", + }, + ) + + +def rank_metrics(rankings: Mapping[str, Sequence[str]]) -> dict[str, Any]: + denominator = len(rankings) + ranks: list[int] = [] + no_result = 0 + for expected, observed in rankings.items(): + if not observed: + no_result += 1 + continue + try: + ranks.append(list(observed).index(expected) + 1) + except ValueError: + continue + return { + "denominator": denominator, + "hit_at_1": round(sum(rank == 1 for rank in ranks) / denominator, 4) if denominator else 0.0, + "hit_at_3": round(sum(rank <= 3 for rank in ranks) / denominator, 4) if denominator else 0.0, + "mrr": round(sum(1 / rank for rank in ranks) / denominator, 4) if denominator else 0.0, + "no_result_count": no_result, + "average_correct_rank": round(sum(ranks) / len(ranks), 4) if ranks else None, + } + + +def evaluate_downstream_retrieval( + *, + parser_id: str, + parser_version: str, + english_result: Mapping[str, Any], + chinese_result: Mapping[str, Any], + model_cache_dir: str | Path, +) -> dict[str, Any]: + """Run the existing multilingual retrieval over existing-pipeline chunks. + + Gold concept ids are consulted only after parser output has been converted + to KnowledgeChunk-shaped records. They never enter parser input, chunk + construction, embedding text, or ranking. + """ + + if str(BACKEND) not in sys.path: + sys.path.insert(0, str(BACKEND)) + from services.cross_language_retrieval import ( + CrossLanguageRetrievalQuery, + SemanticPassage, + rank_chinese_passages, + ) + from services.local_multilingual_embedding import LocalMultilingualEmbeddingBackend + + en_record, en_blocks = parser_result_to_ingestion_contract( + parser_id=parser_id, + parser_version=parser_version, + fixture_id="retrieval_english", + blocks=list(english_result.get("blocks") or []), + language="en", + ) + zh_record, zh_blocks = parser_result_to_ingestion_contract( + parser_id=parser_id, + parser_version=parser_version, + fixture_id="retrieval_chinese", + blocks=list(chinese_result.get("blocks") or []), + language="zh", + ) + english_chunks = build_existing_pipeline_chunks( + parse_record=en_record, + parse_blocks=en_blocks, + source_uid=f"14b-{parser_id}-en", + language="en", + ) + chinese_chunks = build_existing_pipeline_chunks( + parse_record=zh_record, + parse_blocks=zh_blocks, + source_uid=f"14b-{parser_id}-zh", + language="zh", + ) + passages = [ + SemanticPassage( + source_uid=chunk["source_uid"], + chunk_uid=chunk["chunk_uid"], + content=chunk["text"], + language="zh", + source_status="active", + quality_status=chunk.get("quality_status") or "native_text_ok", + content_hash=chunk["content_hash"], + page_number=chunk.get("page_number"), + block_uid=chunk.get("parse_block_uid") or "", + heading_path=chunk.get("source_section") or "", + ) + for chunk in chinese_chunks + ] + backend = LocalMultilingualEmbeddingBackend(model_cache_dir=model_cache_dir) + rankings: dict[str, list[str]] = {} + representation_cache: dict[str, list[float]] = {} + for concept in RETRIEVAL_CONCEPTS: + english_chunk = next( + ( + chunk + for chunk in english_chunks + if concept.english_term.casefold() in str(chunk.get("text") or "").casefold() + ), + None, + ) + if english_chunk is None: + rankings[concept.concept_id] = [] + continue + query = CrossLanguageRetrievalQuery( + english_candidate_uid=f"14b-{parser_id}-{concept.concept_id}", + canonical_english_term=concept.english_term, + normalized_english_term=concept.english_term.casefold(), + english_context=str(english_chunk.get("text") or ""), + discipline=concept.discipline, + allowed_chinese_source_uids=(f"14b-{parser_id}-zh",), + top_k=3, + retrieval_budget=100, + ) + results = rank_chinese_passages( + query, + passages, + backend, + representation_cache=representation_cache, + ) + observed: list[str] = [] + by_uid = {chunk["chunk_uid"]: chunk for chunk in chinese_chunks} + for result in results: + content = str(by_uid.get(result.chunk_uid, {}).get("text") or "") + matched = [ + item.concept_id + for item in RETRIEVAL_CONCEPTS + if item.chinese_term in content + ] + observed.append(matched[0] if len(matched) == 1 else f"ambiguous:{result.chunk_uid}") + rankings[concept.concept_id] = observed + metrics = rank_metrics(rankings) + return { + **metrics, + "parser_id": parser_id, + "parser_version": parser_version, + "english_chunk_count": len(english_chunks), + "chinese_chunk_count": len(chinese_chunks), + "rankings": rankings, + "model_id": backend.model_id, + "model_revision": backend.model_revision, + "backend_id": backend.backend_id, + "gold_used_in_parser": False, + "gold_used_in_retrieval": False, + "gold_used_post_ranking": True, + "external_api_used": False, + } + + +def _aggregate_value(rows: Sequence[Mapping[str, Any]], field_name: str) -> float | None: + values = [float(row[field_name]) for row in rows if row.get(field_name) is not None] + return round(statistics.fmean(values), 4) if values else None + + +def aggregate_parser( + parser_id: str, + document_scores: Sequence[Mapping[str, Any]], + retrieval: Mapping[str, Any], + *, + license_gate: str, +) -> dict[str, Any]: + rows = [row for row in document_scores if row.get("parser_id") == parser_id] + succeeded = sum(bool(row.get("parse_success")) for row in rows) + by_fixture = {str(row.get("fixture_id")): row for row in rows} + + def fixture_value(fixture_id: str, field_name: str) -> Any: + return by_fixture.get(fixture_id, {}).get(field_name) + + return { + "parser_id": parser_id, + "document_count": len(rows), + "parse_success_rate": round(succeeded / len(rows), 4) if rows else 0.0, + "reading_order_accuracy": _aggregate_value(rows, "reading_order_accuracy"), + "heading_definition_integrity": _aggregate_value(rows, "heading_definition_integrity"), + "page_provenance_completeness": _aggregate_value(rows, "page_provenance_completeness"), + "block_provenance_completeness": _aggregate_value(rows, "block_provenance_completeness"), + "bbox_provenance_completeness": _aggregate_value(rows, "bbox_provenance_completeness"), + "table_retention": _aggregate_value(rows, "table_retention"), + "formula_retention": _aggregate_value(rows, "formula_retention"), + "header_footer_filter_rate": _aggregate_value(rows, "header_footer_filter_rate"), + "duplicate_block_count": sum(int(row.get("duplicate_block_count") or 0) for row in rows), + "median_runtime_ms": round(statistics.median([float(row.get("runtime_ms") or 0) for row in rows]), 2) if rows else None, + "peak_rss_mb": max((float(row.get("peak_rss_mb") or 0) for row in rows), default=0.0), + "retrieval_hit_at_1": retrieval.get("hit_at_1"), + "retrieval_hit_at_3": retrieval.get("hit_at_3"), + "retrieval_mrr": retrieval.get("mrr"), + "retrieval_english_chunk_count": retrieval.get("english_chunk_count"), + "retrieval_chinese_chunk_count": retrieval.get("chinese_chunk_count"), + # Aggregate means are useful for comparison but may conceal a + # catastrophic document class. Keep every routing-critical fixture + # explicit so a high average cannot silently authorize production. + "critical_fixture_gates": { + "two_column_reading_order_accuracy": fixture_value( + "two_column_born_digital", "reading_order_accuracy" + ), + "scanned_english_parse_success": fixture_value( + "scanned_english", "parse_success" + ), + "scanned_chinese_parse_success": fixture_value( + "scanned_chinese", "parse_success" + ), + "mixed_layout_table_retention": fixture_value( + "mixed_layout_blocker", "table_retention" + ), + "simple_table_retention": fixture_value( + "simple_table", "table_retention" + ), + "formula_retention": fixture_value("raster_formula", "formula_retention"), + "repeated_header_footer_filter_rate": fixture_value( + "repeated_header_footer", "header_footer_filter_rate" + ), + }, + "license_gate": license_gate, + } + + +def select_candidate(aggregates: Sequence[Mapping[str, Any]]) -> dict[str, Any]: + by_id = {str(item.get("parser_id")): dict(item) for item in aggregates} + baseline = by_id.get("baseline_native_tesseract_formula_region", {}) + baseline_hit3 = float(baseline.get("retrieval_hit_at_3") or 0) + + def eligible(item: Mapping[str, Any]) -> bool: + return ( + item.get("parser_id") != "baseline_native_tesseract_formula_region" + and item.get("license_gate") == "pass" + and float(item.get("parse_success_rate") or 0) >= 0.95 + and float(item.get("reading_order_accuracy") or 0) >= 0.9 + and float(item.get("heading_definition_integrity") or 0) >= 0.9 + and float(item.get("bbox_provenance_completeness") or 0) >= 0.9 + and float(item.get("retrieval_hit_at_3") or 0) >= baseline_hit3 + ) + + candidates = [dict(item) for item in aggregates if eligible(item)] + candidates.sort( + key=lambda item: ( + float(item.get("retrieval_hit_at_3") or 0), + float(item.get("heading_definition_integrity") or 0), + float(item.get("reading_order_accuracy") or 0), + float(item.get("bbox_provenance_completeness") or 0), + -float(item.get("median_runtime_ms") or math.inf), + ), + reverse=True, + ) + selected = candidates[0] if candidates else None + gates = dict((selected or {}).get("critical_fixture_gates") or {}) + recommended_scope: list[str] = [] + excluded_scope: list[str] = [] + composition_requirements = [ + "per_document_quality_routing", + "current_native_parser_fallback", + ] + if selected: + if bool(gates.get("scanned_english_parse_success")) and bool( + gates.get("scanned_chinese_parse_success") + ): + recommended_scope.append("scanned_pdf") + if float(gates.get("simple_table_retention") or 0) >= 1.0: + recommended_scope.append("simple_table_pdf") + if float(gates.get("two_column_reading_order_accuracy") or 0) < 0.9: + excluded_scope.append("multi_column_pdf") + if float(gates.get("mixed_layout_table_retention") or 0) < 1.0: + excluded_scope.append("mixed_layout_table_pdf") + if float(gates.get("formula_retention") or 0) < 1.0: + excluded_scope.append("formula_pdf_without_existing_formula_region") + composition_requirements.append("existing_formula_region") + return { + "selected_parser_id": selected.get("parser_id") if selected else None, + "selected_role": ( + "conditional_complex_document_parser_candidate" + if selected + else "no_candidate_selected" + ), + "fallback_parser_id": "baseline_native_tesseract_formula_region", + "simple_document_policy": "retain_current_native_parser", + "recommended_scope": recommended_scope, + "excluded_scope": excluded_scope, + "composition_requirements": composition_requirements, + # Task 14B selects an integration candidate only. Production routing + # remains unchanged until the next, separately verified adapter task. + "production_adapter_authorized": False, + "mineru_eligible": eligible(by_id.get("mineru", {})), + "reason": ( + "Selected a conditional integration candidate; failed document-class gates remain excluded and production routing is unchanged." + if selected + else "No candidate passed all quality, provenance, retrieval and license gates." + ), + } + + +def sanitize_artifact(value: Any) -> Any: + if isinstance(value, dict): + return { + str(key): sanitize_artifact(child) + for key, child in value.items() + if str(key).casefold() not in BODY_KEYS + } + if isinstance(value, list): + return [sanitize_artifact(child) for child in value] + if isinstance(value, tuple): + return [sanitize_artifact(child) for child in value] + if isinstance(value, Path): + return "" + if isinstance(value, str): + return SECRET_RE.sub("[REDACTED]", LOCAL_PATH_RE.sub("", value)) + return value + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for part in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(part) + return digest.hexdigest() + + +def write_json(path: Path, payload: Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + json.dumps(sanitize_artifact(payload), ensure_ascii=False, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + + +def write_metric_csv(path: Path, rows: Sequence[Mapping[str, Any]]) -> None: + fields = ( + "parser_id", + "fixture_id", + "parse_success", + "reading_order_accuracy", + "heading_definition_integrity", + "page_provenance_completeness", + "block_provenance_completeness", + "bbox_provenance_completeness", + "table_retention", + "formula_retention", + "header_footer_filter_rate", + "duplicate_block_count", + "runtime_ms", + "peak_rss_mb", + ) + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w", encoding="utf-8", newline="") as handle: + writer = csv.DictWriter( + handle, + fieldnames=fields, + extrasaction="ignore", + lineterminator="\n", + ) + writer.writeheader() + writer.writerows(rows) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="Task 14B controlled parser benchmark utilities") + parser.add_argument("--validate-json", default="") + args = parser.parse_args(argv) + if args.validate_json: + payload = json.loads(Path(args.validate_json).read_text(encoding="utf-8")) + if payload.get("schema_version") != SCHEMA_VERSION: + raise SystemExit("invalid benchmark schema version") + print(json.dumps({"valid": True, "schema_version": SCHEMA_VERSION}, sort_keys=True)) + return 0 + parser.error("one command is required") + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/evaluations/run_controlled_parser_benchmark_14b.py b/scripts/evaluations/run_controlled_parser_benchmark_14b.py new file mode 100644 index 0000000..76b6c94 --- /dev/null +++ b/scripts/evaluations/run_controlled_parser_benchmark_14b.py @@ -0,0 +1,364 @@ +#!/usr/bin/env python3 +"""Run the Task 14B parser comparison in isolated local runtimes.""" + +from __future__ import annotations + +import argparse +import json +import os +import platform +import shutil +import subprocess +import sys +import time +from pathlib import Path +from typing import Any + + +# The retrieval backend uses Hugging Face tokenizers before the runner makes +# small local Git subprocess calls for audit metadata. Declare the safe fork +# behavior in the parent too, not only in parser probe children. +os.environ.setdefault("TOKENIZERS_PARALLELISM", "false") + + +ROOT = Path(__file__).resolve().parents[2] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from scripts.evaluations.open_source_parser_eval import controlled_benchmark_14b as benchmark # noqa: E402 + + +PROBE = ROOT / "scripts" / "evaluations" / "open_source_parser_eval" / "candidate_probe_14b.py" + + +def _run( + command: list[str], + *, + env: dict[str, str], + timeout: int, +) -> dict[str, Any]: + started = time.perf_counter() + try: + completed = subprocess.run( + command, + cwd=str(ROOT), + env=env, + check=False, + shell=False, + capture_output=True, + text=True, + timeout=timeout, + ) + return { + "ok": completed.returncode == 0, + "returncode": completed.returncode, + "duration_ms": round((time.perf_counter() - started) * 1000, 2), + "stdout": str(benchmark.sanitize_artifact(completed.stdout))[-1200:], + "stderr": str(benchmark.sanitize_artifact(completed.stderr))[-1200:], + } + except subprocess.TimeoutExpired: + return { + "ok": False, + "returncode": None, + "duration_ms": round((time.perf_counter() - started) * 1000, 2), + "stdout": "", + "stderr": "timeout", + } + + +def _probe_command( + parser_id: str, + fixture: benchmark.BenchmarkFixture, + output_path: Path, + args: argparse.Namespace, +) -> tuple[list[str], dict[str, str]]: + common = [ + str(PROBE), + "--parser", + parser_id, + "--input", + str(fixture.path), + "--fixture-id", + fixture.fixture_id, + "--output", + str(output_path), + "--runtime-root", + str(Path(args.runtime_root) / "probe-runtime" / parser_id), + ] + env = os.environ.copy() + env.update( + { + "HF_HUB_OFFLINE": "1", + "TRANSFORMERS_OFFLINE": "1", + "TOKENIZERS_PARALLELISM": "false", + "PYTHONHASHSEED": "0", + } + ) + if parser_id == "baseline_native_tesseract_formula_region": + return [args.project_python, *common], env + if parser_id == "docling": + env.update( + { + "HOME": str(Path(args.runtime_root) / "docling-home"), + "XDG_CACHE_HOME": str(Path(args.runtime_root) / "docling-cache"), + } + ) + return [ + args.conda, + "run", + "-n", + args.docling_env, + "python", + *common, + "--model-root", + args.docling_model_root, + ], env + env.update( + { + "HOME": args.mineru_home, + "XDG_CACHE_HOME": args.mineru_cache, + "HF_HOME": str(Path(args.mineru_cache) / "huggingface"), + "MINERU_MODEL_SOURCE": "local", + "MINERU_DEVICE_MODE": "cpu", + } + ) + return [args.conda, "run", "-n", args.mineru_env, "python", *common], env + + +def run_probe( + parser_id: str, + fixture: benchmark.BenchmarkFixture, + args: argparse.Namespace, +) -> dict[str, Any]: + output_path = Path(args.runtime_root) / "normalized" / parser_id / f"{fixture.fixture_id}.json" + output_path.parent.mkdir(parents=True, exist_ok=True) + command, env = _probe_command(parser_id, fixture, output_path, args) + process = _run(command, env=env, timeout=args.timeout_seconds) + if output_path.exists(): + try: + payload = json.loads(output_path.read_text(encoding="utf-8")) + except Exception as exc: # noqa: BLE001 + payload = { + "parser_id": parser_id, + "fixture_id": fixture.fixture_id, + "blocks": [], + "errors": [{"code": "PROBE_OUTPUT_INVALID", "message": str(exc)}], + } + else: + payload = { + "parser_id": parser_id, + "fixture_id": fixture.fixture_id, + "blocks": [], + "errors": [ + { + "code": "PROBE_PROCESS_FAILED", + "message": process.get("stderr") or process.get("stdout") or "missing output", + } + ], + } + payload.setdefault("parse_duration_ms", process["duration_ms"]) + payload.setdefault("peak_rss_mb", 0) + payload.setdefault("warnings", []) + payload.setdefault("network", {"external_request_count": 0, "external_hosts": []}) + if int(payload["network"].get("external_request_count") or 0) > 0: + payload.setdefault("errors", []).append( + {"code": "EXTERNAL_NETWORK_OBSERVED", "message": "Parser emitted an external network endpoint."} + ) + return payload + + +def _license_gate(parser_id: str) -> str: + if parser_id in {"baseline_native_tesseract_formula_region", "docling"}: + return "pass" + if parser_id == "mineru": + return "blocked_nonstandard_license" + return "unknown" + + +def _fixture_manifest(fixtures: list[benchmark.BenchmarkFixture]) -> list[dict[str, Any]]: + return [ + { + "fixture_id": fixture.fixture_id, + "filename": fixture.filename, + "privacy_classification": fixture.privacy_classification, + "language": fixture.language, + "purpose": fixture.purpose, + "page_count": _page_count(fixture.path), + "source_hash": benchmark.sha256_file(fixture.path), + } + for fixture in fixtures + ] + + +def _page_count(path: Path) -> int: + import fitz + + with fitz.open(path) as document: + return len(document) + + +def run_benchmark(args: argparse.Namespace) -> dict[str, Any]: + runtime_root = Path(args.runtime_root) + runtime_root.mkdir(parents=True, exist_ok=True) + fixtures = benchmark.build_controlled_fixture_set(runtime_root / "fixtures") + results: dict[tuple[str, str], dict[str, Any]] = {} + document_scores: list[dict[str, Any]] = [] + for parser_id in benchmark.PARSER_IDS: + for fixture in fixtures: + result = run_probe(parser_id, fixture, args) + results[(parser_id, fixture.fixture_id)] = result + document_scores.append(benchmark.score_document(fixture, result)) + print( + json.dumps( + { + "parser_id": parser_id, + "fixture_id": fixture.fixture_id, + "success": not bool(result.get("errors")), + "duration_ms": result.get("parse_duration_ms"), + }, + sort_keys=True, + ), + flush=True, + ) + + retrieval: dict[str, dict[str, Any]] = {} + for parser_id in benchmark.PARSER_IDS: + english = results[(parser_id, "retrieval_english")] + chinese = results[(parser_id, "retrieval_chinese")] + parser_version = str(english.get("parser_version") or chinese.get("parser_version") or "") + if english.get("errors") or chinese.get("errors"): + retrieval[parser_id] = { + **benchmark.rank_metrics({item.concept_id: [] for item in benchmark.RETRIEVAL_CONCEPTS}), + "parser_id": parser_id, + "parser_version": parser_version, + "rankings": {item.concept_id: [] for item in benchmark.RETRIEVAL_CONCEPTS}, + "error": "retrieval_parse_input_unavailable", + "external_api_used": False, + } + continue + retrieval[parser_id] = benchmark.evaluate_downstream_retrieval( + parser_id=parser_id, + parser_version=parser_version, + english_result=english, + chinese_result=chinese, + model_cache_dir=args.embedding_cache, + ) + + aggregates = [ + benchmark.aggregate_parser( + parser_id, + document_scores, + retrieval[parser_id], + license_gate=_license_gate(parser_id), + ) + for parser_id in benchmark.PARSER_IDS + ] + selection = benchmark.select_candidate(aggregates) + external_requests = sum( + int((result.get("network") or {}).get("external_request_count") or 0) + for result in results.values() + ) + summary = { + "task": "14B", + "schema_version": benchmark.SCHEMA_VERSION, + "created_at": benchmark.utc_now(), + "baseline_commit": _git("rev-parse", "HEAD"), + "branch": _git("branch", "--show-current"), + "platform": { + "system": platform.system(), + "machine": platform.machine(), + "python": sys.version.split()[0], + }, + "fixture_count": len(fixtures), + "fixtures": _fixture_manifest(fixtures), + "document_scores": document_scores, + "retrieval": retrieval, + "aggregates": aggregates, + "selection": selection, + "safety": { + "fixtures_private": False, + "external_api_used": False, + "external_parser_request_count": external_requests, + "real_provider_requests": 0, + "real_credentials_read": False, + "production_parser_changed": False, + "production_adapter_changed": False, + "incident_database_accessed": False, + }, + } + output = Path(args.json_output) + benchmark.write_json(output, summary) + benchmark.write_metric_csv(Path(args.csv_output), document_scores) + benchmark.write_json( + Path(args.selection_output), + { + "task": "14B", + "schema_version": "parser-selection-manifest-14b@1.0.0", + "baseline_commit": summary["baseline_commit"], + "parsers": [ + { + "parser_id": aggregate["parser_id"], + "parser_version": next( + ( + result.get("parser_version") + for (candidate, _), result in results.items() + if candidate == aggregate["parser_id"] and result.get("parser_version") + ), + "", + ), + "license_gate": aggregate["license_gate"], + } + for aggregate in aggregates + ], + "selection": selection, + "production_adapter_authorized": selection["production_adapter_authorized"], + }, + ) + return summary + + +def _git(*args: str) -> str: + completed = subprocess.run( + ["git", *args], cwd=str(ROOT), check=True, capture_output=True, text=True + ) + return completed.stdout.strip() + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description="Run Task 14B controlled parser benchmark") + parser.add_argument("--runtime-root", required=True) + parser.add_argument("--project-python", required=True) + parser.add_argument("--conda", default=shutil.which("conda") or "conda") + parser.add_argument("--docling-env", default="lexibridge-eval-docling") + parser.add_argument("--docling-model-root", required=True) + parser.add_argument("--mineru-env", default="lexibridge-eval-mineru") + parser.add_argument("--mineru-home", required=True) + parser.add_argument("--mineru-cache", required=True) + parser.add_argument("--embedding-cache", required=True) + parser.add_argument("--json-output", required=True) + parser.add_argument("--csv-output", required=True) + parser.add_argument("--selection-output", required=True) + parser.add_argument("--timeout-seconds", type=int, default=300) + return parser + + +def main(argv: list[str] | None = None) -> int: + args = build_parser().parse_args(argv) + summary = run_benchmark(args) + print( + json.dumps( + { + "schema_version": summary["schema_version"], + "fixture_count": summary["fixture_count"], + "selected_parser_id": summary["selection"]["selected_parser_id"], + "external_api_used": summary["safety"]["external_api_used"], + "real_provider_requests": summary["safety"]["real_provider_requests"], + }, + sort_keys=True, + ) + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_controlled_parser_benchmark_14b.py b/tests/test_controlled_parser_benchmark_14b.py new file mode 100644 index 0000000..868ded2 --- /dev/null +++ b/tests/test_controlled_parser_benchmark_14b.py @@ -0,0 +1,318 @@ +from __future__ import annotations + +from types import SimpleNamespace + +from scripts.evaluations.open_source_parser_eval import controlled_benchmark_14b as benchmark + + +def _block( + block_id: str, + text: str, + *, + block_type: str = "paragraph", + page_number: int | None = 1, + bbox: dict | None = None, + reading_order: int = 1, +): + return { + "block_id": block_id, + "block_type": block_type, + "text": text, + "page_number": page_number, + "bbox": bbox if bbox is not None else {"x0": 10, "y0": 10, "x1": 90, "y1": 30}, + "reading_order": reading_order, + "provenance": {"source_item_ref": block_id}, + } + + +def test_mineru_v2_normalizer_preserves_page_bbox_types_and_order(): + pages = [[ + { + "type": "title", + "content": {"title_content": [{"type": "text", "content": "Electric Charge"}], "level": 2}, + "bbox": [10, 20, 210, 50], + }, + { + "type": "paragraph", + "content": {"paragraph_content": [{"type": "text", "content": "A conserved property of matter."}]}, + "bbox": [10, 60, 310, 100], + }, + ]] + + blocks = benchmark.normalize_mineru_content_list_v2(pages, fixture_id="synthetic") + + assert [block["block_type"] for block in blocks] == ["heading", "paragraph"] + assert [block["page_number"] for block in blocks] == [1, 1] + assert [block["reading_order"] for block in blocks] == [1, 2] + assert blocks[0]["bbox"] == {"x0": 10.0, "y0": 20.0, "x1": 210.0, "y1": 50.0} + assert blocks[0]["provenance"]["source_parser"] == "mineru" + + +def test_controlled_fixture_set_is_synthetic_bilingual_and_covers_all_metrics(tmp_path): + import fitz + + fixtures = benchmark.build_controlled_fixture_set(tmp_path / "fixtures") + by_id = {fixture.fixture_id: fixture for fixture in fixtures} + + assert len(fixtures) == 11 + assert all(fixture.privacy_classification == "SYNTHETIC" for fixture in fixtures) + assert all(fixture.path.is_file() and fixture.path.suffix == ".pdf" for fixture in fixtures) + assert {fixture.language for fixture in fixtures} >= {"en", "zh"} + assert "two_column_born_digital" in by_id + assert "simple_table" in by_id + assert "raster_formula" in by_id + assert by_id["repeated_header_footer"].expected_noise + assert by_id["retrieval_english"].expected_heading_definition_pairs + assert by_id["retrieval_chinese"].expected_heading_definition_pairs + with fitz.open(by_id["retrieval_chinese"].path) as document: + chinese_text = "\n".join(page.get_text() for page in document).casefold() + assert not any( + concept.english_term.casefold() in chinese_text + for concept in benchmark.RETRIEVAL_CONCEPTS + ) + + +def test_metric_contract_covers_required_quality_dimensions(): + fixture = SimpleNamespace( + fixture_id="heading_definition", + expected_anchors=( + SimpleNamespace(text="Electric Charge", order=1), + SimpleNamespace(text="A conserved property", order=2), + ), + expected_heading_definition_pairs=(("Electric Charge", "A conserved property"),), + expected_noise=("Repeated course header", "Page 1"), + expected_table_rows=0, + expected_table_cols=0, + expected_formula_count=0, + ) + result = { + "parser_id": "candidate", + "errors": [], + "parse_duration_ms": 12.0, + "peak_rss_mb": 42.0, + "blocks": [ + _block("h1", "Electric Charge", block_type="heading", reading_order=1), + _block("p1", "A conserved property of matter.", reading_order=2), + ], + } + + metrics = benchmark.score_document(fixture, result) + + assert metrics["parse_success"] is True + assert metrics["reading_order_accuracy"] == 1.0 + assert metrics["heading_definition_integrity"] == 1.0 + assert metrics["page_provenance_completeness"] == 1.0 + assert metrics["block_provenance_completeness"] == 1.0 + assert metrics["bbox_provenance_completeness"] == 1.0 + assert metrics["header_footer_filter_rate"] == 1.0 + assert metrics["duplicate_block_count"] == 0 + assert metrics["runtime_ms"] == 12.0 + assert metrics["peak_rss_mb"] == 42.0 + + +def test_heading_definition_integrity_requires_same_or_adjacent_block(): + pair = ("Electric Charge", "A conserved property") + adjacent = [ + _block("h", "Electric Charge", block_type="heading", reading_order=1), + _block("p", "A conserved property of matter.", reading_order=2), + ] + split = adjacent + [ + _block("noise", "Unrelated chapter", block_type="heading", reading_order=2), + ] + split[1]["reading_order"] = 3 + + assert benchmark.heading_definition_integrity(adjacent, (pair,)) == 1.0 + assert benchmark.heading_definition_integrity(split, (pair,)) == 0.0 + + +def test_duplicate_metric_ignores_empty_and_counts_repeated_body_text(): + blocks = [ + _block("a", "Repeated definition"), + _block("b", " repeated definition ", reading_order=2), + _block("c", ""), + ] + + assert benchmark.duplicate_block_count(blocks) == 1 + + +def test_parser_blocks_reuse_existing_knowledge_chunk_builder(): + parse_record, parse_blocks = benchmark.parser_result_to_ingestion_contract( + parser_id="docling", + parser_version="2.117.0", + fixture_id="retrieval_zh", + blocks=[ + _block("h-charge", "电荷", block_type="heading", reading_order=1), + _block("p-charge", "物质的一种守恒属性。", reading_order=2), + ], + language="zh", + ) + + chunks = benchmark.build_existing_pipeline_chunks( + parse_record=parse_record, + parse_blocks=parse_blocks, + source_uid="source-zh-docling", + language="zh", + ) + + assert len(chunks) == 1 + assert chunks[0]["source_section"] == "电荷" + assert chunks[0]["page_number"] == 1 + assert chunks[0]["parse_block_uid"] == "h-charge" + assert "parser_backend_docling" in chunks[0]["quality_flags"] + + +def test_retrieval_metrics_are_rank_based_and_not_parser_claims(): + rankings = { + "charge": ["charge", "field", "potential"], + "field": ["potential", "charge", "field"], + "missing": [], + } + + metrics = benchmark.rank_metrics(rankings) + + assert metrics == { + "denominator": 3, + "hit_at_1": 0.3333, + "hit_at_3": 0.6667, + "mrr": 0.4444, + "no_result_count": 1, + "average_correct_rank": 2.0, + } + + +def test_selection_is_scoped_rejects_nonstandard_license_and_does_not_authorize_production(): + aggregates = [ + { + "parser_id": "baseline_native_tesseract_formula_region", + "parse_success_rate": 1.0, + "reading_order_accuracy": 0.8, + "heading_definition_integrity": 0.7, + "bbox_provenance_completeness": 0.2, + "retrieval_hit_at_3": 0.75, + "license_gate": "pass", + "critical_fixture_gates": { + "two_column_reading_order_accuracy": 1.0, + "scanned_english_parse_success": False, + "scanned_chinese_parse_success": False, + "mixed_layout_table_retention": 0.0, + "simple_table_retention": 0.0, + "formula_retention": 0.0, + "repeated_header_footer_filter_rate": 1.0, + }, + }, + { + "parser_id": "docling", + "parse_success_rate": 1.0, + "reading_order_accuracy": 1.0, + "heading_definition_integrity": 1.0, + "bbox_provenance_completeness": 1.0, + "retrieval_hit_at_3": 0.875, + "license_gate": "pass", + "critical_fixture_gates": { + "two_column_reading_order_accuracy": 0.82, + "scanned_english_parse_success": True, + "scanned_chinese_parse_success": True, + "mixed_layout_table_retention": 0.0, + "simple_table_retention": 1.0, + "formula_retention": 0.0, + "repeated_header_footer_filter_rate": 1.0, + }, + }, + { + "parser_id": "mineru", + "parse_success_rate": 1.0, + "reading_order_accuracy": 1.0, + "heading_definition_integrity": 1.0, + "bbox_provenance_completeness": 1.0, + "retrieval_hit_at_3": 1.0, + "license_gate": "blocked_nonstandard_license", + "critical_fixture_gates": { + "two_column_reading_order_accuracy": 1.0, + "scanned_english_parse_success": True, + "scanned_chinese_parse_success": True, + "mixed_layout_table_retention": 1.0, + "simple_table_retention": 1.0, + "formula_retention": 1.0, + "repeated_header_footer_filter_rate": 1.0, + }, + }, + ] + + selected = benchmark.select_candidate(aggregates) + + assert selected["selected_parser_id"] == "docling" + assert selected["selected_role"] == "conditional_complex_document_parser_candidate" + assert selected["fallback_parser_id"] == "baseline_native_tesseract_formula_region" + assert selected["mineru_eligible"] is False + assert selected["production_adapter_authorized"] is False + assert selected["recommended_scope"] == ["scanned_pdf", "simple_table_pdf"] + assert selected["excluded_scope"] == [ + "multi_column_pdf", + "mixed_layout_table_pdf", + "formula_pdf_without_existing_formula_region", + ] + assert "existing_formula_region" in selected["composition_requirements"] + + +def test_aggregate_preserves_critical_fixture_gates_instead_of_hiding_them_in_average(): + scores = [ + { + "parser_id": "docling", + "fixture_id": "two_column_born_digital", + "parse_success": True, + "reading_order_accuracy": 0.82, + "heading_definition_integrity": None, + "page_provenance_completeness": 1.0, + "block_provenance_completeness": 1.0, + "bbox_provenance_completeness": 1.0, + "table_retention": None, + "formula_retention": None, + "header_footer_filter_rate": None, + "duplicate_block_count": 0, + "runtime_ms": 100, + "peak_rss_mb": 500, + }, + { + "parser_id": "docling", + "fixture_id": "raster_formula", + "parse_success": True, + "reading_order_accuracy": None, + "heading_definition_integrity": None, + "page_provenance_completeness": 1.0, + "block_provenance_completeness": 1.0, + "bbox_provenance_completeness": 1.0, + "table_retention": None, + "formula_retention": 0.0, + "header_footer_filter_rate": None, + "duplicate_block_count": 0, + "runtime_ms": 100, + "peak_rss_mb": 500, + }, + ] + + aggregate = benchmark.aggregate_parser( + "docling", + scores, + {"hit_at_1": 0.5, "hit_at_3": 0.6, "mrr": 0.55, "chinese_chunk_count": 11}, + license_gate="pass", + ) + + assert aggregate["critical_fixture_gates"]["two_column_reading_order_accuracy"] == 0.82 + assert aggregate["critical_fixture_gates"]["formula_retention"] == 0.0 + assert aggregate["retrieval_chinese_chunk_count"] == 11 + + +def test_sanitized_artifact_rejects_paths_source_bodies_and_secrets(): + payload = { + "path": "file:///synthetic/private.pdf", + "authorization": "Bearer secret-token", + "request_body": "private source body", + "result": {"parser_id": "docling", "fixture_id": "synthetic"}, + } + + sanitized = benchmark.sanitize_artifact(payload) + + assert "file://" not in str(sanitized) + assert "secret-token" not in str(sanitized) + assert "request_body" not in sanitized + assert sanitized["result"]["parser_id"] == "docling"