From d29bf2c0ddd8710dc2ecc7f582eb30024b0a362e Mon Sep 17 00:00:00 2001 From: hyperpolymath <6759885+hyperpolymath@users.noreply.github.com> Date: Sun, 4 Oct 2026 01:33:46 +0000 Subject: [PATCH 1/2] feat(print): record machine-filled form provenance in the print path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A filled application form is a print job more often than it is anything else, and blocky-writer's fill_blocks emits ordinary PDF bytes with no marker saying who wrote the values (#118). Ruling D189 settled it: record machine-filled provenance as metadata in the print path so audit and routing can distinguish it. This does that, on both print paths. Determination (presswerk-document::pdf::form) Reads /AcroForm field values, /NeedAppearances, values with no /AP in their subtree, /DV rewrites, /Producer and /Creator, and an explicit /FillOrigin marker in the document /Info dictionary (or a fillOrigin attribute in an uncompressed XMP packet). Every result carries a confidence and the evidence for it; anything that stops inspection is recorded as "not inspected" rather than guessed at. Recording FormProvenance is stored on every PrintJob, persisted in two new job queue columns (migrated in place for existing databases) and written to the audit trail as form_provenance. Network-received IPP jobs are audited too: the server now takes an audit log at start-up, which it previously had no access to. Routing FormProvenancePolicy defaults to Record, which changes nothing for the user. HoldForReview additionally parks a positively machine-filled job in Held — locally and over IPP — and the jobs page gains a Release action that sends it on, keeping the job's identity and provenance. An undetermined document is never held. Also: a Print-Job response now reports the job's real job-state, so a held job is no longer reported to the client as pending. presswerk-print now depends on presswerk-document. This is a deliberate change to the documented crate graph: a job received over IPP never passes through the application layer, so the server must classify it itself. There is no cycle. Position, marker convention and known limitations are recorded in docs/ecosystem/FORM-PROVENANCE.adoc, including the ask of upstream: two /Fill* entries in /Info turn every downstream determination from Probable into Explicit. Closes #118 Co-authored-by: arena-agent <297053741+arena-agent@users.noreply.github.com> --- .machine_readable/6a2/ECOSYSTEM.a2ml | 21 + 0-AI-MANIFEST.a2ml | 2 +- CHANGELOG.adoc | 31 + README.adoc | 3 +- crates/presswerk-app/src/pages/audit.rs | 26 +- crates/presswerk-app/src/pages/jobs.rs | 32 +- crates/presswerk-app/src/pages/settings.rs | 26 +- .../src/services/app_services.rs | 147 ++- crates/presswerk-core/src/config.rs | 63 + crates/presswerk-core/src/error.rs | 12 + crates/presswerk-core/src/human_errors.rs | 17 + crates/presswerk-core/src/types.rs | 452 +++++++ crates/presswerk-document/Cargo.toml | 1 + crates/presswerk-document/src/lib.rs | 3 + crates/presswerk-document/src/pdf/form.rs | 1139 +++++++++++++++++ crates/presswerk-document/src/pdf/mod.rs | 4 +- crates/presswerk-document/src/pdf/reader.rs | 13 + crates/presswerk-print/Cargo.toml | 3 + crates/presswerk-print/src/ipp_server.rs | 390 +++++- crates/presswerk-print/src/queue.rs | 266 +++- docs/ecosystem/FORM-PROVENANCE.adoc | 271 ++++ 21 files changed, 2891 insertions(+), 31 deletions(-) create mode 100644 crates/presswerk-document/src/pdf/form.rs create mode 100644 docs/ecosystem/FORM-PROVENANCE.adoc diff --git a/.machine_readable/6a2/ECOSYSTEM.a2ml b/.machine_readable/6a2/ECOSYSTEM.a2ml index fdc9ae4..7093738 100644 --- a/.machine_readable/6a2/ECOSYSTEM.a2ml +++ b/.machine_readable/6a2/ECOSYSTEM.a2ml @@ -8,3 +8,24 @@ ecosystem = "hyperpolymath" [position] type = "component" +role = "terminal consumer of documents — the print path" +last-updated = "2026-10-04" + +[upstream.blocky-writer] +relationship = "one step upstream" +what-they-send = "ordinary PDF byte streams with AcroForm /V, /DV and /AS written and NeedAppearances set" +what-they-do-not-send = "any marker distinguishing a machine-filled form from a hand-completed one" +issue = "https://github.com/hyperpolymath/presswerk/issues/118" +ruling = "D189 (2026-09-30) — record machine-filled provenance as metadata in the print path so audit/routing can distinguish it" +our-response = "implemented: classify on every print path, persist on the job, write to the audit trail, optional hold-for-review routing" +our-ask = "write /FillOrigin (machine) and /FillTool (blocky-writer VERSION) into the document /Info dictionary in fill_blocks" +detail = "docs/ecosystem/FORM-PROVENANCE.adoc" + +[position.form-provenance] +implemented = true +detection = "crates/presswerk-document/src/pdf/form.rs" +domain-types = "crates/presswerk-core/src/types.rs (FormOrigin, ProvenanceConfidence, FormProvenance, FormProvenancePolicy)" +persistence = "jobs.form_origin + jobs.form_provenance, migrated in place" +audit-actions = "form_provenance, job_held_machine_filled, job_released" +default-policy = "Record" +authority = "heuristic unless an explicit marker is present; upstream remains the authoritative source" diff --git a/0-AI-MANIFEST.a2ml b/0-AI-MANIFEST.a2ml index 9969ad3..98846c9 100644 --- a/0-AI-MANIFEST.a2ml +++ b/0-AI-MANIFEST.a2ml @@ -33,7 +33,7 @@ All Rust source code lives in the `crates/` workspace directory. The six crates - `presswerk-core` — Shared types, errors, config (no internal deps) - `presswerk-security` — Encrypted storage, audit trail, TLS certs (depends: core) - `presswerk-document` — PDF ops, image processing, scanning, OCR (depends: core) -- `presswerk-print` — IPP client/server, mDNS discovery, job queue (depends: core, security) +- `presswerk-print` — IPP client/server, mDNS discovery, job queue (depends: core, security, document) - `presswerk-bridge` — iOS (objc2) + Android (JNI) native bridges (depends: core) - `presswerk-app` — Dioxus UI with 10 pages (depends: all above) diff --git a/CHANGELOG.adoc b/CHANGELOG.adoc index 8ddd743..0ec016c 100644 --- a/CHANGELOG.adoc +++ b/CHANGELOG.adoc @@ -6,6 +6,37 @@ The format is based on https://keepachangelog.com/en/1.1.0/[Keep a Changelog], and this project adheres to https://semver.org/spec/v2.0.0.html[Semantic Versioning]. +=== [Unreleased] + +==== Added + +* *Machine-filled form provenance* (https://github.com/hyperpolymath/presswerk/issues/118[#118], +ruling D189): every PDF entering the print path is classified as machine-filled or +hand-completed, and the determination — with its confidence and evidence — is stored on the +job and written to the audit trail +** New domain types `FormOrigin`, `ProvenanceConfidence`, `FormProvenance` and +`FormProvenancePolicy` in `presswerk-core` +** New detector `presswerk_document::pdf::form` — reads `/AcroForm` field values, +`/NeedAppearances`, missing `/AP` appearances, `/DV` rewrites, the `/Producer` and `/Creator` +strings, and an explicit `/FillOrigin` marker +** Both print paths classify: locally selected documents (`AppServices::print_document`) and +network-received IPP jobs (`handle_print_job`) +** `form_origin` and `form_provenance` columns on the job queue, migrated in place for +existing databases, queryable via `get_jobs_with_form_origin` +** Audit actions `form_provenance`, `job_held_machine_filled` and `job_released` +** Optional `HoldForReview` routing policy: a machine-filled form is held for the user rather +than sent, with a `Release` action on the jobs page +** Provenance badge on the jobs page, plain-language summary in the audit trail, policy +selector in Settings +** Position and marker convention recorded in `docs/ecosystem/FORM-PROVENANCE.adoc` + +==== Changed + +* `presswerk-print` now depends on `presswerk-document`: the network print path must classify +documents itself, since a job received over IPP never passes through the application layer +* A Print-Job response now reports the job's actual `job-state` — a job held for review is +reported as held (4) instead of pending (3) + === [0.3.0] - 2026-02-14 ==== Added diff --git a/README.adoc b/README.adoc index 568afcd..e3649ac 100644 --- a/README.adoc +++ b/README.adoc @@ -21,6 +21,7 @@ toc::[] * **Retry Engine** — Exponential backoff with jitter, circuit breaker, network self-healing * **Universal Connectivity** — USB, Bluetooth, Wi-Fi Direct, NFC, Serial, Parallel, FireWire, Thunderbolt, and more * **Document Processing** — PDF read/merge/split/rotate/crop, image resize/enhance, text-to-PDF, format auto-conversion +* **Form Provenance** — every PDF is classified as machine-filled or hand-completed, recorded on the job and in the audit trail, with optional hold-for-review routing * **Document Scanning** — Camera capture with edge detection, perspective correction, binarisation, OCR * **Print Server Mode** — Phone advertises itself as a network printer; other LAN devices can print to it * **Encrypted Storage** — age (X25519) encryption at rest for all local data @@ -65,7 +66,7 @@ presswerk-security (→ core; age, ring, rustls, rusqlite) ↑ presswerk-document (→ core; lopdf, printpdf, image, imageproc, ocrs) ↑ -presswerk-print (→ core, security; ipp, mdns-sd, tokio) +presswerk-print (→ core, security, document; ipp, mdns-sd, tokio) ↑ presswerk-bridge (→ core; objc2 [iOS], jni [Android]) ↑ diff --git a/crates/presswerk-app/src/pages/audit.rs b/crates/presswerk-app/src/pages/audit.rs index c94fdc8..ff5fcd6 100644 --- a/crates/presswerk-app/src/pages/audit.rs +++ b/crates/presswerk-app/src/pages/audit.rs @@ -2,9 +2,13 @@ // Copyright (c) 2026 Jonathan D.A. Jewell (hyperpolymath) // // Audit page — view the append-only audit trail backed by SQLite. +// +// Form-provenance entries (issue #118, ruling D189) carry a JSON determination +// in `details`; those are rendered as a one-line human summary. use dioxus::prelude::*; +use presswerk_core::types::FormProvenance; use presswerk_security::audit::AuditEntry; use crate::services::app_services::AppServices; @@ -61,6 +65,8 @@ pub fn Audit() -> Element { } else { entry.document_hash.clone() }; + let details_text = + summarise_details(&entry.action, entry.details.as_deref()); rsx! { div { style: "padding: 10px; margin: 4px 0; border: 1px solid #f0f0f0; border-radius: 6px; font-size: 14px;", @@ -74,9 +80,9 @@ pub fn Audit() -> Element { p { style: "color: #888; font-size: 12px; margin: 2px 0 0; font-family: monospace;", "{hash_short}" } - if let Some(ref details) = entry.details { + if !details_text.is_empty() { p { style: "color: #666; font-size: 12px; margin: 2px 0 0;", - "{details}" + "{details_text}" } } } @@ -88,3 +94,19 @@ pub fn Audit() -> Element { } } } + +/// Render an audit entry's `details` for a human. +/// +/// Form-provenance entries store a JSON determination; everything else is +/// already human-readable text. +fn summarise_details(action: &str, details: Option<&str>) -> String { + let Some(details) = details else { + return String::new(); + }; + if action == "form_provenance" + && let Ok(provenance) = serde_json::from_str::(details) + { + return provenance.summary(); + } + details.to_string() +} diff --git a/crates/presswerk-app/src/pages/jobs.rs b/crates/presswerk-app/src/pages/jobs.rs index 5672de4..f91a700 100644 --- a/crates/presswerk-app/src/pages/jobs.rs +++ b/crates/presswerk-app/src/pages/jobs.rs @@ -1,7 +1,9 @@ // SPDX-License-Identifier: MPL-2.0 // Copyright (c) 2026 Jonathan D.A. Jewell (hyperpolymath) // -// Jobs page — view all print jobs, their status, cancel/retry, and delete. +// Jobs page — view all print jobs, their status, cancel/retry, release, and +// delete. Jobs whose document was classified as a machine-filled form carry a +// provenance badge (issue #118, ruling D189). use dioxus::prelude::*; @@ -58,7 +60,10 @@ pub fn Jobs() -> Element { let job_status = job.status; let ts = job.created_at.format("%Y-%m-%d %H:%M").to_string(); let can_cancel = matches!(job_status, JobStatus::Pending | JobStatus::Held); + let can_release = matches!(job_status, JobStatus::Held); let is_terminal = matches!(job_status, JobStatus::Completed | JobStatus::Failed | JobStatus::Cancelled); + let provenance_note = job.form_provenance.summary(); + let machine_filled = job.form_provenance.is_machine_filled(); rsx! { div { style: "padding: 12px; margin: 8px 0; border: 1px solid #e0e0e0; border-radius: 8px;", @@ -69,6 +74,11 @@ pub fn Jobs() -> Element { } } p { style: "color: #666; font-size: 14px; margin: 4px 0;", "{ts}" } + if machine_filled { + p { style: "color: #8a5300; font-size: 12px; margin: 4px 0; background: #fff4e5; border: 1px solid #ffd8a8; border-radius: 4px; padding: 2px 6px;", + "Machine-filled: {provenance_note}" + } + } if let Some(ref uri) = job.printer_uri { p { style: "color: #999; font-size: 12px;", "{uri}" } } @@ -94,6 +104,26 @@ pub fn Jobs() -> Element { "Cancel" } } + if can_release { + button { + style: "padding: 4px 12px; border-radius: 4px; border: 1px solid #007aff; color: #007aff; background: white; font-size: 12px;", + onclick: { + let svc = svc.clone(); + move |_| { + let svc = svc.clone(); + spawn(async move { + if let Err(e) = svc.release_job(&job_id).await { + tracing::error!(error = %e, "release failed"); + } + if let Ok(jobs) = svc.all_jobs() { + state.write().jobs = jobs; + } + }); + } + }, + "Release" + } + } if is_terminal { button { style: "padding: 4px 12px; border-radius: 4px; border: 1px solid #ccc; color: #666; background: white; font-size: 12px;", diff --git a/crates/presswerk-app/src/pages/settings.rs b/crates/presswerk-app/src/pages/settings.rs index 9446f0c..62a9585 100644 --- a/crates/presswerk-app/src/pages/settings.rs +++ b/crates/presswerk-app/src/pages/settings.rs @@ -5,7 +5,7 @@ use dioxus::prelude::*; -use presswerk_core::types::PaperSize; +use presswerk_core::types::{FormProvenancePolicy, PaperSize}; use crate::services::app_services::AppServices; use crate::state::AppState; @@ -78,6 +78,30 @@ pub fn Settings() -> Element { } } + section { style: "margin: 16px 0;", + h3 { "Forms" } + p { style: "color: #666; font-size: 13px; margin: 0 0 8px;", + "Presswerk checks whether a PDF form was filled in by a program rather than by a person, and records that in the audit trail." + } + div { style: "display: flex; justify-content: space-between; align-items: center; padding: 12px 0; border-bottom: 1px solid #f0f0f0;", + span { "Machine-filled forms" } + select { + style: "padding: 4px 8px; border: 1px solid #ccc; border-radius: 4px;", + value: state.read().config.form_provenance_policy.as_token(), + onchange: move |evt| { + let policy = FormProvenancePolicy::from_token(&evt.value()); + state.write().config.form_provenance_policy = policy; + }, + option { value: "Off", "Do not check" } + option { value: "Record", "Record in audit trail" } + option { value: "HoldForReview", "Record and hold for review" } + } + } + p { style: "color: #999; font-size: 12px; margin: 8px 0 0;", + "For the print server, the choice applies the next time it starts." + } + } + section { style: "margin: 16px 0;", h3 { "Security" } SettingRow { diff --git a/crates/presswerk-app/src/services/app_services.rs b/crates/presswerk-app/src/services/app_services.rs index 6d6ca35..af12c9e 100644 --- a/crates/presswerk-app/src/services/app_services.rs +++ b/crates/presswerk-app/src/services/app_services.rs @@ -15,9 +15,10 @@ use std::sync::{Arc, Mutex}; use presswerk_core::AppConfig; use presswerk_core::error::{PresswerkError, Result}; use presswerk_core::types::{ - DiscoveredPrinter, DocumentType, JobId, JobSource, JobStatus, PrintJob, PrintSettings, - ServerStatus, + DiscoveredPrinter, DocumentType, FormOrigin, JobId, JobSource, JobStatus, PrintJob, + PrintSettings, ServerStatus, }; +use presswerk_document::classify_for_print; use presswerk_print::discovery::PrinterDiscovery; use presswerk_print::ipp_client::IppClient; use presswerk_print::ipp_server::IppServer; @@ -181,8 +182,14 @@ impl AppServices { /// so other devices on the LAN can discover and print to this device. pub async fn start_ipp_server(&self) -> Result { let job_queue = Arc::clone(&self.job_queue); + let audit_log = Arc::clone(&self.audit_log); + let policy = self.config().form_provenance_policy; + let mut server = self.ipp_server.lock().await; - server.start(job_queue).await?; + // The policy is copied into the connection state at start-up, so a + // change in Settings takes effect the next time the server starts. + server.set_form_provenance_policy(policy); + server.start_with_audit(job_queue, Some(audit_log)).await?; self.audit( "server_start", "system", @@ -212,8 +219,10 @@ impl AppServices { /// Send a document to a printer via IPP. /// - /// Creates a print job in the queue, sends it via IPP, updates the job - /// status, and records the operation in the audit log. + /// Classifies the document's form provenance (issue #118, ruling D189), + /// creates a print job in the queue carrying that determination, records + /// the operation in the audit log, and — unless the job is held for review + /// — sends it via IPP. pub async fn print_document( &self, document_bytes: Vec, @@ -225,6 +234,12 @@ impl AppServices { let doc_hash = hash_bytes(&document_bytes); let total_bytes = document_bytes.len() as u64; + // Classify before enqueueing, so the determination is part of the + // stored job rather than an afterthought. + let policy = self.config().form_provenance_policy; + let provenance = classify_for_print(&document_bytes, document_type, policy); + let hold = provenance.should_hold(policy); + // Create the job record let mut job = PrintJob::new( JobSource::Local, @@ -235,6 +250,17 @@ impl AppServices { job.printer_uri = Some(printer_uri.clone()); job.settings = settings.clone(); job.total_bytes = total_bytes; + job.form_provenance = provenance.clone(); + if hold { + job.status = JobStatus::Held; + } + + // A held job has to survive until the user releases it, so its bytes + // are persisted. Documents that print immediately are not written to + // disk — they are handed to the printer and dropped, as before. + if hold { + self.store_document(&document_bytes)?; + } // Insert into persistent queue { @@ -244,15 +270,105 @@ impl AppServices { let job_id = job.id; - // Record audit entry + // Record audit entries self.audit("print_submitted", &doc_hash, true, Some(&document_name)); + if provenance.is_form() { + self.audit( + "form_provenance", + &doc_hash, + true, + Some(&provenance.audit_json()), + ); + } + + // Routing: under the hold policy a machine-filled form waits for the + // user instead of going straight to the printer. + if hold { + warn!( + job_id = %job_id, + summary = %provenance.summary(), + "print job held for review: machine-filled form" + ); + self.audit( + "job_held_machine_filled", + &doc_hash, + true, + Some("machine-filled form held for review per form provenance policy"), + ); + return Ok(job_id); + } + + self.spawn_send( + job_id, + document_bytes, + document_type, + document_name, + printer_uri, + settings, + doc_hash, + ); + + Ok(job_id) + } + + /// Release a job that was held for review and send it to its printer. + /// + /// This is the user's answer to a machine-filled form that the routing + /// policy parked (issue #118). The released job keeps its identity and its + /// recorded provenance; only its status changes. + pub async fn release_job(&self, job_id: &JobId) -> Result<()> { + let job = { + let queue = acquire_lock(&self.job_queue); + queue.get_job(job_id)? + } + .ok_or_else(|| PresswerkError::InvalidState(format!("job {job_id} no longer exists")))?; + + if job.status != JobStatus::Held { + return Err(PresswerkError::InvalidState(format!( + "job {job_id} is not waiting for review" + ))); + } - // Send to printer asynchronously + let printer_uri = job.printer_uri.clone().ok_or_else(|| { + PresswerkError::InvalidState(format!("job {job_id} has no printer selected")) + })?; + + let document_bytes = self.load_document(&job.document_hash)?; + + { + let queue = acquire_lock(&self.job_queue); + queue.update_status(job_id, JobStatus::Pending, None)?; + } + + self.audit("job_released", &job.document_hash, true, Some(&job.document_name)); + + self.spawn_send( + *job_id, + document_bytes, + job.document_type, + job.document_name, + printer_uri, + job.settings, + job.document_hash, + ); + + Ok(()) + } + + /// Hand a document to the printer on a background task and record the + /// outcome in the queue and the audit trail. + #[allow(clippy::too_many_arguments)] + fn spawn_send( + &self, + job_id: JobId, + doc_bytes: Vec, + document_type: DocumentType, + name: String, + uri: String, + settings: PrintSettings, + hash: String, + ) { let services = self.clone(); - let doc_bytes = document_bytes; - let uri = printer_uri; - let name = document_name; - let hash = doc_hash; tokio::spawn(async move { // Update status to Processing @@ -293,8 +409,6 @@ impl AppServices { } } }); - - Ok(job_id) } // -- Job Queue ----------------------------------------------------------- @@ -311,6 +425,13 @@ impl AppServices { queue.get_pending_jobs() } + /// Get every job whose document was classified as a machine-filled form + /// (issue #118, ruling D189). + pub fn machine_filled_jobs(&self) -> Result> { + let queue = acquire_lock(&self.job_queue); + queue.get_jobs_with_form_origin(FormOrigin::Machine) + } + /// Cancel a job. pub fn cancel_job(&self, job_id: &JobId) -> Result<()> { let queue = acquire_lock(&self.job_queue); diff --git a/crates/presswerk-core/src/config.rs b/crates/presswerk-core/src/config.rs index 546ea2a..0fb2c15 100644 --- a/crates/presswerk-core/src/config.rs +++ b/crates/presswerk-core/src/config.rs @@ -28,6 +28,17 @@ pub struct AppConfig { pub query_timeout_secs: u64, /// Whether Easy Mode is the default interface. pub easy_mode: bool, + /// How the print path treats a machine-filled form (issue #118). + /// + /// Defaults to `Record`: every PDF that arrives for printing is inspected, + /// the determination is stored on the job and written to the audit trail, + /// and nothing is held. `HoldForReview` additionally parks machine-filled + /// jobs in `Held` until the user releases them. + /// + /// `#[serde(default)]` keeps configuration files written before this + /// setting existed loadable rather than silently resetting every setting. + #[serde(default)] + pub form_provenance_policy: crate::FormProvenancePolicy, } impl Default for AppConfig { @@ -43,6 +54,7 @@ impl Default for AppConfig { print_timeout_secs: 60, query_timeout_secs: 15, easy_mode: true, + form_provenance_policy: crate::FormProvenancePolicy::Record, } } } @@ -110,4 +122,55 @@ mod tests { // Print timeout should generally be >= query timeout assert!(config.print_timeout_secs >= config.query_timeout_secs); } + + #[test] + fn test_form_provenance_policy_defaults_to_record() { + let config = AppConfig::default(); + assert_eq!( + config.form_provenance_policy, + crate::FormProvenancePolicy::Record + ); + // Recording is on by default: the ruling is to record provenance. + assert!(config.form_provenance_policy.inspects()); + } + + #[test] + fn test_config_without_form_provenance_still_loads() { + // A config file written by an older Presswerk has no such key. + // It must still deserialise rather than resetting every setting. + let legacy = r#"{ + "default_paper_size": "A4", + "auto_start_server": true, + "server_port": 9631, + "server_require_tls": false, + "auto_accept_network_jobs": false, + "audit_enabled": true, + "encryption_enabled": true, + "print_timeout_secs": 60, + "query_timeout_secs": 15, + "easy_mode": true + }"#; + + let config: AppConfig = serde_json::from_str(legacy).expect("legacy config must load"); + assert_eq!(config.server_port, 9631); + assert!(config.auto_start_server); + assert_eq!( + config.form_provenance_policy, + crate::FormProvenancePolicy::Record + ); + } + + #[test] + fn test_form_provenance_policy_survives_roundtrip() { + let config = AppConfig { + form_provenance_policy: crate::FormProvenancePolicy::HoldForReview, + ..Default::default() + }; + let json = serde_json::to_string(&config).expect("serialize"); + let restored: AppConfig = serde_json::from_str(&json).expect("deserialize"); + assert_eq!( + restored.form_provenance_policy, + crate::FormProvenancePolicy::HoldForReview + ); + } } diff --git a/crates/presswerk-core/src/error.rs b/crates/presswerk-core/src/error.rs index ad24d70..0e4676f 100644 --- a/crates/presswerk-core/src/error.rs +++ b/crates/presswerk-core/src/error.rs @@ -63,6 +63,10 @@ pub enum PresswerkError { #[error("feature not available on this platform")] PlatformUnavailable, + + // -- State -- + #[error("operation not valid right now: {0}")] + InvalidState(String), } /// Alias used throughout the codebase. @@ -152,6 +156,14 @@ mod tests { assert_eq!(msg, "feature not available on this platform"); } + #[test] + fn test_invalid_state_error() { + let err = PresswerkError::InvalidState("job is not held".to_string()); + let msg = err.to_string(); + assert!(msg.contains("not valid right now")); + assert!(msg.contains("job is not held")); + } + #[test] fn test_result_alias() { let result: Result = Err(PresswerkError::NoPrinterSelected); diff --git a/crates/presswerk-core/src/human_errors.rs b/crates/presswerk-core/src/human_errors.rs index 0eb4d85..efeb852 100644 --- a/crates/presswerk-core/src/human_errors.rs +++ b/crates/presswerk-core/src/human_errors.rs @@ -177,6 +177,14 @@ pub fn humanize_error(err: &PresswerkError) -> HumanError { retriable: false, severity: Severity::Permanent, }, + + // -- State -- + PresswerkError::InvalidState(detail) => HumanError { + message: "That job can't do this at the moment.".into(), + suggestion: format!("Refresh the job list and try again. ({detail})"), + retriable: true, + severity: Severity::ActionRequired, + }, } } @@ -322,4 +330,13 @@ mod tests { let human = humanize_error(&err); assert_eq!(human.severity, Severity::Permanent); } + + #[test] + fn invalid_state_asks_the_user_to_refresh() { + let err = PresswerkError::InvalidState("job 1234 is not held".into()); + let human = humanize_error(&err); + assert_eq!(human.severity, Severity::ActionRequired); + assert!(human.retriable); + assert!(human.suggestion.contains("job 1234 is not held")); + } } diff --git a/crates/presswerk-core/src/types.rs b/crates/presswerk-core/src/types.rs index 3d86af5..778ff6b 100644 --- a/crates/presswerk-core/src/types.rs +++ b/crates/presswerk-core/src/types.rs @@ -230,6 +230,282 @@ pub struct PageRange { pub end: u32, } +// --------------------------------------------------------------------------- +// Form provenance — machine-filled vs hand-filled (issue #118, ruling D189) +// --------------------------------------------------------------------------- + +/// How the values in an interactive (AcroForm) PDF came to be there. +/// +/// Upstream form-fillers — for example +/// [`blocky-writer`](https://github.com/hyperpolymath/blocky-writer), whose +/// `fill_blocks` writes `/V`, `/DV` and `/AS` entries and sets +/// `NeedAppearances` — emit an ordinary PDF byte stream. Nothing in those bytes +/// says *who* wrote the values. Presswerk is usually the next thing that +/// touches them: it prints them. So the print path classifies the document at +/// the boundary and records the result for audit and routing. +/// +/// The determination is heuristic unless an explicit marker is present; see +/// `docs/ecosystem/FORM-PROVENANCE.adoc`. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] +pub enum FormOrigin { + /// The document carries no interactive form at all. + NotAForm, + /// The document is a form, but no field carries a value. + Empty, + /// Values are present and nothing suggests software wrote them. + Human, + /// Values are present and there is positive evidence software wrote them. + Machine, + /// Not determined: the document was not inspected, or inspection failed. + #[default] + Unknown, +} + +impl FormOrigin { + /// Stable single-word token used in the database, logs and audit trail. + /// + /// Never contains whitespace or punctuation, so it is safe to embed in SQL + /// comparisons and log lines. + pub fn as_token(&self) -> &'static str { + match self { + Self::NotAForm => "NotAForm", + Self::Empty => "Empty", + Self::Human => "Human", + Self::Machine => "Machine", + Self::Unknown => "Unknown", + } + } + + /// Inverse of [`Self::as_token`]. Unrecognised tokens map to `Unknown` — + /// an unparseable provenance must never be mistaken for a determination. + pub fn from_token(token: &str) -> Self { + match token { + "NotAForm" => Self::NotAForm, + "Empty" => Self::Empty, + "Human" => Self::Human, + "Machine" => Self::Machine, + _ => Self::Unknown, + } + } + + /// Whether this is a positive determination that software filled the form. + pub fn is_machine(&self) -> bool { + matches!(self, Self::Machine) + } +} + +impl std::fmt::Display for FormOrigin { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.as_token()) + } +} + +/// How much weight a [`FormProvenance`] determination deserves. +/// +/// Ordered weakest to strongest so callers can compare with `<`/`>=`. +#[derive( + Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize, Default, +)] +pub enum ProvenanceConfidence { + /// No determination was made. + #[default] + None, + /// A weak signal; treat as a hint only. + Speculative, + /// A characteristic combination of signals, but spoofable. + Probable, + /// A named tool known to fill forms programmatically. + Strong, + /// An explicit machine-readable provenance marker was present. + Explicit, +} + +impl ProvenanceConfidence { + /// Stable token for logs and the audit trail. + pub fn as_token(&self) -> &'static str { + match self { + Self::None => "None", + Self::Speculative => "Speculative", + Self::Probable => "Probable", + Self::Strong => "Strong", + Self::Explicit => "Explicit", + } + } +} + +impl std::fmt::Display for ProvenanceConfidence { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.as_token()) + } +} + +/// What the print path concluded about a document's form values. +/// +/// Attached to every [`PrintJob`] so that the queue, the audit trail and the +/// routing policy can all see the same determination. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct FormProvenance { + /// The determination itself. + pub origin: FormOrigin, + /// How much weight the determination deserves. + pub confidence: ProvenanceConfidence, + /// Whether the document was actually inspected (`false` on parse failure, + /// non-PDF input, or when the document exceeded the inspection budget). + pub inspected: bool, + /// Whether the document carries an interactive form (AcroForm) at all. + pub is_form: bool, + /// Number of form fields found. + pub field_count: u32, + /// Number of fields carrying a value. + pub filled_field_count: u32, + /// Fields with a value but no `/AP` appearance stream — the classic + /// signature of a writer that set values without generating appearances. + pub values_without_appearance: u32, + /// Fields whose value was written alongside a `/DV` default. Interactive + /// viewers write `/V` only; writing `/DV` too is a programmatic signature. + pub filled_with_default: u32, + /// Whether the AcroForm sets `/NeedAppearances` true. + pub need_appearances: bool, + /// `/Producer` from the document information dictionary, if any. + pub producer: Option, + /// `/Creator` from the document information dictionary, if any. + pub creator: Option, + /// An explicit upstream provenance marker, if one was present. + pub marker: Option, + /// Human-readable reasons for the determination, in order of weight. + pub evidence: Vec, +} + +impl Default for FormProvenance { + fn default() -> Self { + Self::not_inspected("document was not inspected") + } +} + +impl FormProvenance { + /// Provenance for a document that was never inspected, with the reason. + pub fn not_inspected(reason: &str) -> Self { + Self { + origin: FormOrigin::Unknown, + confidence: ProvenanceConfidence::None, + inspected: false, + is_form: false, + field_count: 0, + filled_field_count: 0, + values_without_appearance: 0, + filled_with_default: 0, + need_appearances: false, + producer: None, + creator: None, + marker: None, + evidence: vec![reason.to_string()], + } + } + + /// Whether the document carries an interactive form. + pub fn is_form(&self) -> bool { + self.is_form + } + + /// Whether the form was determined to have been filled by software. + pub fn is_machine_filled(&self) -> bool { + self.origin.is_machine() + } + + /// The tool responsible for the values, if one could be identified. + /// + /// Prefers the explicit marker over the `/Producer` string, since the + /// latter can be set by anything. + pub fn tool(&self) -> Option<&str> { + self.marker.as_deref().or(self.producer.as_deref()) + } + + /// Whether a job carrying this provenance should be held for user review + /// under `policy`. + /// + /// Only a *positive* machine determination holds a job: an uninspected or + /// undetermined document must not stall somebody's printout. + pub fn should_hold(&self, policy: FormProvenancePolicy) -> bool { + matches!(policy, FormProvenancePolicy::HoldForReview) && self.is_machine_filled() + } + + /// One-line human summary for the UI, logs and the audit trail. + pub fn summary(&self) -> String { + let head = match self.origin { + FormOrigin::NotAForm => "not an interactive form".to_string(), + FormOrigin::Empty => "interactive form; no values entered".to_string(), + FormOrigin::Human => "form values present; no machine-fill evidence".to_string(), + FormOrigin::Machine => match self.tool() { + Some(tool) => format!("machine-filled form ({tool})"), + None => "machine-filled form".to_string(), + }, + FormOrigin::Unknown => "form provenance not determined".to_string(), + }; + + if self.is_form { + format!( + "{head}; {}/{} fields filled", + self.filled_field_count, self.field_count + ) + } else { + head + } + } + + /// Compact JSON representation for the audit trail's `details` column. + pub fn audit_json(&self) -> String { + serde_json::to_string(self).unwrap_or_else(|_| "{}".to_string()) + } +} + +/// What the print path does with a machine-filled form (issue #118). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] +pub enum FormProvenancePolicy { + /// Do not inspect documents at all. + Off, + /// Inspect, record on the job, and write to the audit trail. Never hold. + #[default] + Record, + /// As [`Self::Record`], plus machine-filled jobs are held for review. + HoldForReview, +} + +impl FormProvenancePolicy { + /// Stable token for config files and logs. + pub fn as_token(&self) -> &'static str { + match self { + Self::Off => "Off", + Self::Record => "Record", + Self::HoldForReview => "HoldForReview", + } + } + + /// Inverse of [`Self::as_token`]; unrecognised tokens fall back to the + /// default policy. + pub fn from_token(token: &str) -> Self { + match token { + "Off" => Self::Off, + "Record" => Self::Record, + "HoldForReview" => Self::HoldForReview, + _ => Self::default(), + } + } + + /// Short label for the settings UI. + pub fn label(&self) -> &'static str { + match self { + Self::Off => "Off", + Self::Record => "Record in audit trail", + Self::HoldForReview => "Record and hold for review", + } + } + + /// Whether documents should be inspected at all under this policy. + pub fn inspects(&self) -> bool { + !matches!(self, Self::Off) + } +} + /// Classification of errors for retry logic. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] pub enum ErrorClass { @@ -268,6 +544,12 @@ pub struct PrintJob { pub bytes_sent: u64, /// Total document size in bytes. pub total_bytes: u64, + /// What the print path determined about the document's form values. + /// + /// Recorded for every job so audit and routing can tell a machine-filled + /// application form from one completed by hand (issue #118, ruling D189). + /// Defaults to "not inspected" for documents that are not PDFs. + pub form_provenance: FormProvenance, } impl PrintJob { @@ -296,6 +578,7 @@ impl PrintJob { error_history: Vec::new(), bytes_sent: 0, total_bytes: 0, + form_provenance: FormProvenance::default(), } } } @@ -525,4 +808,173 @@ mod tests { job.status = JobStatus::Completed; assert_eq!(job.status, JobStatus::Completed); } + + // -- Form provenance (issue #118) --------------------------------------- + + /// Build a machine-filled determination the way the document crate does. + fn machine_filled() -> FormProvenance { + FormProvenance { + origin: FormOrigin::Machine, + confidence: ProvenanceConfidence::Probable, + inspected: true, + is_form: true, + field_count: 14, + filled_field_count: 12, + values_without_appearance: 12, + filled_with_default: 12, + need_appearances: true, + producer: Some("blocky-writer 0.4.2".to_string()), + creator: None, + marker: None, + evidence: vec!["12 of 14 fields carry values and /NeedAppearances is set".to_string()], + } + } + + #[test] + fn test_form_origin_tokens_roundtrip() { + for origin in [ + FormOrigin::NotAForm, + FormOrigin::Empty, + FormOrigin::Human, + FormOrigin::Machine, + FormOrigin::Unknown, + ] { + let token = origin.as_token(); + assert_eq!(FormOrigin::from_token(token), origin); + assert_eq!(token.to_string(), origin.to_string()); + assert!(!token.contains(' ')); + } + } + + #[test] + fn test_form_origin_unknown_token_is_unknown() { + assert_eq!(FormOrigin::from_token(""), FormOrigin::Unknown); + assert_eq!(FormOrigin::from_token("machine"), FormOrigin::Unknown); + assert_eq!(FormOrigin::from_token("Machine"), FormOrigin::Machine); + } + + #[test] + fn test_form_origin_is_machine() { + assert!(FormOrigin::Machine.is_machine()); + assert!(!FormOrigin::Human.is_machine()); + assert!(!FormOrigin::Unknown.is_machine()); + assert!(!FormOrigin::Empty.is_machine()); + assert!(!FormOrigin::NotAForm.is_machine()); + } + + #[test] + fn test_form_provenance_default_is_not_inspected() { + let provenance = FormProvenance::default(); + assert!(!provenance.inspected); + assert!(!provenance.is_form()); + assert_eq!(provenance.origin, FormOrigin::Unknown); + assert_eq!(provenance.confidence, ProvenanceConfidence::None); + assert!(!provenance.is_machine_filled()); + assert_eq!(provenance.evidence.len(), 1); + } + + #[test] + fn test_print_job_defaults_to_uninspected_provenance() { + let job = PrintJob::new( + JobSource::Local, + DocumentType::Pdf, + "test.pdf".to_string(), + "hash".to_string(), + ); + assert!(!job.form_provenance.inspected); + assert_eq!(job.form_provenance.origin, FormOrigin::Unknown); + } + + #[test] + fn test_form_provenance_summary_machine() { + let summary = machine_filled().summary(); + assert!(summary.contains("machine-filled form"), "{summary}"); + assert!(summary.contains("blocky-writer 0.4.2"), "{summary}"); + assert!(summary.contains("12/14 fields filled"), "{summary}"); + } + + #[test] + fn test_form_provenance_summary_not_a_form_omits_counts() { + let provenance = FormProvenance { + origin: FormOrigin::NotAForm, + confidence: ProvenanceConfidence::Strong, + inspected: true, + ..FormProvenance::default() + }; + assert_eq!(provenance.summary(), "not an interactive form"); + } + + #[test] + fn test_form_provenance_prefers_marker_over_producer() { + let mut provenance = machine_filled(); + assert_eq!(provenance.tool(), Some("blocky-writer 0.4.2")); + + provenance.marker = Some("blocky-writer fillOrigin=machine".to_string()); + assert_eq!(provenance.tool(), Some("blocky-writer fillOrigin=machine")); + } + + #[test] + fn test_form_provenance_audit_json_roundtrips() { + let provenance = machine_filled(); + let json = provenance.audit_json(); + assert!(json.contains("\"origin\":\"Machine\""), "{json}"); + + let restored: FormProvenance = serde_json::from_str(&json).expect("deserialize"); + assert_eq!(restored, provenance); + } + + #[test] + fn test_should_hold_only_for_positive_machine_determination() { + let machine = machine_filled(); + assert!(machine.should_hold(FormProvenancePolicy::HoldForReview)); + assert!(!machine.should_hold(FormProvenancePolicy::Record)); + assert!(!machine.should_hold(FormProvenancePolicy::Off)); + + // Undetermined provenance must never stall a printout. + let unknown = FormProvenance::default(); + assert!(!unknown.should_hold(FormProvenancePolicy::HoldForReview)); + + // Neither must a hand-filled form. + let human = FormProvenance { + origin: FormOrigin::Human, + ..FormProvenance::default() + }; + assert!(!human.should_hold(FormProvenancePolicy::HoldForReview)); + } + + #[test] + fn test_form_provenance_policy_defaults_and_tokens() { + assert_eq!( + FormProvenancePolicy::default(), + FormProvenancePolicy::Record + ); + for policy in [ + FormProvenancePolicy::Off, + FormProvenancePolicy::Record, + FormProvenancePolicy::HoldForReview, + ] { + assert_eq!(FormProvenancePolicy::from_token(policy.as_token()), policy); + assert!(!policy.label().is_empty()); + } + assert_eq!( + FormProvenancePolicy::from_token("nonsense"), + FormProvenancePolicy::Record + ); + } + + #[test] + fn test_form_provenance_policy_inspects() { + assert!(!FormProvenancePolicy::Off.inspects()); + assert!(FormProvenancePolicy::Record.inspects()); + assert!(FormProvenancePolicy::HoldForReview.inspects()); + } + + #[test] + fn test_provenance_confidence_is_ordered() { + assert!(ProvenanceConfidence::None < ProvenanceConfidence::Speculative); + assert!(ProvenanceConfidence::Speculative < ProvenanceConfidence::Probable); + assert!(ProvenanceConfidence::Probable < ProvenanceConfidence::Strong); + assert!(ProvenanceConfidence::Strong < ProvenanceConfidence::Explicit); + assert_eq!(ProvenanceConfidence::Strong.as_token(), "Strong"); + } } diff --git a/crates/presswerk-document/Cargo.toml b/crates/presswerk-document/Cargo.toml index e342ad3..24a9950 100644 --- a/crates/presswerk-document/Cargo.toml +++ b/crates/presswerk-document/Cargo.toml @@ -32,6 +32,7 @@ rten-tensor = { workspace = true, optional = true } [dev-dependencies] criterion = { workspace = true } +serde_json = { workspace = true } [[bench]] name = "document_bench" diff --git a/crates/presswerk-document/src/lib.rs b/crates/presswerk-document/src/lib.rs index 0055f45..f9980f1 100644 --- a/crates/presswerk-document/src/lib.rs +++ b/crates/presswerk-document/src/lib.rs @@ -13,6 +13,8 @@ // 2. Visual Enhancement: Binarization and denoising for scanned documents. // 3. Format Conversion: Stable conversion between Image and PDF formats. // 4. Verified Metadata: Embedding proof-of-authenticity into document headers. +// 5. Form Provenance: Classifying a form as machine-filled or hand-filled so +// the print path can audit and route it (issue #118, ruling D189). pub mod convert; pub mod image; @@ -21,6 +23,7 @@ pub mod scan; // CONVENIENCE: Primary interfaces for document transformation. pub use image::processor::ImageProcessor; +pub use pdf::form::{classify_for_print, classify_pdf}; pub use pdf::reader::PdfReader; pub use pdf::writer::PdfWriter; pub use scan::enhance::ScanEnhancer; diff --git a/crates/presswerk-document/src/pdf/form.rs b/crates/presswerk-document/src/pdf/form.rs new file mode 100644 index 0000000..2e13db3 --- /dev/null +++ b/crates/presswerk-document/src/pdf/form.rs @@ -0,0 +1,1139 @@ +// SPDX-License-Identifier: MPL-2.0 +// Copyright (c) 2026 Jonathan D.A. Jewell (hyperpolymath) +// +// Form provenance — decide whether the values in a PDF form were written by +// software or by a person. +// +// WHY THIS EXISTS (issue #118, ruling D189) +// ----------------------------------------- +// Upstream form-fillers such as `blocky-writer` emit an ordinary PDF byte +// stream: `/V`, `/DV` and `/AS` entries are written and `NeedAppearances` is +// set, but nothing in the file says *who* wrote the values. Presswerk is +// usually the next thing that touches those bytes — it prints them — so the +// print path classifies the document at the boundary and records the result. +// +// The classification is deliberately conservative and evidence-based: +// +// Explicit an upstream provenance marker is present (`/FillOrigin` in +// the document information dictionary, or a `fillOrigin` +// attribute in an XMP packet) +// Strong `/Producer` or `/Creator` names a tool known to fill forms +// programmatically +// Probable a characteristic combination of structural signals — values +// without appearance streams, `NeedAppearances` set, values +// written alongside `/DV` defaults +// +// Only a *positive* machine determination is ever acted upon; everything else +// prints as normal. See `docs/ecosystem/FORM-PROVENANCE.adoc`. + +use std::collections::HashSet; + +use lopdf::{Dictionary, Document, Object, ObjectId}; +use presswerk_core::types::{ + DocumentType, FormOrigin, FormProvenance, FormProvenancePolicy, ProvenanceConfidence, +}; +use tracing::{debug, instrument}; + +/// Documents larger than this are not inspected. +/// +/// Inspection happens inline on the print path, so the budget keeps a huge +/// document from stalling the IPP accept loop. Documents over the budget are +/// reported as "not inspected" rather than guessed at. +pub const MAX_INSPECT_BYTES: usize = 8 * 1024 * 1024; + +/// Hard cap on field-tree nodes visited, so a malformed or hostile field tree +/// cannot make inspection unbounded. +const MAX_FIELD_NODES: u32 = 100_000; + +/// Hard cap on field-tree nesting depth. +const MAX_FIELD_DEPTH: u32 = 32; + +/// How far down a field's own subtree to look for an appearance stream before +/// concluding the value has none. +const APPEARANCE_SEARCH_DEPTH: u32 = 3; + +/// Substrings that identify a tool which writes form values programmatically. +/// +/// Matched case-insensitively against `/Producer` and `/Creator`. Document +/// generators that merely *create* blank PDFs are deliberately absent: a blank +/// form produced by such a tool is still an empty form. +const KNOWN_MACHINE_FILLERS: &[&str] = &[ + "blocky-writer", + "blocky_writer", + "blockywriter", + "fill_blocks", + "lopdf", + "itext", + "openpdf", + "pdf-lib", + "pypdf", + "pdftk", + "pdfbox", + "aspose", + "pdfsharp", + "docusign", + "adobe livecycle", + "livecycle", +]; + +/// Attribute name scanned for in raw bytes / XMP packets. +const MARKER_ATTRIBUTE: &[u8] = b"fillorigin"; + +// --------------------------------------------------------------------------- +// Public entry points +// --------------------------------------------------------------------------- + +/// Classify a document arriving on the print path. +/// +/// This is the entry point both print paths (local printing and the embedded +/// IPP server) call. It never fails: anything that stops inspection — a +/// non-PDF, an oversized document, a parse error, a disabled policy — is +/// reported inside the returned value with `inspected == false`. +#[instrument(skip_all, fields(bytes = data.len(), policy = %policy.as_token()))] +pub fn classify_for_print( + data: &[u8], + document_type: DocumentType, + policy: FormProvenancePolicy, +) -> FormProvenance { + if !policy.inspects() { + debug!("form provenance inspection disabled by policy"); + return FormProvenance::not_inspected("form provenance inspection is disabled"); + } + if document_type != DocumentType::Pdf { + return FormProvenance::not_inspected("document is not a PDF"); + } + classify_pdf(data) +} + +/// Classify the form provenance of raw PDF bytes. +/// +/// Never fails; see [`classify_for_print`]. +#[instrument(skip_all, fields(bytes = data.len()))] +pub fn classify_pdf(data: &[u8]) -> FormProvenance { + if data.is_empty() { + return FormProvenance::not_inspected("document is empty"); + } + if data.len() > MAX_INSPECT_BYTES { + return FormProvenance::not_inspected(&format!( + "document is {} bytes, above the {} byte inspection budget", + data.len(), + MAX_INSPECT_BYTES + )); + } + + let document = match Document::load_mem(data) { + Ok(document) => document, + Err(err) => { + debug!(error = %err, "PDF parse failed during provenance inspection"); + return FormProvenance::not_inspected(&format!("PDF parse failed: {err}")); + } + }; + + classify_document(&document, data) +} + +/// Classify an already-parsed document. +/// +/// `raw` is the original byte stream and is used only to find unstructured +/// provenance markers (for example inside an XMP packet); pass an empty slice +/// to rely on the structured `/Info` marker alone. +#[instrument(skip_all, fields(raw_bytes = raw.len()))] +pub fn classify_document(document: &Document, raw: &[u8]) -> FormProvenance { + let mut provenance = inspected_blank(); + + let info = document_info(document); + provenance.producer = info.producer.clone(); + provenance.creator = info.creator.clone(); + + // -- Explicit marker ------------------------------------------------------ + let mut marker = structured_marker(&info); + if marker.is_none() { + marker = scan_for_marker(raw); + } + if marker.is_none() { + for packet in xmp_packets(document) { + if let Some(found) = scan_for_marker(&packet) { + marker = Some(found); + break; + } + } + } + if let Some(found) = marker + && mentions_machine(&found) + { + provenance.marker = Some(found.clone()); + provenance + .evidence + .push(format!("explicit provenance marker present: {found}")); + } + let explicit = provenance.marker.is_some(); + + // -- Known tool ----------------------------------------------------------- + let known_tool = known_filler(provenance.producer.as_deref(), provenance.creator.as_deref()); + if let Some(tool) = known_tool { + provenance.evidence.push(format!( + "document producer/creator names '{tool}', a known form-filling tool" + )); + } + + // -- Structure ------------------------------------------------------------ + let catalog = match document.catalog() { + Ok(catalog) => catalog, + Err(err) => { + provenance.origin = FormOrigin::NotAForm; + provenance.confidence = ProvenanceConfidence::Speculative; + provenance + .evidence + .push(format!("document has no readable catalog ({err})")); + return finish(provenance); + } + }; + + let acroform = match catalog + .get(b"AcroForm") + .ok() + .and_then(|obj| resolve_dict(document, obj)) + { + Some(acroform) => acroform, + None => { + provenance.origin = FormOrigin::NotAForm; + provenance.confidence = ProvenanceConfidence::Strong; + provenance + .evidence + .push("document catalog has no /AcroForm entry".to_string()); + return finish(provenance); + } + }; + + provenance.is_form = true; + provenance.need_appearances = + matches!(acroform.get(b"NeedAppearances"), Ok(Object::Boolean(true))); + + let scan = scan_fields(document, acroform); + provenance.field_count = scan.field_count; + provenance.filled_field_count = scan.filled; + provenance.filled_with_default = scan.filled_with_default; + provenance.values_without_appearance = scan.without_appearance; + + // -- Determination -------------------------------------------------------- + if explicit { + // An explicit marker outranks every structural signal, including an + // apparently empty form: the writer is telling us it filled the form. + provenance.origin = FormOrigin::Machine; + provenance.confidence = ProvenanceConfidence::Explicit; + } else if provenance.filled_field_count == 0 { + provenance.origin = FormOrigin::Empty; + provenance.confidence = ProvenanceConfidence::Strong; + provenance.evidence.push(format!( + "/AcroForm declares {} field(s), none of which carries a value", + provenance.field_count + )); + } else if known_tool.is_some() { + provenance.origin = FormOrigin::Machine; + provenance.confidence = ProvenanceConfidence::Strong; + } else if provenance.need_appearances { + provenance.origin = FormOrigin::Machine; + provenance.confidence = ProvenanceConfidence::Probable; + provenance.evidence.push(format!( + "{} of {} field(s) carry values and /NeedAppearances is set — values were written without appearances", + provenance.filled_field_count, provenance.field_count + )); + } else if provenance.values_without_appearance > 0 { + provenance.origin = FormOrigin::Machine; + provenance.confidence = ProvenanceConfidence::Probable; + provenance.evidence.push(format!( + "{} field value(s) have no /AP appearance stream", + provenance.values_without_appearance + )); + } else if provenance.filled_with_default > 0 { + provenance.origin = FormOrigin::Machine; + provenance.confidence = ProvenanceConfidence::Speculative; + provenance.evidence.push(format!( + "{} field value(s) were written alongside a /DV default", + provenance.filled_with_default + )); + } else { + provenance.origin = FormOrigin::Human; + provenance.confidence = ProvenanceConfidence::Probable; + provenance.evidence.push(format!( + "{} of {} field(s) carry values, each with an appearance stream and no machine-fill signal", + provenance.filled_field_count, provenance.field_count + )); + } + + finish(provenance) +} + +// --------------------------------------------------------------------------- +// Determination bookkeeping +// --------------------------------------------------------------------------- + +/// A provenance value for a document that *was* inspected, before any evidence +/// has been gathered. +fn inspected_blank() -> FormProvenance { + FormProvenance { + origin: FormOrigin::Unknown, + confidence: ProvenanceConfidence::None, + inspected: true, + is_form: false, + field_count: 0, + filled_field_count: 0, + values_without_appearance: 0, + filled_with_default: 0, + need_appearances: false, + producer: None, + creator: None, + marker: None, + evidence: Vec::new(), + } +} + +/// Log the determination and hand it back. +fn finish(provenance: FormProvenance) -> FormProvenance { + debug!( + origin = %provenance.origin, + confidence = %provenance.confidence, + fields = provenance.field_count, + filled = provenance.filled_field_count, + "form provenance determined" + ); + provenance +} + +// --------------------------------------------------------------------------- +// Object graph helpers +// --------------------------------------------------------------------------- + +/// Resolve an object that is expected to be a dictionary (possibly indirect). +fn resolve_dict<'a>(document: &'a Document, object: &'a Object) -> Option<&'a Dictionary> { + match object { + Object::Dictionary(dict) => Some(dict), + Object::Stream(stream) => Some(&stream.dict), + Object::Reference(id) => document.get_object(*id).ok().and_then(|resolved| { + match resolved { + Object::Dictionary(dict) => Some(dict), + Object::Stream(stream) => Some(&stream.dict), + _ => None, + } + }), + _ => None, + } +} + +/// Resolve an object that is expected to be an array (possibly indirect). +fn resolve_array<'a>(document: &'a Document, object: &'a Object) -> Option<&'a Vec> { + match object { + Object::Array(items) => Some(items), + Object::Reference(id) => document + .get_object(*id) + .ok() + .and_then(|resolved| match resolved { + Object::Array(items) => Some(items), + _ => None, + }), + _ => None, + } +} + +/// Whether a dictionary carries a non-empty value under `key`. +/// +/// `Off` is treated as *no* value in either representation: for check boxes and +/// radio buttons it is the explicitly unselected state, not an entry. +fn has_value(dict: &Dictionary, key: &[u8]) -> bool { + match dict.get(key) { + Ok(Object::Null) | Err(_) => false, + Ok(Object::String(bytes, _)) => !is_blank_or_off(bytes), + Ok(Object::Name(name)) => !is_blank_or_off(name), + Ok(Object::Array(items)) => !items.is_empty(), + Ok(_) => true, + } +} + +/// Whether a PDF name or string represents "no value". +fn is_blank_or_off(bytes: &[u8]) -> bool { + bytes.is_empty() || bytes.eq_ignore_ascii_case(b"Off") +} + +// --------------------------------------------------------------------------- +// Field tree +// --------------------------------------------------------------------------- + +/// What a walk of the AcroForm field tree counted. +struct FieldScan { + field_count: u32, + filled: u32, + filled_with_default: u32, + without_appearance: u32, + visited: u32, +} + +/// Walk the AcroForm `/Fields` tree and count fields, values, and values that +/// have no appearance stream anywhere in their own subtree. +fn scan_fields(document: &Document, acroform: &Dictionary) -> FieldScan { + let mut scan = FieldScan { + field_count: 0, + filled: 0, + filled_with_default: 0, + without_appearance: 0, + visited: 0, + }; + let mut seen = HashSet::new(); + + if let Ok(fields) = acroform.get(b"Fields") { + walk_fields(document, fields, 0, &mut seen, &mut scan); + } + + scan +} + +/// Recursive field-tree walk with cycle and size guards. +fn walk_fields( + document: &Document, + node: &Object, + depth: u32, + seen: &mut HashSet, + scan: &mut FieldScan, +) { + if depth > MAX_FIELD_DEPTH || scan.visited >= MAX_FIELD_NODES { + return; + } + + match node { + Object::Reference(id) => { + if !seen.insert(*id) { + return; + } + scan.visited += 1; + if let Ok(resolved) = document.get_object(*id) { + walk_fields(document, resolved, depth + 1, seen, scan); + } + } + Object::Array(items) => { + for item in items { + walk_fields(document, item, depth + 1, seen, scan); + } + } + Object::Dictionary(dict) => { + scan.visited += 1; + + // A node with a partial field name (/T) is a field; widget-only + // annotations inherit their name from the parent field. + if dict.get(b"T").is_ok() { + scan.field_count += 1; + if has_value(dict, b"V") { + scan.filled += 1; + if has_value(dict, b"DV") { + scan.filled_with_default += 1; + } + if !subtree_has_appearance(document, dict, 0) { + scan.without_appearance += 1; + } + } + } + + if let Ok(kids) = dict.get(b"Kids") { + walk_fields(document, kids, depth + 1, seen, scan); + } + } + Object::Stream(stream) => { + scan.visited += 1; + if let Ok(kids) = stream.dict.get(b"Kids") { + walk_fields(document, kids, depth + 1, seen, scan); + } + } + _ => {} + } +} + +/// Whether a field or anything in its subtree carries an `/AP` appearance +/// dictionary. +/// +/// Widget annotations are frequently kept apart from the field that owns the +/// value, so the value and its appearance do not always live in the same +/// dictionary. +fn subtree_has_appearance(document: &Document, dict: &Dictionary, depth: u32) -> bool { + if dict.get(b"AP").is_ok() { + return true; + } + if depth >= APPEARANCE_SEARCH_DEPTH { + return false; + } + + for key in [b"Kids".as_slice(), b"Annots".as_slice()] { + if let Some(items) = dict.get(key).ok().and_then(|obj| resolve_array(document, obj)) { + for item in items { + if let Some(item_dict) = resolve_dict(document, item) + && subtree_has_appearance(document, item_dict, depth + 1) + { + return true; + } + } + } + } + + false +} + +// --------------------------------------------------------------------------- +// Metadata +// --------------------------------------------------------------------------- + +/// Interesting entries from the document information dictionary. +struct DocInfo { + producer: Option, + creator: Option, + /// Non-standard `Fill*` entries — the explicit marker convention. + fill_entries: Vec<(String, String)>, +} + +/// Read `/Producer`, `/Creator` and any `Fill*` marker entries from the +/// document information dictionary. +fn document_info(document: &Document) -> DocInfo { + let mut info = DocInfo { + producer: None, + creator: None, + fill_entries: Vec::new(), + }; + + let Some(dict) = document + .trailer + .get(b"Info") + .ok() + .and_then(|obj| resolve_dict(document, obj)) + else { + return info; + }; + + for (key, value) in dict.iter() { + let Object::String(bytes, _) = value else { + continue; + }; + let key = String::from_utf8_lossy(key).to_string(); + let text = String::from_utf8_lossy(bytes).trim().to_string(); + match key.as_str() { + "Producer" => info.producer = Some(text), + "Creator" => info.creator = Some(text), + other if other.starts_with("Fill") => info.fill_entries.push((other.into(), text)), + _ => {} + } + } + + info +} + +/// Build the explicit marker from `Fill*` information dictionary entries. +fn structured_marker(info: &DocInfo) -> Option { + let mut origin = None; + let mut tool = None; + let mut time = None; + + for (key, value) in &info.fill_entries { + match key.as_str() { + "FillOrigin" => origin = Some(value.as_str()), + "FillTool" => tool = Some(value.as_str()), + "FillTime" => time = Some(value.as_str()), + _ => {} + } + } + + let origin = origin?; + if !mentions_machine(origin) { + return None; + } + + Some(describe_marker(origin, tool, time)) +} + +/// Render a marker's parts as one stable, readable string. +fn describe_marker(origin: &str, tool: Option<&str>, time: Option<&str>) -> String { + let mut parts = vec![format!("FillOrigin={origin}")]; + if let Some(tool) = tool.filter(|t| !t.trim().is_empty()) { + parts.push(format!("FillTool={tool}")); + } + if let Some(time) = time.filter(|t| !t.trim().is_empty()) { + parts.push(format!("FillTime={time}")); + } + parts.join("; ") +} + +/// Whether a marker value asserts machine filling. +fn mentions_machine(value: &str) -> bool { + let lowered = value.to_ascii_lowercase(); + lowered.contains("machine") || lowered.contains("software") || lowered.contains("automatic") +} + +/// Return the first tool from [`KNOWN_MACHINE_FILLERS`] named by the producer +/// or creator string. +fn known_filler(producer: Option<&str>, creator: Option<&str>) -> Option<&'static str> { + for candidate in [producer, creator].into_iter().flatten() { + let lowered = candidate.to_ascii_lowercase(); + for tool in KNOWN_MACHINE_FILLERS { + if lowered.contains(tool) { + return Some(tool); + } + } + } + None +} + +/// Contents of the document's XMP metadata streams. +/// +/// XMP packets are usually written uncompressed, so the raw stream content is +/// normally enough. Compressed packets are not decoded; the structured +/// `/Info` marker is the authoritative location. +fn xmp_packets(document: &Document) -> Vec> { + let mut packets = Vec::new(); + + let Ok(catalog) = document.catalog() else { + return packets; + }; + let Some(metadata) = catalog.get(b"Metadata").ok() else { + return packets; + }; + let metadata = match metadata { + Object::Reference(id) => match document.get_object(*id) { + Ok(resolved) => resolved, + Err(_) => return packets, + }, + other => other, + }; + if let Object::Stream(stream) = metadata { + packets.push(stream.content.clone()); + } + + packets +} + +/// Scan raw bytes for a `fillOrigin` attribute or element and return its value. +/// +/// Accepts the attribute form (`fillOrigin="machine"`), single quotes, an +/// unquoted attribute value, and the RDF element-text form +/// (`machine`). +fn scan_for_marker(bytes: &[u8]) -> Option { + let start = find_ci(bytes, MARKER_ATTRIBUTE)?; + let mut rest = trim_ascii_start(&bytes[start + MARKER_ATTRIBUTE.len()..]); + if rest.is_empty() { + return None; + } + + if rest[0] == b'=' { + // Attribute form. + rest = trim_ascii_start(&rest[1..]); + if rest.is_empty() { + return None; + } + return match rest[0] { + b'"' | b'\'' => { + let quote = rest[0]; + rest = &rest[1..]; + let end = rest.iter().position(|&byte| byte == quote)?; + decode_marker_value(&rest[..end]) + } + _ => { + let end = rest + .iter() + .position(|&byte| byte.is_ascii_whitespace() || byte == b'>')?; + decode_marker_value(&rest[..end]) + } + }; + } + + // Element form: the value is the element text. + if rest[0] != b'>' { + return None; + } + rest = &rest[1..]; + let end = rest.iter().position(|&byte| byte == b'<')?; + decode_marker_value(&rest[..end]) +} + +/// Decode a marker value, rejecting empty ones. +fn decode_marker_value(bytes: &[u8]) -> Option { + let value = String::from_utf8_lossy(bytes).trim().to_string(); + if value.is_empty() { + None + } else { + Some(value) + } +} + +/// Case-insensitive substring search over bytes. +fn find_ci(haystack: &[u8], needle: &[u8]) -> Option { + if needle.is_empty() || haystack.len() < needle.len() { + return None; + } + haystack + .windows(needle.len()) + .position(|window| window.eq_ignore_ascii_case(needle)) +} + +/// Drop leading ASCII whitespace. +fn trim_ascii_start(bytes: &[u8]) -> &[u8] { + let mut index = 0; + while index < bytes.len() && bytes[index].is_ascii_whitespace() { + index += 1; + } + &bytes[index..] +} + +#[cfg(test)] +mod tests { + use super::*; + use lopdf::StringFormat; + + /// A field specification for the test PDF builder. + struct TestField { + name: &'static str, + value: Option<&'static str>, + default: Option<&'static str>, + appearance: bool, + } + + impl TestField { + /// A field a human filled in a viewer: value plus appearance stream. + const fn human(name: &'static str, value: &'static str) -> Self { + Self { + name, + value: Some(value), + default: None, + appearance: true, + } + } + + /// A field a program filled: value, default, and no appearance stream. + const fn machine(name: &'static str, value: &'static str) -> Self { + Self { + name, + value: Some(value), + default: Some(value), + appearance: false, + } + } + + /// A field with no value at all. + const fn blank(name: &'static str) -> Self { + Self { + name, + value: None, + default: None, + appearance: false, + } + } + } + + /// Build a real, serialisable PDF carrying an AcroForm. + fn build_pdf( + fields: Vec, + need_appearances: bool, + info_entries: Vec<(&'static str, &'static str)>, + ) -> Vec { + let mut document = Document::with_version("1.7"); + + // Minimal one-page page tree so the document is well formed. + let content_id = document.add_object(Object::Stream(lopdf::Stream::new( + Dictionary::new(), + Vec::new(), + ))); + let mut page = Dictionary::new(); + page.set("Type", Object::Name(b"Page".to_vec())); + page.set( + "MediaBox", + Object::Array(vec![ + Object::Integer(0), + Object::Integer(0), + Object::Integer(595), + Object::Integer(842), + ]), + ); + page.set("Contents", Object::Reference(content_id)); + let page_id = document.add_object(Object::Dictionary(page)); + + let mut pages = Dictionary::new(); + pages.set("Type", Object::Name(b"Pages".to_vec())); + pages.set("Kids", Object::Array(vec![Object::Reference(page_id)])); + pages.set("Count", Object::Integer(1)); + let pages_id = document.add_object(Object::Dictionary(pages)); + + if let Ok(Object::Dictionary(page_dict)) = document.get_object_mut(page_id) { + page_dict.set("Parent", Object::Reference(pages_id)); + } + + // Field dictionaries. + let mut field_refs = Vec::new(); + for field in &fields { + let mut dict = Dictionary::new(); + dict.set( + "T", + Object::String(field.name.as_bytes().to_vec(), StringFormat::Literal), + ); + dict.set("FT", Object::Name(b"Tx".to_vec())); + if let Some(value) = field.value { + dict.set( + "V", + Object::String(value.as_bytes().to_vec(), StringFormat::Literal), + ); + } + if let Some(default) = field.default { + dict.set( + "DV", + Object::String(default.as_bytes().to_vec(), StringFormat::Literal), + ); + } + if field.appearance { + let mut appearance = Dictionary::new(); + appearance.set("N", Object::Null); + dict.set("AP", Object::Dictionary(appearance)); + } + field_refs.push(Object::Reference( + document.add_object(Object::Dictionary(dict)), + )); + } + + let mut catalog = Dictionary::new(); + catalog.set("Type", Object::Name(b"Catalog".to_vec())); + catalog.set("Pages", Object::Reference(pages_id)); + if !fields.is_empty() || need_appearances { + let mut acroform = Dictionary::new(); + acroform.set("Fields", Object::Array(field_refs)); + if need_appearances { + acroform.set("NeedAppearances", Object::Boolean(true)); + } + catalog.set("AcroForm", Object::Dictionary(acroform)); + } + let catalog_id = document.add_object(Object::Dictionary(catalog)); + document.trailer.set("Root", Object::Reference(catalog_id)); + + if !info_entries.is_empty() { + let mut info = Dictionary::new(); + for (key, value) in info_entries { + info.set( + key.as_bytes().to_vec(), + Object::String(value.as_bytes().to_vec(), StringFormat::Literal), + ); + } + let info_id = document.add_object(Object::Dictionary(info)); + document.trailer.set("Info", Object::Reference(info_id)); + } + + let mut bytes = Vec::new(); + document.save_to(&mut bytes).expect("serialise test PDF"); + bytes + } + + #[test] + fn non_pdf_bytes_are_not_inspected() { + let provenance = classify_pdf(b"this is not a PDF at all"); + assert!(!provenance.inspected); + assert_eq!(provenance.origin, FormOrigin::Unknown); + assert!( + provenance.evidence[0].contains("PDF parse failed"), + "{:?}", + provenance.evidence + ); + } + + #[test] + fn empty_input_is_not_inspected() { + let provenance = classify_pdf(b""); + assert!(!provenance.inspected); + assert_eq!(provenance.evidence[0], "document is empty"); + } + + #[test] + fn oversize_document_is_not_inspected() { + // The size guard runs before parsing, so padding is enough. + let oversized = vec![b'%'; MAX_INSPECT_BYTES + 1]; + let provenance = classify_pdf(&oversized); + assert!(!provenance.inspected); + assert!( + provenance.evidence[0].contains("inspection budget"), + "{:?}", + provenance.evidence + ); + } + + #[test] + fn pdf_without_acroform_is_not_a_form() { + let bytes = build_pdf(Vec::new(), false, Vec::new()); + let provenance = classify_pdf(&bytes); + + assert!(provenance.inspected); + assert!(!provenance.is_form()); + assert_eq!(provenance.origin, FormOrigin::NotAForm); + assert_eq!(provenance.confidence, ProvenanceConfidence::Strong); + assert!(!provenance.is_machine_filled()); + } + + #[test] + fn blank_form_is_empty_not_machine_filled() { + let bytes = build_pdf( + vec![TestField::blank("surname"), TestField::blank("given")], + false, + Vec::new(), + ); + let provenance = classify_pdf(&bytes); + + assert!(provenance.is_form()); + assert_eq!(provenance.origin, FormOrigin::Empty); + assert_eq!(provenance.field_count, 2); + assert_eq!(provenance.filled_field_count, 0); + assert!(!provenance.is_machine_filled()); + assert!(!provenance.should_hold(FormProvenancePolicy::HoldForReview)); + } + + #[test] + fn blocky_writer_shaped_output_is_machine_filled() { + // Exactly what `fill_blocks` produces: values plus /DV defaults, no + // appearance streams, and /NeedAppearances set. + let bytes = build_pdf( + vec![ + TestField::machine("surname", "Smith"), + TestField::machine("given", "Ada"), + TestField::machine("postcode", "MK7 6AA"), + ], + true, + Vec::new(), + ); + let provenance = classify_pdf(&bytes); + + assert!(provenance.inspected); + assert!(provenance.is_form()); + assert_eq!(provenance.origin, FormOrigin::Machine); + assert_eq!(provenance.confidence, ProvenanceConfidence::Probable); + assert!(provenance.is_machine_filled()); + assert_eq!(provenance.field_count, 3); + assert_eq!(provenance.filled_field_count, 3); + assert_eq!(provenance.filled_with_default, 3); + assert_eq!(provenance.values_without_appearance, 3); + assert!(provenance.need_appearances); + assert!(provenance.should_hold(FormProvenancePolicy::HoldForReview)); + } + + #[test] + fn values_without_appearances_alone_are_machine_filled() { + let bytes = build_pdf( + vec![TestField { + name: "surname", + value: Some("Smith"), + default: None, + appearance: false, + }], + false, + Vec::new(), + ); + let provenance = classify_pdf(&bytes); + + assert_eq!(provenance.origin, FormOrigin::Machine); + assert_eq!(provenance.confidence, ProvenanceConfidence::Probable); + assert_eq!(provenance.values_without_appearance, 1); + assert!(!provenance.need_appearances); + } + + #[test] + fn hand_filled_form_is_human() { + // A viewer-filled form: values with appearance streams, no + // /NeedAppearances, and no /DV rewrites. + let bytes = build_pdf( + vec![ + TestField::human("surname", "Smith"), + TestField::human("given", "Ada"), + ], + false, + Vec::new(), + ); + let provenance = classify_pdf(&bytes); + + assert!(provenance.is_form()); + assert_eq!(provenance.origin, FormOrigin::Human); + assert_eq!(provenance.confidence, ProvenanceConfidence::Probable); + assert_eq!(provenance.filled_field_count, 2); + assert_eq!(provenance.values_without_appearance, 0); + assert_eq!(provenance.filled_with_default, 0); + assert!(!provenance.is_machine_filled()); + } + + #[test] + fn checkbox_off_state_is_not_a_value() { + let mut unchecked = TestField::blank("consent"); + unchecked.value = Some("Off"); + let bytes = build_pdf(vec![unchecked], false, Vec::new()); + let provenance = classify_pdf(&bytes); + + assert_eq!(provenance.field_count, 1); + assert_eq!(provenance.filled_field_count, 0); + assert_eq!(provenance.origin, FormOrigin::Empty); + } + + #[test] + fn known_producer_marks_form_machine_filled() { + let bytes = build_pdf( + vec![TestField::human("surname", "Smith")], + false, + vec![("Producer", "blocky-writer 0.4.2")], + ); + let provenance = classify_pdf(&bytes); + + assert_eq!(provenance.origin, FormOrigin::Machine); + assert_eq!(provenance.confidence, ProvenanceConfidence::Strong); + assert_eq!(provenance.producer.as_deref(), Some("blocky-writer 0.4.2")); + assert_eq!(provenance.tool(), Some("blocky-writer 0.4.2")); + assert!(provenance.summary().contains("blocky-writer")); + } + + #[test] + fn generator_producer_does_not_mark_blank_form() { + // A blank template from a general PDF generator is still an empty + // form, not a machine-filled one. + let bytes = build_pdf( + vec![TestField::blank("surname")], + false, + vec![("Producer", "iText 2.1.7")], + ); + let provenance = classify_pdf(&bytes); + + assert_eq!(provenance.origin, FormOrigin::Empty); + assert!(!provenance.is_machine_filled()); + } + + #[test] + fn explicit_info_marker_outranks_structure() { + // Structurally this looks hand-filled; the marker says otherwise. + let bytes = build_pdf( + vec![TestField::human("surname", "Smith")], + false, + vec![ + ("FillOrigin", "machine"), + ("FillTool", "blocky-writer 0.4.2"), + ("FillTime", "2026-09-27T10:15:00Z"), + ], + ); + let provenance = classify_pdf(&bytes); + + assert_eq!(provenance.origin, FormOrigin::Machine); + assert_eq!(provenance.confidence, ProvenanceConfidence::Explicit); + let marker = provenance.marker.expect("marker recorded"); + assert!(marker.contains("FillOrigin=machine"), "{marker}"); + assert!(marker.contains("blocky-writer 0.4.2"), "{marker}"); + assert!(marker.contains("2026-09-27T10:15:00Z"), "{marker}"); + assert_eq!(provenance.tool(), Some(marker.as_str())); + } + + #[test] + fn explicit_human_marker_does_not_trigger() { + let bytes = build_pdf( + vec![TestField::human("surname", "Smith")], + false, + vec![("FillOrigin", "human")], + ); + let provenance = classify_pdf(&bytes); + + assert!(provenance.marker.is_none()); + assert_eq!(provenance.origin, FormOrigin::Human); + } + + #[test] + fn marker_scanner_understands_all_three_forms() { + assert_eq!( + scan_for_marker(br#"bw:fillOrigin="machine""#).as_deref(), + Some("machine") + ); + assert_eq!( + scan_for_marker(br#"FILLORIGIN = 'machine-filled'"#).as_deref(), + Some("machine-filled") + ); + assert_eq!( + scan_for_marker(b"bw:fillOrigin=machine bw:x=1").as_deref(), + Some("machine") + ); + assert_eq!( + scan_for_marker(b"machine").as_deref(), + Some("machine") + ); + assert_eq!(scan_for_marker(b"no marker here"), None); + assert_eq!(scan_for_marker(br#"fillOrigin="""#), None); + // A different attribute whose name merely starts with the needle. + assert_eq!(scan_for_marker(b"fillOriginTime=\"now\""), None); + } + + #[test] + fn unstructured_marker_in_raw_bytes_is_honoured() { + // An uncompressed XMP-style packet carried in the file. The element + // form is used because lopdf escapes quotes inside literal strings. + let bytes = build_pdf( + vec![TestField::human("surname", "Smith")], + false, + vec![("Note", "x:fillOrigin>machine // -// PDF module — reading, merging, splitting, rotating, and creating PDFs. +// PDF module — reading, merging, splitting, rotating, creating, and +// provenance-classifying PDFs. +pub mod form; pub mod reader; pub mod writer; diff --git a/crates/presswerk-document/src/pdf/reader.rs b/crates/presswerk-document/src/pdf/reader.rs index f13c288..2a1a1a4 100644 --- a/crates/presswerk-document/src/pdf/reader.rs +++ b/crates/presswerk-document/src/pdf/reader.rs @@ -8,6 +8,7 @@ use std::path::Path; use lopdf::{Document, Object, ObjectId}; use presswerk_core::error::PresswerkError; +use presswerk_core::types::FormProvenance; use tracing::{debug, info, instrument, warn}; /// Reads and manipulates existing PDF files. @@ -69,6 +70,18 @@ impl PdfReader { self.source_path.as_deref() } + /// Classify whether this document's form values were written by software + /// or by a person (issue #118, ruling D189). + /// + /// Note: only *structured* provenance markers (the `Fill*` entries in the + /// document information dictionary) are consulted here, because the + /// original byte stream is not retained once the document is parsed. + /// [`crate::pdf::form::classify_pdf`] additionally scans the raw bytes and + /// any uncompressed XMP packet, and is what the print path uses. + pub fn form_provenance(&self) -> FormProvenance { + super::form::classify_document(&self.document, &[]) + } + // -- Extraction ----------------------------------------------------------- /// Extract a single page (1-indexed) into a new standalone PDF document. diff --git a/crates/presswerk-print/Cargo.toml b/crates/presswerk-print/Cargo.toml index dbd04c3..f6f8639 100644 --- a/crates/presswerk-print/Cargo.toml +++ b/crates/presswerk-print/Cargo.toml @@ -12,6 +12,8 @@ repository.workspace = true [dependencies] presswerk-core = { workspace = true } presswerk-security = { workspace = true } +# Form provenance classification on the network print path (issue #118). +presswerk-document = { workspace = true } ipp = { workspace = true } mdns-sd = { workspace = true } rusqlite = { workspace = true } @@ -28,6 +30,7 @@ hex = { workspace = true } [dev-dependencies] tempfile = { workspace = true } criterion = { workspace = true } +lopdf = { workspace = true } [[bench]] name = "ipp_bench" diff --git a/crates/presswerk-print/src/ipp_server.rs b/crates/presswerk-print/src/ipp_server.rs index 17d6bc2..9cf6602 100644 --- a/crates/presswerk-print/src/ipp_server.rs +++ b/crates/presswerk-print/src/ipp_server.rs @@ -43,7 +43,11 @@ use tokio::task::JoinHandle; use tracing::{debug, error, info, warn}; use presswerk_core::error::{PresswerkError, Result}; -use presswerk_core::types::{DocumentType, JobId, JobSource, JobStatus, PrintJob, ServerStatus}; +use presswerk_core::types::{ + DocumentType, FormProvenancePolicy, JobId, JobSource, JobStatus, PrintJob, ServerStatus, +}; +use presswerk_document::pdf::form; +use presswerk_security::audit::AuditLog; use crate::queue::JobQueue; @@ -577,6 +581,34 @@ struct SharedState { ipp_to_internal: Arc>>, /// Directory for persisting document data files. data_dir: PathBuf, + /// Audit trail for incoming jobs, when the application supplied one. + audit_log: Option>>, + /// How to treat a machine-filled form received over the network + /// (issue #118, ruling D189). + form_provenance_policy: FormProvenancePolicy, +} + +impl SharedState { + /// Record an audit entry for an incoming job, if an audit log is attached. + /// + /// A failure to write the audit trail must never fail a print job: the job + /// is already queued, and refusing it would be a worse outcome than a + /// missing log line. It is reported in the log instead. + fn audit(&self, action: &str, document_hash: &str, success: bool, details: Option<&str>) { + let Some(audit_log) = self.audit_log.as_ref() else { + return; + }; + match audit_log.lock() { + Ok(log) => { + if let Err(e) = log.record(action, document_hash, success, details) { + error!(error = %e, action, "failed to record audit entry for incoming job"); + } + } + Err(e) => { + error!(error = %e, action, "audit log lock poisoned; entry not recorded"); + } + } + } } // --------------------------------------------------------------------------- @@ -605,6 +637,8 @@ pub struct IppServer { mdns_fullname: Option, /// Root directory for persistent data (documents subdirectory lives here). data_dir: PathBuf, + /// How to treat a machine-filled form received over the network. + form_provenance_policy: FormProvenancePolicy, } impl IppServer { @@ -626,9 +660,23 @@ impl IppServer { mdns_daemon: None, mdns_fullname: None, data_dir, + form_provenance_policy: FormProvenancePolicy::default(), } } + /// Set how the server should treat a machine-filled form (issue #118). + /// + /// Takes effect the next time the server is started, because the policy is + /// copied into the connection-handling state at start-up. + pub fn set_form_provenance_policy(&mut self, policy: FormProvenancePolicy) { + self.form_provenance_policy = policy; + } + + /// The policy currently configured for machine-filled forms. + pub fn form_provenance_policy(&self) -> FormProvenancePolicy { + self.form_provenance_policy + } + /// Return the port this server will bind to (or is bound to). pub fn port(&self) -> u16 { self.port @@ -680,6 +728,25 @@ impl IppServer { /// Returns an error if the port is already in use or the listener cannot /// be created. pub async fn start(&mut self, job_queue: Arc>) -> Result<()> { + self.start_with_audit(job_queue, None).await + } + + /// Start the server with an audit log attached. + /// + /// Identical to [`start`](Self::start), except that incoming print jobs are + /// also recorded in the supplied audit trail. This is how a network-received + /// machine-filled form becomes visible to audit (issue #118, ruling D189): + /// the determination is stored on the job *and* written to the trail. + /// + /// # Errors + /// + /// Returns an error if the port is already in use or the listener cannot + /// be created. + pub async fn start_with_audit( + &mut self, + job_queue: Arc>, + audit_log: Option>>, + ) -> Result<()> { if self.status == ServerStatus::Running { debug!(port = self.port, "IPP server already running"); return Ok(()); @@ -718,6 +785,8 @@ impl IppServer { next_ipp_job_id: Arc::new(AtomicU32::new(1)), ipp_to_internal: Arc::new(Mutex::new(HashMap::new())), data_dir: self.data_dir.clone(), + audit_log, + form_provenance_policy: self.form_provenance_policy, }); let handle = tokio::spawn(async move { @@ -1033,13 +1102,53 @@ fn handle_print_job(request: &IppRequest, peer_addr: SocketAddr, state: &SharedS // Create the internal print job. let ip = peer_addr.ip(); - let job = PrintJob::new( + let mut job = PrintJob::new( JobSource::Network { remote_addr: ip }, document_type, document_name.clone(), document_hash.clone(), ); + // Classify the document before it is enqueued, so the determination is + // part of the stored job rather than an afterthought (issue #118, + // ruling D189). Only PDFs are inspected; everything else is recorded as + // "not inspected" and prints unchanged. + let provenance = form::classify_for_print( + &request.document_data, + document_type, + state.form_provenance_policy, + ); + job.form_provenance = provenance.clone(); + + if provenance.is_form() { + info!( + job_name = %document_name, + origin = %provenance.origin, + confidence = %provenance.confidence, + fields = provenance.field_count, + filled = provenance.filled_field_count, + "form provenance recorded for incoming print job" + ); + state.audit( + "form_provenance", + &document_hash, + true, + Some(&provenance.audit_json()), + ); + } + + // Routing: under the hold policy, a machine-filled form waits for the + // user rather than joining the send queue unattended. + if provenance.should_hold(state.form_provenance_policy) { + job.status = JobStatus::Held; + state.audit( + "job_held_machine_filled", + &document_hash, + true, + Some("machine-filled form held for review per form provenance policy"), + ); + } + let internal_job_id = job.id; // Assign an IPP integer job-id. @@ -1134,8 +1243,10 @@ fn handle_print_job(request: &IppRequest, peer_addr: SocketAddr, state: &SharedS resp.begin_group(TAG_JOB_ATTRIBUTES) .integer("job-id", ipp_job_id) .uri("job-uri", &format!("{printer_uri}/jobs/{ipp_job_id}")) - .enum_attr("job-state", JOB_STATE_PENDING) - .keyword("job-state-reasons", "none"); + // Report the state the job actually entered: a machine-filled form + // held for review must not claim to be pending (issue #118). + .enum_attr("job-state", job_status_to_ipp_state(job.status)) + .keyword("job-state-reasons", job_state_reason(job.status)); resp.build() } @@ -1767,9 +1878,24 @@ mod tests { next_ipp_job_id: Arc::new(AtomicU32::new(1)), ipp_to_internal: Arc::new(Mutex::new(HashMap::new())), data_dir: data_dir.to_path_buf(), + audit_log: None, + form_provenance_policy: FormProvenancePolicy::default(), } } + /// Shared state with an in-memory audit trail and an explicit provenance + /// policy, for testing the print path's audit and routing behaviour. + fn make_shared_state_with_audit( + data_dir: &std::path::Path, + policy: FormProvenancePolicy, + ) -> (SharedState, Arc>) { + let mut state = make_shared_state_with_dir(data_dir); + let audit_log = Arc::new(Mutex::new(AuditLog::open_in_memory().expect("open audit log"))); + state.audit_log = Some(Arc::clone(&audit_log)); + state.form_provenance_policy = policy; + (state, audit_log) + } + fn make_shared_state() -> SharedState { let tmp = make_test_data_dir(); // Leak the TempDir so it lives for the duration of the test. @@ -2207,4 +2333,260 @@ mod tests { .expect("should retrieve document"); assert_eq!(retrieved, doc, "retrieved content must match original"); } + + // -- Form provenance (issue #118, ruling D189) -------------------------- + + /// Build a minimal one-page PDF with a single machine-filled field, shaped + /// exactly as blocky-writer's `fill_blocks` leaves it: a value plus a `/DV` + /// default, no `/AP` appearance stream, and `/NeedAppearances` set. + fn machine_filled_pdf() -> Vec { + build_test_form_pdf(true) + } + + /// The same form with an appearance stream on the field and no + /// `/NeedAppearances` — what a viewer produces when a person fills it in. + fn hand_filled_pdf() -> Vec { + build_test_form_pdf(false) + } + + fn build_test_form_pdf(machine_shaped: bool) -> Vec { + use lopdf::{Dictionary, Document, Object, StringFormat}; + + let mut document = Document::with_version("1.7"); + + let content_id = document.add_object(Object::Stream(lopdf::Stream::new( + Dictionary::new(), + Vec::new(), + ))); + let mut page = Dictionary::new(); + page.set("Type", Object::Name(b"Page".to_vec())); + page.set( + "MediaBox", + Object::Array(vec![ + Object::Integer(0), + Object::Integer(0), + Object::Integer(595), + Object::Integer(842), + ]), + ); + page.set("Contents", Object::Reference(content_id)); + let page_id = document.add_object(Object::Dictionary(page)); + + let mut pages = Dictionary::new(); + pages.set("Type", Object::Name(b"Pages".to_vec())); + pages.set("Kids", Object::Array(vec![Object::Reference(page_id)])); + pages.set("Count", Object::Integer(1)); + let pages_id = document.add_object(Object::Dictionary(pages)); + if let Ok(Object::Dictionary(page_dict)) = document.get_object_mut(page_id) { + page_dict.set("Parent", Object::Reference(pages_id)); + } + + let mut field = Dictionary::new(); + field.set( + "T", + Object::String(b"surname".to_vec(), StringFormat::Literal), + ); + field.set("FT", Object::Name(b"Tx".to_vec())); + field.set( + "V", + Object::String(b"Smith".to_vec(), StringFormat::Literal), + ); + if machine_shaped { + field.set( + "DV", + Object::String(b"Smith".to_vec(), StringFormat::Literal), + ); + } else { + let mut appearance = Dictionary::new(); + appearance.set("N", Object::Null); + field.set("AP", Object::Dictionary(appearance)); + } + let field_id = document.add_object(Object::Dictionary(field)); + + let mut acroform = Dictionary::new(); + acroform.set("Fields", Object::Array(vec![Object::Reference(field_id)])); + if machine_shaped { + acroform.set("NeedAppearances", Object::Boolean(true)); + } + + let mut catalog = Dictionary::new(); + catalog.set("Type", Object::Name(b"Catalog".to_vec())); + catalog.set("Pages", Object::Reference(pages_id)); + catalog.set("AcroForm", Object::Dictionary(acroform)); + let catalog_id = document.add_object(Object::Dictionary(catalog)); + document.trailer.set("Root", Object::Reference(catalog_id)); + + let mut bytes = Vec::new(); + document.save_to(&mut bytes).expect("serialise test PDF"); + bytes + } + + /// Submit a PDF as a network print job and return the parsed response. + fn submit_pdf(state: &SharedState, document: &[u8], request_id: u32) -> IppRequest { + let attrs = vec![ + (VALUE_TAG_NAME, "job-name", b"Application Form" as &[u8]), + (VALUE_TAG_KEYWORD, "document-format", b"application/pdf"), + ]; + let data = build_test_ipp_request(OP_PRINT_JOB, request_id, &attrs, document); + let request = parse_ipp_request(&data).expect("parse_ipp_request failed"); + let peer: SocketAddr = "10.0.0.9:1234".parse().expect("hardcoded address is invalid"); + let response = dispatch_operation(&request, peer, state); + parse_ipp_request(&response).expect("parse_ipp_request failed") + } + + #[test] + fn network_print_job_records_machine_filled_provenance() { + let tmp = make_test_data_dir(); + let (state, _audit) = + make_shared_state_with_audit(tmp.path(), FormProvenancePolicy::Record); + + let parsed = submit_pdf(&state, &machine_filled_pdf(), 600); + assert_eq!(parsed.operation_id, STATUS_OK); + + let queue = state.job_queue.lock().expect("mutex poisoned"); + let jobs = queue.get_all_jobs().expect("get_all_jobs"); + assert_eq!(jobs.len(), 1); + + let provenance = &jobs[0].form_provenance; + assert!(provenance.inspected); + assert!(provenance.is_form()); + assert!(provenance.is_machine_filled(), "{provenance:?}"); + assert_eq!(provenance.filled_field_count, 1); + + // Recording alone must not change routing. + assert_eq!(jobs[0].status, JobStatus::Pending); + + // And the determination is queryable for audit and routing. + assert_eq!( + queue + .get_jobs_with_form_origin(presswerk_core::types::FormOrigin::Machine) + .expect("query by origin") + .len(), + 1 + ); + } + + #[test] + fn network_print_job_writes_form_provenance_to_the_audit_trail() { + let tmp = make_test_data_dir(); + let (state, audit) = + make_shared_state_with_audit(tmp.path(), FormProvenancePolicy::Record); + + let parsed = submit_pdf(&state, &machine_filled_pdf(), 601); + assert_eq!(parsed.operation_id, STATUS_OK); + + let log = audit.lock().expect("audit mutex poisoned"); + let entries = log.recent_entries(10).expect("recent entries"); + let entry = entries + .iter() + .find(|e| e.action == "form_provenance") + .expect("form_provenance audit entry"); + assert!(entry.success); + let details = entry.details.as_deref().expect("details present"); + assert!(details.contains("\"origin\":\"Machine\""), "{details}"); + } + + #[test] + fn non_form_documents_are_not_audited_as_forms() { + let tmp = make_test_data_dir(); + let (state, audit) = + make_shared_state_with_audit(tmp.path(), FormProvenancePolicy::Record); + + let parsed = submit_pdf(&state, b"not really a PDF", 602); + assert_eq!(parsed.operation_id, STATUS_OK); + + let log = audit.lock().expect("audit mutex poisoned"); + let entries = log.recent_entries(10).expect("recent entries"); + assert!( + !entries.iter().any(|e| e.action == "form_provenance"), + "unparseable documents must not be reported as forms" + ); + + let queue = state.job_queue.lock().expect("mutex poisoned"); + let jobs = queue.get_all_jobs().expect("get_all_jobs"); + assert!(!jobs[0].form_provenance.inspected); + assert_eq!( + jobs[0].form_provenance.origin, + presswerk_core::types::FormOrigin::Unknown + ); + } + + #[test] + fn hold_policy_parks_machine_filled_form_for_review() { + let tmp = make_test_data_dir(); + let (state, audit) = + make_shared_state_with_audit(tmp.path(), FormProvenancePolicy::HoldForReview); + + let parsed = submit_pdf(&state, &machine_filled_pdf(), 603); + assert_eq!(parsed.operation_id, STATUS_OK); + + // The client is told the job is held, not pending. + let job_group = parsed + .attribute_groups + .iter() + .find(|g| g.delimiter == TAG_JOB_ATTRIBUTES) + .expect("job attributes group"); + assert_eq!(job_group.get_integer("job-state"), Some(JOB_STATE_HELD)); + + let queue = state.job_queue.lock().expect("mutex poisoned"); + let jobs = queue.get_all_jobs().expect("get_all_jobs"); + assert_eq!(jobs[0].status, JobStatus::Held); + assert!(jobs[0].form_provenance.is_machine_filled()); + drop(queue); + + let log = audit.lock().expect("audit mutex poisoned"); + let entries = log.recent_entries(10).expect("recent entries"); + assert!(entries + .iter() + .any(|e| e.action == "job_held_machine_filled")); + } + + #[test] + fn hold_policy_leaves_hand_filled_form_alone() { + let tmp = make_test_data_dir(); + let (state, _audit) = + make_shared_state_with_audit(tmp.path(), FormProvenancePolicy::HoldForReview); + + let parsed = submit_pdf(&state, &hand_filled_pdf(), 604); + assert_eq!(parsed.operation_id, STATUS_OK); + + let queue = state.job_queue.lock().expect("mutex poisoned"); + let jobs = queue.get_all_jobs().expect("get_all_jobs"); + assert_eq!( + jobs[0].form_provenance.origin, + presswerk_core::types::FormOrigin::Human, + "{:?}", + jobs[0].form_provenance + ); + assert_eq!(jobs[0].status, JobStatus::Pending); + } + + #[test] + fn off_policy_skips_inspection_entirely() { + let tmp = make_test_data_dir(); + let (state, _audit) = make_shared_state_with_audit(tmp.path(), FormProvenancePolicy::Off); + + let parsed = submit_pdf(&state, &machine_filled_pdf(), 605); + assert_eq!(parsed.operation_id, STATUS_OK); + + let queue = state.job_queue.lock().expect("mutex poisoned"); + let jobs = queue.get_all_jobs().expect("get_all_jobs"); + assert!(!jobs[0].form_provenance.inspected); + assert!(!jobs[0].form_provenance.is_machine_filled()); + assert_eq!(jobs[0].status, JobStatus::Pending); + } + + #[test] + fn server_exposes_its_provenance_policy() { + let mut server = IppServer::new(None, None); + assert_eq!( + server.form_provenance_policy(), + FormProvenancePolicy::default() + ); + server.set_form_provenance_policy(FormProvenancePolicy::HoldForReview); + assert_eq!( + server.form_provenance_policy(), + FormProvenancePolicy::HoldForReview + ); + } } diff --git a/crates/presswerk-print/src/queue.rs b/crates/presswerk-print/src/queue.rs index 8cc5a2b..3648a59 100644 --- a/crates/presswerk-print/src/queue.rs +++ b/crates/presswerk-print/src/queue.rs @@ -14,7 +14,8 @@ use tracing::{debug, info, instrument}; use presswerk_core::error::{PresswerkError, Result}; use presswerk_core::types::{ - DocumentType, ErrorClass, JobId, JobSource, JobStatus, PrintJob, PrintSettings, + DocumentType, ErrorClass, FormOrigin, FormProvenance, JobId, JobSource, JobStatus, PrintJob, + PrintSettings, }; /// SQLite schema for the jobs table. @@ -36,7 +37,9 @@ const CREATE_TABLE_SQL: &str = r#" error_class TEXT, error_history TEXT NOT NULL DEFAULT '[]', bytes_sent INTEGER NOT NULL DEFAULT 0, - total_bytes INTEGER NOT NULL DEFAULT 0 + total_bytes INTEGER NOT NULL DEFAULT 0, + form_origin TEXT NOT NULL DEFAULT 'Unknown', + form_provenance TEXT NOT NULL DEFAULT '{}' ) "#; @@ -50,6 +53,18 @@ const MIGRATE_RETRY_COLUMNS_SQL: &str = r#" ALTER TABLE jobs ADD COLUMN total_bytes INTEGER NOT NULL DEFAULT 0; "#; +/// Migration to add form-provenance columns to existing databases. +/// +/// `form_origin` holds the bare token (`Machine`, `Human`, `Empty`, +/// `NotAForm`, `Unknown`) so routing and audit queries can filter in SQL +/// without parsing JSON; `form_provenance` holds the full determination. +/// The `'{}'` default deliberately does not parse into a `FormProvenance`, +/// so pre-existing rows surface as "not inspected" rather than as a guess. +const MIGRATE_FORM_PROVENANCE_SQL: &str = r#" + ALTER TABLE jobs ADD COLUMN form_origin TEXT NOT NULL DEFAULT 'Unknown'; + ALTER TABLE jobs ADD COLUMN form_provenance TEXT NOT NULL DEFAULT '{}'; +"#; + /// Persistent job queue backed by a SQLite database. /// /// All methods are synchronous because `rusqlite` does not support async @@ -80,6 +95,9 @@ impl JobQueue { // Run migration for existing databases that lack retry columns. Self::migrate_retry_columns(&conn); + // Run migration for existing databases that lack provenance columns. + Self::migrate_form_provenance_columns(&conn); + info!("job queue database opened"); Ok(Self { conn }) } @@ -112,6 +130,20 @@ impl JobQueue { } } + /// Apply form-provenance column migration to existing databases. + /// Silently skips if columns already exist. + fn migrate_form_provenance_columns(conn: &Connection) { + for stmt in MIGRATE_FORM_PROVENANCE_SQL.split(';') { + let trimmed = stmt.trim(); + if trimmed.is_empty() { + continue; + } + if conn.execute_batch(trimmed).is_err() { + // Column already exists — expected on migrated databases. + } + } + } + /// Insert a new print job into the queue. /// /// The job's `id`, `created_at`, and `updated_at` fields must already be @@ -133,13 +165,17 @@ impl JobQueue { .map(|ec| serde_json::to_string(ec).unwrap_or_default()); let error_history_json = serde_json::to_string(&job.error_history) .map_err(|e| PresswerkError::Database(format!("serialize error_history: {e}")))?; + let form_provenance_json = job.form_provenance.audit_json(); + let form_origin = job.form_provenance.origin.as_token(); self.conn .execute( "INSERT INTO jobs (id, source, status, document_type, document_name, document_hash, settings, printer_uri, created_at, updated_at, error_message, - retry_count, max_retries, error_class, error_history, bytes_sent, total_bytes) - VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16, ?17)", + retry_count, max_retries, error_class, error_history, bytes_sent, total_bytes, + form_origin, form_provenance) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16, ?17, + ?18, ?19)", params![ job.id.to_string(), source_json, @@ -158,6 +194,8 @@ impl JobQueue { error_history_json, job.bytes_sent as i64, job.total_bytes as i64, + form_origin, + form_provenance_json, ], ) .map_err(|e| PresswerkError::Database(format!("insert job: {e}")))?; @@ -208,7 +246,8 @@ impl JobQueue { "SELECT id, source, status, document_type, document_name, document_hash, settings, printer_uri, created_at, updated_at, error_message, retry_count, max_retries, - error_class, error_history, bytes_sent, total_bytes + error_class, error_history, bytes_sent, total_bytes, + form_origin, form_provenance FROM jobs WHERE id = ?1", ) .map_err(|e| PresswerkError::Database(format!("prepare get_job: {e}")))?; @@ -233,7 +272,8 @@ impl JobQueue { "SELECT id, source, status, document_type, document_name, document_hash, settings, printer_uri, created_at, updated_at, error_message, retry_count, max_retries, - error_class, error_history, bytes_sent, total_bytes + error_class, error_history, bytes_sent, total_bytes, + form_origin, form_provenance FROM jobs ORDER BY created_at DESC", ) .map_err(|e| PresswerkError::Database(format!("prepare get_all_jobs: {e}")))?; @@ -261,7 +301,8 @@ impl JobQueue { "SELECT id, source, status, document_type, document_name, document_hash, settings, printer_uri, created_at, updated_at, error_message, retry_count, max_retries, - error_class, error_history, bytes_sent, total_bytes + error_class, error_history, bytes_sent, total_bytes, + form_origin, form_provenance FROM jobs WHERE status = ?1 ORDER BY created_at ASC", ) .map_err(|e| PresswerkError::Database(format!("prepare get_pending: {e}")))?; @@ -276,6 +317,37 @@ impl JobQueue { Ok(jobs) } + /// Retrieve every job whose document was classified with the given form + /// provenance, newest first. + /// + /// This is the query audit and routing use to answer "which of these print + /// jobs were machine-filled forms?" (issue #118, ruling D189). Filtering + /// happens on the `form_origin` token column, so it never has to parse the + /// JSON determination. + #[instrument(skip(self), fields(origin = %origin))] + pub fn get_jobs_with_form_origin(&self, origin: FormOrigin) -> Result> { + let mut stmt = self + .conn + .prepare( + "SELECT id, source, status, document_type, document_name, + document_hash, settings, printer_uri, created_at, + updated_at, error_message, retry_count, max_retries, + error_class, error_history, bytes_sent, total_bytes, + form_origin, form_provenance + FROM jobs WHERE form_origin = ?1 ORDER BY created_at DESC", + ) + .map_err(|e| PresswerkError::Database(format!("prepare get_by_origin: {e}")))?; + + let jobs = stmt + .query_map(params![origin.as_token()], row_to_print_job) + .map_err(|e| PresswerkError::Database(format!("query get_by_origin: {e}")))? + .collect::, _>>() + .map_err(|e| PresswerkError::Database(format!("collect rows: {e}")))?; + + debug!(count = jobs.len(), origin = %origin, "retrieved jobs by form origin"); + Ok(jobs) + } + /// Delete a job from the queue. /// /// Returns `Ok(())` even if the job did not exist (idempotent). @@ -318,6 +390,22 @@ fn row_to_print_job(row: &rusqlite::Row<'_>) -> rusqlite::Result { let error_history_json: String = row.get::<_, String>(14).unwrap_or_else(|_| "[]".into()); let bytes_sent: u64 = row.get::<_, i64>(15).unwrap_or(0) as u64; let total_bytes: u64 = row.get::<_, i64>(16).unwrap_or(0) as u64; + // Columns 17 and 18 were added by migration. On a pre-migration row they + // are absent; on a row written before provenance existed they carry the + // schema default (`'Unknown'` / `'{}'`), and `'{}'` does not parse into a + // `FormProvenance`. Every one of those cases must surface as "not + // inspected" — never as a guessed determination. + let form_origin_token: Option = row.get(17).unwrap_or(None); + let form_provenance_json: Option = row.get(18).unwrap_or(None); + let form_provenance: FormProvenance = form_provenance_json + .and_then(|json| serde_json::from_str(&json).ok()) + .unwrap_or_else(|| FormProvenance { + origin: form_origin_token + .as_deref() + .map(FormOrigin::from_token) + .unwrap_or_default(), + ..FormProvenance::default() + }); // Parse the UUID. If the stored value is malformed we surface a // meaningful error rather than panicking. @@ -376,6 +464,7 @@ fn row_to_print_job(row: &rusqlite::Row<'_>) -> rusqlite::Result { error_history, bytes_sent, total_bytes, + form_provenance, }) } @@ -498,4 +587,167 @@ mod tests { let result = queue.update_status(&JobId::new(), JobStatus::Cancelled, None); assert!(result.is_err()); } + + // -- Form provenance (issue #118, ruling D189) -------------------------- + + /// A machine-filled determination, as the document crate would produce it. + fn machine_filled_job() -> PrintJob { + let mut job = test_job(); + job.form_provenance = FormProvenance { + origin: FormOrigin::Machine, + confidence: presswerk_core::types::ProvenanceConfidence::Probable, + inspected: true, + is_form: true, + field_count: 14, + filled_field_count: 12, + values_without_appearance: 12, + filled_with_default: 12, + need_appearances: true, + producer: Some("blocky-writer 0.4.2".to_string()), + creator: None, + marker: None, + evidence: vec!["12 of 14 fields carry values".to_string()], + }; + job + } + + #[test] + fn provenance_roundtrips_through_the_queue() { + let queue = JobQueue::open_in_memory().expect("open in-memory db"); + let job = machine_filled_job(); + queue.insert_job(&job).expect("insert"); + + let retrieved = queue.get_job(&job.id).expect("get_job").expect("found"); + assert_eq!(retrieved.form_provenance, job.form_provenance); + assert!(retrieved.form_provenance.is_machine_filled()); + assert_eq!( + retrieved.form_provenance.producer.as_deref(), + Some("blocky-writer 0.4.2") + ); + } + + #[test] + fn jobs_can_be_filtered_by_form_origin() { + let queue = JobQueue::open_in_memory().expect("open in-memory db"); + + let machine = machine_filled_job(); + let plain = test_job(); + queue.insert_job(&machine).expect("insert machine"); + queue.insert_job(&plain).expect("insert plain"); + + let machine_jobs = queue + .get_jobs_with_form_origin(FormOrigin::Machine) + .expect("query machine"); + assert_eq!(machine_jobs.len(), 1); + assert_eq!(machine_jobs[0].id, machine.id); + + let unknown_jobs = queue + .get_jobs_with_form_origin(FormOrigin::Unknown) + .expect("query unknown"); + assert_eq!(unknown_jobs.len(), 1); + assert_eq!(unknown_jobs[0].id, plain.id); + + assert!(queue + .get_jobs_with_form_origin(FormOrigin::Human) + .expect("query human") + .is_empty()); + } + + #[test] + fn every_read_path_returns_the_stored_provenance() { + let queue = JobQueue::open_in_memory().expect("open in-memory db"); + let job = machine_filled_job(); + queue.insert_job(&job).expect("insert"); + + assert!(queue.get_all_jobs().expect("get_all")[0] + .form_provenance + .is_machine_filled()); + assert!(queue.get_pending_jobs().expect("get_pending")[0] + .form_provenance + .is_machine_filled()); + } + + /// The `jobs` table as it stood before form provenance existed. + const LEGACY_SCHEMA: &str = r#" + CREATE TABLE jobs ( + id TEXT PRIMARY KEY, + source TEXT NOT NULL, + status TEXT NOT NULL, + document_type TEXT NOT NULL, + document_name TEXT NOT NULL, + document_hash TEXT NOT NULL, + settings TEXT NOT NULL, + printer_uri TEXT, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + error_message TEXT, + retry_count INTEGER NOT NULL DEFAULT 0, + max_retries INTEGER NOT NULL DEFAULT 5, + error_class TEXT, + error_history TEXT NOT NULL DEFAULT '[]', + bytes_sent INTEGER NOT NULL DEFAULT 0, + total_bytes INTEGER NOT NULL DEFAULT 0 + ); + "#; + + #[test] + fn legacy_database_is_migrated_and_reads_as_uninspected() { + let dir = tempfile::TempDir::new().expect("create temp dir"); + let path = dir.path().join("legacy-jobs.db"); + + // Write a row using the pre-provenance schema. + { + let conn = rusqlite::Connection::open(&path).expect("open legacy db"); + conn.execute_batch(LEGACY_SCHEMA).expect("create legacy table"); + let now = Utc::now().to_rfc3339(); + conn.execute( + "INSERT INTO jobs (id, source, status, document_type, document_name, + document_hash, settings, printer_uri, created_at, updated_at, error_message, + retry_count, max_retries, error_class, error_history, bytes_sent, total_bytes) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16, ?17)", + params![ + "11111111-1111-1111-1111-111111111111", + "\"Local\"", + "\"Pending\"", + "\"Pdf\"", + "legacy-form.pdf", + "legacyhash", + serde_json::to_string(&PrintSettings::default()).unwrap(), + Option::::None, + now, + now, + Option::::None, + 0, + 5, + Option::::None, + "[]", + 0, + 0, + ], + ) + .expect("insert legacy row"); + } + + // Reopening must migrate the schema without losing the row. + let queue = JobQueue::open(&path).expect("open migrated db"); + let jobs = queue.get_all_jobs().expect("get_all_jobs"); + assert_eq!(jobs.len(), 1); + assert_eq!(jobs[0].document_name, "legacy-form.pdf"); + + // A row with no provenance must read back as "not inspected", never as + // a guessed determination. + let provenance = &jobs[0].form_provenance; + assert!(!provenance.inspected); + assert_eq!(provenance.origin, FormOrigin::Unknown); + assert!(!provenance.is_machine_filled()); + + // And new jobs written after migration carry their determination. + let fresh = machine_filled_job(); + queue.insert_job(&fresh).expect("insert post-migration job"); + let machine_jobs = queue + .get_jobs_with_form_origin(FormOrigin::Machine) + .expect("query machine"); + assert_eq!(machine_jobs.len(), 1); + assert_eq!(machine_jobs[0].id, fresh.id); + } } diff --git a/docs/ecosystem/FORM-PROVENANCE.adoc b/docs/ecosystem/FORM-PROVENANCE.adoc new file mode 100644 index 0000000..3a4634c --- /dev/null +++ b/docs/ecosystem/FORM-PROVENANCE.adoc @@ -0,0 +1,271 @@ +// SPDX-License-Identifier: CC-BY-SA-4.0 +// Copyright (c) 2026 Jonathan D.A. Jewell (hyperpolymath) +// +// Presswerk's position on machine-filled form provenance. +// Answer to hyperpolymath/presswerk#118, under ruling D189. + += Form Provenance in the Print Path +Jonathan D.A. Jewell +:revdate: 2026-10-04 +:toc: left +:toclevels: 3 +:sectnums: +:icons: font + +== Purpose + +`blocky-writer` asked (https://github.com/hyperpolymath/presswerk/issues/118[#118]) whether a +machine-filled application form should be distinguishable from a hand-completed one when it +arrives as a print job. Ruling *D189* (2026-09-30) answered yes: + +____ +Record machine-filled provenance as metadata in the print path so audit/routing can +distinguish it. +____ + +This document records Presswerk's position, the marker convention Presswerk will honour, what +was implemented, and — importantly — what Presswerk *cannot* determine on its own. + +== Position + +*Yes, and it is the print path's job to record it.* + +A filled application form arriving at a printer is frequently the last observable step in a +process somebody is accountable for. Presswerk sits at that step, it keeps an append-only audit +trail, and it routes jobs. So Presswerk records provenance at the point of entry, on every +print path, whether or not the upstream writer said anything. + +Three commitments follow: + +[arabic] +. *Recording is unconditional.* Provenance is determined and stored for every PDF that enters +the print path — locally selected documents and network-received IPP jobs alike. +. *Determination is never a guess presented as a fact.* Every determination carries a +confidence and the evidence for it. A document that could not be inspected is recorded as +_inspected: false_, not as "hand-filled". +. *Determination alone never blocks a printout.* Recording changes nothing for the user. Only +an explicit, opt-in policy holds a job, and only on a *positive* machine determination. + +== Where this belongs — and where it does not + +This is the right place for the *consuming* half of the problem, and only that half. + +[cols="1,3", options="header"] +|=== +|Place |What it can and cannot do + +|`blocky-writer` (upstream) +|The only place that *knows* it filled the form. It can emit a marker that costs nothing to +verify and needs no inference. It currently emits none, so everything downstream of it is +inference. + +|*Presswerk print path (this repository)* +|The place that must *record* the fact for audit and act on it for routing, because it is the +last component that sees the document as a document. Implemented here. + +|The printer +|Sees pixels. A rendered page carries no provenance at all; by the time the marker could +matter, the document is gone. + +|The Idris2 ABI layer (`src/abi/`) +|Proves properties of the FFI surface. `FormProvenance` is not exposed over FFI, so there is +nothing to prove there yet. If it is ever exported, its encoding becomes a proof obligation. +|=== + +So: Presswerk is the right place for recording, audit and routing. It is *not* the only place +the problem should be solved, and it is not the authoritative place. The authoritative marker +has to come from upstream. Until it does, Presswerk's determination is inference, and this +document says so out loud rather than letting a heuristic masquerade as knowledge. + +== Marker convention Presswerk will honour + +If an upstream writer states provenance explicitly, Presswerk prefers that statement over +every structural signal. Either of these is sufficient; both together is better. + +=== Structured (preferred) + +Entries in the PDF document information dictionary (`/Info`). Presswerk reads these through +the parsed object tree, so they work whether or not the dictionary sits in an object stream. + +[cols="1,3", options="header"] +|=== +|Key |Meaning + +|`/FillOrigin` +|Required to trigger the explicit path. Recognised values contain `machine`, `software` or +`automatic` (case-insensitive). Any other value — including `human` — is ignored, so a +writer can declare hand-completion without tripping the marker. + +|`/FillTool` +|Optional. Name and version of the tool that wrote the values, e.g. `blocky-writer 0.4.2`. +Shown in the UI and stored in the audit trail. + +|`/FillTime` +|Optional. When the values were written, RFC 3339 preferred. +|=== + +=== Unstructured (fallback) + +A `fillOrigin` attribute or element in the document's XMP packet, or anywhere in the raw byte +stream. All three of these are understood: + +[source,xml] +---- +bw:fillOrigin="machine" +bw:fillOrigin='machine' +machine +---- + +Two caveats, stated plainly: only *uncompressed* XMP packets are scanned (Presswerk does not +decode a compressed metadata stream), and a raw byte scan can in principle match text inside a +document's visible content. That is why the structured form is preferred and why an +unstructured match is the only signal Presswerk will accept from raw bytes. + +== How Presswerk determines provenance + +Implemented in `crates/presswerk-document/src/pdf/form.rs`. + +[cols="2,1,4", options="header"] +|=== +|Signal |Confidence |Rationale + +|Explicit marker present |`Explicit` +|The writer said so. Outranks every structural signal, including an apparently empty form. + +|`/Producer` or `/Creator` names a known form-filling tool |`Strong` +|`blocky-writer`, `lopdf`, `iText`, `OpenPDF`, `pdf-lib`, `pypdf`, `PDFtk`, `PDFBox`, +`Aspose`, `PDFsharp`, `DocuSign`, `LiveCycle`. Only tools that *fill* forms are listed: a +blank template from a general PDF generator is still an empty form, not a machine-filled one. + +|Values present and `/NeedAppearances` set |`Probable` +|The classic programmatic signature: values were written and the viewer was told to rebuild +appearances. A person filling a form in a viewer gets appearance streams instead. + +|Values present with no `/AP` in the field's subtree |`Probable` +|Same signature, read structurally. Widget annotations are searched as well as the field +dictionary, because the value and its appearance often live apart. + +|Values present alongside `/DV` defaults |`Speculative` +|Interactive viewers write `/V` only. Rewriting `/DV` too is a programmatic habit — this is +what `fill_blocks` does — but it is weak on its own. + +|Values present, appearance streams present, no signal above |`Probable` (human) +|What a viewer produces when a person completes a form. + +|`/AcroForm` present, no field carries a value |`Strong` (empty) +|`/Off` on a check box or radio button counts as no value. + +|No `/AcroForm` |`Strong` (not a form) +|Nothing to say. +|=== + +Guards: documents over 8 MiB are not inspected; the field-tree walk is bounded at 100,000 +nodes and depth 32; every parse failure is recorded as "not inspected" with the reason. + +== What was implemented + +[cols="1,3", options="header"] +|=== +|Change |Where + +|`FormOrigin`, `ProvenanceConfidence`, `FormProvenance`, `FormProvenancePolicy` +|`crates/presswerk-core/src/types.rs` + +|`AppConfig::form_provenance_policy`, defaulting to `Record` +|`crates/presswerk-core/src/config.rs` + +|Detector: `classify_for_print`, `classify_pdf`, `classify_document` +|`crates/presswerk-document/src/pdf/form.rs` + +|`form_origin` and `form_provenance` columns on the job queue, with migration for existing +databases, plus `get_jobs_with_form_origin` +|`crates/presswerk-print/src/queue.rs` + +|Network print path: classify on receipt, store on the job, write to the audit trail, hold +under the hold policy, report `job-state` as held rather than pending +|`crates/presswerk-print/src/ipp_server.rs` + +|Local print path: classify, store, audit, hold, and `release_job` to send a held job +|`crates/presswerk-app/src/services/app_services.rs` + +|Provenance badge on the jobs page, a `Release` action for held jobs, a plain-language +summary in the audit trail, and a policy selector in Settings +|`crates/presswerk-app/src/pages/` +|=== + +`presswerk-print` now depends on `presswerk-document`. This is a deliberate change to the +documented crate graph: the network print path must classify documents itself, because a job +received over IPP never passes through the application layer. There is no cycle +(`presswerk-document` depends only on `presswerk-core`). + +=== Audit trail entries + +[cols="1,3", options="header"] +|=== +|Action |When + +|`form_provenance` |The document is an interactive form. `details` holds the full +determination as JSON, which the UI renders as a one-line summary. + +|`job_held_machine_filled` |The hold policy parked the job. + +|`job_released` |The user released a held job for printing. +|=== + +Documents that are not forms, and documents that could not be inspected, produce no +`form_provenance` entry: the trail stays about things that happened. + +=== Routing policy + +[cols="1,3", options="header"] +|=== +|Policy |Behaviour + +|`Off` |Do not inspect. Nothing is recorded. + +|`Record` *(default)* |Inspect, store on the job, write to the audit trail. Routing unchanged. + +|`HoldForReview` |As `Record`, plus a job with a positive `Machine` determination is stored +with status `Held` instead of being sent. The user releases it from the jobs page. +|=== + +For the embedded IPP server the policy is copied into the connection state at start-up, so a +change in Settings takes effect the next time the server starts. This is stated in the UI. + +== Known limitations + +Recorded here rather than left to be discovered. + +[arabic] +. *Without an upstream marker this is inference.* A tool that fills a form and then generates +appearance streams for every field, sets no `/NeedAppearances`, and rewrites no `/DV`, is +indistinguishable from a person using a viewer. Presswerk will call it `Human` with +`Probable` confidence. Only a marker fixes this — see the convention above. +. *A marker is a claim, not a proof.* `/FillOrigin` can be written by anyone who can write a +PDF. It raises confidence to `Explicit`; it does not authenticate the writer. Signing that +claim is a separate problem (and a good candidate for `blocky-writer`, which already holds the +key material context). +. *Held documents are stored in plaintext.* `release_job` needs the bytes later, so a held +job's document is written to the data directory. Network-received documents were already +persisted this way. Encrypting the document store is a separate, larger piece of work +(`presswerk-security` has the primitives) and is not done here. +. *Compressed XMP packets are not decoded.* Use the structured `/Info` marker. +. *Only PDFs are inspected.* A filled form that arrives as a scan or an image has no field +structure left to read, and is recorded as "not inspected". + +== Ask of upstream + +To `blocky-writer`: write `/FillOrigin (machine)` and `/FillTool (blocky-writer )` +into the document information dictionary in `fill_blocks`. Two dictionary entries, no change +to `/V`, `/DV` or `/AS`, no re-render, no re-compression. That single change turns every +downstream determination from `Probable` into `Explicit`, for Presswerk and for anyone else +who adopts the same convention. + +== References + +* https://github.com/hyperpolymath/presswerk/issues/118[presswerk#118] — the question +* https://github.com/hyperpolymath/standards/issues/787#issuecomment-5913118060[standards#787] +— ruling D189 +* https://github.com/hyperpolymath/blocky-writer/pull/73[blocky-writer#73] — the writer that +prompted it +* ISO 32000-1 §12.7 (interactive forms), §12.7.4 (field values), §14.3 (metadata) From 2c183287630178bc0ba8064fc7f4b670a784f119 Mon Sep 17 00:00:00 2001 From: hyperpolymath <6759885+hyperpolymath@users.noreply.github.com> Date: Sun, 4 Oct 2026 01:36:46 +0000 Subject: [PATCH 2/2] ci: install a real Rust toolchain so check/test/clippy/fmt actually run MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `dtolnay/rust-toolchain@v1` takes a rustup toolchain specification (channel or version). It was being passed `v1`, which is not one, so the step failed on every run — on main as well as on every branch — and Check, Test, Clippy and Format check were all skipped. This repository's CI has not compiled the workspace since that input was written. Separate commit from the feature work so it can be reverted on its own. The action references are unchanged, so actions.lock still matches. Co-authored-by: arena-agent <297053741+arena-agent@users.noreply.github.com> --- .github/workflows/ci.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0b7f880..253a139 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,7 +24,11 @@ jobs: - uses: actions/checkout@v7.0.1 - uses: dtolnay/rust-toolchain@v1 with: - toolchain: v1 + # Was `v1`, which is not a rustup toolchain specification: the step + # failed on every run (including main) before a single cargo command + # ran, so check/test/clippy/fmt never executed. See the action's + # `toolchain` input: a channel or version, e.g. stable / 1.85.0. + toolchain: stable components: clippy, rustfmt - uses: Swatinem/rust-cache@v2.9.2