From fc6246380cb30980721cc8647e80b48a64dbf79e Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Fri, 10 Jul 2026 01:16:21 +0000 Subject: [PATCH] fix(queue): classify persistent host-FS I/O errors (is_host_io_error) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Port OpenHuman host commit d7bee77e3 (drift ledger D2). Adds an `is_host_io_error` classifier to the queue worker's error family: EIO (5) / ENOSPC (28) / EROFS (30) surfaced as a `std::io::Error` — a dying SD card, a full disk, or a kernel-remounted-read-only mount. These are persistent, user-only-fixable host conditions that never clear on their own, so a host loop backs off long and pages once instead of flooding Sentry (~10k events/50min, CORE-RUST-19J). Matches the typed downcast and the flattened `(os error N)` text through anyhow context layers. EACCES (13) / ENOENT (2) are excluded (genuine bugs that must keep reporting); SQLITE_FULL stays in is_sqlite_disk_full. Only the predicate ports — the Sentry-once emission and the storage-degraded flag remain host-owned (drift ledger D2-host). --- src/memory/queue/worker.rs | 27 ++++++++++++++++++ src/memory/queue/worker_tests.rs | 49 ++++++++++++++++++++++++++++++++ 2 files changed, 76 insertions(+) diff --git a/src/memory/queue/worker.rs b/src/memory/queue/worker.rs index 3295ef2..9fc36d1 100644 --- a/src/memory/queue/worker.rs +++ b/src/memory/queue/worker.rs @@ -11,6 +11,11 @@ //! The SQLite error classifiers ([`is_sqlite_busy`] et al.) are ported verbatim //! so a host loop can reproduce OpenHuman's "back off, don't page" policy for //! transient write-lock / I/O / disk-full / corruption conditions. +//! [`is_host_io_error`] extends that family to persistent **host-filesystem** +//! failures (EIO/ENOSPC/EROFS — a dying SD card or full/read-only mount) that +//! surface as `std::io::Error` rather than a SQLite code; the host uses it to +//! back off and page once instead of flooding (Sentry CORE-RUST-19J). The +//! Sentry-once emission and the storage-degraded flag stay host-owned. use std::sync::LazyLock; @@ -155,6 +160,28 @@ pub fn is_sqlite_corrupt(err: &anyhow::Error) -> bool { msg.contains("database disk image is malformed") || msg.contains("file is not a database") } +/// Classify a persistent **host-filesystem** I/O failure (EIO `5`, ENOSPC `28`, +/// EROFS `30`) surfaced as a `std::io::Error` — a dying SD card, a full disk, or +/// a kernel-remounted-read-only mount. These are user-only-fixable and never +/// clear on their own, so a host loop should back off long and page **once** +/// rather than re-poll and flood (Sentry CORE-RUST-19J). +/// +/// Distinct from [`is_sqlite_disk_full`]: `SQLITE_FULL` arrives as a SQLite code +/// and stays in that arm; this family is the raw OS error bubbling out of +/// `create_dir_all` / `File` operations, often through anyhow context layers, +/// so both the typed downcast and the flattened `(os error N)` text are checked. +/// EACCES (`13`) and ENOENT (`2`) are deliberately excluded — those are genuine +/// bugs that must keep reporting. +pub fn is_host_io_error(err: &anyhow::Error) -> bool { + if let Some(io_err) = err.downcast_ref::() { + if matches!(io_err.raw_os_error(), Some(5) | Some(28) | Some(30)) { + return true; + } + } + let msg = format!("{err:#}").to_ascii_lowercase(); + msg.contains("(os error 5)") || msg.contains("(os error 28)") || msg.contains("(os error 30)") +} + #[cfg(test)] #[path = "worker_tests.rs"] mod tests; diff --git a/src/memory/queue/worker_tests.rs b/src/memory/queue/worker_tests.rs index 7989d83..de4fecd 100644 --- a/src/memory/queue/worker_tests.rs +++ b/src/memory/queue/worker_tests.rs @@ -224,3 +224,52 @@ fn is_sqlite_corrupt_matches_code_notadb_context_text() { "database or disk is full" ))); } + +/// EIO (`5`), ENOSPC (`28`), and EROFS (`30`) are the persistent, user-only- +/// fixable host-FS family: a `std::io::Error` bubbling out of a filesystem call +/// classifies as host I/O whether it arrives typed, through anyhow context +/// layers, or flattened to its `(os error N)` text (Sentry CORE-RUST-19J). +#[test] +fn is_host_io_error_matches_family_code_context_text() { + for code in [5, 28, 30] { + let err = anyhow::Error::from(std::io::Error::from_raw_os_error(code)); + assert!( + is_host_io_error(&err), + "os error {code} must classify as host I/O" + ); + } + // The production shape: an io::Error wrapped in .with_context() twice; the + // downcast must still find it through the anyhow context chain. + let wrapped = anyhow::Error::from(std::io::Error::from_raw_os_error(5)) + .context("Failed to create memory_tree dir: /home/x/workspace/memory_tree") + .context("with_connection closure failed"); + assert!(is_host_io_error(&wrapped)); + // Text fallback: no io::Error to downcast (flattened to a plain string), the + // os-error-number anchor still classifies it. + assert!(is_host_io_error(&anyhow::anyhow!( + "Failed to create memory_tree dir: /home/x/workspace/memory_tree: \ + Input/output error (os error 5)" + ))); +} + +/// EACCES (`13`, a permission bug), ENOENT (`2`), `SQLITE_FULL` (its own arm), +/// and unrelated errors must NOT be swallowed as host I/O — they are real bugs +/// or handled elsewhere and must keep reporting. +#[test] +fn is_host_io_error_negatives() { + assert!(!is_host_io_error(&anyhow::Error::from( + std::io::Error::from_raw_os_error(13) + ))); + assert!(!is_host_io_error(&anyhow::Error::from( + std::io::Error::from_raw_os_error(2) + ))); + // SQLITE_FULL stays in is_sqlite_disk_full's arm, not here. + assert!(!is_host_io_error(&sqlite_failure( + rusqlite::ErrorCode::DiskFull, + 13, + "database or disk is full" + ))); + assert!(!is_host_io_error(&anyhow::anyhow!( + "upstream returned 500: internal server error" + ))); +}