From 8afb332edfe8294e252b9e6b4a1156d83f339c68 Mon Sep 17 00:00:00 2001 From: AJ Slater Date: Tue, 25 Aug 2026 20:29:18 -0700 Subject: [PATCH 1/2] fix(watcher): don't delete a library that is only unmounted MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A dropped network share, an ejected volume, or a docker bind mount that didn't come up presents as an empty or missing directory rather than an error, so every comic in the library looks deleted at once. Acting on that removes every row and cascades away every bookmark and reading position in the library — for files that are perfectly fine and will be back as soon as the mount is. The poller has refused to scan a library in that state for a long time, by three separate checks. The watcher had none: it already holds the events, so it deleted. The delete phase's existence check can't help either, because while the mount is gone the files genuinely are unreachable. The watcher now consults the same checks before acting on any task that carries deletes. Adds and modifies are left alone; they can't destroy anything. Those checks move to ``codex.librarian.fs.mounted`` so both scanners share one definition of what a vanished library looks like, rather than one of them growing a defense the other never hears about. Co-Authored-By: Claude Fable 5 --- codex/librarian/fs/mounted.py | 31 +++++ codex/librarian/fs/poller/poller.py | 20 +-- codex/librarian/fs/watcher/watcher.py | 38 +++++- tests/test_watcher_unmount_guard.py | 172 ++++++++++++++++++++++++++ 4 files changed, 243 insertions(+), 18 deletions(-) create mode 100644 codex/librarian/fs/mounted.py create mode 100644 tests/test_watcher_unmount_guard.py diff --git a/codex/librarian/fs/mounted.py b/codex/librarian/fs/mounted.py new file mode 100644 index 000000000..25ed50d97 --- /dev/null +++ b/codex/librarian/fs/mounted.py @@ -0,0 +1,31 @@ +""" +Recognize a library root that isn't really there. + +A dropped network share, an ejected volume, or a docker bind mount that +didn't come up presents as an empty (or missing) directory rather than an +error. Every comic in the library then looks deleted at once, and acting +on that removes the rows and cascades their bookmarks away — for files +that are perfectly fine and will be back as soon as the mount is. + +The delete-phase existence check cannot help here: while the mount is +gone the files genuinely are unreachable. The only defense is to notice +the shape of the failure and refuse to act, which is what both scanners +do with this. +""" + +from pathlib import Path + +#: Docker bind mounts of a missing host path can be seeded with this file +#: so an unmounted volume is distinguishable from an empty library. +DOCKER_UNMOUNTED_FN = "DOCKER_UNMOUNTED_VOLUME" + + +def unmounted_reason(root: Path) -> str: + """Return why this library root looks unmounted, or "" if it looks fine.""" + if not root.is_dir(): + return "is not there" + if (root / DOCKER_UNMOUNTED_FN).exists(): + return "looks like an unmounted docker volume" + if not any(root.iterdir()): + return "is empty. Suspect unmounted" + return "" diff --git a/codex/librarian/fs/poller/poller.py b/codex/librarian/fs/poller/poller.py index b4c8277e3..ce4ec8845 100644 --- a/codex/librarian/fs/poller/poller.py +++ b/codex/librarian/fs/poller/poller.py @@ -10,6 +10,7 @@ from humanize import naturaldelta from codex.librarian.fs.import_task import build_import_task +from codex.librarian.fs.mounted import unmounted_reason from codex.librarian.fs.poller.snapshot import DatabaseSnapshot, DiskSnapshot from codex.librarian.fs.poller.snapshot_diff import SnapshotDiff from codex.librarian.fs.poller.status import FSPollStatus @@ -19,7 +20,6 @@ from codex.models import Library from codex.views.const import EPOCH_START -DOCKER_UNMOUNTED_FN = "DOCKER_UNMOUNTED_VOLUME" _DIR_NOT_FOUND_TIMEOUT = 15 * 60 _LIBRARY_ONLY = ( "path", @@ -67,32 +67,20 @@ def stop(self) -> None: # Timeout computation # ####################### - def _get_poll_timeout(self, library: Library) -> float | None: # noqa: PLR0911 + def _get_poll_timeout(self, library: Library) -> float | None: """ Compute seconds until this library's next scheduled poll. Returns None to wait forever (manual poll only). """ watch_path = Path(library.path) - unmounted_marker = watch_path / DOCKER_UNMOUNTED_FN if not library.poll: self.log.info(f"Library {library.path} waiting for manual poll.") return None - if not watch_path.is_dir(): - self.log.warning(f"Library {library.path} not found. Not polling.") - return _DIR_NOT_FOUND_TIMEOUT - - if unmounted_marker.exists(): - warning = f"Library {library.path} looks like an unmounted docker volume. Not polling." - self.log.warning(warning) - return _DIR_NOT_FOUND_TIMEOUT - - if not tuple(watch_path.iterdir()): - self.log.warning( - f"{library.path} is empty. Suspect unmounted. Not polling." - ) + if reason := unmounted_reason(watch_path): + self.log.warning(f"Library {library.path} {reason}. Not polling.") return _DIR_NOT_FOUND_TIMEOUT if library.update_in_progress: diff --git a/codex/librarian/fs/watcher/watcher.py b/codex/librarian/fs/watcher/watcher.py index 12b175e9f..52e8c5945 100644 --- a/codex/librarian/fs/watcher/watcher.py +++ b/codex/librarian/fs/watcher/watcher.py @@ -9,6 +9,7 @@ from codex.librarian.fs.filters import is_ignored_path, match_comic from codex.librarian.fs.import_task import build_import_task +from codex.librarian.fs.mounted import unmounted_reason from codex.librarian.fs.watcher.events import process_changes from codex.librarian.fs.watcher.status import FSWatcherRestartStatus from codex.librarian.threads import NamedThread @@ -127,8 +128,41 @@ def _process_changes(self, changes: set[tuple[Change, str]]) -> None: for library_pk, events in events_by_library.items(): task = build_import_task(library_pk, events) - if task is not None: - self.librarian_queue.put(task) + if task is None: + continue + if self._is_a_vanished_library(task): + continue + self.librarian_queue.put(task) + + def _is_a_vanished_library(self, task) -> bool: + """ + Whether this task's deletes are really an unmounted library. + + A dropped share or volume presents every comic in the library as + deleted at once. The poller refuses to scan a library in that + state; the watcher already holds the events, so it has to refuse + to act on them. Only deletes are worth checking — an add or a + modify against a missing mount can't do damage. + """ + if not (task.files_deleted or task.dirs_deleted or task.covers_deleted): + return False + root = self._library_root(task.library_id) + if root is None: + return False + reason = unmounted_reason(root) + if not reason: + return False + self.log.warning( + f"Library {root} {reason}. Ignoring the deletes it just reported." + ) + return True + + def _library_root(self, library_pk: int) -> Path | None: + """Return a watched library's root path.""" + for path, pk in self._library_paths.items(): + if pk == library_pk: + return Path(path) + return None def _get_extant_paths(self, paths: list[str]) -> list[str]: extant_paths = [] diff --git a/tests/test_watcher_unmount_guard.py b/tests/test_watcher_unmount_guard.py new file mode 100644 index 000000000..87d3c7838 --- /dev/null +++ b/tests/test_watcher_unmount_guard.py @@ -0,0 +1,172 @@ +""" +A library that isn't really there must not have its comics deleted. + +A dropped network share, an ejected volume, or a docker bind mount that +didn't come up presents as an empty or missing directory, so every comic +under it looks deleted at once. The poller refuses to scan in that state; +the watcher already holds the events, so it has to refuse to act on them. +The delete-phase existence check cannot help — while the mount is gone +the files really are unreachable. +""" + +from __future__ import annotations + +import shutil +from pathlib import Path +from typing import Any, Final, override + +from django.test import TestCase +from loguru import logger +from watchfiles import Change + +from codex.librarian.fs.mounted import DOCKER_UNMOUNTED_FN, unmounted_reason +from codex.librarian.fs.watcher.watcher import LibraryWatcherThread +from codex.librarian.scribe.importer.tasks import ImportTask + +_ROOT: Final = Path("/tmp/codex.tests.unmount") # noqa: S108 +_LIBRARY_PK: Final = 1 + + +def _double(stub: object) -> Any: + """Pass a test double through a concretely-typed seam.""" + return stub + + +class _ListQueue: + """Records what the watcher queues.""" + + def __init__(self, items: list) -> None: + self.items = items + + def put(self, item) -> None: + self.items.append(item) + + +def _watcher() -> LibraryWatcherThread: + """Build a watcher without its threading machinery.""" + watcher = LibraryWatcherThread.__new__(LibraryWatcherThread) + watcher.log = _double(logger) + watcher._library_paths = {str(_ROOT): _LIBRARY_PK} # noqa: SLF001 + return watcher + + +def _delete_task() -> ImportTask: + return ImportTask( + library_id=_LIBRARY_PK, + files_deleted=frozenset({str(_ROOT / "a.cbz")}), + ) + + +class UnmountedReasonTests(TestCase): + """The shared check both scanners consult.""" + + @override + def setUp(self) -> None: + shutil.rmtree(_ROOT, ignore_errors=True) + _ROOT.mkdir(parents=True) + + @override + def tearDown(self) -> None: + shutil.rmtree(_ROOT, ignore_errors=True) + + def test_a_populated_directory_looks_mounted(self) -> None: + (_ROOT / "a.cbz").write_text("comic") + + assert not unmounted_reason(_ROOT) + + def test_a_missing_directory_is_flagged(self) -> None: + shutil.rmtree(_ROOT) + + assert "not there" in unmounted_reason(_ROOT) + + def test_an_empty_directory_is_flagged(self) -> None: + assert "empty" in unmounted_reason(_ROOT) + + def test_the_docker_marker_is_flagged(self) -> None: + (_ROOT / DOCKER_UNMOUNTED_FN).write_text("") + + assert "docker" in unmounted_reason(_ROOT) + + +class WatcherUnmountGuardTests(TestCase): + """Deletes from a vanished library never reach the queue.""" + + @override + def setUp(self) -> None: + shutil.rmtree(_ROOT, ignore_errors=True) + _ROOT.mkdir(parents=True) + + @override + def tearDown(self) -> None: + shutil.rmtree(_ROOT, ignore_errors=True) + + def test_deletes_are_dropped_when_the_library_is_empty(self) -> None: + """An empty root means the mount is gone, not that every comic is.""" + assert _watcher()._is_a_vanished_library(_delete_task()) # noqa: SLF001 + + def test_deletes_are_dropped_when_the_root_is_missing(self) -> None: + shutil.rmtree(_ROOT) + + assert _watcher()._is_a_vanished_library(_delete_task()) # noqa: SLF001 + + def test_real_deletes_still_pass(self) -> None: + """A library with other comics still in it is really deleting one.""" + (_ROOT / "b.cbz").write_text("comic") + + assert not _watcher()._is_a_vanished_library(_delete_task()) # noqa: SLF001 + + def test_a_task_without_deletes_is_never_blocked(self) -> None: + """Adds and modifies can't destroy anything, so they are not checked.""" + task = ImportTask( + library_id=_LIBRARY_PK, + files_modified=frozenset({str(_ROOT / "a.cbz")}), + ) + + assert not _watcher()._is_a_vanished_library(task) # noqa: SLF001 + + def test_an_unknown_library_is_not_blocked(self) -> None: + """Without a root to check, the guard stays out of the way.""" + task = ImportTask( + library_id=999, files_deleted=frozenset({str(_ROOT / "a.cbz")}) + ) + + assert not _watcher()._is_a_vanished_library(task) # noqa: SLF001 + + +class WatcherProcessChangesTests(TestCase): + """The guard is wired into the path that queues the work.""" + + @override + def setUp(self) -> None: + shutil.rmtree(_ROOT, ignore_errors=True) + _ROOT.mkdir(parents=True) + + @override + def tearDown(self) -> None: + shutil.rmtree(_ROOT, ignore_errors=True) + + @staticmethod + def _run(queue: list) -> None: + watcher = _watcher() + watcher.librarian_queue = _double(_ListQueue(queue)) + watcher._process_changes( # noqa: SLF001 + {(Change.deleted, str(_ROOT / "a.cbz"))} + ) + + def test_an_empty_library_queues_nothing(self) -> None: + """The whole library looking deleted never reaches the importer.""" + queued: list = [] + + self._run(queued) + + assert not queued + + def test_a_populated_library_queues_the_delete(self) -> None: + """A real delete is still reported.""" + (_ROOT / "b.cbz").write_text("comic") + queued: list = [] + + self._run(queued) + + assert len(queued) == 1 + assert queued[0].files_deleted == {str(_ROOT / "a.cbz")} From cae7cd5867adea77ce9c8449358ee6672c34418f Mon Sep 17 00:00:00 2001 From: AJ Slater Date: Tue, 25 Aug 2026 20:29:18 -0700 Subject: [PATCH 2/2] docs(news): unmounted library guard in v2.2.11 Co-Authored-By: Claude Fable 5 --- NEWS.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/NEWS.md b/NEWS.md index 1090e6f25..53f5e0e0f 100644 --- a/NEWS.md +++ b/NEWS.md @@ -23,6 +23,9 @@ border-radius: 128px; a CBR conversion even when a library scan lands in the middle of one. - Editing a comic's tags twice in a row no longer fails the second edit with a "no such file" error when renaming is on. + - A watched library on a network share or removable volume that goes missing + no longer has all its comics deleted. Polling already refused to scan in + that state; watching now refuses to act on it too. - Comics are never deleted from the database while their files are still on disk, so a misread filesystem event can no longer take a comic's bookmarks and read progress with it.