From 301e7ef9f1b0b1d5bb18622cd568da230b2f30c6 Mon Sep 17 00:00:00 2001 From: colombod Date: Mon, 24 Aug 2026 13:38:16 +0000 Subject: [PATCH] refactor(storage): isolate blob, queue, and identity storage behind backend-neutral protocols MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Transplant PR #76's storage-API isolation onto PR #78's hardened durable queue so the two stack cleanly (78 -> 76). Every store is now reached only through a backend-neutral Protocol + factory; no consumer constructs a concrete backend or touches an on-disk path. queue_manager/ FileSystemQueueManager implements the QueueManager Protocol. Built on #78's authoritative body (the durable per-record cursor model): the class body is byte-identical to #78's queue_manager.py apart from the class rename and its self- references. protocol.py carries #78's Record/Batch verbatim; factory.create_queue_manager is the only queue backend selector. blob_store/ FileSystemBlobStore implements the BlobStore Protocol. write() returns a BlobReference (uri + size + last_modified); the store gains scan()/list() (async BlobReference iterators) and a fenced delete(uri, if_unmodified=ref). Adds settings.blob_backend. identity_store/ FileSystemIdentityStore implements the IdentityStore Protocol; the commit-order and fail-closed-load contract lives in the protocol. The backing path is private -- callers use exists(). Also folds in the blob-key fix: process_event includes tool_call_id in the blob-key node_id, so two same-millisecond parallel events no longer collide on one blob and silently overwrite each other. Consumers (registry, main, blob_processor, pipeline) build stores via the factories and pass URIs/references, never paths. The boot-reclaim dry-run log derives the blob path from the QueueManager's own queues_dir (fixing a latent mismatch when it differs from settings.queues_path); the identity first-boot warnings no longer echo the store path. The sole remaining config-path read outside the storage layer is registry.queues_dir_path -- the resolver the WriterLease boot detector uses precisely because it must not construct a QueueManager. Full non-neo4j suite: 2082 passed. Neo4j subsets (queue durability, blob ingest, identity auth): green. 🤖 Generated with [Amplifier](https://github.com/microsoft/amplifier) Co-Authored-By: Amplifier <240397093+microsoft-amplifier@users.noreply.github.com> --- context_intelligence_server/blob_processor.py | 4 +- context_intelligence_server/blob_store.py | 237 -------------- .../blob_store/__init__.py | 29 ++ .../blob_store/factory.py | 46 +++ .../blob_store/filesystem.py | 298 +++++++++++++++++ .../blob_store/protocol.py | 109 +++++++ context_intelligence_server/config.py | 6 + .../identity_store/__init__.py | 29 ++ .../identity_store/factory.py | 49 +++ .../filesystem.py} | 97 +++--- .../identity_store/protocol.py | 117 +++++++ context_intelligence_server/main.py | 36 +- context_intelligence_server/pipeline.py | 11 +- .../queue_manager/__init__.py | 31 ++ .../queue_manager/factory.py | 35 ++ .../filesystem.py} | 60 +--- .../queue_manager/protocol.py | 125 +++++++ context_intelligence_server/registry.py | 12 +- tests/conftest.py | 1 + tests/integration/test_crash_recovery.py | 6 +- tests/neo4j/test_concurrent_flush.py | 6 +- tests/neo4j/test_handler_flush_concurrency.py | 4 +- tests/neo4j/test_oom_regression.py | 4 +- tests/neo4j/test_orphan_visibility.py | 4 +- tests/neo4j/test_queues_actions.py | 4 +- .../neo4j/test_steady_state_reclaim_neo4j.py | 6 +- tests/routers/test_queues.py | 4 +- tests/test_blob_processor.py | 21 +- tests/test_blob_store.py | 308 ++++++++++++++---- tests/test_boot_safety.py | 50 +-- tests/test_concurrent_append.py | 6 +- tests/test_drain_lifecycle_logging.py | 8 +- tests/test_durable_append_framing.py | 43 +-- tests/test_identity_store.py | 48 +-- tests/test_m2_service_auth.py | 18 +- tests/test_main.py | 10 +- tests/test_pipeline.py | 191 ++++++++--- tests/test_queue_manager.py | 20 +- tests/test_registry.py | 20 +- tests/test_steady_state_reclaim.py | 44 +-- tests/test_writer_lease.py | 14 +- 41 files changed, 1539 insertions(+), 632 deletions(-) delete mode 100644 context_intelligence_server/blob_store.py create mode 100644 context_intelligence_server/blob_store/__init__.py create mode 100644 context_intelligence_server/blob_store/factory.py create mode 100644 context_intelligence_server/blob_store/filesystem.py create mode 100644 context_intelligence_server/blob_store/protocol.py create mode 100644 context_intelligence_server/identity_store/__init__.py create mode 100644 context_intelligence_server/identity_store/factory.py rename context_intelligence_server/{identity_store.py => identity_store/filesystem.py} (71%) create mode 100644 context_intelligence_server/identity_store/protocol.py create mode 100644 context_intelligence_server/queue_manager/__init__.py create mode 100644 context_intelligence_server/queue_manager/factory.py rename context_intelligence_server/{queue_manager.py => queue_manager/filesystem.py} (97%) create mode 100644 context_intelligence_server/queue_manager/protocol.py diff --git a/context_intelligence_server/blob_processor.py b/context_intelligence_server/blob_processor.py index bd7d8c25..5be7e33c 100644 --- a/context_intelligence_server/blob_processor.py +++ b/context_intelligence_server/blob_processor.py @@ -96,8 +96,8 @@ async def process_event_data( key = f"{node_id}__{field_name}" try: - uri = await blob_store.write(session_id, key, value) - data[field_name] = {"$blob_ref": uri} + ref = await blob_store.write(session_id, key, value) + data[field_name] = {"$blob_ref": ref.uri} except Exception as exc: # noqa: BLE001 logger.warning( "blob_offload_failed session=%s field=%s node=%s: %s", diff --git a/context_intelligence_server/blob_store.py b/context_intelligence_server/blob_store.py deleted file mode 100644 index 94511781..00000000 --- a/context_intelligence_server/blob_store.py +++ /dev/null @@ -1,237 +0,0 @@ -"""AsyncDiskBlobStore — async, disk-backed blob storage with ci-blob:// URIs. - -Disk layout: - //blobs/.json - -URI scheme: - ci-blob:/// - -All filesystem I/O is wrapped with ``asyncio.to_thread`` to keep the event -loop non-blocking. -""" - -from __future__ import annotations - -import asyncio -import json -import os -import shutil -import tempfile -from pathlib import Path -from typing import Any, Protocol, cast, runtime_checkable - -_SCHEME = "ci-blob://" - - -# --------------------------------------------------------------------------- -# BlobStore protocol -# --------------------------------------------------------------------------- - - -@runtime_checkable -class BlobStore(Protocol): - """Protocol for a session-scoped, URI-addressable blob store.""" - - async def write( - self, session_id: str, key: str, value: dict[str, Any] | list[Any] - ) -> str: - """Persist *value* as JSON and return a ``ci-blob://`` URI.""" - ... - - async def read(self, uri: str) -> dict[str, Any] | list[Any]: - """Resolve *uri* and return the stored value. - - Raises: - ValueError: If *uri* does not match the ``ci-blob://`` scheme. - FileNotFoundError: If no blob exists at the resolved path. - """ - ... - - async def list(self, session_id: str) -> list[str]: - """Return all blob URIs for *session_id*, sorted lexicographically.""" - ... - - async def dump(self, uri: str, dest_dir: Path | str | None = None) -> str: - """Copy the blob file addressed by *uri* to *dest_dir*. - - Args: - uri: ``ci-blob://`` URI identifying the blob to copy. - dest_dir: Destination directory. Defaults to - ``Path(tempfile.gettempdir()) / 'ci-blobs'``. - - Returns: - The destination file path as a string. - - Raises: - ValueError: If *uri* is not a valid ``ci-blob://`` URI. - FileNotFoundError: If no blob exists at the resolved path. - """ - ... - - -# --------------------------------------------------------------------------- -# AsyncDiskBlobStore -# --------------------------------------------------------------------------- - - -class AsyncDiskBlobStore: - """Async, disk-backed implementation of :class:`BlobStore`. - - Args: - root: Root directory under which all session blobs are stored. - """ - - def __init__(self, root: Path | str) -> None: - self._root = Path(root) - - # ------------------------------------------------------------------ - # Internal helpers - # ------------------------------------------------------------------ - - def _make_uri(self, session_id: str, key: str) -> str: - """Return the canonical ``ci-blob://`` URI for a session/key pair.""" - return f"{_SCHEME}{session_id}/{key}" - - def _parse_uri(self, uri: str) -> tuple[str, str]: - """Parse a ``ci-blob://`` URI into ``(session_id, key)``. - - Raises: - ValueError: If *uri* is not a valid ``ci-blob://`` URI. - """ - if not uri.startswith(_SCHEME): - raise ValueError( - f"Invalid URI scheme — expected '{_SCHEME}...', got: {uri!r}" - ) - remainder = uri[len(_SCHEME) :] - # remainder must be "/" — both parts non-empty - if "/" not in remainder: - raise ValueError(f"URI missing key component: {uri!r}") - session_id, _, key = remainder.partition("/") - if not session_id or not key: - raise ValueError(f"URI has empty session_id or key: {uri!r}") - return session_id, key - - def _blob_path(self, session_id: str, key: str) -> Path: - """Return the filesystem path for a given session/key blob.""" - return self._root / session_id / "blobs" / f"{key}.json" - - # ------------------------------------------------------------------ - # Public accessors (mirror of internal helpers for external callers) - # ------------------------------------------------------------------ - - def parse_uri(self, uri: str) -> tuple[str, str]: - """Public alias for :meth:`_parse_uri`.""" - return self._parse_uri(uri) - - def blob_path(self, session_id: str, key: str) -> Path: - """Public alias for :meth:`_blob_path`.""" - return self._blob_path(session_id, key) - - # ------------------------------------------------------------------ - # Async API - # ------------------------------------------------------------------ - - async def write( - self, session_id: str, key: str, value: dict[str, Any] | list[Any] - ) -> str: - """Persist *value* as JSON and return a ``ci-blob://`` URI. - - Creates the directory ``//blobs/`` if needed. - - Returns: - A ``ci-blob:///`` URI. - """ - path = self._blob_path(session_id, key) - - def _write() -> None: - path.parent.mkdir(parents=True, exist_ok=True) - data = json.dumps(value) - tmp_fd, tmp_name = tempfile.mkstemp( - dir=str(path.parent), prefix=f"{key}.", suffix=".tmp" - ) - try: - with os.fdopen(tmp_fd, "w", encoding="utf-8") as f: - f.write(data) - f.flush() - os.fsync(f.fileno()) - os.replace(tmp_name, path) - except BaseException: - try: - os.unlink(tmp_name) - except FileNotFoundError: - pass - raise - - await asyncio.to_thread(_write) - return self._make_uri(session_id, key) - - async def read(self, uri: str) -> dict[str, Any] | list[Any]: - """Return the blob addressed by *uri*. - - The session_id is resolved from the URI itself — callers do not - supply it separately (avoids the bundle footgun where the wrong - session_id is passed). - - Raises: - ValueError: If *uri* is not a valid ``ci-blob://`` URI. - FileNotFoundError: If no blob exists at the resolved path. - """ - session_id, key = self._parse_uri(uri) - path = self._blob_path(session_id, key) - - def _read() -> dict[str, Any] | list[Any]: - try: - return cast( - dict[str, Any] | list[Any], - json.loads(path.read_text(encoding="utf-8")), - ) - except FileNotFoundError: - raise FileNotFoundError(f"Blob not found: {uri!r} (path: {path})") - - return await asyncio.to_thread(_read) - - async def list(self, session_id: str) -> list[str]: - """Return all blob URIs for *session_id*, sorted lexicographically. - - Returns an empty list if the session directory does not exist. - """ - blobs_dir = self._root / session_id / "blobs" - - def _list() -> list[str]: - if not blobs_dir.exists(): - return [] - keys = sorted(p.stem for p in blobs_dir.glob("*.json")) - return [self._make_uri(session_id, key) for key in keys] - - return await asyncio.to_thread(_list) - - async def dump(self, uri: str, dest_dir: Path | str | None = None) -> str: - """Copy the blob file addressed by *uri* to *dest_dir*. - - Args: - uri: ``ci-blob://`` URI identifying the blob to copy. - dest_dir: Destination directory. Defaults to - ``Path(tempfile.gettempdir()) / 'ci-blobs'``. - - Returns: - The destination file path as a string. - - Raises: - ValueError: If *uri* is not a valid ``ci-blob://`` URI. - FileNotFoundError: If no blob exists at the resolved path. - """ - session_id, key = self._parse_uri(uri) - src = self._blob_path(session_id, key) - - if dest_dir is None: - dest_dir_path = Path(tempfile.gettempdir()) / "ci-blobs" - else: - dest_dir_path = Path(dest_dir) - - def _copy() -> str: - if not src.exists(): - raise FileNotFoundError(f"Blob not found: {uri!r}") - dest_dir_path.mkdir(parents=True, exist_ok=True) - return str(shutil.copy2(src, dest_dir_path)) - - return await asyncio.to_thread(_copy) diff --git a/context_intelligence_server/blob_store/__init__.py b/context_intelligence_server/blob_store/__init__.py new file mode 100644 index 00000000..0a9ecdac --- /dev/null +++ b/context_intelligence_server/blob_store/__init__.py @@ -0,0 +1,29 @@ +"""blob_store \u2014 session-scoped, URI-addressable blob storage. + +The public surface is the backend-neutral :class:`BlobStore` Protocol plus +:class:`BlobReference` / :class:`BlobNotFoundError` and the +:func:`create_blob_store` factory. Consumers should depend on these, never on +a concrete backend class. + +Package layout: + protocol.py BlobStore Protocol, BlobReference, BlobNotFoundError \u2014 the + backend-neutral seam (no filesystem imports). + filesystem.py FileSystemBlobStore \u2014 the disk-backed implementation. + factory.py create_blob_store(settings) \u2014 the ONLY place a backend is + selected and the ONLY place (besides config.py) that reads + settings.blob_path. +""" + +from __future__ import annotations + +from .factory import create_blob_store +from .filesystem import FileSystemBlobStore +from .protocol import BlobNotFoundError, BlobReference, BlobStore + +__all__ = [ + "BlobNotFoundError", + "BlobReference", + "BlobStore", + "FileSystemBlobStore", + "create_blob_store", +] diff --git a/context_intelligence_server/blob_store/factory.py b/context_intelligence_server/blob_store/factory.py new file mode 100644 index 00000000..adebde1b --- /dev/null +++ b/context_intelligence_server/blob_store/factory.py @@ -0,0 +1,46 @@ +"""Config-driven BlobStore factory \u2014 the ONLY place a blob-store backend is selected. + +This is the single seam through which the concrete backend is chosen. Adding +a new backend (e.g. Azure) means: one new module implementing +:class:`~.protocol.BlobStore`, one new branch here, and a config value \u2014 +zero changes to :mod:`context_intelligence_server.registry` or any consumer. + +This module (and :mod:`~context_intelligence_server.config`) are the only +places ``settings.blob_path`` is read \u2014 the on-disk root is a filesystem- +backend concern, resolved here and handed to the concrete backend at +construction time. Callers only ever see the :class:`~.protocol.BlobStore` +Protocol. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from .filesystem import FileSystemBlobStore +from .protocol import BlobStore + +if TYPE_CHECKING: + from context_intelligence_server.config import Settings + + +def create_blob_store(settings: Settings) -> BlobStore: + """Build the configured :class:`~.protocol.BlobStore` backend. + + Reads ``settings.blob_backend`` (default ``"filesystem"``) to select the + implementation: + + - ``"filesystem"``: :class:`~.filesystem.FileSystemBlobStore` rooted at + ``settings.blob_path``. + - ``"azure"``: not yet implemented. + - anything else: rejected as an unknown backend. + + Raises: + NotImplementedError: If ``blob_backend == "azure"`` (not yet built). + ValueError: If ``blob_backend`` names an unknown backend. + """ + backend = settings.blob_backend + if backend == "filesystem": + return FileSystemBlobStore(root=settings.blob_path) + if backend == "azure": + raise NotImplementedError("azure blob backend not yet implemented") + raise ValueError(f"Unknown blob_backend: {backend!r}") diff --git a/context_intelligence_server/blob_store/filesystem.py b/context_intelligence_server/blob_store/filesystem.py new file mode 100644 index 00000000..505508e2 --- /dev/null +++ b/context_intelligence_server/blob_store/filesystem.py @@ -0,0 +1,298 @@ +"""FileSystemBlobStore \u2014 async, disk-backed blob storage with ci-blob:// URIs. + +Disk layout: + //blobs/.json + +URI scheme: + ci-blob:/// + +All filesystem I/O is wrapped with ``asyncio.to_thread`` to keep the event +loop non-blocking. + +This is a concrete implementation of the :class:`~context_intelligence_server.blob_store.protocol.BlobStore` +Protocol. No ``Path``, on-disk layout, ``dest_dir``, or ``os.*`` detail +appears in the Protocol or in any value it returns \u2014 those details are +private to this class (and, later, an Azure equivalent). +""" + +from __future__ import annotations + +import asyncio +import json +import os +import shutil +import tempfile +from collections.abc import AsyncIterator +from pathlib import Path +from typing import Any, cast + +from .protocol import BlobNotFoundError, BlobReference + +_SCHEME = "ci-blob://" + + +class FileSystemBlobStore: + """Async, disk-backed implementation of :class:`~.protocol.BlobStore`. + + Args: + root: Root directory under which all session blobs are stored. + """ + + def __init__(self, root: Path | str) -> None: + self._root = Path(root) + + # ------------------------------------------------------------------ + # Internal helpers + # ------------------------------------------------------------------ + + def _make_uri(self, session_id: str, key: str) -> str: + """Return the canonical ``ci-blob://`` URI for a session/key pair.""" + return f"{_SCHEME}{session_id}/{key}" + + def _parse_uri(self, uri: str) -> tuple[str, str]: + """Parse a ``ci-blob://`` URI into ``(session_id, key)``. + + Raises: + ValueError: If *uri* is not a valid ``ci-blob://`` URI. + """ + if not uri.startswith(_SCHEME): + raise ValueError( + f"Invalid URI scheme \u2014 expected '{_SCHEME}...', got: {uri!r}" + ) + remainder = uri[len(_SCHEME) :] + # remainder must be "/" \u2014 both parts non-empty + if "/" not in remainder: + raise ValueError(f"URI missing key component: {uri!r}") + session_id, _, key = remainder.partition("/") + if not session_id or not key: + raise ValueError(f"URI has empty session_id or key: {uri!r}") + return session_id, key + + def _blob_path(self, session_id: str, key: str) -> Path: + """Return the filesystem path for a given session/key blob.""" + return self._root / session_id / "blobs" / f"{key}.json" + + # ------------------------------------------------------------------ + # Public accessors (mirror of internal helpers for external callers) + # ------------------------------------------------------------------ + + def parse_uri(self, uri: str) -> tuple[str, str]: + """Public alias for :meth:`_parse_uri`.""" + return self._parse_uri(uri) + + def blob_path(self, session_id: str, key: str) -> Path: + """Public alias for :meth:`_blob_path`.""" + return self._blob_path(session_id, key) + + # ------------------------------------------------------------------ + # Async API + # ------------------------------------------------------------------ + + async def write( + self, session_id: str, key: str, value: dict[str, Any] | list[Any] + ) -> BlobReference: + """Persist *value* as JSON and return a :class:`BlobReference`. + + Creates the directory ``//blobs/`` if needed. + ``last_modified`` is the storage mtime (from the same ``stat`` call + that produces ``size``) \u2014 never a writer-clock timestamp. + """ + path = self._blob_path(session_id, key) + + def _write() -> os.stat_result: + path.parent.mkdir(parents=True, exist_ok=True) + data = json.dumps(value) + tmp_fd, tmp_name = tempfile.mkstemp( + dir=str(path.parent), prefix=f"{key}.", suffix=".tmp" + ) + try: + with os.fdopen(tmp_fd, "w", encoding="utf-8") as f: + f.write(data) + f.flush() + os.fsync(f.fileno()) + os.replace(tmp_name, path) + except BaseException: + try: + os.unlink(tmp_name) + except FileNotFoundError: + pass + raise + return path.stat() + + st = await asyncio.to_thread(_write) + return BlobReference( + uri=self._make_uri(session_id, key), + session_id=session_id, + key=key, + size=st.st_size, + last_modified=st.st_mtime, + ) + + async def read(self, uri: str) -> dict[str, Any] | list[Any]: + """Return the blob addressed by *uri*. + + The session_id is resolved from the URI itself \u2014 callers do not + supply it separately (avoids the bundle footgun where the wrong + session_id is passed). + + Raises: + ValueError: If *uri* is not a valid ``ci-blob://`` URI. + BlobNotFoundError: If no blob exists for *uri*. Subclasses + ``FileNotFoundError`` for back-compat; the message carries + the URI only \u2014 never the on-disk path. + """ + session_id, key = self._parse_uri(uri) + path = self._blob_path(session_id, key) + + def _read() -> dict[str, Any] | list[Any]: + try: + return cast( + dict[str, Any] | list[Any], + json.loads(path.read_text(encoding="utf-8")), + ) + except FileNotFoundError: + raise BlobNotFoundError(f"Blob not found: {uri!r}") from None + + return await asyncio.to_thread(_read) + + async def list(self, session_id: str) -> AsyncIterator[BlobReference]: + """Stream all blob references for *session_id*. + + Yields nothing if the session's blobs directory does not exist. + The scandir + per-entry stat work is offloaded to a thread in small + units (per-entry), so the event loop stays responsive and references + stream out incrementally rather than blocking on the whole walk. + """ + blobs_dir = self._root / session_id / "blobs" + + def _list_entries() -> list[tuple[str, int, float]]: + if not blobs_dir.exists(): + return [] + entries: list[tuple[str, int, float]] = [] + with os.scandir(blobs_dir) as it: + for entry in it: + if not entry.name.endswith(".json"): + continue + st = entry.stat() + key = entry.name[: -len(".json")] + entries.append((key, st.st_size, st.st_mtime)) + entries.sort(key=lambda e: e[0]) + return entries + + entries = await asyncio.to_thread(_list_entries) + for key, size, last_modified in entries: + yield BlobReference( + uri=self._make_uri(session_id, key), + session_id=session_id, + key=key, + size=size, + last_modified=last_modified, + ) + + async def scan(self) -> AsyncIterator[BlobReference]: + """Stream all blob references across ALL sessions. + + Walks ``/*/blobs/*.json`` \u2014 session-dir enumeration and each + session's blob-dir scan are offloaded to a thread in small units + (never one giant ``to_thread`` for the whole tree), so references + stream out as they are discovered instead of materializing the + entire store in memory before yielding anything. + """ + + def _list_session_dirs() -> list[str]: + if not self._root.exists(): + return [] + with os.scandir(self._root) as it: + return sorted(entry.name for entry in it if entry.is_dir()) + + session_ids = await asyncio.to_thread(_list_session_dirs) + for session_id in session_ids: + async for ref in self.list(session_id): + yield ref + + async def delete( + self, uri: str, if_unmodified: BlobReference | None = None + ) -> bool: + """Delete the blob addressed by *uri*. + + Idempotent: returns ``False`` (never raises) if the blob is already + absent, ``True`` if it existed and was removed. + + Args: + uri: The ``ci-blob://`` URI to delete. + if_unmodified: When ``None`` (default), unconditional delete \u2014 + unlinks and returns ``True``, or ``False`` if already absent. + When provided, this is a **fenced compare-and-delete**: the + blob is re-``stat``'d (inside the same thread hop, right + before the unlink, to minimise the TOCTOU window) and the + delete only proceeds if ``st_mtime``/``st_size`` still match + *if_unmodified* \u2014 i.e. nothing rewrote the blob since it was + observed (e.g. by ``scan()``). If the blob is missing, or it + changed, the delete is refused and ``False`` is returned \u2014 + the blob is left untouched on disk. + """ + session_id, key = self._parse_uri(uri) + path = self._blob_path(session_id, key) + + def _delete() -> bool: + if if_unmodified is None: + try: + os.unlink(path) + return True + except FileNotFoundError: + return False + + # Fenced compare-and-delete: stat first, unlink only if unchanged. + try: + st = path.stat() + except FileNotFoundError: + return False + if ( + st.st_mtime != if_unmodified.last_modified + or st.st_size != if_unmodified.size + ): + # Blob was rewritten since it was observed \u2014 refuse to delete. + return False + try: + os.unlink(path) + return True + except FileNotFoundError: + # Deleted concurrently between our stat and unlink. + return False + + return await asyncio.to_thread(_delete) + + async def dump(self, uri: str, dest_dir: Path | str | None = None) -> str: + """Copy the blob file addressed by *uri* to *dest_dir*. + + Disk-only helper \u2014 NOT part of the :class:`~.protocol.BlobStore` + Protocol (no production caller; kept as a concrete convenience for + external tooling that needs a local export). + + Args: + uri: ``ci-blob://`` URI identifying the blob to copy. + dest_dir: Destination directory. Defaults to + ``Path(tempfile.gettempdir()) / 'ci-blobs'``. + + Returns: + The destination file path as a string. + + Raises: + ValueError: If *uri* is not a valid ``ci-blob://`` URI. + FileNotFoundError: If no blob exists at the resolved path. + """ + session_id, key = self._parse_uri(uri) + src = self._blob_path(session_id, key) + + if dest_dir is None: + dest_dir_path = Path(tempfile.gettempdir()) / "ci-blobs" + else: + dest_dir_path = Path(dest_dir) + + def _copy() -> str: + if not src.exists(): + raise FileNotFoundError(f"Blob not found: {uri!r}") + dest_dir_path.mkdir(parents=True, exist_ok=True) + return str(shutil.copy2(src, dest_dir_path)) + + return await asyncio.to_thread(_copy) diff --git a/context_intelligence_server/blob_store/protocol.py b/context_intelligence_server/blob_store/protocol.py new file mode 100644 index 00000000..a48c1eaa --- /dev/null +++ b/context_intelligence_server/blob_store/protocol.py @@ -0,0 +1,109 @@ +"""BlobStore Protocol \u2014 the backend-neutral seam. + +The only identity that crosses the boundary is the ``ci-blob:///`` +URI, carried by :class:`BlobReference`. No ``Path``, on-disk layout, ``dest_dir``, +or ``os.*`` detail appears here or in any value the Protocol returns \u2014 that is +private to a concrete backend (:class:`~context_intelligence_server.blob_store.filesystem.FileSystemBlobStore` +and, later, an Azure equivalent). + +This module is backend-neutral by construction: it imports nothing from +``os``, ``pathlib``, or any filesystem library. +""" + +from __future__ import annotations + +from collections.abc import AsyncIterator +from dataclasses import dataclass +from typing import Any, Protocol, runtime_checkable + +# --------------------------------------------------------------------------- +# BlobNotFoundError \u2014 backend-neutral missing-blob exception (guard #6) +# --------------------------------------------------------------------------- + + +class BlobNotFoundError(FileNotFoundError): + """Raised when a blob addressed by a ``ci-blob://`` URI does not exist. + + Subclasses :class:`FileNotFoundError` so existing ``except + FileNotFoundError`` callers keep working unchanged (zero caller churn). + The message carries the URI ONLY \u2014 never an on-disk path, container, or + account \u2014 so a future Azure backend can raise the same type/message + shape and no caller (or log line) ever learns which backend is in use. + """ + + +# --------------------------------------------------------------------------- +# BlobReference \u2014 cheap handle: identity + metadata, NO payload, NO Path +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class BlobReference: + """Cheap handle \u2014 identity + metadata, NO payload, NO Path. + + This is what ``scan()``/``list()`` return and what everything except a + payload read passes around. It is what gets serialized on the graph (as + its ``.uri``). + """ + + uri: str # ci-blob:/// \u2014 the ONLY address callers use + session_id: str + key: str + size: int # content length in bytes + last_modified: float # epoch seconds: disk st_mtime || azure Last-Modified + + +# --------------------------------------------------------------------------- +# BlobStore protocol +# --------------------------------------------------------------------------- + + +@runtime_checkable +class BlobStore(Protocol): + """Protocol for a session-scoped, URI-addressable blob store. + + 100% backend-neutral: the only identity that crosses the boundary is the + ``ci-blob://`` URI (carried by :class:`BlobReference`). No ``Path``, no + on-disk layout, no ``dest_dir``, no ``os.*`` \u2014 ever. + """ + + async def write( + self, session_id: str, key: str, value: dict[str, Any] | list[Any] + ) -> BlobReference: + """Persist *value* as JSON and return a :class:`BlobReference`.""" + ... + + def list(self, session_id: str) -> AsyncIterator[BlobReference]: + """Stream all blob references for *session_id* (one session).""" + ... + + def scan(self) -> AsyncIterator[BlobReference]: + """Stream all blob references across ALL sessions.""" + ... + + async def delete( + self, uri: str, if_unmodified: BlobReference | None = None + ) -> bool: + """Delete the blob addressed by *uri*. Idempotent: returns False if absent. + + Args: + uri: The ``ci-blob://`` URI to delete. + if_unmodified: When provided, this is a **fenced (compare-and-delete)** + delete \u2014 the store re-checks the blob's current metadata against + *if_unmodified* (disk: mtime + size; Azure: ``If-Match`` ETag) and + refuses (returns ``False``, does NOT delete) if the blob changed + since *if_unmodified* was observed (e.g. by a `scan()`). When + ``None`` (default), this is the unconditional idempotent delete. + """ + ... + + async def read(self, uri: str) -> dict[str, Any] | list[Any]: + """Resolve *uri* and return the stored value (the sole payload path). + + Raises: + ValueError: If *uri* does not match the ``ci-blob://`` scheme. + BlobNotFoundError: If no blob exists for *uri*. Subclasses + ``FileNotFoundError`` for back-compat; the message carries the + URI only \u2014 never an on-disk path, container, or account. + """ + ... diff --git a/context_intelligence_server/config.py b/context_intelligence_server/config.py index 6c776c9b..87dd5e0a 100644 --- a/context_intelligence_server/config.py +++ b/context_intelligence_server/config.py @@ -624,6 +624,12 @@ def _validate_neo4j_max_connection_lifetime(cls, v: float) -> float: # ------------------------------------------------------------------------- # Storage paths # ------------------------------------------------------------------------- + # blob_backend selects the BlobStore implementation via + # context_intelligence_server.blob_store.create_blob_store(); "filesystem" + # is the only backend implemented today. blob_path is that filesystem + # backend's own root -- read only by the factory (and here, at + # declaration) and otherwise meaningless to any other backend. + blob_backend: str = "filesystem" blob_path: str = "/data/blobs" queues_path: str = "/data/queues" diff --git a/context_intelligence_server/identity_store/__init__.py b/context_intelligence_server/identity_store/__init__.py new file mode 100644 index 00000000..1cf81b75 --- /dev/null +++ b/context_intelligence_server/identity_store/__init__.py @@ -0,0 +1,29 @@ +"""identity_store -- durable, write-through key -> contributor-identity map. + +The public surface is the backend-neutral :class:`IdentityStore` Protocol plus +the :func:`create_identity_store` factory. Consumers should depend on these, +never on a concrete backend class. + +Package layout: + protocol.py IdentityStore Protocol -- the backend-neutral seam (no + filesystem imports). AUTH-CRITICAL commit-order and + fail-closed-load guarantees are documented here. + filesystem.py FileSystemIdentityStore -- the JSON-file-backed + implementation. + factory.py create_identity_store(settings, kind) -- the ONLY place a + backend is selected and the ONLY place (besides config.py) + that reads settings.entra_identities_store_path / + settings.api_keys_store_path. +""" + +from __future__ import annotations + +from .factory import create_identity_store +from .filesystem import FileSystemIdentityStore +from .protocol import IdentityStore + +__all__ = [ + "FileSystemIdentityStore", + "IdentityStore", + "create_identity_store", +] diff --git a/context_intelligence_server/identity_store/factory.py b/context_intelligence_server/identity_store/factory.py new file mode 100644 index 00000000..e2d64c66 --- /dev/null +++ b/context_intelligence_server/identity_store/factory.py @@ -0,0 +1,49 @@ +"""Config-driven IdentityStore factory -- the ONLY place a backend is selected. + +This is the single seam through which the concrete backend is chosen. Adding +a new backend (e.g. Azure) means: one new module implementing +:class:`~.protocol.IdentityStore`, one new branch here -- zero changes to +:mod:`context_intelligence_server.main` or any other consumer. + +This module (and :mod:`~context_intelligence_server.config`) are the only +places ``settings.entra_identities_store_path`` / ``settings.api_keys_store_path`` +are read -- the on-disk location is a filesystem-backend concern, resolved +here and handed to the concrete backend at construction time. Callers only +ever see the :class:`~.protocol.IdentityStore` Protocol. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import TYPE_CHECKING + +from .filesystem import FileSystemIdentityStore +from .protocol import IdentityStore + +if TYPE_CHECKING: + from context_intelligence_server.config import Settings + + +def create_identity_store(settings: Settings, kind: str) -> IdentityStore: + """Build an :class:`~.protocol.IdentityStore` rooted at the configured path for *kind*. + + Construction only -- callers are responsible for ``load()`` / ``seed()`` + and any auth-mode wiring (this mirrors + ``blob_store.factory.create_blob_store``: a single-backend, config-reading + seam that keeps the store paths out of consumers such as ``main.py``). + + Args: + settings: The active ``Settings``. + kind: ``"entra"`` for the OID identity map, ``"api_key"`` for the + SHA-256 digest keystore. + + Raises: + ValueError: If *kind* is neither ``"entra"`` nor ``"api_key"``. + """ + if kind == "entra": + path = settings.entra_identities_store_path + elif kind == "api_key": + path = settings.api_keys_store_path + else: + raise ValueError(f"Unknown identity store kind: {kind!r}") + return FileSystemIdentityStore(Path(path)) diff --git a/context_intelligence_server/identity_store.py b/context_intelligence_server/identity_store/filesystem.py similarity index 71% rename from context_intelligence_server/identity_store.py rename to context_intelligence_server/identity_store/filesystem.py index a7da96e7..c18d92d6 100644 --- a/context_intelligence_server/identity_store.py +++ b/context_intelligence_server/identity_store/filesystem.py @@ -1,19 +1,29 @@ -"""Durable identity-map store for the Context Intelligence Server. +"""FileSystemIdentityStore -- disk-backed identity map with atomic writes. -Each ``IdentityStore`` wraps ONE JSON file and keeps an in-process dict -(``_data``) that IS the live source of truth for the single-replica process. -A second derived dict, ``flat_dict``, exposes ``{key: contributor_id}`` and is -kept in-sync with ``_data`` via in-place mutations so that any object holding a -reference to ``flat_dict`` always sees the latest state without a restart. +Each ``FileSystemIdentityStore`` wraps ONE JSON file and keeps an in-process +dict (``_data``) that IS the live source of truth for the single-replica +process. A second derived dict, ``flat_dict``, exposes ``{key: contributor_id}`` +and is kept in-sync with ``_data`` via in-place mutations so that any object +holding a reference to ``flat_dict`` always sees the latest state without a +restart. -**Commit order (ROB F2 — NON-NEGOTIABLE)** +This is a concrete implementation of the +:class:`~context_intelligence_server.identity_store.protocol.IdentityStore` +Protocol. No ``Path``, on-disk layout, or ``os.*`` detail appears in the +Protocol or in any value it returns -- those details are private to this +class (and, later, an Azure equivalent). + +**Commit order (ROB F2 -- NON-NEGOTIABLE)** On every mutation (put / delete): 1. Build the new data dict (do NOT touch ``_data`` yet). -2. Serialize and write to a tempfile **in the same directory** as the target file. -3. ``os.replace()`` the tempfile onto the target (atomic rename on POSIX / Azure Files). -4. **ONLY IF the above succeeds**: update ``_data`` and ``flat_dict`` in-place. +2. Serialize and write to a tempfile **in the same directory** as the target + file. +3. ``os.replace()`` the tempfile onto the target (atomic rename on POSIX / + Azure Files). +4. **ONLY IF the above succeeds**: update ``_data`` and ``flat_dict`` + in-place. If the file write raises for any reason, ``_data`` and ``flat_dict`` are **unchanged** and the exception propagates to the caller (who returns 5xx). @@ -23,13 +33,13 @@ On ``load()``: -- Missing file → empty dict (normal first boot). No log, no raise. -- Corrupt / torn / partial / invalid-JSON file → **empty dict + a LOUD - ``logger.error`` / ``logger.critical``**. The server MUST NOT crash-loop on - a bad store file. An empty map means "nobody is bound yet" — every auth +- Missing file -> empty dict (normal first boot). No log, no raise. +- Corrupt / torn / partial / invalid-JSON file -> **empty dict + a LOUD + ``logger.error`` / ``logger.critical``**. The server MUST NOT crash-loop on + a bad store file. An empty map means "nobody is bound yet" -- every auth attempt then fails normally until an admin re-populates via the /admin API. -File format (both modes share the same abstraction):: +File format (both modes share the same on-disk shape):: # api-keys.json { @@ -42,6 +52,8 @@ } """ +from __future__ import annotations + import json import logging import os @@ -51,18 +63,19 @@ logger = logging.getLogger(__name__) -class IdentityStore: +class FileSystemIdentityStore: """Durable, write-through identity map backed by a single JSON file. - See module docstring for the commit-order contract and fail-closed guarantees. + See module docstring for the commit-order contract and fail-closed + guarantees. Args: - path: Absolute path to the JSON store file. The parent directory is + path: Absolute path to the JSON store file. The parent directory is created automatically on the first write. """ def __init__(self, path: Path) -> None: - self.path = path + self._path = path # Rich format: {key: {id: ..., display_name?: ...}} self._data: dict[str, dict[str, str]] = {} # Flat derived cache: {key: contributor_id}. @@ -77,23 +90,23 @@ def __init__(self, path: Path) -> None: def load(self) -> None: """Read the file and populate the in-process map. - Missing file → empty dict (normal first boot, no log). - Corrupt / non-dict → empty dict + LOUD error log, never raise. + Missing file -> empty dict (normal first boot, no log). + Corrupt / non-dict -> empty dict + LOUD error log, never raise. """ - if not self.path.exists(): - # Normal first boot — the file hasn't been written yet. + if not self._path.exists(): + # Normal first boot -- the file hasn't been written yet. self._data = {} self._rebuild_flat() return raw: object try: - raw = json.loads(self.path.read_text(encoding="utf-8")) + raw = json.loads(self._path.read_text(encoding="utf-8")) except (json.JSONDecodeError, UnicodeDecodeError, OSError) as exc: logger.error( - "identity_store.load CORRUPT FILE path=%s error=%r — " + "identity_store.load CORRUPT FILE path=%s error=%r -- " "failing CLOSED to empty map. Re-populate via /admin API.", - self.path, + self._path, exc, ) self._data = {} @@ -102,9 +115,9 @@ def load(self) -> None: if not isinstance(raw, dict): logger.critical( - "identity_store.load INVALID FORMAT path=%s got=%r — " + "identity_store.load INVALID FORMAT path=%s got=%r -- " "expected a JSON object at top level. Failing CLOSED to empty map.", - self.path, + self._path, type(raw).__name__, ) self._data = {} @@ -118,15 +131,15 @@ def load(self) -> None: self._rebuild_flat() def put(self, key: str, value: dict[str, str]) -> None: - """Upsert *key* → *value*. + """Upsert *key* -> *value*. - Commit order (F2): write tempfile → os.replace → update in-process. + Commit order (F2): write tempfile -> os.replace -> update in-process. Raises on file-write failure; in-process state is UNCHANGED. """ new_data = dict(self._data) new_data[key] = value self._write_atomic(new_data) - # File write succeeded — now update in-process state. + # File write succeeded -- now update in-process state. self._data[key] = value contributor_id = value.get("id", "") if contributor_id: @@ -137,13 +150,13 @@ def put(self, key: str, value: dict[str, str]) -> None: def delete(self, key: str) -> None: """Remove *key* from the store. - Commit order (F2): write tempfile → os.replace → update in-process. + Commit order (F2): write tempfile -> os.replace -> update in-process. Raises on file-write failure; in-process state is UNCHANGED. No-op if *key* is not present. """ new_data = {k: v for k, v in self._data.items() if k != key} self._write_atomic(new_data) - # File write succeeded — now update in-process state. + # File write succeeded -- now update in-process state. self._data.pop(key, None) self.flat_dict.pop(key, None) @@ -166,12 +179,12 @@ def seed(self, data: dict[str, dict[str, str]]) -> None: except Exception as exc: logger.warning( "identity_store.seed: could not write seed to %s: %r " - "— in-memory map is live but the file is not yet persisted. " + "-- in-memory map is live but the file is not yet persisted. " "The next mutation via /admin API will persist the file.", - self.path, + self._path, exc, ) - # Update in-memory regardless — data is from durable config. + # Update in-memory regardless -- data is from durable config. self._data = dict(data) self._rebuild_flat() @@ -186,6 +199,10 @@ def items(self): # type: ignore[override] def __len__(self) -> int: return len(self._data) + def exists(self) -> bool: + """Whether the store has ever been persisted to its backing file.""" + return self._path.exists() + # ------------------------------------------------------------------ # Internals # ------------------------------------------------------------------ @@ -203,7 +220,7 @@ def _rebuild_flat(self) -> None: self.flat_dict[key] = contributor_id def _write_atomic(self, data: dict[str, dict[str, str]]) -> None: - """Write *data* atomically to ``self.path``. + """Write *data* atomically to ``self._path``. Steps: 1. Create parent directory (parents=True, exist_ok=True). @@ -216,15 +233,15 @@ def _write_atomic(self, data: dict[str, dict[str, str]]) -> None: Raises the underlying OS/IO exception so the caller (put/delete) knows the write failed and leaves in-process state unchanged. """ - self.path.parent.mkdir(parents=True, exist_ok=True) - tmp_fd, tmp_str = tempfile.mkstemp(dir=str(self.path.parent), suffix=".tmp") + self._path.parent.mkdir(parents=True, exist_ok=True) + tmp_fd, tmp_str = tempfile.mkstemp(dir=str(self._path.parent), suffix=".tmp") tmp_path = Path(tmp_str) try: with os.fdopen(tmp_fd, "w", encoding="utf-8") as fh: json.dump(data, fh, indent=2, ensure_ascii=False) fh.flush() os.fsync(fh.fileno()) - os.replace(str(tmp_path), str(self.path)) + os.replace(str(tmp_path), str(self._path)) except Exception: # Best-effort cleanup of the tempfile before propagating. try: diff --git a/context_intelligence_server/identity_store/protocol.py b/context_intelligence_server/identity_store/protocol.py new file mode 100644 index 00000000..3182eb69 --- /dev/null +++ b/context_intelligence_server/identity_store/protocol.py @@ -0,0 +1,117 @@ +"""IdentityStore Protocol -- the backend-neutral seam. + +Each ``IdentityStore`` wraps a durable ``key -> {"id": contributor_id, ...}`` +map and keeps an in-process, live-mutated view of it (:attr:`flat_dict`) so +that any object holding a reference (e.g. an auth resolver's keystore) always +sees the latest state without a restart. No ``Path``, on-disk layout, or +``os.*`` detail appears here or in any value the Protocol returns -- that is +private to a concrete backend +(:class:`~context_intelligence_server.identity_store.filesystem.FileSystemIdentityStore` +and, later, an Azure equivalent). + +This is AUTH-CRITICAL surface. The following guarantees are part of the +Protocol's contract and every backend MUST uphold them: + +**Commit order (ROB F2 -- NON-NEGOTIABLE)** + +On every mutation (``put`` / ``delete``): + +1. The durable store is written FIRST (whatever "durable" means for the + backend -- a file, a blob, etc.). +2. **ONLY IF** that write succeeds does in-process state (``flat_dict`` and + any internal map) get updated, IN-PLACE, so existing references to + ``flat_dict`` observe the change immediately. +3. If the durable write fails, in-process state is **UNCHANGED** and the + exception propagates to the caller (who returns 5xx). The durable store + and in-process memory are never out of sync. + +**Fail-CLOSED load()** + +On ``load()``: + +- Missing / never-persisted store -> empty map (normal first boot). No log, + no raise. +- Corrupt / unreadable store -> empty map + a LOUD error log. The server + MUST NOT crash-loop on a bad store. An empty map means "nobody is bound + yet" -- every auth attempt then fails normally until an admin re-populates + via the /admin API. + +**flat_dict is a live, shared view** + +``flat_dict`` is ``{key: contributor_id}``, mutated IN-PLACE on every +``put`` / ``delete`` / ``seed`` / ``load`` so any object holding a reference +to it always sees the latest state. + +This module is backend-neutral by construction: it imports nothing from +``os``, ``pathlib``, or any filesystem library. +""" + +from __future__ import annotations + +from collections.abc import ItemsView +from typing import Protocol, runtime_checkable + + +@runtime_checkable +class IdentityStore(Protocol): + """Protocol for a durable, write-through identity map. + + 100% backend-neutral: no ``Path``, no on-disk layout, no ``os.*`` -- + ever. + """ + + flat_dict: dict[str, str] + """Live derived view: ``{key: contributor_id}``. Shared BY REFERENCE with + consumers (e.g. auth resolvers) so mutations are visible with no restart.""" + + def load(self) -> None: + """Read the durable store and populate the in-process map. + + Fail-closed: missing store -> empty map (silent, normal first boot); + corrupt store -> empty map + a loud error log. Never raises. + """ + ... + + def put(self, key: str, value: dict[str, str]) -> None: + """Upsert ``key`` -> ``value``. + + Commit order (F2): durable write -> in-process update. Raises on + durable-write failure; in-process state is left unchanged. + """ + ... + + def delete(self, key: str) -> None: + """Remove ``key`` from the store. + + Commit order (F2): durable write -> in-process update. Raises on + durable-write failure; in-process state is left unchanged. No-op if + ``key`` is not present. + """ + ... + + def seed(self, data: dict[str, dict[str, str]]) -> None: + """Bulk-seed from config on first boot. + + Unlike ``put()`` (which enforces F2 write-before-memory strictly), + in-process state is updated even if the durable write fails -- the + data came from durable config, so memory-ahead-of-durable-store is + safe (a restart re-seeds from config again). A warning is logged if + the durable write fails. + """ + ... + + def get(self, key: str) -> dict[str, str] | None: + """Return the value for ``key``, or ``None`` if not present.""" + ... + + def items(self) -> ItemsView[str, dict[str, str]]: + """Iterate over ``(key, value)`` pairs in the store.""" + ... + + def __len__(self) -> int: + """Return the number of entries currently in the store.""" + ... + + def exists(self) -> bool: + """Whether the store has ever been persisted to its backing store.""" + ... diff --git a/context_intelligence_server/main.py b/context_intelligence_server/main.py index 912c3bb4..c1ffb22e 100644 --- a/context_intelligence_server/main.py +++ b/context_intelligence_server/main.py @@ -13,7 +13,6 @@ from dataclasses import dataclass from datetime import datetime from functools import partial -from pathlib import Path from typing import Any from fastapi import Depends, FastAPI, HTTPException, Request @@ -32,13 +31,16 @@ require_read, require_write, ) -from context_intelligence_server.blob_store import AsyncDiskBlobStore +from context_intelligence_server.blob_store import create_blob_store from context_intelligence_server.config import Neo4jClientConfig, Settings, get_settings from context_intelligence_server.idempotency import ( EventIdempotencyCache, KeyedAsyncLocks, ) -from context_intelligence_server.identity_store import IdentityStore +from context_intelligence_server.identity_store import ( + IdentityStore, + create_identity_store, +) from context_intelligence_server.logging_config import setup_logging from context_intelligence_server.models import ( CypherRequest, @@ -458,7 +460,7 @@ async def _boot_reclaim() -> None: logger.warning( "boot_reclaimed reason=%s path=%s session=%s bytes=%d action=dry_run", c.reason, - Path(settings.queues_path) / f"{key}.log", + qm.queues_dir / f"{key}.log", key, c.size, ) @@ -843,9 +845,9 @@ def create_asgi_app( if s.auth_mode == "entra": # Build and load the entra identity store. - entra_store = IdentityStore(Path(s.entra_identities_store_path)) + entra_store = create_identity_store(s, "entra") entra_store.load() - if not entra_store.path.exists(): + if not entra_store.exists(): # First boot: seed from config, converting flat {oid: contributor_id} # to the rich {oid: {"id": contributor_id}} format IdentityStore expects. config_map = s.build_identity_map() @@ -862,9 +864,8 @@ def create_asgi_app( "entra identity map is EMPTY at startup (0 bound oids) — server " "is UP and serving, but every delegated (human) token will " "receive 403 until identities are onboarded. Bind the first user " - "with an IdentityAdmin-role token via PUT /admin/identities/{oid} " - "(store=%s). This is expected on a fresh /data volume.", - s.entra_identities_store_path, + "with an IdentityAdmin-role token via PUT /admin/identities/{oid}. " + "This is expected on a fresh /data volume." ) # Disjointness invariant: each oid belongs to exactly one identity @@ -899,9 +900,9 @@ def create_asgi_app( admin_api_key_digest = None else: # Build and load the API-key store. - key_store = IdentityStore(Path(s.api_keys_store_path)) + key_store = create_identity_store(s, "api_key") key_store.load() - if not key_store.path.exists(): + if not key_store.exists(): # First boot: seed from config, converting flat {sha256: contributor_id} # to the rich {sha256: {"id": contributor_id}} format. config_ks = s.build_keystore() @@ -919,9 +920,7 @@ def create_asgi_app( "static keystore is EMPTY at startup (0 bound keys) — server " "is UP but fail-CLOSED; every request will 401 until keys are " "onboarded. Add the first key with the admin token via " - "PUT /admin/keys/{sha256hash} (store=%s). Expected on a fresh " - "/data volume.", - s.api_keys_store_path, + "PUT /admin/keys/{sha256hash}. Expected on a fresh /data volume." ) else: logger.warning( @@ -931,8 +930,7 @@ def create_asgi_app( "/admin API is unreachable without an admin key: every token " "401s at the middleware before require_admin runs). Set " "admin_api_key/admin_api_key_sha256 to enable runtime " - "onboarding, or add api_keys in config and restart. (store=%s)", - s.api_keys_store_path, + "onboarding, or add api_keys in config and restart." ) # Pass key_store.flat_dict (the LIVE dict) so the resolver sees any @@ -1155,14 +1153,14 @@ async def post_events( @app.get("/blobs/{session_id}", dependencies=[Depends(require_read)]) async def list_blobs(session_id: str) -> JSONResponse: - blob_store = AsyncDiskBlobStore(root=_settings.blob_path) - uris = await blob_store.list(session_id) + blob_store = create_blob_store(_settings) + uris = [ref.uri async for ref in blob_store.list(session_id)] return JSONResponse(content={"session_id": session_id, "blobs": uris}) @app.get("/blobs/{session_id}/{key}", dependencies=[Depends(require_read)]) async def get_blob(session_id: str, key: str) -> JSONResponse: - blob_store = AsyncDiskBlobStore(root=_settings.blob_path) + blob_store = create_blob_store(_settings) uri = f"ci-blob://{session_id}/{key}" try: content = await blob_store.read(uri) diff --git a/context_intelligence_server/pipeline.py b/context_intelligence_server/pipeline.py index 7310fd9c..d114cb13 100644 --- a/context_intelligence_server/pipeline.py +++ b/context_intelligence_server/pipeline.py @@ -173,7 +173,16 @@ async def process_event( data.get("timestamp") if isinstance(data, dict) else None ) if session_id and timestamp and worker.services.blob_store: - node_id = make_node_id(session_id, event, timestamp) + # The blob-key node_id MUST match handlers/data_layer_1/default.py's + # event_node_id (same session_id + event + timestamp + tool_call_id), + # otherwise two distinct same-millisecond events (e.g. parallel tool + # calls in the same batch) collide on an identical blob key and the + # second write silently overwrites the first, while the first Event + # node's $blob_ref still points at that URI (now holding the wrong + # payload). tool_call_id is present on ALL event types that carry it, + # not just tool:*, so it is safe to read unconditionally here. + disambiguator = data.get("tool_call_id") if isinstance(data, dict) else None + node_id = make_node_id(session_id, event, timestamp, disambiguator) await process_event_data( data, worker.services.blob_store, session_id, node_id ) diff --git a/context_intelligence_server/queue_manager/__init__.py b/context_intelligence_server/queue_manager/__init__.py new file mode 100644 index 00000000..3c0ce8a2 --- /dev/null +++ b/context_intelligence_server/queue_manager/__init__.py @@ -0,0 +1,31 @@ +"""queue_manager — durable, per-session append-only queue for the event-write pipeline. + +The public surface is the backend-neutral :class:`QueueManager` Protocol plus +the :class:`Batch` / :class:`Record` value types and the +:func:`create_queue_manager` factory. Consumers depend on these, never on a +concrete backend class. + +Package layout: + protocol.py QueueManager Protocol, Batch, Record — the backend-neutral + seam (no filesystem imports). + filesystem.py FileSystemQueueManager — the on-disk implementation, plus + the on-disk-only Verdict classification enum. + factory.py create_queue_manager(settings) — the ONLY place a backend is + selected and the ONLY place (besides config.py) that reads + settings.queues_path. +""" + +from __future__ import annotations + +from .factory import create_queue_manager +from .filesystem import FileSystemQueueManager, Verdict +from .protocol import Batch, QueueManager, Record + +__all__ = [ + "Batch", + "FileSystemQueueManager", + "QueueManager", + "Record", + "Verdict", + "create_queue_manager", +] diff --git a/context_intelligence_server/queue_manager/factory.py b/context_intelligence_server/queue_manager/factory.py new file mode 100644 index 00000000..fadf960e --- /dev/null +++ b/context_intelligence_server/queue_manager/factory.py @@ -0,0 +1,35 @@ +"""Config-driven QueueManager factory — the ONLY place a queue backend is selected. + +This is the single seam through which the concrete backend is chosen. Adding +a new backend (e.g. Azure) means: one new module implementing +:class:`~.protocol.QueueManager`, one new branch here, and a config value — +zero changes to :mod:`context_intelligence_server.registry` or any consumer. + +This module (and :mod:`~context_intelligence_server.config`) are the only +places ``settings.queues_path`` is read — the on-disk root is a filesystem- +backend concern, resolved here and handed to the concrete backend at +construction time. Callers only ever see the :class:`~.protocol.QueueManager` +Protocol. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import TYPE_CHECKING + +from .filesystem import FileSystemQueueManager +from .protocol import QueueManager + +if TYPE_CHECKING: + from context_intelligence_server.config import Settings + + +def create_queue_manager(settings: Settings) -> QueueManager: + """Build the durable ``QueueManager`` from config. + + Single backend today (on-disk), so this is a thin config-reading seam + rather than a multi-backend dispatcher — but it keeps ``settings.queues_path`` + out of consumers (mirrors ``blob_store.factory.create_blob_store`` and + ``identity_store.factory.create_identity_store``). + """ + return FileSystemQueueManager(queues_dir=Path(settings.queues_path)) diff --git a/context_intelligence_server/queue_manager.py b/context_intelligence_server/queue_manager/filesystem.py similarity index 97% rename from context_intelligence_server/queue_manager.py rename to context_intelligence_server/queue_manager/filesystem.py index 301bc4d6..c77f7135 100644 --- a/context_intelligence_server/queue_manager.py +++ b/context_intelligence_server/queue_manager/filesystem.py @@ -29,6 +29,7 @@ from typing import Any, TypeVar from context_intelligence_server.config import get_settings +from context_intelligence_server.queue_manager.protocol import Batch, Record logger = logging.getLogger(__name__) @@ -42,52 +43,6 @@ _SCAN_CHUNK_BYTES = 1 << 20 -@dataclass(frozen=True) -class Record: - """One log record and the byte range the QUEUE assigned it. - - ``start``/``end`` are opaque cursor values PRODUCED BY THE QUEUE and only - ever handed back to it (``commit``). Callers MUST NOT compute them and - MUST NOT assume ``end - start == len(raw) + 1`` -- that relationship is - the queue's private framing invariant (module docstring), not a public - contract. - """ - - raw: bytes # WITHOUT the terminator, exactly as ``lines`` is today - start: int - end: int - - -@dataclass(frozen=True) -class Batch: - """A contiguous batch of log records read from a session's append-only log. - - Attributes: - session_id: The session the records belong to. - records: Queue-produced ``Record``s -- each carries its own opaque - ``start``/``end`` cursor. The queue produces these offsets; a - caller (the registry) only ever hands them back via ``commit``. - start_offset: Byte position in the log where this batch begins. - end_offset: Byte position in the log AFTER the last returned record. - This is the value passed to ``commit``. When no complete records - are available, ``end_offset == start_offset``. - """ - - session_id: str - records: list[Record] - start_offset: int - end_offset: int - - @property - def lines(self) -> list[bytes]: - """Raw record payloads, terminator-stripped -- the pre-Record view. - - Derived from ``records`` so the two can never disagree. Retained - because ~90 call sites across main.py and 12 test files read it. - """ - return [r.raw for r in self.records] - - class Verdict(str, Enum): """Boot-safety classifier verdict. @@ -188,8 +143,11 @@ async def _await_uninterrupted(coro: Coroutine[Any, Any, _T]) -> _T: return result -class QueueManager: - """Manages per-session append-only queues on disk.""" +class FileSystemQueueManager: + """Manages per-session append-only queues on disk. + + Implements :class:`~context_intelligence_server.queue_manager.protocol.QueueManager`. + """ def __init__(self, queues_dir: Path): self._dir = Path(queues_dir) @@ -395,7 +353,7 @@ def _write_all(fd: int, data: bytes) -> None: def _discard_partial(fd: int, start: int, path: Path) -> None: """Newline-terminate a partial write; never truncates -- queue bytes are never removed.""" try: - QueueManager._write_all(fd, b"\n") + FileSystemQueueManager._write_all(fd, b"\n") except OSError: logger.exception( "append_partial_terminate_failed path=%s start=%d " @@ -436,7 +394,7 @@ def _heal_one(path: Path) -> tuple[int, bool]: skip the tail; next boot retries). Raises ``OSError`` on any failure; the caller catches it per file. """ - end = QueueManager._last_complete_end(path) + end = FileSystemQueueManager._last_complete_end(path) size = path.stat().st_size if size <= end: return 0, False @@ -455,7 +413,7 @@ def _heal_one(path: Path) -> tuple[int, bool]: flags = os.O_WRONLY | os.O_CREAT | os.O_TRUNC | getattr(os, "O_BINARY", 0) fd = os.open(quarantine, flags, 0o644) try: - QueueManager._write_all(fd, data) + FileSystemQueueManager._write_all(fd, data) finally: os.close(fd) diff --git a/context_intelligence_server/queue_manager/protocol.py b/context_intelligence_server/queue_manager/protocol.py new file mode 100644 index 00000000..769993a8 --- /dev/null +++ b/context_intelligence_server/queue_manager/protocol.py @@ -0,0 +1,125 @@ +"""QueueManager Protocol — the backend-neutral seam. + +A ``QueueManager`` manages a durable, per-session append-only queue for the +event-write pipeline. Consumers depend on this Protocol plus the ``Batch`` / +``Record`` value types, never on a concrete backend class. No ``Path``, +on-disk layout, or ``os.*`` detail appears here or in any value the Protocol +returns — those are private to a concrete backend +(:class:`~context_intelligence_server.queue_manager.filesystem.FileSystemQueueManager`, +and, later, an Azure equivalent). +""" + +from __future__ import annotations + +from collections.abc import Callable +from dataclasses import dataclass +from typing import Any, Protocol, runtime_checkable + + +@dataclass(frozen=True) +class Record: + """One log record and the byte range the QUEUE assigned it. + + ``start``/``end`` are opaque cursor values PRODUCED BY THE QUEUE and only + ever handed back to it (``commit``). Callers MUST NOT compute them and + MUST NOT assume ``end - start == len(raw) + 1`` -- that relationship is + the queue's private framing invariant (module docstring), not a public + contract. + """ + + raw: bytes # WITHOUT the terminator, exactly as ``lines`` is today + start: int + end: int + + +@dataclass(frozen=True) +class Batch: + """A contiguous batch of log records read from a session's append-only log. + + Attributes: + session_id: The session the records belong to. + records: Queue-produced ``Record``s -- each carries its own opaque + ``start``/``end`` cursor. The queue produces these offsets; a + caller (the registry) only ever hands them back via ``commit``. + start_offset: Byte position in the log where this batch begins. + end_offset: Byte position in the log AFTER the last returned record. + This is the value passed to ``commit``. When no complete records + are available, ``end_offset == start_offset``. + """ + + session_id: str + records: list[Record] + start_offset: int + end_offset: int + + @property + def lines(self) -> list[bytes]: + """Raw record payloads, terminator-stripped -- the pre-Record view. + + Derived from ``records`` so the two can never disagree. Retained + because ~90 call sites across main.py and 12 test files read it. + """ + return [r.raw for r in self.records] + + +@runtime_checkable +class QueueManager(Protocol): + """Durable, per-session append-only queue. + + The method set mirrors the on-disk backend's public surface. A backend + reports its own queue root via ``queues_dir``; every other on-disk detail + stays private to the implementation. + """ + + @property + def queues_dir(self) -> Any: ... + + async def heal_torn_tails(self) -> dict[str, int]: ... + + async def append(self, session_id: str, raw: bytes) -> None: ... + + async def read_batch(self, session_id: str, max_items: int) -> Batch: ... + + async def commit(self, session_id: str, new_offset: int) -> None: ... + + async def dead_letter(self, session_id: str, raw: bytes, error: str) -> None: ... + + async def delete_drained(self, session_id: str) -> bool: ... + + async def compact_committed_prefix( + self, session_id: str, min_prefix_bytes: int = 0 + ) -> int: ... + + async def read_dead_letters(self, session_id: str) -> list[dict]: ... + + async def read_first_line(self, key: str) -> bytes | None: ... + + async def classify_session( + self, key: str, head_is_resumable: Callable[[bytes], bool] + ) -> Any: ... + + async def reclaim(self, c: Any, is_owned: Callable[[], bool]) -> bool: ... + + async def reclaim_orphans( + self, before_ts: float, enabled: bool = True + ) -> dict[str, int]: ... + + async def active_sessions(self) -> list[str]: ... + + async def recover(self) -> list[str]: ... + + async def derive_all_stats(self) -> dict[str, Any]: ... + + async def spool_stats(self) -> dict[str, int]: ... + + async def dead_letter_keys(self) -> list[str]: ... + + async def purge_dead_letters(self, worker_key: str) -> int: ... + + async def expire_dead_letters( + self, now: float, retention_seconds: float, enabled: bool + ) -> dict[str, int]: ... + + async def recovery_seed_counts(self) -> tuple[int, int]: ... + + async def recovery_reconcile_dead(self) -> int: ... diff --git a/context_intelligence_server/registry.py b/context_intelligence_server/registry.py index 3f215584..cbef0304 100644 --- a/context_intelligence_server/registry.py +++ b/context_intelligence_server/registry.py @@ -10,14 +10,18 @@ from pathlib import Path from typing import Any -from context_intelligence_server.blob_store import AsyncDiskBlobStore +from context_intelligence_server.blob_store import create_blob_store from context_intelligence_server.config import get_settings from context_intelligence_server.neo4j_store import ( Neo4jGraphStore, build_bounded_neo4j_driver, ) from context_intelligence_server.pipeline import process_event, setup_handlers -from context_intelligence_server.queue_manager import Batch, QueueManager +from context_intelligence_server.queue_manager import ( + Batch, + QueueManager, + create_queue_manager, +) from context_intelligence_server.services import HookStateService from context_intelligence_server.status import EventRecord, ring_buffer @@ -109,7 +113,7 @@ def _ensure_infra(self) -> None: """ if self._queue_manager is None: settings = get_settings() - self._queue_manager = QueueManager(queues_dir=Path(settings.queues_path)) + self._queue_manager = create_queue_manager(settings) self._write_semaphore = asyncio.Semaphore(settings.write_concurrency) self._max_delivery_attempts = settings.max_delivery_attempts @@ -817,7 +821,7 @@ def get_or_create( """ if session_id not in self._workers: settings = get_settings() - blob_store = AsyncDiskBlobStore(root=settings.blob_path) + blob_store = create_blob_store(settings) _admin = settings.resolve_neo4j_admin() neo4j_store = Neo4jGraphStore( uri=_admin.url, diff --git a/tests/conftest.py b/tests/conftest.py index 3fddf0cf..43336efc 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -106,6 +106,7 @@ def safe_settings(tmp_path: Any) -> Generator[None, None, None]: _real = _Settings() class _SettingsProxy: + blob_backend: str = _real.blob_backend blob_path: str = _real.blob_path queues_path: str = str(tmp_path / "queues") # Redirect identity-store paths so the registry proxy never touches the diff --git a/tests/integration/test_crash_recovery.py b/tests/integration/test_crash_recovery.py index 61fb544a..aeae3f1a 100644 --- a/tests/integration/test_crash_recovery.py +++ b/tests/integration/test_crash_recovery.py @@ -9,7 +9,7 @@ from unittest.mock import AsyncMock, patch from context_intelligence_server import registry as registry_module -from context_intelligence_server.queue_manager import QueueManager +from context_intelligence_server.queue_manager import FileSystemQueueManager, QueueManager from context_intelligence_server.registry import SessionRegistry, SessionWorker from context_intelligence_server.services import HookStateService @@ -23,7 +23,7 @@ async def test_no_loss_after_crash_mid_drain() -> None: registry_module.get_settings() ) # queues_path patched to tmp_path by safe_settings sid = "crash-sess" - qm = QueueManager(queues_dir=Path(settings.queues_path)) + qm = FileSystemQueueManager(queues_dir=Path(settings.queues_path)) K = 5 for i in range(K): @@ -94,7 +94,7 @@ async def test_offset_never_advances_over_undurable_data() -> None: buffer-restore-on-failure (neo4j_store.py:686-696).""" settings = registry_module.get_settings() sid = "flush-fail-sess" - qm = QueueManager(queues_dir=Path(settings.queues_path)) + qm = FileSystemQueueManager(queues_dir=Path(settings.queues_path)) await qm.append(sid, _line("e0", "/ws", {"session_id": sid})) reg = SessionRegistry() diff --git a/tests/neo4j/test_concurrent_flush.py b/tests/neo4j/test_concurrent_flush.py index 470aa83d..d130a6a7 100644 --- a/tests/neo4j/test_concurrent_flush.py +++ b/tests/neo4j/test_concurrent_flush.py @@ -25,7 +25,7 @@ Neo4jGraphStore, ensure_neo4j_schema, ) -from context_intelligence_server.queue_manager import QueueManager +from context_intelligence_server.queue_manager import FileSystemQueueManager, QueueManager from context_intelligence_server.registry import SessionRegistry, SessionWorker from context_intelligence_server.services import HookStateService from context_intelligence_server.utils import make_node_id @@ -226,7 +226,7 @@ async def test_durable_drain_multi_writer_zero_loss( await driver.close() reg = SessionRegistry() - reg._queue_manager = QueueManager(queues_dir=tmp_path / "queues") + reg._queue_manager = FileSystemQueueManager(queues_dir=tmp_path / "queues") reg._write_semaphore = asyncio.Semaphore(2) reg._max_delivery_attempts = 5 qm = reg._queue_manager @@ -327,7 +327,7 @@ async def test_durable_poison_isolation_no_contamination( await driver.close() reg = SessionRegistry() - reg._queue_manager = QueueManager(queues_dir=tmp_path / "queues") + reg._queue_manager = FileSystemQueueManager(queues_dir=tmp_path / "queues") reg._write_semaphore = asyncio.Semaphore(2) reg._max_delivery_attempts = 2 qm = reg._queue_manager diff --git a/tests/neo4j/test_handler_flush_concurrency.py b/tests/neo4j/test_handler_flush_concurrency.py index 2695d7e8..6da04c9a 100644 --- a/tests/neo4j/test_handler_flush_concurrency.py +++ b/tests/neo4j/test_handler_flush_concurrency.py @@ -20,7 +20,7 @@ from context_intelligence_server.handlers.data_layer_2.session import SessionHandler from context_intelligence_server.neo4j_store import Neo4jGraphStore, ensure_neo4j_schema from context_intelligence_server.pipeline import process_event, setup_handlers -from context_intelligence_server.queue_manager import QueueManager +from context_intelligence_server.queue_manager import FileSystemQueueManager, QueueManager from context_intelligence_server.registry import SessionRegistry, SessionWorker from context_intelligence_server.services import HookStateService @@ -83,7 +83,7 @@ def _line(event: str, workspace: str, data: dict[str, Any]) -> bytes: def _build_registry(queues_dir: Path, *, write_concurrency: int = 8) -> SessionRegistry: reg = SessionRegistry() - reg._queue_manager = QueueManager(queues_dir=queues_dir) + reg._queue_manager = FileSystemQueueManager(queues_dir=queues_dir) reg._write_semaphore = asyncio.Semaphore(write_concurrency) reg._max_delivery_attempts = 3 return reg diff --git a/tests/neo4j/test_oom_regression.py b/tests/neo4j/test_oom_regression.py index fad5e0e8..94f7b0ea 100644 --- a/tests/neo4j/test_oom_regression.py +++ b/tests/neo4j/test_oom_regression.py @@ -310,7 +310,7 @@ async def test_finalization_path_freezes_then_restart_then_drains( from unittest.mock import AsyncMock from context_intelligence_server.pipeline import setup_handlers - from context_intelligence_server.queue_manager import QueueManager + from context_intelligence_server.queue_manager import FileSystemQueueManager, QueueManager from context_intelligence_server.registry import SessionRegistry, SessionWorker from context_intelligence_server.services import HookStateService @@ -324,7 +324,7 @@ async def test_finalization_path_freezes_then_restart_then_drains( # fixture) patches into get_settings().queues_path for every new # SessionRegistry() instance constructed in this test. queues_dir = tmp_path / "queues" - qm = QueueManager(queues_dir=queues_dir) + qm = FileSystemQueueManager(queues_dir=queues_dir) async def _make( rows: int, byts: int diff --git a/tests/neo4j/test_orphan_visibility.py b/tests/neo4j/test_orphan_visibility.py index b416a8a2..188da47b 100644 --- a/tests/neo4j/test_orphan_visibility.py +++ b/tests/neo4j/test_orphan_visibility.py @@ -44,7 +44,7 @@ from neo4j import AsyncGraphDatabase from context_intelligence_server.neo4j_store import Neo4jGraphStore -from context_intelligence_server.queue_manager import QueueManager +from context_intelligence_server.queue_manager import FileSystemQueueManager, QueueManager from context_intelligence_server.registry import SessionRegistry, SessionWorker from context_intelligence_server.services import HookStateService from context_intelligence_server.status import build_status_response @@ -145,7 +145,7 @@ async def test_finalization_orphan_surfaces_on_status( # guarantees this QueueManager and the registry's queue_manager point at # the same on-disk queue. queues_dir = tmp_path / "queues" - qm = QueueManager(queues_dir=queues_dir) + qm = FileSystemQueueManager(queues_dir=queues_dir) # ----------------------------------------------------------------------- # Seed the durable queue in exact order (line counts are load-bearing). diff --git a/tests/neo4j/test_queues_actions.py b/tests/neo4j/test_queues_actions.py index 6276bbc1..59e7c41a 100644 --- a/tests/neo4j/test_queues_actions.py +++ b/tests/neo4j/test_queues_actions.py @@ -34,7 +34,7 @@ Neo4jGraphStore, ensure_neo4j_schema, ) -from context_intelligence_server.queue_manager import QueueManager +from context_intelligence_server.queue_manager import FileSystemQueueManager, QueueManager from context_intelligence_server.registry import SessionRegistry, SessionWorker from context_intelligence_server.services import HookStateService @@ -96,7 +96,7 @@ async def test_replay_rewrites_through_real_drainer( # worker is pre-registered, so get_or_create never builds a settings-derived # store). reg = SessionRegistry() - reg._queue_manager = QueueManager(queues_dir=tmp_path / "queues") + reg._queue_manager = FileSystemQueueManager(queues_dir=tmp_path / "queues") reg._write_semaphore = asyncio.Semaphore(1) reg._max_delivery_attempts = 3 qm = reg._queue_manager diff --git a/tests/neo4j/test_steady_state_reclaim_neo4j.py b/tests/neo4j/test_steady_state_reclaim_neo4j.py index 0bb98a35..5c07ce0e 100644 --- a/tests/neo4j/test_steady_state_reclaim_neo4j.py +++ b/tests/neo4j/test_steady_state_reclaim_neo4j.py @@ -21,7 +21,7 @@ from neo4j import AsyncGraphDatabase from context_intelligence_server.neo4j_store import Neo4jGraphStore, ensure_neo4j_schema -from context_intelligence_server.queue_manager import QueueManager +from context_intelligence_server.queue_manager import FileSystemQueueManager, QueueManager from context_intelligence_server.registry import SessionRegistry, SessionWorker from context_intelligence_server.services import HookStateService @@ -69,7 +69,7 @@ def _seq_event(i: int, sid: str) -> bytes: def _build_registry(queues_dir: Path) -> SessionRegistry: reg = SessionRegistry() - reg._queue_manager = QueueManager(queues_dir=queues_dir) + reg._queue_manager = FileSystemQueueManager(queues_dir=queues_dir) reg._write_semaphore = asyncio.Semaphore(8) reg._max_delivery_attempts = 3 return reg @@ -271,7 +271,7 @@ async def test_boot_recovered_session_compacts_before_dry_exit( queues_dir = tmp_path / "queues" reg = SessionRegistry() - reg._queue_manager = QueueManager(queues_dir=queues_dir) + reg._queue_manager = FileSystemQueueManager(queues_dir=queues_dir) reg._write_semaphore = asyncio.Semaphore(8) reg._max_delivery_attempts = 3 qm = reg.queue_manager diff --git a/tests/routers/test_queues.py b/tests/routers/test_queues.py index c8909e57..8a218891 100644 --- a/tests/routers/test_queues.py +++ b/tests/routers/test_queues.py @@ -9,7 +9,7 @@ import pytest from context_intelligence_server.main import registry -from context_intelligence_server.queue_manager import QueueManager +from context_intelligence_server.queue_manager import FileSystemQueueManager, QueueManager def _point_registry_at(tmp_path: Path) -> QueueManager: @@ -17,7 +17,7 @@ def _point_registry_at(tmp_path: Path) -> QueueManager: Returns the QueueManager so tests can seed dead-letter records directly. """ - qm = QueueManager(queues_dir=tmp_path / "queues") + qm = FileSystemQueueManager(queues_dir=tmp_path / "queues") registry._queue_manager = qm registry._write_semaphore = asyncio.Semaphore(2) registry._max_delivery_attempts = 5 diff --git a/tests/test_blob_processor.py b/tests/test_blob_processor.py index 82ee196c..43ae8074 100644 --- a/tests/test_blob_processor.py +++ b/tests/test_blob_processor.py @@ -29,6 +29,15 @@ _lift_raw_fields, process_event_data, ) +from context_intelligence_server.blob_store import BlobReference + + +def _ref(uri: str) -> BlobReference: + """A BlobReference for a mocked write() return (only .uri is read here).""" + session_id, _, key = uri.removeprefix("ci-blob://").partition("/") + return BlobReference( + uri=uri, session_id=session_id, key=key, size=0, last_modified=0.0 + ) # --------------------------------------------------------------------------- @@ -55,7 +64,7 @@ async def test_process_event_data_mutates_in_place() -> None: original_id = id(data) blob_store = AsyncMock() - blob_store.write = AsyncMock(return_value="ci-blob://sess/node__raw") + blob_store.write = AsyncMock(return_value=_ref("ci-blob://sess/node__raw")) await process_event_data(data, blob_store, "sess", "node") @@ -74,7 +83,7 @@ async def test_process_event_data_returns_none() -> None: """process_event_data returns None.""" data: dict[str, Any] = {"result": {"answer": 42}} blob_store = AsyncMock() - blob_store.write = AsyncMock(return_value="ci-blob://sess/node__result") + blob_store.write = AsyncMock(return_value=_ref("ci-blob://sess/node__result")) result = await process_event_data(data, blob_store, "sess", "node") @@ -96,8 +105,8 @@ async def test_blob_ref_substitution_on_successful_write() -> None: # Use a function-based side_effect so the returned URI always matches # the actual key argument, regardless of BLOB_FIELDS frozenset iteration order. - async def _write(session_id: str, key: str, value: object) -> str: - return f"ci-blob://{session_id}/{key}" + async def _write(session_id: str, key: str, value: object) -> BlobReference: + return _ref(f"ci-blob://{session_id}/{key}") blob_store.write = AsyncMock(side_effect=_write) @@ -134,7 +143,7 @@ async def test_absent_fields_are_skipped() -> None: """Fields in BLOB_FIELDS that are absent from data are not added.""" data: dict[str, Any] = {"other_field": "untouched"} blob_store = AsyncMock() - blob_store.write = AsyncMock(return_value="ci-blob://sess/node__something") + blob_store.write = AsyncMock(return_value=_ref("ci-blob://sess/node__something")) await process_event_data(data, blob_store, "sess", "node") @@ -158,7 +167,7 @@ async def test_none_fields_are_skipped() -> None: "messages": None, } blob_store = AsyncMock() - blob_store.write = AsyncMock(return_value="ci-blob://sess/node__x") + blob_store.write = AsyncMock(return_value=_ref("ci-blob://sess/node__x")) await process_event_data(data, blob_store, "sess", "node") diff --git a/tests/test_blob_store.py b/tests/test_blob_store.py index 4a3a31ad..13884795 100644 --- a/tests/test_blob_store.py +++ b/tests/test_blob_store.py @@ -1,14 +1,14 @@ -"""Tests for AsyncDiskBlobStore — Write, Read, List, Dump. +"""Tests for FileSystemBlobStore — Write, Read, List, Scan, Delete, Dump. -15 tests covering: +Covers: 1. write/read roundtrip -2. URI format +2. BlobReference.uri format 3. directory structure creation 4. URI-based session_id resolution 5. missing blob raises FileNotFoundError 6. invalid URI raises ValueError 7. empty list for missing session -8. correct URI listing +8. correct BlobReference listing (async iterator) 9. session isolation 10. asyncio.to_thread delegation verification 11. dump() copies blob to specified dest_dir @@ -16,19 +16,26 @@ 13. dump() missing blob raises FileNotFoundError 14. dump() delegates copy2 via asyncio.to_thread 15. BlobStore protocol conformance +16. scan() yields BlobReference across multiple sessions +17. delete() is idempotent (True then False) and removes the blob +18. list()/write() BlobReference has correct uri/size/last_modified """ from __future__ import annotations import asyncio import json +import os from pathlib import Path from unittest.mock import patch import pytest - -from context_intelligence_server.blob_store import AsyncDiskBlobStore, BlobStore - +from context_intelligence_server.blob_store import ( + BlobNotFoundError, + BlobReference, + BlobStore, + FileSystemBlobStore, +) # --------------------------------------------------------------------------- # Fixtures @@ -36,9 +43,13 @@ @pytest.fixture -def store(tmp_path: Path) -> AsyncDiskBlobStore: - """Return a fresh AsyncDiskBlobStore rooted at a temporary directory.""" - return AsyncDiskBlobStore(root=tmp_path) +def store(tmp_path: Path) -> FileSystemBlobStore: + """Return a fresh FileSystemBlobStore rooted at a temporary directory.""" + return FileSystemBlobStore(root=tmp_path) + + +async def _list_uris(store: FileSystemBlobStore, session_id: str) -> list[str]: + return [ref.uri async for ref in store.list(session_id)] # --------------------------------------------------------------------------- @@ -46,23 +57,28 @@ def store(tmp_path: Path) -> AsyncDiskBlobStore: # --------------------------------------------------------------------------- -async def test_write_read_roundtrip(store: AsyncDiskBlobStore) -> None: +async def test_write_read_roundtrip(store: FileSystemBlobStore) -> None: """Data written can be read back unchanged.""" payload = {"event": "tool_call", "tool": "bash", "args": ["ls"]} - uri = await store.write("session-abc", "tool_call_01", payload) - result = await store.read(uri) + ref = await store.write("session-abc", "tool_call_01", payload) + result = await store.read(ref.uri) assert result == payload # --------------------------------------------------------------------------- -# 2. URI format +# 2. BlobReference.uri format # --------------------------------------------------------------------------- -async def test_uri_format(store: AsyncDiskBlobStore) -> None: - """write() returns a ci-blob:/// URI.""" - uri = await store.write("session-xyz", "my_key", {"x": 1}) - assert uri == "ci-blob://session-xyz/my_key" +async def test_uri_format(store: FileSystemBlobStore) -> None: + """write() returns a BlobReference whose .uri is ci-blob:///.""" + ref = await store.write("session-xyz", "my_key", {"x": 1}) + assert isinstance(ref, BlobReference) + assert ref.uri == "ci-blob://session-xyz/my_key" + assert ref.session_id == "session-xyz" + assert ref.key == "my_key" + assert ref.size > 0 + assert ref.last_modified > 0 # --------------------------------------------------------------------------- @@ -71,7 +87,7 @@ async def test_uri_format(store: AsyncDiskBlobStore) -> None: async def test_directory_structure_creation( - store: AsyncDiskBlobStore, tmp_path: Path + store: FileSystemBlobStore, tmp_path: Path ) -> None: """write() creates //blobs/.json on disk.""" await store.write("session-123", "blob_key", {"data": "value"}) @@ -87,17 +103,17 @@ async def test_directory_structure_creation( async def test_uri_based_session_id_resolution( - store: AsyncDiskBlobStore, tmp_path: Path + store: FileSystemBlobStore, tmp_path: Path ) -> None: """read() resolves the session_id from the URI, not from a parameter.""" session_id = "session-uri-resolve" key = "my_blob" payload = {"resolved": True} - uri = await store.write(session_id, key, payload) + ref = await store.write(session_id, key, payload) # Confirm URI contains session_id - assert session_id in uri + assert session_id in ref.uri # read must successfully resolve session_id from URI - result = await store.read(uri) + result = await store.read(ref.uri) assert result == payload @@ -106,7 +122,7 @@ async def test_uri_based_session_id_resolution( # --------------------------------------------------------------------------- -async def test_missing_blob_raises_file_not_found(store: AsyncDiskBlobStore) -> None: +async def test_missing_blob_raises_file_not_found(store: FileSystemBlobStore) -> None: """read() raises FileNotFoundError for a URI pointing to a non-existent blob.""" uri = "ci-blob://session-missing/nonexistent_key" with pytest.raises(FileNotFoundError): @@ -118,7 +134,7 @@ async def test_missing_blob_raises_file_not_found(store: AsyncDiskBlobStore) -> # --------------------------------------------------------------------------- -async def test_invalid_uri_raises_value_error(store: AsyncDiskBlobStore) -> None: +async def test_invalid_uri_raises_value_error(store: FileSystemBlobStore) -> None: """read() raises ValueError for URIs that don't match the ci-blob:// scheme.""" with pytest.raises(ValueError): await store.read("not-a-ci-blob-uri") @@ -135,30 +151,35 @@ async def test_invalid_uri_raises_value_error(store: AsyncDiskBlobStore) -> None # --------------------------------------------------------------------------- -async def test_empty_list_for_missing_session(store: AsyncDiskBlobStore) -> None: - """list() returns an empty list when no blobs exist for the session.""" - result = await store.list("session-does-not-exist") +async def test_empty_list_for_missing_session(store: FileSystemBlobStore) -> None: + """list() yields nothing when no blobs exist for the session.""" + result = await _list_uris(store, "session-does-not-exist") assert result == [] # --------------------------------------------------------------------------- -# 8. Correct URI listing +# 8. Correct BlobReference listing (async iterator) # --------------------------------------------------------------------------- -async def test_correct_uri_listing(store: AsyncDiskBlobStore) -> None: - """list() returns all blob URIs for a session, sorted.""" +async def test_correct_uri_listing(store: FileSystemBlobStore) -> None: + """list() yields all blob references for a session, sorted by key.""" session_id = "session-list" await store.write(session_id, "key_b", {"b": 2}) await store.write(session_id, "key_a", {"a": 1}) await store.write(session_id, "key_c", {"c": 3}) - uris = await store.list(session_id) - assert uris == [ + refs = [ref async for ref in store.list(session_id)] + assert [r.uri for r in refs] == [ "ci-blob://session-list/key_a", "ci-blob://session-list/key_b", "ci-blob://session-list/key_c", ] + for r in refs: + assert isinstance(r, BlobReference) + assert r.session_id == session_id + assert r.size > 0 + assert r.last_modified > 0 # --------------------------------------------------------------------------- @@ -166,14 +187,14 @@ async def test_correct_uri_listing(store: AsyncDiskBlobStore) -> None: # --------------------------------------------------------------------------- -async def test_session_isolation(store: AsyncDiskBlobStore) -> None: - """list() only returns URIs for the requested session, not other sessions.""" +async def test_session_isolation(store: FileSystemBlobStore) -> None: + """list() only returns references for the requested session, not other sessions.""" await store.write("session-alpha", "blob_1", {"alpha": True}) await store.write("session-beta", "blob_2", {"beta": True}) await store.write("session-alpha", "blob_3", {"alpha2": True}) - alpha_uris = await store.list("session-alpha") - beta_uris = await store.list("session-beta") + alpha_uris = await _list_uris(store, "session-alpha") + beta_uris = await _list_uris(store, "session-beta") assert all("session-alpha" in u for u in alpha_uris) assert all("session-beta" in u for u in beta_uris) @@ -188,7 +209,7 @@ async def test_session_isolation(store: AsyncDiskBlobStore) -> None: async def test_asyncio_to_thread_delegation(tmp_path: Path) -> None: """All filesystem I/O is delegated to asyncio.to_thread for non-blocking I/O.""" - store = AsyncDiskBlobStore(root=tmp_path) + store = FileSystemBlobStore(root=tmp_path) to_thread_calls: list[str] = [] original_to_thread = asyncio.to_thread @@ -200,7 +221,8 @@ async def tracking_to_thread(func, *args, **kwargs): # type: ignore[no-untyped- with patch("asyncio.to_thread", side_effect=tracking_to_thread): await store.write("sess", "k", {"v": 1}) await store.read("ci-blob://sess/k") - await store.list("sess") + async for _ in store.list("sess"): + pass assert len(to_thread_calls) >= 3, ( f"Expected at least 3 asyncio.to_thread calls (write, read, list), " @@ -214,16 +236,16 @@ async def tracking_to_thread(func, *args, **kwargs): # type: ignore[no-untyped- async def test_dump_copy_to_specified_dest_dir( - store: AsyncDiskBlobStore, tmp_path: Path + store: FileSystemBlobStore, tmp_path: Path ) -> None: """dump() copies the blob file to the specified dest_dir and returns the path.""" session_id = "session-dump-copy" key = "blob_to_copy" payload = {"copy": "me"} - uri = await store.write(session_id, key, payload) + ref = await store.write(session_id, key, payload) dest_dir = tmp_path / "my_dest" - result = await store.dump(uri, dest_dir=dest_dir) + result = await store.dump(ref.uri, dest_dir=dest_dir) result_path = Path(result) assert result_path.exists() @@ -236,15 +258,15 @@ async def test_dump_copy_to_specified_dest_dir( # --------------------------------------------------------------------------- -async def test_dump_default_dest_dir(store: AsyncDiskBlobStore) -> None: +async def test_dump_default_dest_dir(store: FileSystemBlobStore) -> None: """dump() uses Path(tempfile.gettempdir()) / 'ci-blobs' when dest_dir is None.""" import tempfile session_id = "session-dump-default" key = "default_blob" - uri = await store.write(session_id, key, {"default": True}) + ref = await store.write(session_id, key, {"default": True}) - result = await store.dump(uri) + result = await store.dump(ref.uri) expected_dir = Path(tempfile.gettempdir()) / "ci-blobs" result_path = Path(result) @@ -258,7 +280,7 @@ async def test_dump_default_dest_dir(store: AsyncDiskBlobStore) -> None: async def test_dump_missing_blob_raises_file_not_found( - store: AsyncDiskBlobStore, + store: FileSystemBlobStore, ) -> None: """dump() raises FileNotFoundError with 'Blob not found' message for missing blob.""" uri = "ci-blob://session-nonexistent/missing_blob" @@ -272,12 +294,12 @@ async def test_dump_missing_blob_raises_file_not_found( async def test_dump_uses_asyncio_to_thread_for_copy2( - store: AsyncDiskBlobStore, tmp_path: Path + store: FileSystemBlobStore, tmp_path: Path ) -> None: """dump() delegates shutil.copy2 to asyncio.to_thread for non-blocking I/O.""" session_id = "session-dump-thread" key = "thread_blob" - uri = await store.write(session_id, key, {"thread": True}) + ref = await store.write(session_id, key, {"thread": True}) dest_dir = tmp_path / "thread_dest" to_thread_calls: list[str] = [] @@ -288,7 +310,7 @@ async def tracking_to_thread(func, *args, **kwargs): # type: ignore[no-untyped- return await original_to_thread(func, *args, **kwargs) with patch("asyncio.to_thread", side_effect=tracking_to_thread): - await store.dump(uri, dest_dir=dest_dir) + await store.dump(ref.uri, dest_dir=dest_dir) assert len(to_thread_calls) >= 1, ( f"Expected at least 1 asyncio.to_thread call for dump(), " @@ -301,8 +323,8 @@ async def tracking_to_thread(func, *args, **kwargs): # type: ignore[no-untyped- # --------------------------------------------------------------------------- -def test_blob_store_protocol_conformance(store: AsyncDiskBlobStore) -> None: - """AsyncDiskBlobStore conforms to the BlobStore protocol.""" +def test_blob_store_protocol_conformance(store: FileSystemBlobStore) -> None: + """FileSystemBlobStore conforms to the BlobStore protocol.""" assert isinstance(store, BlobStore) @@ -312,18 +334,20 @@ def test_blob_store_protocol_conformance(store: AsyncDiskBlobStore) -> None: async def test_write_is_atomic_no_torn_file_on_failure( - store: AsyncDiskBlobStore, tmp_path: Path + store: FileSystemBlobStore, tmp_path: Path ) -> None: """A failure during os.replace leaves no torn final file and no temp siblings.""" session_id = "sess-atomic" key = "k1" - with patch( - "context_intelligence_server.blob_store.os.replace", - side_effect=OSError("simulated replace failure"), + with ( + patch( + "context_intelligence_server.blob_store.filesystem.os.replace", + side_effect=OSError("simulated replace failure"), + ), + pytest.raises(OSError), ): - with pytest.raises(OSError): - await store.write(session_id, key, {"v": 1}) + await store.write(session_id, key, {"v": 1}) final_path = store.blob_path(session_id, key) # No torn file observable at the final path. @@ -335,16 +359,180 @@ async def test_write_is_atomic_no_torn_file_on_failure( async def test_write_replaces_atomically_on_success( - store: AsyncDiskBlobStore, + store: FileSystemBlobStore, ) -> None: """On success the final file has the exact JSON, no temp remains, URI is correct.""" session_id = "sess-atomic" key = "k2" - uri = await store.write(session_id, key, {"v": 1}) + ref = await store.write(session_id, key, {"v": 1}) - assert uri == "ci-blob://sess-atomic/k2" + assert ref.uri == "ci-blob://sess-atomic/k2" final_path = store.blob_path(session_id, key) assert final_path.read_text(encoding="utf-8") == '{"v": 1}' # No leftover temp files. assert list(final_path.parent.glob("*.tmp")) == [] + + +# --------------------------------------------------------------------------- +# 16. scan() yields BlobReference across multiple sessions +# --------------------------------------------------------------------------- + + +async def test_scan_yields_references_across_sessions( + store: FileSystemBlobStore, +) -> None: + """scan() streams a BlobReference for every blob across ALL sessions.""" + await store.write("session-scan-a", "k1", {"a": 1}) + await store.write("session-scan-a", "k2", {"a": 2}) + await store.write("session-scan-b", "k1", {"b": 1}) + + refs = [ref async for ref in store.scan()] + uris = {r.uri for r in refs} + assert uris == { + "ci-blob://session-scan-a/k1", + "ci-blob://session-scan-a/k2", + "ci-blob://session-scan-b/k1", + } + for r in refs: + assert isinstance(r, BlobReference) + assert r.size > 0 + assert r.last_modified > 0 + + +async def test_scan_empty_store_yields_nothing(store: FileSystemBlobStore) -> None: + """scan() over an empty store yields no references.""" + refs = [ref async for ref in store.scan()] + assert refs == [] + + +# --------------------------------------------------------------------------- +# 17. delete() is idempotent and removes the blob +# --------------------------------------------------------------------------- + + +async def test_delete_idempotent_true_then_false(store: FileSystemBlobStore) -> None: + """delete() returns True the first time (blob existed), False thereafter.""" + ref = await store.write("session-delete", "to_delete", {"gone": "soon"}) + + first = await store.delete(ref.uri) + assert first is True + + # The blob is actually removed from disk. + with pytest.raises(FileNotFoundError): + await store.read(ref.uri) + + second = await store.delete(ref.uri) + assert second is False + + +async def test_delete_missing_blob_returns_false(store: FileSystemBlobStore) -> None: + """delete() on a never-written blob returns False, never raises.""" + result = await store.delete("ci-blob://never-existed/nope") + assert result is False + + +# --------------------------------------------------------------------------- +# BlobNotFoundError — neutral missing-blob error (guard #6) +# --------------------------------------------------------------------------- + + +async def test_missing_blob_raises_blob_not_found_error_no_path_leak( + store: FileSystemBlobStore, tmp_path: Path +) -> None: + """read() of a missing uri raises BlobNotFoundError (a FileNotFoundError + subclass, for back-compat) whose message carries the uri only — never the + on-disk path/root. + """ + uri = "ci-blob://session-missing/nonexistent_key" + + with pytest.raises(BlobNotFoundError) as exc_info: + await store.read(uri) + + # Back-compat: existing `except FileNotFoundError` callers still catch it. + assert isinstance(exc_info.value, FileNotFoundError) + + message = str(exc_info.value) + assert uri in message + # No on-disk path fragment or root leaks into the message. + assert "path" not in message.lower() + assert str(tmp_path) not in message + + +# --------------------------------------------------------------------------- +# Fenced (compare-and-delete) delete — guard #1 +# --------------------------------------------------------------------------- + + +async def test_fenced_delete_succeeds_when_unchanged( + store: FileSystemBlobStore, +) -> None: + """delete(uri, if_unmodified=ref) removes the blob when it has not + changed since ref was observed (e.g. by scan()/list()).""" + ref = await store.write("session-fence", "unchanged_key", {"v": 1}) + + result = await store.delete(ref.uri, if_unmodified=ref) + assert result is True + + with pytest.raises(BlobNotFoundError): + await store.read(ref.uri) + + +async def test_fenced_delete_refuses_when_rewritten( + store: FileSystemBlobStore, +) -> None: + """delete(uri, if_unmodified=stale_ref) returns False and leaves the + (new) blob on disk when the blob was rewritten after stale_ref was + observed. + + Deterministic (not sleep-based): the rewrite uses a longer JSON payload + so the size differs regardless of filesystem mtime granularity, and the + file's mtime is also forced forward via os.utime so both the size AND + mtime comparisons independently detect the change. + """ + session_id, key = "session-fence-stale", "rewritten_key" + + stale_ref = await store.write(session_id, key, {"v": 1}) + + # Rewrite with a longer payload -> different size, independent of mtime + # resolution/granularity on the filesystem. + new_ref = await store.write(session_id, key, {"v": 1, "extra": "x" * 64}) + assert new_ref.size != stale_ref.size + + # Force the mtime to be unambiguously different too (belt-and-suspenders + # against any filesystem where sizes could coincidentally collide). + path = store.blob_path(session_id, key) + new_mtime = stale_ref.last_modified + 100.0 + os.utime(path, (new_mtime, new_mtime)) + + result = await store.delete(stale_ref.uri, if_unmodified=stale_ref) + assert result is False + + # The (new) blob survives on disk, untouched. + survived = await store.read(stale_ref.uri) + assert survived == {"v": 1, "extra": "x" * 64} + + +async def test_fenced_delete_missing_blob_returns_false( + store: FileSystemBlobStore, +) -> None: + """delete(uri, if_unmodified=ref) on an already-absent blob returns False.""" + ref = await store.write("session-fence-missing", "gone_key", {"v": 1}) + assert await store.delete(ref.uri) is True # unconditional delete first + + result = await store.delete(ref.uri, if_unmodified=ref) + assert result is False + + +async def test_unconditional_delete_still_idempotent( + store: FileSystemBlobStore, +) -> None: + """Unconditional delete(uri) (if_unmodified=None, the default) is + unchanged: True then False, idempotent.""" + ref = await store.write("session-fence-uncond", "plain_key", {"v": 1}) + + first = await store.delete(ref.uri) + assert first is True + + second = await store.delete(ref.uri) + assert second is False diff --git a/tests/test_boot_safety.py b/tests/test_boot_safety.py index 60c1361d..488bfd64 100644 --- a/tests/test_boot_safety.py +++ b/tests/test_boot_safety.py @@ -23,7 +23,7 @@ import context_intelligence_server.main as main_module from context_intelligence_server.config import Settings from context_intelligence_server.main import _head_is_resumable, lifespan -from context_intelligence_server.queue_manager import QueueManager, Verdict +from context_intelligence_server.queue_manager import FileSystemQueueManager, QueueManager, Verdict from context_intelligence_server.registry import SessionRegistry, SessionWorker from context_intelligence_server.services import HookStateService from context_intelligence_server.status import boot_state @@ -45,7 +45,7 @@ def _line(workspace: str = "/ws", event: str = "tool_use", **data: Any) -> bytes async def _qm(tmp_path: Path) -> QueueManager: - return QueueManager(queues_dir=tmp_path) + return FileSystemQueueManager(queues_dir=tmp_path) def _seed_log(tmp_path: Path, key: str, content: bytes) -> Path: @@ -177,7 +177,7 @@ async def test_classify_unparseable_offset_large_still_resets(tmp_path: Path) -> qm = await _qm(tmp_path) settings = Settings(reclaim_redrain_max_bytes=1) # would have forced "large" with patch( - "context_intelligence_server.queue_manager.get_settings", + "context_intelligence_server.queue_manager.filesystem.get_settings", return_value=settings, ): c = await qm.classify_session("k4", _head_is_resumable) @@ -194,7 +194,7 @@ async def test_classify_offset_past_eof_large_still_deletes(tmp_path: Path) -> N qm = await _qm(tmp_path) settings = Settings(reclaim_redrain_max_bytes=1) # force "large" with patch( - "context_intelligence_server.queue_manager.get_settings", + "context_intelligence_server.queue_manager.filesystem.get_settings", return_value=settings, ): c = await qm.classify_session("k4b", _head_is_resumable) @@ -294,7 +294,7 @@ async def test_boot_reclaim_survives_garbage_log_and_real_drain_dead_letters_it( garbage = b"not json at all\n" _seed_log(tmp_path, "garbage-key", garbage) - qm = QueueManager(queues_dir=tmp_path) + qm = FileSystemQueueManager(queues_dir=tmp_path) monkeypatch.setattr(main_module.registry, "_queue_manager", qm) monkeypatch.setattr(main_module._settings, "reclaim_enabled", True) monkeypatch.setattr(main_module._settings, "crash_recovery_respawn_limit", 8) @@ -609,7 +609,7 @@ async def test_g4_topup_read_batch_guard_skips_corrupt_key( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: monkeypatch.setattr( - main_module.registry, "_queue_manager", QueueManager(queues_dir=tmp_path) + main_module.registry, "_queue_manager", FileSystemQueueManager(queues_dir=tmp_path) ) qm = main_module.registry.queue_manager _seed_log(tmp_path, "bad", _line()) @@ -653,7 +653,7 @@ async def test_all_four_crash_triggers_reach_ready_with_reclaim_disabled( main_module.app.state.schema_ready = True # schema is out of scope here monkeypatch.setattr( - main_module.registry, "_queue_manager", QueueManager(queues_dir=tmp_path) + main_module.registry, "_queue_manager", FileSystemQueueManager(queues_dir=tmp_path) ) monkeypatch.setattr(main_module._settings, "reclaim_enabled", False) monkeypatch.setattr(main_module._settings, "crash_recovery_respawn_limit", 8) @@ -678,7 +678,7 @@ async def test_gate1_live_worker_key_never_reclaimed( _seed_log(tmp_path, "owned", line) _seed_offset(tmp_path, "owned", str(len(line))) # fully_drained shape - qm = QueueManager(queues_dir=tmp_path) + qm = FileSystemQueueManager(queues_dir=tmp_path) monkeypatch.setattr(main_module.registry, "_queue_manager", qm) monkeypatch.setattr(main_module._settings, "reclaim_enabled", True) monkeypatch.setattr(main_module._settings, "crash_recovery_respawn_limit", 8) @@ -697,7 +697,7 @@ async def test_key_reclaimed_after_worker_removed_drains_from_zero( _seed_log(tmp_path, "was-owned", line) _seed_offset(tmp_path, "was-owned", str(len(line))) - qm = QueueManager(queues_dir=tmp_path) + qm = FileSystemQueueManager(queues_dir=tmp_path) monkeypatch.setattr(main_module.registry, "_queue_manager", qm) monkeypatch.setattr(main_module._settings, "reclaim_enabled", True) monkeypatch.setattr(main_module._settings, "crash_recovery_respawn_limit", 8) @@ -724,7 +724,7 @@ async def test_recovered_drainer_population_bounded_and_makes_progress( bound, and cumulative distinct dispatches exceed the ceiling (forward progress).""" ceiling = 3 n_sessions = 9 - qm = QueueManager(queues_dir=tmp_path) + qm = FileSystemQueueManager(queues_dir=tmp_path) for i in range(n_sessions): await qm.append(f"sess-{i}", _line(session_id=f"sess-{i}")) @@ -885,7 +885,7 @@ async def test_boot_reclaim_orphan_offset_survives_with_reclaim_disabled( `reclaim_orphans` itself, not just gate a telemetry counter.""" (tmp_path / "orphan.offset").write_text("5", encoding="utf-8") - qm = QueueManager(queues_dir=tmp_path) + qm = FileSystemQueueManager(queues_dir=tmp_path) monkeypatch.setattr(main_module.registry, "_queue_manager", qm) monkeypatch.setattr(main_module._settings, "reclaim_enabled", False) monkeypatch.setattr(main_module._settings, "crash_recovery_respawn_limit", 8) @@ -903,7 +903,7 @@ async def test_boot_reclaim_orphan_offset_survives_with_reclaim_disabled( async def test_dry_exit_fires_for_recovered_drainer_over_drained_log( tmp_path: Path, ) -> None: - qm = QueueManager(queues_dir=tmp_path) + qm = FileSystemQueueManager(queues_dir=tmp_path) reg = SessionRegistry() reg._queue_manager = qm line = _line() @@ -922,7 +922,7 @@ async def test_dry_exit_fires_for_recovered_drainer_over_drained_log( async def test_dry_exit_negative_control_live_created_worker_never_exits( tmp_path: Path, ) -> None: - qm = QueueManager(queues_dir=tmp_path) + qm = FileSystemQueueManager(queues_dir=tmp_path) reg = SessionRegistry() reg._queue_manager = qm line = _line() @@ -950,7 +950,7 @@ async def test_dry_exit_negative_control_recovered_that_drained_still_exits( """The RED test proving the field is NOT last_event_time in disguise: a recovered worker that DID process >=1 record must still exit once it runs dry (v1.2's rejected fix excluded exactly this population).""" - qm = QueueManager(queues_dir=tmp_path) + qm = FileSystemQueueManager(queues_dir=tmp_path) reg = SessionRegistry() reg._queue_manager = qm line = _line() @@ -968,7 +968,7 @@ async def test_dry_exit_negative_control_recovered_that_drained_still_exits( async def test_d1_no_strand_after_recheck_await(tmp_path: Path) -> None: """A POST landing during the recheck's await must not strand the drainer's client -- the flag is re-read after the await, aborting the exit.""" - qm = QueueManager(queues_dir=tmp_path) + qm = FileSystemQueueManager(queues_dir=tmp_path) reg = SessionRegistry() reg._queue_manager = qm worker = _make_worker("sess-race", live_event_seen=False) @@ -1090,7 +1090,7 @@ async def test_no_phantom_reclaim_for_dead_only_key( """A correctly-finalized session retains only a .dead.jsonl; classify must never see this key at all (it iterates *.log stems only).""" _seed_dead(tmp_path, "finalized", "kept\n") - qm = QueueManager(queues_dir=tmp_path) + qm = FileSystemQueueManager(queues_dir=tmp_path) monkeypatch.setattr(main_module.registry, "_queue_manager", qm) monkeypatch.setattr(main_module._settings, "reclaim_enabled", True) monkeypatch.setattr(main_module._settings, "crash_recovery_respawn_limit", 8) @@ -1113,7 +1113,7 @@ async def test_reclaim_disabled_classifies_but_deletes_nothing( line = _line() _seed_log(tmp_path, "would-resume", line) - qm = QueueManager(queues_dir=tmp_path) + qm = FileSystemQueueManager(queues_dir=tmp_path) monkeypatch.setattr(main_module.registry, "_queue_manager", qm) monkeypatch.setattr(main_module._settings, "reclaim_enabled", False) monkeypatch.setattr(main_module._settings, "crash_recovery_respawn_limit", 8) @@ -1132,7 +1132,7 @@ async def test_reclaim_enabled_true_deletes_what_dry_run_named( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: _seed_log(tmp_path, "target", b"") - qm = QueueManager(queues_dir=tmp_path) + qm = FileSystemQueueManager(queues_dir=tmp_path) monkeypatch.setattr(main_module.registry, "_queue_manager", qm) monkeypatch.setattr(main_module._settings, "crash_recovery_respawn_limit", 8) @@ -1159,7 +1159,7 @@ async def test_boot_reclaim_auto_reclaims_drained_log_under_shipped_defaults( """A fully-drained log (committed == complete_end == size) is the same evidence delete_drained already acts on unconditionally -- it must be reclaimed at boot even under shipped defaults (reclaim_enabled=False).""" - qm = QueueManager(queues_dir=tmp_path) + qm = FileSystemQueueManager(queues_dir=tmp_path) line = _line() await qm.append("drained-key", line) await qm.commit("drained-key", len(line)) @@ -1192,7 +1192,7 @@ async def test_boot_reclaim_risky_verdicts_stay_gated_under_shipped_defaults( _seed_log(tmp_path, "reset-key", line * 2) _seed_offset(tmp_path, "reset-key", "garbage") # small -> RESET_OFFSET - qm = QueueManager(queues_dir=tmp_path) + qm = FileSystemQueueManager(queues_dir=tmp_path) monkeypatch.setattr(main_module.registry, "_queue_manager", qm) monkeypatch.setattr(main_module._settings, "reclaim_enabled", False) monkeypatch.setattr(main_module._settings, "crash_recovery_respawn_limit", 8) @@ -1204,7 +1204,7 @@ async def test_boot_reclaim_risky_verdicts_stay_gated_under_shipped_defaults( # other test's get_settings.cache_clear() changing the shared singleton. threshold_settings = Settings(reclaim_redrain_max_bytes=150) with patch( - "context_intelligence_server.queue_manager.get_settings", + "context_intelligence_server.queue_manager.filesystem.get_settings", return_value=threshold_settings, ): # Confirm the verdicts are what this test claims before asserting. @@ -1232,14 +1232,14 @@ async def test_boot_reclaim_large_unparseable_offset_never_deletes_log( _seed_log(tmp_path, "big-unparseable", big) _seed_offset(tmp_path, "big-unparseable", "not-a-number") - qm = QueueManager(queues_dir=tmp_path) + qm = FileSystemQueueManager(queues_dir=tmp_path) monkeypatch.setattr(main_module.registry, "_queue_manager", qm) monkeypatch.setattr(main_module._settings, "reclaim_enabled", True) monkeypatch.setattr(main_module._settings, "crash_recovery_respawn_limit", 8) threshold_settings = Settings(reclaim_redrain_max_bytes=1) # old "large" cliff with patch( - "context_intelligence_server.queue_manager.get_settings", + "context_intelligence_server.queue_manager.filesystem.get_settings", return_value=threshold_settings, ): await main_module._boot_reclaim() @@ -1255,7 +1255,7 @@ async def test_boot_reclaim_drained_log_with_live_worker_is_skipped( ) -> None: """The has_worker guard runs BEFORE classify -- a live worker's log is never touched by boot reclaim, DRAINED or not.""" - qm = QueueManager(queues_dir=tmp_path) + qm = FileSystemQueueManager(queues_dir=tmp_path) line = _line() await qm.append("live-drained-key", line) await qm.commit("live-drained-key", len(line)) @@ -1480,7 +1480,7 @@ async def test_merged_head_session_resumes_and_drains_behind_the_poison_line( tmp_path, "merged-head", str(len(good_head)) ) # merged is first-uncommitted - qm = QueueManager(queues_dir=tmp_path) + qm = FileSystemQueueManager(queues_dir=tmp_path) c1 = await qm.classify_session("merged-head", _head_is_resumable) assert c1.verdict is Verdict.RESUMABLE assert c1.reason == "fallback_workspace" diff --git a/tests/test_concurrent_append.py b/tests/test_concurrent_append.py index 9cd6f248..406559d0 100644 --- a/tests/test_concurrent_append.py +++ b/tests/test_concurrent_append.py @@ -15,7 +15,7 @@ import os from pathlib import Path -from context_intelligence_server.queue_manager import QueueManager +from context_intelligence_server.queue_manager import FileSystemQueueManager, QueueManager _SMALL_SIZE = 64 _LARGE_SIZE = 1_500_000 # > 1 MiB, mixed in with small records @@ -79,7 +79,7 @@ async def test_concurrent_appends_many_sessions_no_tear_or_merge_or_loss( """>=8 sessions x >=50 records each, all interleaved concurrently, plus concurrent appends to the SAME session and several >1 MiB payloads mixed with small ones.""" - qm = QueueManager(queues_dir=tmp_path / "queues") + qm = FileSystemQueueManager(queues_dir=tmp_path / "queues") num_sessions = 10 records_per_session = 60 @@ -105,7 +105,7 @@ async def test_concurrent_appends_many_sessions_no_tear_or_merge_or_loss( async def test_concurrent_appends_single_session_hammered(tmp_path: Path) -> None: """Worst-case contention: many concurrent tasks writing ONE session's file.""" - qm = QueueManager(queues_dir=tmp_path / "queues") + qm = FileSystemQueueManager(queues_dir=tmp_path / "queues") session_id = "hot-session" num_records = 300 diff --git a/tests/test_drain_lifecycle_logging.py b/tests/test_drain_lifecycle_logging.py index 209512bc..f3ae3f76 100644 --- a/tests/test_drain_lifecycle_logging.py +++ b/tests/test_drain_lifecycle_logging.py @@ -22,7 +22,7 @@ import pytest import context_intelligence_server.main as main_module -from context_intelligence_server.queue_manager import QueueManager +from context_intelligence_server.queue_manager import FileSystemQueueManager, QueueManager from context_intelligence_server.registry import SessionRegistry from context_intelligence_server.writer_lease import WriterLease, WriterLeaseConflict from tests.test_drain_supervision import ( @@ -324,7 +324,7 @@ async def test_g5_dead_letter_write_oserror_logs_error_and_reraises( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: - qm = QueueManager(queues_dir=tmp_path / "queues") + qm = FileSystemQueueManager(queues_dir=tmp_path / "queues") sid = "d3-g5-dead-letter" injected = OSError(errno.EIO, "Input/output error") monkeypatch.setattr(qm, "_write_record", MagicMock(side_effect=injected)) @@ -374,7 +374,7 @@ def _fake_stat(self: Path, *a: object, **kw: object) -> object: return orig_stat(self, *a, **kw) # type: ignore[misc] monkeypatch.setattr(Path, "stat", _fake_stat) - qm = QueueManager(queues_dir=qdir) + qm = FileSystemQueueManager(queues_dir=qdir) with caplog.at_level(logging.WARNING, logger=LOGGER_NAME): result = await qm.reclaim_orphans( @@ -416,7 +416,7 @@ def _fake_stat(self: Path, *a: object, **kw: object) -> object: return orig_stat(self, *a, **kw) # type: ignore[misc] monkeypatch.setattr(Path, "stat", _fake_stat) - qm = QueueManager(queues_dir=qdir) + qm = FileSystemQueueManager(queues_dir=qdir) with caplog.at_level(logging.WARNING, logger=LOGGER_NAME): result = await qm.reclaim_orphans( diff --git a/tests/test_durable_append_framing.py b/tests/test_durable_append_framing.py index 6ec95fea..2725bdd8 100644 --- a/tests/test_durable_append_framing.py +++ b/tests/test_durable_append_framing.py @@ -18,8 +18,9 @@ import pytest -from context_intelligence_server import queue_manager as qm_module -from context_intelligence_server.queue_manager import QueueManager, _KeyGuard +from context_intelligence_server.queue_manager import filesystem as qm_module +from context_intelligence_server.queue_manager import FileSystemQueueManager, QueueManager +from context_intelligence_server.queue_manager.filesystem import _KeyGuard from context_intelligence_server.registry import SessionRegistry pytestmark = pytest.mark.integration @@ -105,7 +106,7 @@ async def test_control_local_o_append_is_atomic(tmp_path: Path) -> None: every line parses even without the guard doing any work. Passing alone is not proof the guard works -- see test_smb_split_write_no_longer_merges_records. """ - qm = QueueManager(tmp_path) + qm = FileSystemQueueManager(tmp_path) records: list[bytes] = [] for i in range(12): @@ -226,7 +227,7 @@ async def test_smb_split_write_no_longer_merges_records( via the shim while a second appender for the same key waits on the guard; both lines land whole, in order, and parse cleanly. """ - qm = QueueManager(tmp_path) + qm = FileSystemQueueManager(tmp_path) real_write = os.write large_first_op = threading.Event() @@ -282,7 +283,7 @@ async def _append_small() -> None: async def test_concurrent_appends_under_smb_shim_all_parse( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, key: str ) -> None: - qm = QueueManager(tmp_path) + qm = FileSystemQueueManager(tmp_path) monkeypatch.setattr(qm_module.os, "write", _SplitWriteOS(os.write)) records: list[bytes] = [] @@ -313,7 +314,7 @@ async def test_concurrent_appends_under_smb_shim_all_parse( async def test_cancel_mid_write_never_releases_the_file_lock( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: - qm = QueueManager(tmp_path) + qm = FileSystemQueueManager(tmp_path) key = SESSION entered_write = threading.Event() @@ -405,7 +406,7 @@ def _on_chunk(fd: int, idx: int, chunk: bytes, total_len: int) -> None: async def test_distinct_keys_append_concurrently( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: - qm = QueueManager(tmp_path) + qm = FileSystemQueueManager(tmp_path) key_a = "key-a" key_b = "key-b" @@ -444,7 +445,7 @@ def _on_chunk(fd: int, idx: int, chunk: bytes, total_len: int) -> None: async def test_heal_torn_tails_truncates_and_quarantines(tmp_path: Path) -> None: - qm = QueueManager(tmp_path) + qm = FileSystemQueueManager(tmp_path) key = "torn-key" log_path = qm._log_path(key) good = b'{"event":"a"}\n' @@ -471,7 +472,7 @@ async def test_heal_torn_tails_truncates_and_quarantines(tmp_path: Path) -> None async def test_heal_torn_tails_on_empty_and_newline_free_files(tmp_path: Path) -> None: - qm = QueueManager(tmp_path) + qm = FileSystemQueueManager(tmp_path) empty_key = "empty-key" nf_key = "newline-free-key" qm._log_path(empty_key).write_bytes(b"") @@ -505,7 +506,7 @@ async def test_heal_torn_tails_on_empty_and_newline_free_files(tmp_path: Path) - async def test_partial_write_failure_discards_the_record( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: - qm = QueueManager(tmp_path) + qm = FileSystemQueueManager(tmp_path) key = "fail-key" real_write = os.write calls = {"n": 0} @@ -540,7 +541,7 @@ def _flaky_write(fd: int, data: Any) -> int: async def test_partial_write_failure_logs_when_newline_terminate_fails( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture ) -> None: - qm = QueueManager(tmp_path) + qm = FileSystemQueueManager(tmp_path) key = "fail-key-2" real_write = os.write calls = {"n": 0} @@ -584,7 +585,7 @@ async def test_discard_partial_never_destroys_a_peer_process_committed_line( completes and closes its own fully-formed, already-acknowledged record. _discard_partial must never remove those bytes. """ - qm = QueueManager(tmp_path) + qm = FileSystemQueueManager(tmp_path) key = "race-key" path = qm._log_path(key) path.write_bytes(b'{"payload":"PRIOR-COMMITTED"}\n') @@ -639,7 +640,7 @@ async def test_discard_partial_single_writer_preserves_prior_records( next record); a subsequent drain dead-letters the fragment rather than crashing. """ - qm = QueueManager(tmp_path) + qm = FileSystemQueueManager(tmp_path) key = "single-writer-key" prior = _event_bytes("prior-committed") await qm.append(key, prior) @@ -683,7 +684,7 @@ def _flaky_write(fd: int, data: Any) -> int: async def test_delete_drained_cannot_race_an_in_flight_append( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: - qm = QueueManager(tmp_path) + qm = FileSystemQueueManager(tmp_path) key = "race-delete-key" entered = threading.Event() @@ -724,7 +725,7 @@ def _on_chunk(fd: int, idx: int, chunk: bytes, total_len: int) -> None: async def test_delete_drained_retains_a_log_with_uncommitted_bytes( tmp_path: Path, caplog: pytest.LogCaptureFixture ) -> None: - qm = QueueManager(tmp_path) + qm = FileSystemQueueManager(tmp_path) key = "uncommitted-key" first = _event_bytes("first") second = _event_bytes("second") @@ -755,7 +756,7 @@ async def test_delete_drained_retains_a_log_with_uncommitted_bytes( async def test_guard_map_is_released_on_delete_drained_and_identity_checked( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: - qm = QueueManager(tmp_path) + qm = FileSystemQueueManager(tmp_path) keys = [f"session-{i}" for i in range(5)] for k in keys: await qm.append(k, _event_bytes("ev")) @@ -811,7 +812,7 @@ def _paused_stat(self: Path, *a: Any, **kw: Any) -> Any: async def test_dead_letter_record_is_framed_under_smb_shim( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: - qm = QueueManager(tmp_path) + qm = FileSystemQueueManager(tmp_path) key = "dl-key" monkeypatch.setattr(qm_module.os, "write", _SplitWriteOS(os.write)) large_raw = _event_bytes("bad:record", filler=300 * 1024) @@ -823,7 +824,7 @@ async def test_dead_letter_record_is_framed_under_smb_shim( async def test_dead_letter_parsing_survives_a_malformed_line(tmp_path: Path) -> None: - qm = QueueManager(tmp_path) + qm = FileSystemQueueManager(tmp_path) key = "dl-malformed-key" dead_path = qm._dead_path(key) good = json.dumps({"ts": 1.0, "error": "e", "payload": "ok"}) @@ -849,7 +850,7 @@ async def test_guard_survives_a_delete_that_races_a_parked_appender( survive while appender A still holds a reference, so a subsequent appender B is served by the SAME guard rather than a disjoint one. """ - qm = QueueManager(tmp_path) + qm = FileSystemQueueManager(tmp_path) key = "race-key" seed = _event_bytes("seed") @@ -913,7 +914,7 @@ def _paused_stat(self: Path, *a: Any, **kw: Any) -> Any: async def test_heal_torn_tails_survives_an_oserror_and_still_boots( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture ) -> None: - qm = QueueManager(tmp_path) + qm = FileSystemQueueManager(tmp_path) good = b'{"event":"ok"}\n' torn = b'{"event":"torn","data":"no-terminator-yet' for name in ("a", "b", "c"): @@ -991,7 +992,7 @@ async def test_read_batch_over_a_pre_existing_merged_middle_line( """A pre-existing merged middle line still fails _parse_line -- consuming it (dead-letter, commit past it) is the drainer's job, not this fix's. """ - qm = QueueManager(tmp_path) + qm = FileSystemQueueManager(tmp_path) key = "merged-middle-key" assert _SEEDS is not None merged = (_SEEDS / "seed_corrupt_merged_line_1.0MiB.raw").read_bytes() diff --git a/tests/test_identity_store.py b/tests/test_identity_store.py index 61733c91..bc8cd6e9 100644 --- a/tests/test_identity_store.py +++ b/tests/test_identity_store.py @@ -14,7 +14,7 @@ import pytest -from context_intelligence_server.identity_store import IdentityStore +from context_intelligence_server.identity_store import FileSystemIdentityStore # --------------------------------------------------------------------------- @@ -42,7 +42,7 @@ def _bob_entry() -> dict[str, str]: class TestPutGetRoundtrip: def test_put_then_get_returns_value(self, tmp_path: Path) -> None: """put(key, value) → get(key) returns that value immediately.""" - store = IdentityStore(path=tmp_path / "store.json") + store = FileSystemIdentityStore(path=tmp_path / "store.json") store.load() store.put(FAKE_HASH_A, _alice_entry()) @@ -53,25 +53,25 @@ def test_put_then_get_returns_value(self, tmp_path: Path) -> None: def test_put_persists_to_file_and_loads_fresh(self, tmp_path: Path) -> None: """Round-trip: put → create new store → load → value is present.""" store_path = tmp_path / "store.json" - store = IdentityStore(path=store_path) + store = FileSystemIdentityStore(path=store_path) store.load() store.put(FAKE_HASH_A, _alice_entry()) # New store instance reads from disk - store2 = IdentityStore(path=store_path) + store2 = FileSystemIdentityStore(path=store_path) store2.load() assert store2.get(FAKE_HASH_A) == _alice_entry() def test_get_missing_key_returns_none(self, tmp_path: Path) -> None: """get() returns None for a key that was never put.""" - store = IdentityStore(path=tmp_path / "store.json") + store = FileSystemIdentityStore(path=tmp_path / "store.json") store.load() assert store.get(FAKE_HASH_A) is None def test_delete_removes_key(self, tmp_path: Path) -> None: """delete(key) removes the entry from in-process dict AND file.""" store_path = tmp_path / "store.json" - store = IdentityStore(path=store_path) + store = FileSystemIdentityStore(path=store_path) store.load() store.put(FAKE_HASH_A, _alice_entry()) store.delete(FAKE_HASH_A) @@ -79,13 +79,13 @@ def test_delete_removes_key(self, tmp_path: Path) -> None: assert store.get(FAKE_HASH_A) is None # Verify file is also updated - store2 = IdentityStore(path=store_path) + store2 = FileSystemIdentityStore(path=store_path) store2.load() assert store2.get(FAKE_HASH_A) is None def test_items_returns_all_entries(self, tmp_path: Path) -> None: """items() yields all key-value pairs currently in the store.""" - store = IdentityStore(path=tmp_path / "store.json") + store = FileSystemIdentityStore(path=tmp_path / "store.json") store.load() store.put(FAKE_HASH_A, _alice_entry()) store.put(FAKE_HASH_B, _bob_entry()) @@ -95,7 +95,7 @@ def test_items_returns_all_entries(self, tmp_path: Path) -> None: def test_upsert_overwrites_existing(self, tmp_path: Path) -> None: """put() on an existing key overwrites the value.""" - store = IdentityStore(path=tmp_path / "store.json") + store = FileSystemIdentityStore(path=tmp_path / "store.json") store.load() store.put(FAKE_HASH_A, _alice_entry()) store.put(FAKE_HASH_A, {"id": "alice-updated"}) @@ -105,12 +105,12 @@ def test_upsert_overwrites_existing(self, tmp_path: Path) -> None: def test_sequential_puts_all_persist(self, tmp_path: Path) -> None: """Multiple sequential puts all persist correctly (each write is the full map).""" store_path = tmp_path / "store.json" - store = IdentityStore(path=store_path) + store = FileSystemIdentityStore(path=store_path) store.load() store.put(FAKE_HASH_A, _alice_entry()) store.put(FAKE_HASH_B, _bob_entry()) - store2 = IdentityStore(path=store_path) + store2 = FileSystemIdentityStore(path=store_path) store2.load() assert store2.get(FAKE_HASH_A) == _alice_entry() assert store2.get(FAKE_HASH_B) == _bob_entry() @@ -124,7 +124,7 @@ def test_sequential_puts_all_persist(self, tmp_path: Path) -> None: class TestLoadFailClosed: def test_missing_file_yields_empty_dict(self, tmp_path: Path) -> None: """Missing store file → load() yields empty dict (normal first boot), no raise.""" - store = IdentityStore(path=tmp_path / "nonexistent.json") + store = FileSystemIdentityStore(path=tmp_path / "nonexistent.json") store.load() # must not raise assert store.get(FAKE_HASH_A) is None assert list(store.items()) == [] @@ -136,7 +136,7 @@ def test_corrupt_json_yields_empty_dict_and_logs_error( store_path = tmp_path / "store.json" store_path.write_text("{{{{not valid json at all}}}}", encoding="utf-8") - store = IdentityStore(path=store_path) + store = FileSystemIdentityStore(path=store_path) with caplog.at_level(logging.ERROR): store.load() # must NOT raise @@ -156,7 +156,7 @@ def test_valid_json_but_not_dict_yields_empty_and_logs( json.dumps([{"id": "alice"}]), encoding="utf-8" ) # list, not dict - store = IdentityStore(path=store_path) + store = FileSystemIdentityStore(path=store_path) with caplog.at_level(logging.ERROR): store.load() # must NOT raise @@ -171,7 +171,7 @@ def test_partial_write_torn_file_loads_empty( store_path = tmp_path / "store.json" store_path.write_bytes(b'{"aaa": {"id": "al') # truncated mid-write - store = IdentityStore(path=store_path) + store = FileSystemIdentityStore(path=store_path) with caplog.at_level(logging.ERROR): store.load() # must NOT raise @@ -188,7 +188,7 @@ def test_partial_write_torn_file_loads_empty( class TestWriteFileThenSwapMemory: def test_put_write_failure_leaves_dict_unchanged(self, tmp_path: Path) -> None: """If os.replace raises, the in-process dict is UNCHANGED (F2 contract).""" - store = IdentityStore(path=tmp_path / "store.json") + store = FileSystemIdentityStore(path=tmp_path / "store.json") store.load() # Establish an existing entry store.put(FAKE_HASH_A, _alice_entry()) @@ -208,7 +208,7 @@ def test_put_write_failure_leaves_dict_unchanged(self, tmp_path: Path) -> None: def test_delete_write_failure_leaves_dict_unchanged(self, tmp_path: Path) -> None: """If delete's file write fails, the in-process dict is UNCHANGED.""" - store = IdentityStore(path=tmp_path / "store.json") + store = FileSystemIdentityStore(path=tmp_path / "store.json") store.load() store.put(FAKE_HASH_A, _alice_entry()) @@ -221,7 +221,7 @@ def test_delete_write_failure_leaves_dict_unchanged(self, tmp_path: Path) -> Non def test_failed_write_leaves_no_torn_tempfile(self, tmp_path: Path) -> None: """A failed os.replace must clean up the tempfile — no orphaned .tmp files.""" - store = IdentityStore(path=tmp_path / "store.json") + store = FileSystemIdentityStore(path=tmp_path / "store.json") store.load() with patch("os.replace", side_effect=OSError("simulated disk full")): @@ -235,7 +235,7 @@ def test_failed_write_leaves_no_torn_tempfile(self, tmp_path: Path) -> None: def test_atomic_write_uses_tempfile_in_same_dir(self, tmp_path: Path) -> None: """Writes use a temp file in the same directory (then os.replace).""" store_path = tmp_path / "store.json" - store = IdentityStore(path=store_path) + store = FileSystemIdentityStore(path=store_path) store.load() replaced_from: list[str] = [] @@ -266,13 +266,13 @@ class TestFlatDictLiveReference: def test_flat_dict_empty_on_new_store(self, tmp_path: Path) -> None: """flat_dict is empty after load() with no file.""" - store = IdentityStore(path=tmp_path / "store.json") + store = FileSystemIdentityStore(path=tmp_path / "store.json") store.load() assert store.flat_dict == {} def test_flat_dict_updated_after_put(self, tmp_path: Path) -> None: """flat_dict is updated immediately after put().""" - store = IdentityStore(path=tmp_path / "store.json") + store = FileSystemIdentityStore(path=tmp_path / "store.json") store.load() store.put(FAKE_HASH_A, {"id": "alice"}) @@ -280,7 +280,7 @@ def test_flat_dict_updated_after_put(self, tmp_path: Path) -> None: def test_flat_dict_updated_after_delete(self, tmp_path: Path) -> None: """flat_dict removes key immediately after delete().""" - store = IdentityStore(path=tmp_path / "store.json") + store = FileSystemIdentityStore(path=tmp_path / "store.json") store.load() store.put(FAKE_HASH_A, {"id": "alice"}) store.delete(FAKE_HASH_A) @@ -290,7 +290,7 @@ def test_flat_dict_updated_after_delete(self, tmp_path: Path) -> None: def test_flat_dict_is_same_object_across_puts(self, tmp_path: Path) -> None: """flat_dict is the SAME dict object before and after put() (so a shared reference to flat_dict stays live).""" - store = IdentityStore(path=tmp_path / "store.json") + store = FileSystemIdentityStore(path=tmp_path / "store.json") store.load() flat_ref = store.flat_dict # capture the reference @@ -309,7 +309,7 @@ def test_flat_dict_populated_from_file_on_load(self, tmp_path: Path) -> None: json.dumps({FAKE_HASH_A: {"id": "alice"}, FAKE_HASH_B: {"id": "bob"}}), encoding="utf-8", ) - store = IdentityStore(path=store_path) + store = FileSystemIdentityStore(path=store_path) store.load() assert store.flat_dict[FAKE_HASH_A] == "alice" diff --git a/tests/test_m2_service_auth.py b/tests/test_m2_service_auth.py index a6f9781f..024454e0 100644 --- a/tests/test_m2_service_auth.py +++ b/tests/test_m2_service_auth.py @@ -123,18 +123,20 @@ def _service_claims(roles: list[str], appid: str = FAKE_APPID) -> dict[str, Any] class _MockBlobStore: - """Mock for AsyncDiskBlobStore — returns empty list, never touches filesystem.""" + """Mock BlobStore — empty list, never touches filesystem.""" - def __init__(self, root: Any) -> None: - pass - - async def list(self, session_id: str) -> list[str]: - return [] + async def list(self, session_id: str) -> AsyncGenerator[Any, None]: + return + yield # unreachable: makes list() an async generator async def read(self, uri: str) -> Any: raise FileNotFoundError(f"mock blob store: not found: {uri}") +def _mock_blob_store_factory(settings: Any) -> _MockBlobStore: + return _MockBlobStore() + + class _MockNeo4jResult: """Async-iterable result mock that yields a fixed list of rows.""" @@ -377,7 +379,7 @@ async def test_cap_sr_r_reader_read_capable( private_key, asgi = service_asgi token = _sign_jwt(private_key, _service_claims(roles=["Reader"])) - monkeypatch.setattr(main_module, "AsyncDiskBlobStore", _MockBlobStore) + monkeypatch.setattr(main_module, "create_blob_store", _mock_blob_store_factory) async with _make_client(asgi) as c: resp = await c.get( @@ -456,7 +458,7 @@ async def test_cap_sc_r_contributor_read_capable( private_key, asgi = service_asgi token = _sign_jwt(private_key, _service_claims(roles=["Contributor"])) - monkeypatch.setattr(main_module, "AsyncDiskBlobStore", _MockBlobStore) + monkeypatch.setattr(main_module, "create_blob_store", _mock_blob_store_factory) async with _make_client(asgi) as c: resp = await c.get( diff --git a/tests/test_main.py b/tests/test_main.py index 5dd27c80..1bfa6129 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -163,11 +163,11 @@ async def test_post_events_increments_accepted_counter( monkeypatch: pytest.MonkeyPatch, ) -> None: """A durably-accepted event increments the registry accepted_total.""" - from context_intelligence_server.queue_manager import QueueManager + from context_intelligence_server.queue_manager import FileSystemQueueManager, QueueManager # Point the registry at a tmp queue dir so the durable append is isolated. monkeypatch.setattr( - main_module.registry, "_queue_manager", QueueManager(queues_dir=tmp_path) + main_module.registry, "_queue_manager", FileSystemQueueManager(queues_dir=tmp_path) ) monkeypatch.setattr( main_module.registry, "get_or_create", lambda *args, **kwargs: MagicMock() @@ -1313,11 +1313,11 @@ async def test_lifespan_seeds_counters_from_disk(tmp_path: Path) -> None: zero residual: 1 committed + 1 pending line yields accepted=2, written=1, in_queue=1, residual=0 after reconcile -> seed_counts -> seed_counters. """ - from context_intelligence_server.queue_manager import QueueManager + from context_intelligence_server.queue_manager import FileSystemQueueManager, QueueManager from context_intelligence_server.registry import SessionRegistry # Seed a queue dir with one committed line and one still-pending line. - seed_qm = QueueManager(queues_dir=tmp_path) + seed_qm = FileSystemQueueManager(queues_dir=tmp_path) sid = "sess-seed" line1 = json.dumps({"event": "a", "workspace": "/ws", "data": {}}).encode("utf-8") line2 = json.dumps({"event": "b", "workspace": "/ws", "data": {}}).encode("utf-8") @@ -1328,7 +1328,7 @@ async def test_lifespan_seeds_counters_from_disk(tmp_path: Path) -> None: # Fresh registry reusing the same on-disk queue dir. reg = SessionRegistry() - reg._queue_manager = QueueManager(queues_dir=tmp_path) + reg._queue_manager = FileSystemQueueManager(queues_dir=tmp_path) # Production order: reconcile dead lines BEFORE seeding the counts. await reg.queue_manager.recovery_reconcile_dead() diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index dfe3bffb..36c97d97 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -27,6 +27,7 @@ import pytest + # --------------------------------------------------------------------------- # NOTE: ToolCallHandler stub injection is performed in conftest.py so it # fires before any test module loads, regardless of pytest collection order. @@ -152,8 +153,8 @@ def test_setup_handlers_returns_pipeline_handlers() -> None: def test_setup_handlers_has_default_handler_with_services() -> None: - from context_intelligence_server.handlers.data_layer_1.default import DefaultHandler from context_intelligence_server.pipeline import setup_handlers + from context_intelligence_server.handlers.data_layer_1.default import DefaultHandler from context_intelligence_server.services import HookStateService services = HookStateService(workspace="test") @@ -184,17 +185,17 @@ def test_setup_handlers_enricher_count() -> None: def test_setup_handlers_enricher_order() -> None: """Enrichers must be [SessionHandler, OrchestratorRunHandler, IterationHandler, ContentBlockHandler, ToolCallHandler] in that dispatch order.""" - from context_intelligence_server.handlers.data_layer_2.content_block import ( - ContentBlockHandler, + from context_intelligence_server.pipeline import setup_handlers + from context_intelligence_server.handlers.data_layer_2.session import SessionHandler + from context_intelligence_server.handlers.data_layer_2.orchestrator_run import ( + OrchestratorRunHandler, ) from context_intelligence_server.handlers.data_layer_2.iteration import ( IterationHandler, ) - from context_intelligence_server.handlers.data_layer_2.orchestrator_run import ( - OrchestratorRunHandler, + from context_intelligence_server.handlers.data_layer_2.content_block import ( + ContentBlockHandler, ) - from context_intelligence_server.handlers.data_layer_2.session import SessionHandler - from context_intelligence_server.pipeline import setup_handlers from context_intelligence_server.services import HookStateService services = HookStateService(workspace="test") @@ -209,19 +210,19 @@ def test_setup_handlers_enricher_order() -> None: def test_setup_handlers_l3_enricher_order() -> None: """Layer 3 enrichers must be appended after all Layer 2 enrichers in correct order: [DelegationHandler, SkillLoadHandler, RecipeRunHandler, RecipeStepHandler].""" + from context_intelligence_server.pipeline import setup_handlers from context_intelligence_server.handlers.data_layer_3.delegation import ( DelegationHandler, ) + from context_intelligence_server.handlers.data_layer_3.skill_load import ( + SkillLoadHandler, + ) from context_intelligence_server.handlers.data_layer_3.recipe_run import ( RecipeRunHandler, ) from context_intelligence_server.handlers.data_layer_3.recipe_step import ( RecipeStepHandler, ) - from context_intelligence_server.handlers.data_layer_3.skill_load import ( - SkillLoadHandler, - ) - from context_intelligence_server.pipeline import setup_handlers from context_intelligence_server.services import HookStateService services = HookStateService(workspace="test") @@ -387,46 +388,6 @@ async def test_process_event_terminal_does_not_self_flush( mock_worker.services.graph.flush.assert_not_called() -async def test_process_event_terminal_does_not_self_flush_real_handlers() -> None: - """Wires the real ``setup_handlers(services)`` enrichers (not - ``_StubEnricher``) to verify ``SessionHandler`` never self-flushes; - also asserts the session node was actually written.""" - from context_intelligence_server.pipeline import process_event, setup_handlers - from context_intelligence_server.registry import SessionWorker - from context_intelligence_server.services import HookStateService - - services = HookStateService(workspace="test-real-handlers") - handlers = setup_handlers(services) - worker = SessionWorker( - session_id="sess-real-1", workspace="test-real-handlers", services=services - ) - - real_flush = services.graph.flush - flush_calls: list[None] = [] - - async def _counting_flush() -> None: - flush_calls.append(None) - await real_flush() - - services.graph.flush = _counting_flush # type: ignore[method-assign] - - data = {"session_id": "sess-real-1", "timestamp": "2026-01-01T00:00:00Z"} - await process_event(worker, "session:end", data, handlers) - - assert flush_calls == [], ( - f"SessionHandler._handle_end (via the REAL setup_handlers enrichers) " - f"called graph.flush directly {len(flush_calls)} time(s) -- " - f"process_event must not self-flush; the drainer's gated " - f"_flush_barrier is the sole trigger" - ) - - # Confirm the real SessionHandler actually ran and wrote the node. - node = await services.graph.get_node("sess-real-1") - assert node is not None and node.get("status") == "completed", ( - f"real SessionHandler did not run (mis-wired fixture?): {node}" - ) - - async def test_process_event_non_terminal_does_not_self_flush( mock_worker: MagicMock, pipeline_handlers: Any, @@ -451,7 +412,7 @@ async def test_process_event_default_handler_exception_propagates( mock_worker: MagicMock, default_handler: _StubDefaultHandler, ) -> None: - """A default-handler (step 4) error must PROPAGATE so the drainer + """Phase B2: a default-handler (step 4) error must PROPAGATE so the drainer can dead-letter the line instead of committing the offset past a never-persisted event (no silent loss).""" from context_intelligence_server.pipeline import PipelineHandlers, process_event @@ -465,6 +426,7 @@ async def test_process_event_default_handler_exception_propagates( ) +# NOTE (Task 6): test_process_event_flush_exception_propagates was removed. # process_event no longer flushes at all — the drainer's gated _flush_barrier is # the sole write trigger, so flush-failure-propagation is now a drainer contract # covered by tests/test_registry.py::TestDurableDrainLoop @@ -476,7 +438,7 @@ async def test_process_event_propagates_handler_error( mock_worker: MagicMock, pipeline_handlers: Any, ) -> None: - """A handler error in steps 2-6 must + """Phase B2 (USER DECISION option a): a handler error in steps 2-6 must PROPAGATE, not be swallowed — here ensure_session_node (step 2) raises and process_event must re-raise so the drainer routes the line to dead-letter rather than committing the offset past a never-persisted event.""" @@ -526,8 +488,12 @@ async def test_blob_processing_called_when_all_conditions_met( ) as mock_node_id, ): await process_event(worker, "session:start", data, pipeline_handlers) + # The blob-key node_id must be computed with the SAME disambiguator + # (tool_call_id) that handlers/data_layer_1/default.py uses for the + # Event node id -- otherwise parallel same-millisecond events collide + # on the blob key (see test_blob_node_id_matches_default_handler_event_node_id). mock_node_id.assert_called_once_with( - "sess-123", "session:start", "2024-01-01T00:00:00Z" + "sess-123", "session:start", "2024-01-01T00:00:00Z", None ) mock_process.assert_called_once_with( data, worker.services.blob_store, "sess-123", "test-node-id" @@ -604,6 +570,123 @@ async def test_blob_skip_missing_timestamp_logs_warning( assert "missing timestamp" in caplog.text +# =========================================================================== +# Blob-key collision regression (data integrity) +# +# Root cause: pipeline.py computed the blob-key node_id WITHOUT the +# tool_call_id disambiguator that handlers/data_layer_1/default.py uses for +# the Event node id. Two distinct same-session, same-event, same-millisecond +# events with DIFFERENT tool_call_id (e.g. parallel tool calls) therefore +# minted the SAME blob key, and the second write silently clobbered the +# first via FileSystemBlobStore's os.replace -- while the first Event node's +# $blob_ref still pointed at that (now-overwritten) URI. +# =========================================================================== + + +async def test_parallel_same_millisecond_events_do_not_collide_on_blob_key( + pipeline_handlers: Any, + tmp_path: Any, +) -> None: + """Two events sharing session_id + event name + timestamp (same epoch-ms) + but with DIFFERENT tool_call_id must mint DISTINCT ci-blob:// URIs, and + each blob must read back its own payload -- no silent overwrite.""" + from context_intelligence_server.blob_store import FileSystemBlobStore + from context_intelligence_server.pipeline import process_event + + blob_store = FileSystemBlobStore(root=tmp_path) + + worker = MagicMock() + worker.services.ensure_session_node = AsyncMock() + worker.services.touch_session = AsyncMock() + worker.services.graph = MagicMock() + worker.services.graph.flush = AsyncMock() + worker.services.blob_store = blob_store + + session_id = "sess-parallel" + event_name = "tool_call:end" + timestamp = "2024-06-01T12:00:00.000Z" # fixed -- identical epoch-ms for both + + data_a: dict[str, Any] = { + "session_id": session_id, + "timestamp": timestamp, + "tool_call_id": "call-A", + "result": {"payload": "result-from-call-A"}, + } + data_b: dict[str, Any] = { + "session_id": session_id, + "timestamp": timestamp, + "tool_call_id": "call-B", + "result": {"payload": "result-from-call-B"}, + } + + await process_event(worker, event_name, data_a, pipeline_handlers) + await process_event(worker, event_name, data_b, pipeline_handlers) + + # (c) each event's data[field] == {"$blob_ref": } + assert "$blob_ref" in data_a["result"] + assert "$blob_ref" in data_b["result"] + uri_a = data_a["result"]["$blob_ref"] + uri_b = data_b["result"]["$blob_ref"] + + # (a) distinct URIs -- no collision + assert uri_a != uri_b, ( + f"Blob key collision: both events minted the same URI {uri_a!r} -- " + "the second write silently overwrote the first's blob." + ) + + # (b) both blobs exist and each reads back its OWN distinct payload + read_a = await blob_store.read(uri_a) + read_b = await blob_store.read(uri_b) + assert read_a == {"payload": "result-from-call-A"} + assert read_b == {"payload": "result-from-call-B"} + + +async def test_blob_node_id_matches_default_handler_event_node_id( + pipeline_handlers: Any, +) -> None: + """Pins the invariant: the blob-key node_id pipeline.process_event computes + must EQUAL the event_node_id handlers/data_layer_1/default.py computes for + the same event (make_node_id(session_id, event, timestamp, + data.get("tool_call_id"))). If these ever drift apart again, the + collision this test suite guards against reappears.""" + from context_intelligence_server.pipeline import process_event + from context_intelligence_server.utils import make_node_id + + worker = MagicMock() + worker.services.ensure_session_node = AsyncMock() + worker.services.touch_session = AsyncMock() + worker.services.graph = MagicMock() + worker.services.graph.flush = AsyncMock() + worker.services.blob_store = MagicMock() # truthy blob_store + + session_id = "sess-invariant" + event_name = "tool_call:end" + timestamp = "2024-06-01T12:00:00.000Z" + tool_call_id = "call-invariant" + + data = { + "session_id": session_id, + "timestamp": timestamp, + "tool_call_id": tool_call_id, + } + + expected_event_node_id = make_node_id( + session_id, event_name, timestamp, tool_call_id + ) + + with patch( + "context_intelligence_server.pipeline.process_event_data", + new_callable=AsyncMock, + ) as mock_process: + await process_event(worker, event_name, data, pipeline_handlers) + actual_blob_node_id = mock_process.call_args.args[3] + + assert actual_blob_node_id == expected_event_node_id, ( + "Blob-key node_id has drifted from default.py's event_node_id -- " + "this reintroduces the same-millisecond blob-key collision." + ) + + # =========================================================================== # process_event — touch_session call site # =========================================================================== diff --git a/tests/test_queue_manager.py b/tests/test_queue_manager.py index d7e743e6..33f3790a 100644 --- a/tests/test_queue_manager.py +++ b/tests/test_queue_manager.py @@ -7,7 +7,7 @@ import pytest from context_intelligence_server.queue_manager import ( Batch, - QueueManager, + FileSystemQueueManager, Record, Verdict, ) @@ -15,13 +15,13 @@ @pytest.fixture def qm(tmp_path): - return QueueManager(queues_dir=tmp_path / "queues") + return FileSystemQueueManager(queues_dir=tmp_path / "queues") def test_constructor_creates_queues_dir(tmp_path): target = tmp_path / "nested" / "queues" assert not target.exists() - QueueManager(queues_dir=target) + FileSystemQueueManager(queues_dir=target) assert target.is_dir() @@ -217,12 +217,12 @@ async def test_commit_advances_offset(qm): async def test_commit_persists_across_a_new_instance(tmp_path): qdir = tmp_path / "queues" - qm1 = QueueManager(queues_dir=qdir) + qm1 = FileSystemQueueManager(queues_dir=qdir) await qm1.append("s1", b"a") await qm1.append("s1", b"b") batch = await qm1.read_batch("s1", max_items=1) await qm1.commit("s1", batch.end_offset) - qm2 = QueueManager(queues_dir=qdir) # simulate restart + qm2 = FileSystemQueueManager(queues_dir=qdir) # simulate restart resumed = await qm2.read_batch("s1", max_items=10) assert resumed.lines == [b"b"] @@ -375,9 +375,9 @@ async def test_read_dead_letters_rejects_unsafe_session_id(qm, bad_id): async def test_delete_drained_removes_log_and_offset_keeps_dead(tmp_path) -> None: - from context_intelligence_server.queue_manager import QueueManager + from context_intelligence_server.queue_manager import FileSystemQueueManager - qm = QueueManager(queues_dir=tmp_path) + qm = FileSystemQueueManager(queues_dir=tmp_path) await qm.append("s", b"line") await qm.commit("s", 5) await qm.dead_letter("s", b"bad\n", "boom") @@ -761,7 +761,7 @@ def test_complete_data_end_newline_on_chunk_boundary(qm, tmp_path, monkeypatch): """The backward scan reads fixed non-overlapping windows; a newline landing exactly on a chunk boundary must still be found (regression guard for the streaming rewrite).""" - import context_intelligence_server.queue_manager as qm_mod + import context_intelligence_server.queue_manager.filesystem as qm_mod monkeypatch.setattr(qm_mod, "_SCAN_CHUNK_BYTES", 8) log = tmp_path / "queues" / "s1.log" @@ -776,7 +776,7 @@ def test_complete_data_end_newline_on_chunk_boundary(qm, tmp_path, monkeypatch): def test_count_newlines_matches_naive_across_ranges(qm, tmp_path, monkeypatch): """_count_newlines(start,end) == data[start:end].count(b'\\n') for arbitrary ranges, including across a small chunk size (multi-chunk streaming).""" - import context_intelligence_server.queue_manager as qm_mod + import context_intelligence_server.queue_manager.filesystem as qm_mod monkeypatch.setattr(qm_mod, "_SCAN_CHUNK_BYTES", 4) log = tmp_path / "queues" / "s1.log" @@ -800,7 +800,7 @@ def test_count_newlines_missing_and_empty_range(qm): def test_count_dead_matches_naive_and_streams(qm, tmp_path, monkeypatch): """_count_dead == old data.count(b'\\n') for empty / multi-record / missing, including a newline on a chunk boundary (streamed, not read_bytes).""" - import context_intelligence_server.queue_manager as qm_mod + import context_intelligence_server.queue_manager.filesystem as qm_mod monkeypatch.setattr(qm_mod, "_SCAN_CHUNK_BYTES", 8) dead = tmp_path / "queues" / "s1.dead.jsonl" diff --git a/tests/test_registry.py b/tests/test_registry.py index 901e00e7..f707d973 100644 --- a/tests/test_registry.py +++ b/tests/test_registry.py @@ -15,7 +15,7 @@ import pytest import context_intelligence_server.registry as registry_module -from context_intelligence_server.blob_store import AsyncDiskBlobStore +from context_intelligence_server.blob_store import FileSystemBlobStore from context_intelligence_server.config import get_settings from context_intelligence_server.queue_manager import QueueManager, Record from context_intelligence_server.registry import ( @@ -247,10 +247,10 @@ async def test_worker_has_workspace_attribute( async def test_worker_services_blob_store_is_async_disk_blob_store( self, registry: SessionRegistry ) -> None: - """worker.services.blob_store is an AsyncDiskBlobStore instance.""" + """worker.services.blob_store is an FileSystemBlobStore instance.""" worker = registry.get_or_create("session-1", "/workspace/test") - assert isinstance(worker.services.blob_store, AsyncDiskBlobStore) + assert isinstance(worker.services.blob_store, FileSystemBlobStore) @pytest.mark.asyncio async def test_worker_services_blob_store_root_matches_settings_blob_path( @@ -261,7 +261,7 @@ async def test_worker_services_blob_store_root_matches_settings_blob_path( settings = get_settings() blob_store = worker.services.blob_store - assert isinstance(blob_store, AsyncDiskBlobStore) + assert isinstance(blob_store, FileSystemBlobStore) assert blob_store._root == Path(settings.blob_path) @@ -1990,7 +1990,7 @@ def test_get_or_create_accepts_created_by_kwarg(self) -> None: with ( patch("context_intelligence_server.registry.Neo4jGraphStore") as MockStore, patch( - "context_intelligence_server.registry.AsyncDiskBlobStore" + "context_intelligence_server.registry.create_blob_store" ) as MockBlob, patch( "context_intelligence_server.registry.HookStateService" @@ -2021,7 +2021,7 @@ def test_get_or_create_default_created_by_is_none(self) -> None: with ( patch("context_intelligence_server.registry.Neo4jGraphStore") as MockStore, patch( - "context_intelligence_server.registry.AsyncDiskBlobStore" + "context_intelligence_server.registry.create_blob_store" ) as MockBlob, patch( "context_intelligence_server.registry.HookStateService" @@ -2047,7 +2047,7 @@ class TestSessionOwnershipInvariant: - log nothing at ERROR when the same (or None) created_by arrives; - log an ERROR and preserve the bound id when a different created_by arrives. - Mocking strategy: patch Neo4jGraphStore / AsyncDiskBlobStore / HookStateService + Mocking strategy: patch Neo4jGraphStore / FileSystemBlobStore / HookStateService exactly as TestGetOrCreateCreatedBy does. After the first get_or_create call (which stores a worker whose .services is the mock_svc MagicMock), we manually set mock_svc.graph.created_by = "alice" to simulate the bound state — the real @@ -2069,7 +2069,7 @@ def test_get_or_create_reuse_keeps_bound_created_by( caplog.at_level(logging.ERROR, logger="context_intelligence_server"), patch("context_intelligence_server.registry.Neo4jGraphStore") as MockStore, patch( - "context_intelligence_server.registry.AsyncDiskBlobStore" + "context_intelligence_server.registry.create_blob_store" ) as MockBlob, patch( "context_intelligence_server.registry.HookStateService" @@ -2109,7 +2109,7 @@ def test_get_or_create_reuse_ignores_new_created_by_no_error( caplog.at_level(logging.ERROR, logger="context_intelligence_server"), patch("context_intelligence_server.registry.Neo4jGraphStore") as MockStore, patch( - "context_intelligence_server.registry.AsyncDiskBlobStore" + "context_intelligence_server.registry.create_blob_store" ) as MockBlob, patch( "context_intelligence_server.registry.HookStateService" @@ -2148,7 +2148,7 @@ def test_invariant_violation_is_observed_and_not_overwritten( caplog.at_level(logging.ERROR, logger="context_intelligence_server"), patch("context_intelligence_server.registry.Neo4jGraphStore") as MockStore, patch( - "context_intelligence_server.registry.AsyncDiskBlobStore" + "context_intelligence_server.registry.create_blob_store" ) as MockBlob, patch( "context_intelligence_server.registry.HookStateService" diff --git a/tests/test_steady_state_reclaim.py b/tests/test_steady_state_reclaim.py index 2d99bbce..d11f9c96 100644 --- a/tests/test_steady_state_reclaim.py +++ b/tests/test_steady_state_reclaim.py @@ -22,9 +22,9 @@ import pytest import context_intelligence_server.main as main_module -import context_intelligence_server.queue_manager as queue_manager_module +import context_intelligence_server.queue_manager.filesystem as queue_manager_module from context_intelligence_server.config import Settings -from context_intelligence_server.queue_manager import QueueManager +from context_intelligence_server.queue_manager import FileSystemQueueManager, QueueManager from context_intelligence_server.registry import SessionRegistry, SessionWorker from context_intelligence_server.services import HookStateService @@ -81,7 +81,7 @@ async def test_b_undrained_tail_never_reclaimed_past_committed_c_less_than_tail( ) -> None: """C < E-C: commit 40 of 100 events, compact, and prove events 41..100 survive in order, untouched, with committed rebased to 0.""" - qm = QueueManager(queues_dir=tmp_path) + qm = FileSystemQueueManager(queues_dir=tmp_path) sid = "s-tail-c-lt-tail" events = [_fixed(i) for i in range(100)] for ev in events: @@ -109,7 +109,7 @@ async def test_b_undrained_tail_never_reclaimed_past_committed_c_greater_than_ta tmp_path: Path, ) -> None: """C > E-C: commit 70 of 100 events -- the tail is now the SMALLER side.""" - qm = QueueManager(queues_dir=tmp_path) + qm = FileSystemQueueManager(queues_dir=tmp_path) sid = "s-tail-c-gt-tail" events = [_fixed(i) for i in range(100)] for ev in events: @@ -134,7 +134,7 @@ async def test_b_undrained_tail_never_reclaimed_past_committed_c_greater_than_ta async def test_b_below_min_prefix_bytes_is_a_noop(tmp_path: Path) -> None: """C below min_prefix_bytes bails without touching the file at all.""" - qm = QueueManager(queues_dir=tmp_path) + qm = FileSystemQueueManager(queues_dir=tmp_path) sid = "s-below-threshold" for i in range(10): await qm.append(sid, _fixed(i)) @@ -159,7 +159,7 @@ async def test_c_mid_copy_oserror_is_a_pure_noop( ) -> None: """An OSError raised mid-copy (Precision 1) must not mutate anything and must not escape -- it is caught and the method returns 0.""" - qm = QueueManager(queues_dir=tmp_path) + qm = FileSystemQueueManager(queues_dir=tmp_path) sid = "s-mid-copy-fault" for i in range(9): await qm.append(sid, _fixed(i)) @@ -176,7 +176,7 @@ def _raise(fd: int, data: bytes) -> None: raise OSError("simulated mid-copy failure") monkeypatch.setattr( - queue_manager_module.QueueManager, "_write_all", staticmethod(_raise) + queue_manager_module.FileSystemQueueManager, "_write_all", staticmethod(_raise) ) reclaimed = await qm.compact_committed_prefix(sid, 0) # must not raise @@ -192,7 +192,7 @@ async def test_c_window2_offset_rebased_before_log_replaced_bounded_redrive( """Simulates a crash after the offset was rebased to 0 but before the log was replaced. A reader resuming from this on-disk state must see a bounded re-drive (the committed prefix duplicated) -- never a loss.""" - qm = QueueManager(queues_dir=tmp_path) + qm = FileSystemQueueManager(queues_dir=tmp_path) sid = "s-window2" events = [_fixed(i) for i in range(9)] for ev in events: @@ -220,7 +220,7 @@ async def test_c_control_rejected_log_then_offset_order_loses_data( """CONTROL: applies the alternative log-then-offset ordering by hand and stops mid-window (log replaced, offset not yet rewritten), proving that order silently drops undrained data -- why offset-before-log is used.""" - qm = QueueManager(queues_dir=tmp_path) + qm = FileSystemQueueManager(queues_dir=tmp_path) sid = "s-rejected-order" events = [_fixed(i) for i in range(9)] for ev in events: @@ -264,7 +264,7 @@ async def test_c_control_rejected_log_then_offset_order_loses_data( async def test_i_replace_failure_restores_offset_zero_accounting_drift( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture ) -> None: - qm = QueueManager(queues_dir=tmp_path) + qm = FileSystemQueueManager(queues_dir=tmp_path) sid = "s-replace-fails" events = [_fixed(i) for i in range(9)] for ev in events: @@ -316,7 +316,7 @@ async def test_i_double_replace_failure_logs_restore_failed_honestly( """If the RESTORE itself also fails, the honest (documented) fallback is a logged `compact_restore_failed ... redrive_expected=true` -- never a silent success claim.""" - qm = QueueManager(queues_dir=tmp_path) + qm = FileSystemQueueManager(queues_dir=tmp_path) sid = "s-double-fail" for i in range(9): await qm.append(sid, _fixed(i)) @@ -357,7 +357,7 @@ def _always_raise(src: Any, dst: Any) -> None: async def test_j_large_tail_does_not_block_prefix_reclaim( tmp_path: Path, ) -> None: - qm = QueueManager(queues_dir=tmp_path) + qm = FileSystemQueueManager(queues_dir=tmp_path) sid = "s-huge-tail" # 20 events * 10 bytes = 200 bytes total. for i in range(20): @@ -388,7 +388,7 @@ async def test_j_large_tail_does_not_block_prefix_reclaim( async def test_e_dead_letters_older_than_retention_expired_newer_kept( tmp_path: Path, ) -> None: - qm = QueueManager(queues_dir=tmp_path) + qm = FileSystemQueueManager(queues_dir=tmp_path) now = time.time() retention = 30 * 86400.0 @@ -420,7 +420,7 @@ async def test_e_dead_letters_older_than_retention_expired_newer_kept( async def test_e_dry_run_deletes_nothing_but_still_classifies( tmp_path: Path, caplog: pytest.LogCaptureFixture ) -> None: - qm = QueueManager(queues_dir=tmp_path) + qm = FileSystemQueueManager(queues_dir=tmp_path) now = time.time() retention = 30 * 86400.0 old_path = _seed_dead(tmp_path, "would-expire", _dead_record("p")) @@ -439,7 +439,7 @@ async def test_e_dry_run_deletes_nothing_but_still_classifies( async def test_e_retention_zero_disables_expiry(tmp_path: Path) -> None: - qm = QueueManager(queues_dir=tmp_path) + qm = FileSystemQueueManager(queues_dir=tmp_path) now = time.time() old_path = _seed_dead(tmp_path, "ancient", _dead_record("p")) os.utime(old_path, (now - 10_000_000, now - 10_000_000)) @@ -471,7 +471,7 @@ async def test_k_expiry_is_opt_in_disabled_under_shipped_defaults( assert settings.reclaim_enabled is False assert settings.dead_letter_expiry_enabled is False - qm = QueueManager(queues_dir=tmp_path) + qm = FileSystemQueueManager(queues_dir=tmp_path) now = time.time() old_path = _seed_dead(tmp_path, "shipped-default-no-expire", _dead_record("p")) old_mtime = now - settings.dead_letter_retention_seconds - 3600 @@ -495,7 +495,7 @@ async def test_k_expiry_is_opt_in_disabled_under_shipped_defaults( async def test_k_dead_letter_expiry_enabled_false_still_dry_runs( tmp_path: Path, ) -> None: - qm = QueueManager(queues_dir=tmp_path) + qm = FileSystemQueueManager(queues_dir=tmp_path) now = time.time() old_path = _seed_dead(tmp_path, "would-expire-2", _dead_record("p")) os.utime(old_path, (now - (31 * 86400.0), now - (31 * 86400.0))) @@ -516,7 +516,7 @@ async def test_m_boot_phase_expiry_never_calls_record_purged( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: monkeypatch.setattr( - main_module.registry, "_queue_manager", QueueManager(queues_dir=tmp_path) + main_module.registry, "_queue_manager", FileSystemQueueManager(queues_dir=tmp_path) ) main_module.registry._accepted_total = 0 main_module.registry._written_total = 0 @@ -563,7 +563,7 @@ async def test_m_sweep_tick_expiry_applies_record_purged( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: monkeypatch.setattr( - main_module.registry, "_queue_manager", QueueManager(queues_dir=tmp_path) + main_module.registry, "_queue_manager", FileSystemQueueManager(queues_dir=tmp_path) ) main_module.registry._accepted_total = 0 main_module.registry._written_total = 0 @@ -632,7 +632,7 @@ async def test_f_status_not_blocked_by_an_in_progress_compaction( background thread; /status must still return promptly -- it never acquires guard.file_lock for any key.""" monkeypatch.setattr( - main_module.registry, "_queue_manager", QueueManager(queues_dir=tmp_path) + main_module.registry, "_queue_manager", FileSystemQueueManager(queues_dir=tmp_path) ) qm = main_module.registry.queue_manager sid = "s-status-lock" @@ -669,7 +669,7 @@ async def test_f_status_not_blocked_by_a_concurrent_dead_letter_expiry( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: monkeypatch.setattr( - main_module.registry, "_queue_manager", QueueManager(queues_dir=tmp_path) + main_module.registry, "_queue_manager", FileSystemQueueManager(queues_dir=tmp_path) ) qm = main_module.registry.queue_manager for i in range(50): @@ -694,7 +694,7 @@ async def test_f_status_not_blocked_by_a_concurrent_dead_letter_expiry( @pytest.fixture async def reg_qm(tmp_path: Path): reg = SessionRegistry() - reg._queue_manager = QueueManager(queues_dir=tmp_path) + reg._queue_manager = FileSystemQueueManager(queues_dir=tmp_path) reg._write_semaphore = asyncio.Semaphore(8) reg._max_delivery_attempts = 3 yield reg, reg._queue_manager diff --git a/tests/test_writer_lease.py b/tests/test_writer_lease.py index 760c1082..3f11ee2f 100644 --- a/tests/test_writer_lease.py +++ b/tests/test_writer_lease.py @@ -25,7 +25,7 @@ import pytest from context_intelligence_server.config import Settings from context_intelligence_server.main import lifespan -from context_intelligence_server.queue_manager import QueueManager +from context_intelligence_server.queue_manager import FileSystemQueueManager, QueueManager from context_intelligence_server.status import boot_state from context_intelligence_server.writer_lease import ( WriterLease, @@ -500,7 +500,7 @@ async def test_r1_no_queue_manager_constructed_by_d6( def _fail_init(self: object, queues_dir: Path) -> None: pytest.fail("QueueManager.__init__ must never be called by the detector") - monkeypatch.setattr(QueueManager, "__init__", _fail_init) + monkeypatch.setattr(FileSystemQueueManager, "__init__", _fail_init) lease = WriterLease() await lease.acquire(_settings(), lambda: main_module.registry.queues_dir_path) @@ -535,14 +535,14 @@ async def test_r1_exactly_one_queue_manager_under_concurrent_construction( main_module.registry._queue_manager = None construct_count = 0 - real_init = QueueManager.__init__ + real_init = FileSystemQueueManager.__init__ - def _counting_init(self: QueueManager, queues_dir: Path) -> None: + def _counting_init(self: FileSystemQueueManager, queues_dir: Path) -> None: nonlocal construct_count construct_count += 1 real_init(self, queues_dir) - monkeypatch.setattr(QueueManager, "__init__", _counting_init) + monkeypatch.setattr(FileSystemQueueManager, "__init__", _counting_init) lease = WriterLease() @@ -685,7 +685,7 @@ def _hang() -> None: try: await asyncio.sleep(0.5) - qm = QueueManager(queues_dir=tmp_path / "shared-pool-check") + qm = FileSystemQueueManager(queues_dir=tmp_path / "shared-pool-check") start = time.monotonic() await asyncio.wait_for(qm.append("sid-1", b"hello"), timeout=1.0) elapsed = time.monotonic() - start @@ -941,7 +941,7 @@ async def test_no_collision_with_existing_session_scans(tmp_path: Path) -> None: lease = WriterLease() await lease.acquire(_settings(), lambda: tmp_path) - qm = QueueManager(queues_dir=tmp_path) + qm = FileSystemQueueManager(queues_dir=tmp_path) (tmp_path / "sess-1.log").write_bytes(b'{"event":"x","workspace":"/w"}\n') (tmp_path / "sess-1.offset").write_text("0", encoding="utf-8")