diff --git a/.env.example b/.env.example index 686cc62d58..03cd19f2d9 100644 --- a/.env.example +++ b/.env.example @@ -456,6 +456,21 @@ OPENHUMAN_ANALYTICS_ENABLED=true # (periodic auto-sync disabled). Unset → 24h default. # OPENHUMAN_MEMORY_SYNC_INTERVAL_SECS=86400 +# --------------------------------------------------------------------------- +# Subsystem driver binding ([subsystems.memory] — kernel.md §3.6) +# --------------------------------------------------------------------------- +# Which driver answers for the memory subsystem, and the memory-hook budgets. +# NOTHING READS THESE YET — the config surface exists so bindings can be +# authored ahead of the registry landing. Defaults reproduce today's behaviour. +# Per-driver option tables ([subsystems.memory.drivers.], incl. the +# fail-closed `trust_state`) are config-file only; there is no env form. +# OPENHUMAN_MEMORY_DRIVER=tinycortex +# OPENHUMAN_MEMORY_HOOKS_AUTO_RECALL=true +# OPENHUMAN_MEMORY_HOOKS_AUTO_CAPTURE=true +# OPENHUMAN_MEMORY_HOOKS_MAX_CONTEXT_TOKENS=2000 +# OPENHUMAN_MEMORY_HOOKS_RECALL_MAX_CHARS=1000 +# OPENHUMAN_MEMORY_HOOKS_CAPTURE_MAX_CHARS=500 + # --------------------------------------------------------------------------- # Logging # --------------------------------------------------------------------------- diff --git a/.github/workflows/ci-lite.yml b/.github/workflows/ci-lite.yml index ac3d9bbef3..84f7cfb479 100644 --- a/.github/workflows/ci-lite.yml +++ b/.github/workflows/ci-lite.yml @@ -485,7 +485,7 @@ jobs: # # This asserts the calibration still holds. If it fails, every # projection built on the simulator is suspect until it is fixed. - run: python3 scripts/dep-sim.py --cut-nothing --expect-names 285 + run: python3 scripts/dep-sim.py --cut-nothing --expect-names 286 - name: Guard — new feature-gated test modules must be acknowledged # Self-maintaining coverage: the set of source files that #[cfg]-gate a test on diff --git a/Cargo.lock b/Cargo.lock index 1be5663ac7..c25e84fa65 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4723,6 +4723,7 @@ dependencies = [ "tinybus", "tinychannels", "tinycortex", + "tinycortex-api", "tinyflows", "tinyhumans-sdk", "tinyjuice", @@ -7288,8 +7289,10 @@ dependencies = [ "anyhow", "async-trait", "chrono", + "dirs 5.0.1", "futures", "git2", + "hex", "log", "parking_lot", "rand 0.10.1", @@ -7302,6 +7305,7 @@ dependencies = [ "sha2 0.10.9", "thiserror 2.0.18", "tinyagents", + "tinycortex-api", "tokio", "toml 0.8.23", "tracing", @@ -7309,6 +7313,20 @@ dependencies = [ "walkdir", ] +[[package]] +name = "tinycortex-api" +version = "0.1.1" +dependencies = [ + "anyhow", + "async-trait", + "chrono", + "serde", + "serde_json", + "sha2 0.10.9", + "thiserror 2.0.18", + "uuid 1.23.1", +] + [[package]] name = "tinyflows" version = "0.6.0" diff --git a/Cargo.toml b/Cargo.toml index bf1fb48625..089e38f2d7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -136,7 +136,23 @@ tinyagents = { version = "2.1", features = ["sqlite"] } # aligned to the host pins (=0.40 / 0.21) so one bundled SQLite + one libgit2 # link. The submodule intentionally tracks reviewed upstream main commits; # keep this semver requirement compatible with the vendored crate version. -tinycortex = { version = "0.1", features = ["git-diff", "persona", "sync"] } +tinycortex = { version = "0.1", features = [ + "git-diff", + "obsidian", + "persona", + "sync", + "wiki-git", +] } +# The memory *contract* — value types, the thirteen capability families, the +# `MemoryProvider` driver trait, and the null reference driver. A direct path +# dependency rather than a re-export, because `tinycortex::memory` aliases back +# only `{error, traits, types}`; `capabilities`, `provider`, `null`, `health`, +# `recall`, and `version` are reachable only through the api crate itself. +# Deliberately dependency-light (no rusqlite/git2/reqwest/regex/async runtime) +# so a third-party driver can compile against the contract without pulling in +# the embedded engine. No `[patch.crates-io]` entry is needed: cargo unifies +# this with the `path = "api"` dependency the engine crate already declares. +tinycortex-api = { path = "vendor/tinycortex/api" } tinychannels = { version = "0.1", features = ["relay-websocket"] } # tinybus — the message bus. Owns what `src/core/event_bus/` used to: the typed # pub/sub surface (`EventBus`, `EventHandler`, `SubscriptionHandle`), the diff --git a/app/src-tauri/Cargo.lock b/app/src-tauri/Cargo.lock index 35e3f500f8..1eef37c81e 100644 --- a/app/src-tauri/Cargo.lock +++ b/app/src-tauri/Cargo.lock @@ -5570,6 +5570,7 @@ dependencies = [ "tinyagents", "tinychannels", "tinycortex", + "tinycortex-api", "tinyflows", "tinyhumans-sdk", "tinyjuice", @@ -8950,8 +8951,10 @@ dependencies = [ "anyhow", "async-trait", "chrono", + "dirs 5.0.1", "futures", "git2", + "hex", "log", "parking_lot", "rand 0.10.1", @@ -8964,6 +8967,7 @@ dependencies = [ "sha2 0.10.9", "thiserror 2.0.18", "tinyagents", + "tinycortex-api", "tokio", "toml 0.8.2", "tracing", @@ -8971,6 +8975,20 @@ dependencies = [ "walkdir", ] +[[package]] +name = "tinycortex-api" +version = "0.1.1" +dependencies = [ + "anyhow", + "async-trait", + "chrono", + "serde", + "serde_json", + "sha2 0.10.9", + "thiserror 2.0.18", + "uuid 1.23.1", +] + [[package]] name = "tinyflows" version = "0.6.0" diff --git a/docs/specs/kernel.md b/docs/specs/kernel.md index 958b2d1b24..2f6b74e877 100644 --- a/docs/specs/kernel.md +++ b/docs/specs/kernel.md @@ -124,6 +124,17 @@ The one exception is the **CLI**, which keeps its subcommand arm and reports a * ("memory driver `supermemory` does not support tree summarisation") — same reasoning as the retained `mcp` and `tui` CLI arms. +Implemented for memory at the CLI boundary. `core::cli_capability` resolves the bound driver +directly — no CLI subcommand except `run`/`serve` builds a `CoreContext`, so the ambient +`capability_allowed` gate would always default open — and `core::all::capability_for_parts` / +`sole_capability_for_namespace` supply the **unfiltered** lookup that tells "no such command" +apart from "gated", which every filtered lookup has already collapsed into one absence. Both CLI +paths are covered: the generic `openhuman ` dispatcher and the hand-written +`openhuman memory ` adapter. `core::dispatch` is deliberately untouched — it is the shared +`/rpc` path, where the absence rule above still holds. A genuinely unknown command still reports +unknown namespace / function / subcommand; collapsing the two would make real typos harder to +diagnose. + ### 3.4 Policy is kernel-side and non-bypassable Every subsystem call from product code goes through a kernel-owned **guard decorator**, never to diff --git a/docs/specs/memory-guard-allowlist.md b/docs/specs/memory-guard-allowlist.md new file mode 100644 index 0000000000..c778572608 --- /dev/null +++ b/docs/specs/memory-guard-allowlist.md @@ -0,0 +1,158 @@ +# Memory-guard allowlist + +Every place in the tree that still reaches memory **without** going through +`MemoryGuard`, and why. Produced by M4b; consumed by M4c. + +Pinned by the ratchet in `src/openhuman/memory/bypass_allowlist_tests.rs` +(M4c), which fails **both ways** — when a new unguarded call site appears +(`no_new_memory_driver_bypasses`), and when an allowlisted one is cleaned up +without being struck from the list (`bypass_allowlist_has_no_stale_entries`). +Two further tests stop the lint rotting into a rubber stamp: the scanner must +find a known bypass, and every needle must still match something. + +M4b shipped a provisional, file-keyed version of this guard inside +`memory/ops/guard_tests.rs`. M4c deleted it — two allowlists over one tree must +both be struck on every cleanup, and the one nobody remembers is exactly the +dead-string rot the ratchet exists to prevent. + +## Scope + +The lint scans `src/` for twelve patterns, keyed on `(file, pattern)` so the +failure message names the needle that tripped: + +| Pattern | What it hands out | +| --- | --- | +| `active_memory_client(` | `MemoryClientRef` | +| `global::client_if_ready(` / `global::client(` | `MemoryClientRef` | +| `.memory_handle(` | raw `Arc` | +| `.profile_conn(` | raw `Arc>` | +| `.get_document(` | `pub(crate)` read-one escape hatch | +| `EmbeddedMemoryProvider::new(` / `NullMemoryProvider::new(` | a driver, built outside `binding::for_workspace` | +| `MemoryClient::from_workspace_dir(` | a second engine on the same store | +| `binding::for_workspace(` / `.memory_binding(` | a raw `MemoryBinding` | +| `.unguarded_provider(` | the raw `Arc` off a `MemoryBinding` | + +**By-path test files (`*_tests.rs`, `tests.rs`, `test_support/`) are out of +scope.** Driver tests construct drivers — that is what a driver test *is* — +so allowlisting them would add ~25 entries that can never shrink and would +churn on every new test. Inline `#[cfg(test)] mod tests` blocks are *not* +stripped, because brace-tracking Rust with a line scanner is fragile and +getting it wrong silently hides production sites; the three files affected are +allowlisted with a reason saying so. Comment lines are skipped, so doc-comment +references are not mistaken for calls. + +`global::init(workspace)` is deliberately **not** scanned. It binds the +workspace; it does not read or write memory, and every call site is a +login / active-user-switch / boot / CLI-entry lifecycle event +(`security/credentials/ops.rs`, `desktop/app_state/ops.rs`, +`core/runtime/context.rs`, `core/memory_cli.rs`, `core/subconscious_cli.rs`, +`bin/slack_backfill.rs`, `bin/gmail_backfill_3d.rs`, +`memory/ops/documents.rs`'s `memory_init`, `memory/tinycortex/sync.rs`). + +## What M4b re-pointed + +Four RPC handlers, all in `src/openhuman/memory/ops/`, each of whose contract +twin is a literal one-line delegation to the same host method on the same +store: + +| Handler | Contract method | Driver body | +| --- | --- | --- | +| `documents::doc_put` | `MemoryDocuments::put_document` | `client.put_doc(input)` | +| `kv_graph::kv_set` | `MemoryGraph::kv_put` | `client.kv_set(ns, key, &value)` | +| `tool_memory::tool_rule_list` | `MemoryToolMemory::tool_rules` | `tool_memory_store(memory).list_rules(tool)` | +| `tool_memory::tool_rule_delete` | `MemoryToolMemory::delete_tool_rule` | `tool_memory_store(memory).delete_rule(tool, id)` | + +**Three deltas ride along, and they are the point of the milestone, not +accidents:** + +1. **Tier enforcement.** A write now goes through `ToolOperation::Act`, so a + `readonly` autonomy tier refuses it and the hourly action budget is charged + one unit. Reads take `ToolOperation::Read`, which `SecurityPolicy` answers + `Ok` for unconditionally today. +2. **Error strings gain a method prefix.** The driver wraps host failures + through `host_error(context, error)`, so `""` becomes + `"put_document: "`. Additive context, never a swallowed cause. +3. **Taint may be raised.** `doc_put` still passes `MemoryTaint::Internal`; the + guard's `stamp_taint` promotes it to `ExternalSync` when the turn runs under + a source scope. It can never launder the other direction. + +Redaction is a byte-identical pass-through for an embedded driver, and the +ambient source scope is applied only on `MemoryTree::query_source`, so neither +changes anything here. + +## The allowlist + +### A. Legitimate residents — the driver, the seam, the bind site + +| Path | Reason | +| --- | --- | +| `memory/driver/embedded/mod.rs` | This **is** the driver. Guarding it would be a cycle. | +| `memory/driver/embedded/tool_memory_tests.rs` | Driver tests. | +| `memory/tinycortex/sync.rs` | The engine seam. | +| `memory/global.rs` | The process-global slot itself. | +| `memory/ops/helpers.rs` | Defines `active_memory_client`. | +| `memory/ops/guard.rs`, `guard_tests.rs` | The guarded resolver; matches only in prose and in its own fallback. | +| `memory/ops/provider.rs` (`.unguarded_provider(`) | Health probe on the bound driver; a liveness probe is not product code. | +| `core/cli_capability.rs` (`binding::for_workspace(`) | The CLI's capability gate (`kernel.md` §3.3's one exception to "degradation is absence"). Reads the driver id and advertised capability set only — the same two values `memory.provider_status` already returns over RPC — and never reaches memory content. No CLI subcommand except `run`/`serve` builds a `CoreContext`, so `CoreContext::memory()` resolves to nothing and there is no guard to route through. `core/memory_cli.rs` calls `bound_memory_driver_for` rather than binding itself. | +| `core/subsystems_cli.rs` | The `openhuman subsystems` slot table. Delegates to `memory_subsystem_status` (which itself resolves the binding in `memory/ops/provider.rs`, already allowlisted above), so `subsystems_cli.rs` never touches `binding::for_workspace(` directly — the CLI's command arms go through `bound_memory_driver_for`. | + +### B. Unguardable raw SQLite — `profile_conn()`, out of scope for M4 + +No decorator can wrap an `Arc>`. These reach the +profile / facet tables beneath all seven policy steps. **This is why "the guard +is the only path" is not yet a true invariant.** + +| Path | Sites | +| --- | --- | +| `memory/sync/composio/providers/profile.rs` | 5 | +| `agent/learning/schemas.rs` | 3 | +| `agent/learning/tools.rs` | 1 | +| `agent/learning/startup.rs` | 2 | +| `memory/store/client_tests.rs` | 2 (test) | + +The brief named only the first two files. The other two were found by grep and +are recorded here so M4c starts from the real set. + +### C. Needs a concrete engine type the contract does not expose + +| Path | Reason | +| --- | --- | +| `agent/experience/ops.rs` | `AgentExperienceStore::new` takes `Arc`; the non-`"memory"` subdir branch also builds `UnifiedMemory::new_with_memory_dir` directly — a per-profile store the binding has no concept of. | +| `agent/harness/session/builder/factory.rs` | `.memory_handle()` → `Arc`. | +| `flows/tinyflows/memory_adapter.rs` | Returns `Arc` to satisfy a tinyflows engine trait. The contract has no `Arc` door. | +| `flows/bus.rs` | `resolve_memory() -> Option>`, and carries a `#[cfg(test)] memory_override` seam a guard would bypass. | +| `memory/tool_memory/tools/list.rs`, `tools/put.rs` | Agent tools building `ToolMemoryStore` from `memory_handle()`. Re-pointable in principle via `as_tool_memory()` — **deferred to M5**, which filters the tool surface by capability and would collide with a re-point made now. | +| `memory/ops/tool_memory.rs` (`open_store`) | Still needed by the four handlers left on the client. Shrank; did not disappear. | + +### D. No contract method exists, or the wire shape would change + +| Path | Reason | +| --- | --- | +| `memory/ops/documents.rs` — `namespace_list`, `doc_ingest`, `doc_list`, `doc_delete`, `clear_namespace`, `context_query`, `context_recall`, `memory_*` | Each answers with a `serde_json::Value` / `String` shape with no typed contract twin; `clear_namespace` has no contract method at all; `memory_query_namespace` depends on `query_limit_for_request(client: &MemoryClient, …)`. | +| `memory/ops/kv_graph.rs` — `kv_get`, `kv_delete`, `kv_list_namespace`, `graph_upsert`, `graph_query` | `kv_get` is an O(slice) scan in the driver and returns `MemoryKvRecord`, not `Value`; `kv_delete` has **no** contract method; `graph_query`'s camelCase→typed conversion is documented as new and lossy. | +| `memory/ops/tool_memory.rs` — `tool_rule_put`, `tool_rule_get`, `tool_rules_json`, `tool_rules_for_prompt` | `put_tool_rule` returns unit while the RPC returns the stored rule with a refreshed `updated_at`; the other three have no contract equivalent. | +| `memory/ops/sync.rs` | `client.ingestion_state().snapshot()` — queue telemetry, absent from the contract. | +| `memory/ops/learn.rs` | `list_namespaces() -> Vec` vs the contract's `Vec`, then heavy engine work. | +| `flows/ops.rs` | `clear_namespace` (no contract method) plus a `memory_client_override` test seam. | +| `integrations/composio/schemas.rs` | Passes `&MemoryClientRef` into `user_scopes::save`. | +| `memory/sync/composio/providers/user_scopes.rs`, `types.rs` | Same `&MemoryClientRef` parameter shape. | + +### E. Tests + +`flows/ops_tests.rs`, `flows/tinyflows/memory_node_e2e_tests.rs`, +`integrations/composio/ops_tests.rs`, `core/runtime/context.rs` (its `#[cfg(test)]` +module). + +## Honest scorecard + +Four of the twenty-eight `active_memory_client()` call sites now route through +the guard. Eleven non-test `profile_conn()` sites and twelve non-test +`memory_handle()` sites still hand out raw handles. The defensible claim for M4 +is therefore: + +> Every memory RPC handler whose contract twin is a literal delegation now +> routes through the guard, and every remaining bypass is enumerated here with +> a reason and pinned by a drift guard. + +"Impossible to skip by construction" is **not** true until `memory_handle()` +and `profile_conn()` are gone. diff --git a/scripts/kernel-floor.limits b/scripts/kernel-floor.limits index 81d5f9b674..7833d19beb 100644 --- a/scripts/kernel-floor.limits +++ b/scripts/kernel-floor.limits @@ -13,6 +13,23 @@ # Simulate with: scripts/dep-sim.py --cut # # History +# 307/284/5 2026-08-09 merge upstream/main + PR #5446 memory-subsystem work. +# Measured, not derived: `scripts/kernel-floor.sh flows` +# reports 307/284/5 on this merge. +3 names over the +# 304/281/5 below, and the split matters: +# * +2 INHERITED — `tinybus` + `tinybus-macros`, vendored +# by upstream #5459. The event bus is always-on kernel +# surface, so they are in the `flows` profile there too; +# upstream landed them without moving this number. +# * +1 OURS — `tinycortex-api`, the memory contract crate +# (`MemoryProvider`, `Capabilities`, `NullMemoryProvider`) +# that `MemorySubsystemConfig` -> `MemoryBinding` now +# depends on directly. It was already in the graph +# transitively via `tinycortex`, so no *new* third-party +# code enters the kernel profile — only a new direct edge. +# The -6 packages / -1 native from the tokio-tungstenite +# unification below is preserved, not spent: native builds +# stay at 5 and aws-lc-sys stays absent. # 304/281/5 2026-08-08 local whisper.cpp STT deleted (`whisper-rs` + # `whisper-rs-sys` out of BOTH Cargo worlds, and the # `[patch.crates-io] whisper-rs-sys` fork entries with @@ -130,4 +147,4 @@ # Native: aws-lc-sys libgit2-sys libsqlite3-sys libz-sys # lzma-sys ring. Target after gating is 222 names / 2 native # (libsqlite3-sys, ring) — see docs/plans MIGRATION-PLAN G6. -flows:304:281:5 +flows:307:284:5 diff --git a/src/bin/library_profile/mock.rs b/src/bin/library_profile/mock.rs index 0a6ce86cb2..7d619331c9 100644 --- a/src/bin/library_profile/mock.rs +++ b/src/bin/library_profile/mock.rs @@ -67,6 +67,7 @@ fn model_response(response: ChatResponse) -> ModelResponse { raw: None, resolved_model: None, continue_turn: None, + served_from_cache: false, } } diff --git a/src/core/all.rs b/src/core/all.rs index b95fb6860b..bf96252892 100644 --- a/src/core/all.rs +++ b/src/core/all.rs @@ -10,6 +10,8 @@ use std::sync::OnceLock; use serde_json::{Map, Value}; +use tinycortex_api::capabilities::{Capabilities, Capability}; + use crate::core::ControllerSchema; /// A pinned, boxed future returned by a controller handler. @@ -221,18 +223,50 @@ impl DomainGroup { #[derive(Clone)] struct GroupedController { group: DomainGroup, + /// The memory-driver capability family this controller's surface needs, if + /// any (M5.2, `docs/specs/kernel.md` §3.3). + /// + /// `None` — the overwhelming majority — means "not gated on memory + /// capabilities at all", either because the controller belongs to another + /// domain entirely, or because it is host surface that survives any driver + /// (`people`, `memory.list_files`, `memory.provider_status`), or because + /// its family is MANDATORY and so a gate could never fire. + /// + /// `Some(c)` means the surface is ABSENT when the bound driver does not + /// advertise `c`: unknown-method over `/rpc`, omitted from `/schema`. + /// Absence, not a stub that errors — a registered-but-failing method + /// teaches a model that the capability exists and makes it retry. Same + /// reasoning as the `flows` compile-time gate (see CLAUDE.md) and as + /// `tinycortex_api::capabilities`' module docs. + capability: Option, controller: RegisteredController, } -/// Append `items` to `dst`, tagging each with `group`. This is the single seam -/// that attaches a [`DomainGroup`] to every domain's controllers without the -/// domain modules knowing about groups. +/// Append `items` to `dst`, tagging each with `group` and no capability gate. +/// This is the single seam that attaches a [`DomainGroup`] to every domain's +/// controllers without the domain modules knowing about groups. fn push(dst: &mut Vec, group: DomainGroup, items: Vec) { - dst.extend( - items - .into_iter() - .map(|controller| GroupedController { group, controller }), - ); + push_cap(dst, group, None, items); +} + +/// [`push`] plus a memory-capability gate. +/// +/// Every [`DomainGroup::Memory`] site calls THIS one with an explicit +/// `Option` — including the explicit `None`s — so "which family +/// does this surface need" is a decision recorded at the registration site +/// rather than a default nobody chose. `memory_capability_map_is_exhaustive` +/// in `all_tests.rs` fails if a Memory push site is added without one. +fn push_cap( + dst: &mut Vec, + group: DomainGroup, + capability: Option, + items: Vec, +) { + dst.extend(items.into_iter().map(|controller| GroupedController { + group, + capability, + controller, + })); } /// The [`DomainSet`](crate::core::runtime::DomainSet) of the ambient dispatch @@ -250,6 +284,39 @@ fn group_allowed(group: DomainGroup) -> bool { active_domain_set().is_none_or(|s| s.allows(group)) } +/// Whether the given memory capability family is advertised by the bound +/// driver under the ambient context (M5.2). +/// +/// **Defaults OPEN**, exactly like [`group_allowed`]: `None` is always allowed, +/// and with no ambient context / no bound driver +/// `CoreContext::current_memory_capabilities` returns the full set +/// (`memory::binding::unbound_default_capabilities`). Roughly 4000 unit tests +/// run pre-boot with no bound driver; a deny-by-default here would turn every +/// memory test red at once. Denying is only ever correct AFTER a driver has +/// actually answered `capabilities()`. +/// (`pub(crate)` so the agent-tool post-filter in +/// [`crate::openhuman::tools::ops::all_tools_with_runtime`] gates on the exact +/// same predicate the RPC registry does — one definition, two surfaces.) +pub(crate) fn capability_allowed(capability: Option) -> bool { + match capability { + None => true, + Some(_) => capability_allowed_in( + crate::core::runtime::context::CoreContext::current_memory_capabilities(), + capability, + ), + } +} + +/// [`capability_allowed`] against an already-resolved set. +/// +/// The collect-all paths hoist the lookup out of their filter closure: +/// resolving the set walks `CoreContext -> memory_binding -> RwLock read -> +/// HashMap`, materially heavier than `group_allowed`'s task-local +/// read, and would otherwise run once per controller across the whole registry. +fn capability_allowed_in(caps: Capabilities, capability: Option) -> bool { + capability.is_none_or(|c| caps.contains(c)) +} + /// The global static registry of all controllers, initialized once on first access. static REGISTRY: OnceLock> = OnceLock::new(); @@ -297,10 +364,19 @@ fn cli_adapters() -> &'static [RegisteredCliAdapter] { // feature: with the feature off, `voice::cli::run_standalone_subcommand` // resolves to the facade stub, which returns a "voice disabled" error so // `openhuman voice` fails gracefully instead of the subcommand vanishing. - vec![RegisteredCliAdapter { - namespace: "voice", - handler: crate::openhuman::voice::cli::run_standalone_subcommand, - }] + vec![ + RegisteredCliAdapter { + namespace: "voice", + handler: crate::openhuman::voice::cli::run_standalone_subcommand, + }, + // Bare `openhuman subsystems` prints the slot table; `openhuman + // subsystems status` still routes through the generic namespace + // dispatcher and prints JSON. + RegisteredCliAdapter { + namespace: "subsystems", + handler: crate::core::subsystems_cli::run_subsystems_command, + }, + ] }) } @@ -420,6 +496,19 @@ fn build_registered_controllers() -> Vec { DomainGroup::Platform, crate::openhuman::platform::health::all_health_registered_controllers(), ); + // Kernel subsystem/driver bindings: slot, bound driver, class, health, + // contract version, capabilities (docs/specs/kernel.md §6 item 6). The one + // controller registered from `src/core/` — it is a kernel binding table + // with no `src/openhuman/` family of its own, so it is tagged `Platform` + // rather than earning a `DomainGroup` variant for a single read-only + // function. Consequence: like `health`, it is absent under + // `DomainSet::harness()`, while `memory.provider_status` (a `Memory` + // family method) stays reachable there. + push( + &mut controllers, + DomainGroup::Platform, + crate::core::subsystem::all_subsystems_registered_controllers(), + ); // One-time first-run initialization (Python/spaCy/Node provisioning) push( &mut controllers, @@ -566,9 +655,12 @@ fn build_registered_controllers() -> Vec { crate::openhuman::inference::embeddings::all_embeddings_registered_controllers(), ); // People resolution and interaction scoring - push( + push_cap( &mut controllers, DomainGroup::Memory, + // Host-owned address book + interaction scoring, not a driver family: + // `people` has no `Capability` and survives every bound driver. + None, crate::openhuman::memory::people::all_people_registered_controllers(), ); // Sandbox execution backends (Docker, local jail, policy, cleanup) @@ -636,16 +728,87 @@ fn build_registered_controllers() -> Vec { DomainGroup::Platform, crate::openhuman::tools::registry::all_tool_registry_registered_controllers(), ); - // Document and knowledge graph storage - push( + // Document and knowledge graph storage. The single `memory` RPC namespace + // spans four driver capability families plus two host-only surfaces, so it + // registers as nine tagged pushes rather than one (M5.2). Order matches + // `memory::schemas::all_registered_controllers`, which + // `registered_controller_order_is_pinned_to_the_capability_partition_snapshot` pins. + push_cap( + &mut controllers, + DomainGroup::Memory, + // Core + Recall are MANDATORY families — `Capabilities::validate` + // refuses to bind a driver missing them — so against a *driver's* + // advertised set this gate can never fire. It is tagged anyway, + // because one host decision answers below the driver: + // `CoreContext::memory_capabilities` returns the EMPTY set for a + // deliberate `driver = "null"`, which is how "the operator turned + // memory off" removes the mandatory surface too. `Core` alone stands + // for the pair — the two are always advertised together, and no + // partition here holds only recall methods. + Some(Capability::Core), + crate::openhuman::memory::all_memory_core_recall_registered_controllers(), + ); + push_cap( + &mut controllers, + DomainGroup::Memory, + Some(Capability::Documents), + crate::openhuman::memory::all_memory_documents_registered_controllers(), + ); + push_cap( + &mut controllers, + DomainGroup::Memory, + Some(Capability::Ingest), + crate::openhuman::memory::all_memory_ingest_registered_controllers(), + ); + push_cap( + &mut controllers, + DomainGroup::Memory, + // Plain workspace file I/O through the host, not a driver family. + None, + crate::openhuman::memory::all_memory_files_registered_controllers(), + ); + push_cap( + &mut controllers, + DomainGroup::Memory, + Some(Capability::Graph), + crate::openhuman::memory::all_memory_kv_graph_registered_controllers(), + ); + push_cap( + &mut controllers, + DomainGroup::Memory, + Some(Capability::Sources), + crate::openhuman::memory::all_memory_sync_registered_controllers(), + ); + push_cap( + &mut controllers, + DomainGroup::Memory, + // `learn_all` runs the TREE SUMMARIZER over namespaces, so it belongs + // to Tree, not Ingest — `Capability::Ingest` is `ingest_document` / + // `ingest_chat`, whose RPC surface is `memory.doc_ingest` above. + Some(Capability::Tree), + crate::openhuman::memory::all_memory_learn_registered_controllers(), + ); + push_cap( &mut controllers, DomainGroup::Memory, - crate::openhuman::memory::all_memory_registered_controllers(), + // NEVER gated: `memory.provider_status` is the RPC that REPORTS the + // bound driver's capability set. Gating it on a capability would be + // self-referential and would hide the explanation for every other + // absence in this block. + None, + crate::openhuman::memory::all_memory_provider_registered_controllers(), + ); + push_cap( + &mut controllers, + DomainGroup::Memory, + Some(Capability::ToolMemory), + crate::openhuman::memory::all_memory_tool_memory_registered_controllers(), ); // Long-term goals list (editable list + turn-based enrichment agent) - push( + push_cap( &mut controllers, DomainGroup::Memory, + Some(Capability::Goals), crate::openhuman::memory::goals::all_memory_goals_registered_controllers(), ); // Thread-level goal (Codex-style per-thread completion contract) @@ -655,40 +818,56 @@ fn build_registered_controllers() -> Vec { crate::openhuman::threads::goals::all_thread_goals_registered_controllers(), ); // Memory tree ingestion layer (#707 — canonicalised chunks with provenance) - push( + push_cap( &mut controllers, DomainGroup::Memory, + // DELIBERATE, not inherited: `memory/schema/registry.rs`'s ~25 methods + // span tree, entities, graph and maintenance, and are tagged as ONE + // capability rather than split. Tree and entities are treated here as + // parts of a single encapsulated memory surface, not independently + // degradable families. The visible consequence: a driver advertising + // `entities` but not `tree` still loses `memory_tree.top_entities`. + // Split it only when a real driver needs that distinction. + Some(Capability::Tree), crate::openhuman::memory::tree::all_memory_tree_registered_controllers(), ); // Memory tree retrieval layer (#710 — LLM-callable read tools over the tree) - push( + push_cap( &mut controllers, DomainGroup::Memory, + Some(Capability::Tree), crate::openhuman::memory::tree::all_retrieval_registered_controllers(), ); // Slack → memory-tree ingestion engine (per-message ingest, no bucketing) - push( + push_cap( &mut controllers, DomainGroup::Memory, + // Grouped with the other three sync namespaces rather than `Ingest`: a + // driver that cannot accept synced source items should lose the whole + // source-sync surface coherently, not half of it. + Some(Capability::Sources), crate::openhuman::integrations::composio::providers::slack::all_slack_memory_registered_controllers(), ); // Per-connection memory sync status, controls, and progress (#1136) - push( + push_cap( &mut controllers, DomainGroup::Memory, + Some(Capability::Sources), crate::openhuman::memory::sync::sync_status::all_memory_sync_status_registered_controllers( ), ); // Memory sources — user-configured data connectors registry - push( + push_cap( &mut controllers, DomainGroup::Memory, + Some(Capability::Sources), crate::openhuman::memory::sources::all_memory_sources_registered_controllers(), ); // Memory diff — snapshot-based change tracking for memory sources - push( + push_cap( &mut controllers, DomainGroup::Memory, + Some(Capability::Diff), crate::openhuman::memory::diff::all_memory_diff_registered_controllers(), ); // Referral and growth tracking @@ -776,9 +955,10 @@ fn build_registered_controllers() -> Vec { crate::openhuman::platform::update::all_update_registered_controllers(), ); // Hierarchical knowledge summarization - push( + push_cap( &mut controllers, DomainGroup::Memory, + Some(Capability::Tree), crate::openhuman::memory::tree::all_tree_summarizer_registered_controllers(), ); // Self-learning and user context enrichment @@ -942,9 +1122,10 @@ fn build_internal_only_controllers() -> Vec { /// omitted. With no active context, or under `DomainSet::full()`, this returns /// the complete set (byte-identical to pre-#4796). pub fn all_registered_controllers() -> Vec { + let caps = crate::core::runtime::context::CoreContext::current_memory_capabilities(); registry() .iter() - .filter(|g| group_allowed(g.group)) + .filter(|g| group_allowed(g.group) && capability_allowed_in(caps, g.capability)) .map(|g| g.controller.clone()) .collect() } @@ -957,9 +1138,10 @@ pub fn all_registered_controllers() -> Vec { /// [`all_registered_controllers`], so `/schema` omits gated namespaces /// automatically under `harness()`. pub fn all_controller_schemas() -> Vec { + let caps = crate::core::runtime::context::CoreContext::current_memory_capabilities(); registry() .iter() - .filter(|g| group_allowed(g.group)) + .filter(|g| group_allowed(g.group) && capability_allowed_in(caps, g.capability)) .map(|g| g.controller.schema.clone()) .collect() } @@ -1113,6 +1295,9 @@ pub fn namespace_description(namespace: &str) -> Option<&'static str> { "devices" => Some( "Paired mobile device management — pairing channel creation, listing, and revocation.", ), + "subsystems" => Some( + "Kernel subsystem slots and their bound drivers: class, health, contract version, and advertised capabilities.", + ), "tinyplace" => Some( "tiny.place A2A social-network integration: directory, explorer, and search over the agent network.", ), @@ -1142,6 +1327,86 @@ pub fn rpc_method_from_parts(namespace: &str, function: &str) -> Option .map(|g| g.controller.rpc_method_name()) } +/// The memory-driver capability family a controller's surface requires, looked +/// up in the **UNFILTERED** registry. +/// +/// Returns `None` when no controller with that `(namespace, function)` is +/// registered anywhere — a genuine typo. Returns `Some(None)` when the +/// controller exists and is ungated, and `Some(Some(c))` when it exists and +/// needs family `c`. +/// +/// The `Option>` is the whole point: it is what lets the CLI tell +/// "no such command" apart from "this command exists but the bound driver does +/// not advertise its family". Every *filtered* lookup ([`schema_for_rpc_method`], +/// [`all_controller_schemas`]) collapses those two into one absence, which is +/// correct for `/rpc` and for agent tools (`docs/specs/kernel.md` §3.3) and +/// wrong for a human at a terminal — the CLI is §3.3's one named exception. +/// +/// Scoped to the agent-facing [`registry`] exactly like [`rpc_method_from_parts`], +/// the other lookup that backs CLI routing: an internal-only controller is not +/// CLI-invokable in any configuration, so reporting a capability fact for one +/// would name a cause that is not the reason the command is unavailable. +pub fn capability_for_parts(namespace: &str, function: &str) -> Option> { + registry() + .iter() + .find(|g| { + g.controller.schema.namespace == namespace && g.controller.schema.function == function + }) + .map(|g| g.capability) +} + +/// The memory-driver capability family required by an RPC method, looked up in +/// the **UNFILTERED** registry. +/// +/// This is the method-name counterpart of [`capability_for_parts`]. The raw +/// `openhuman call --method …` CLI form has no namespace/function split, but +/// must still produce the CLI's configuration-fact diagnostic before it +/// dispatches a capability-gated method. +pub fn capability_for_rpc_method(method: &str) -> Option> { + registry() + .iter() + .find(|g| g.controller.rpc_method_name() == method) + .map(|g| g.capability) +} + +/// The capability a whole namespace's surface requires, when every controller +/// in it agrees — looked up in the **UNFILTERED** registry. +/// +/// `None` when the namespace does not exist at all, or when nothing in it is +/// gated, or when its controllers span more than one family. Used for the +/// unknown-namespace case: a namespace whose controllers are ALL gated on one +/// family disappears from the CLI's namespace list entirely, so there is no +/// function name left to look up. +/// +/// Deliberately conservative — it reports a family only when that family is the +/// sole gate across the namespace, so a mixed namespace (like `memory`, which +/// spans four families plus host surface) yields `None` and falls back to the +/// ordinary unknown-namespace message rather than naming one family +/// misleadingly. +pub fn sole_capability_for_namespace(namespace: &str) -> Option { + let mut found: Option = None; + let mut any = false; + for grouped in registry() + .iter() + .filter(|g| g.controller.schema.namespace == namespace) + { + any = true; + match (grouped.capability, found) { + // An ungated member means the namespace does not vanish wholesale + // because of one family, so naming one would be a lie. + (None, _) => return None, + (Some(c), None) => found = Some(c), + (Some(c), Some(prev)) if c == prev => {} + (Some(_), Some(_)) => return None, + } + } + if any { + found + } else { + None + } +} + /// Retrieves the schema for a specific RPC method. /// /// Checks both the agent-facing registry and the internal registry so that @@ -1155,10 +1420,18 @@ pub fn schema_for_rpc_method(method: &str) -> Option { // call with bad params would return the controller's validation error // instead of method-not-found, leaking the hidden RPC surface. No ambient // context ⇒ `group_allowed` is `true` ⇒ unfiltered, identical to pre-#4796. + // + // The memory-capability gate (M5.2) rides here for exactly the same reason: + // a `memory_tree.*` method hidden because the bound driver never advertised + // `tree` must not leak back out through a param-validation error. registry() .iter() .chain(internal_registry().iter()) - .find(|g| g.controller.rpc_method_name() == method && group_allowed(g.group)) + .find(|g| { + g.controller.rpc_method_name() == method + && group_allowed(g.group) + && capability_allowed(g.capability) + }) .map(|g| g.controller.schema.clone()) } @@ -1324,6 +1597,18 @@ pub async fn try_invoke_registered_rpc( ); return None; } + + // Memory-capability gate (M5.2). Deliberately a SECOND block rather than a + // clause folded into the check above, so the two gates log distinguishably: + // an operator seeing an absent `memory_tree.*` needs to know whether it was + // the DomainSet or the bound driver's advertised capability set. + if !capability_allowed(grouped.capability) { + log::debug!( + "[rpc][capability-gate] method '{method}' suppressed — memory capability {:?} not advertised by the bound driver", + grouped.capability + ); + return None; + } let handler = grouped.controller.handler; // Establish the ambient CoreContext for the duration of the handler so diff --git a/src/core/all_tests.rs b/src/core/all_tests.rs index 83a7da9cd7..88d6322c68 100644 --- a/src/core/all_tests.rs +++ b/src/core/all_tests.rs @@ -30,6 +30,7 @@ fn grouped(controllers: Vec) -> Vec { .into_iter() .map(|controller| GroupedController { group: DomainGroup::Platform, + capability: None, controller, }) .collect() @@ -122,6 +123,7 @@ fn namespace_description_known_namespaces() { assert!(namespace_description("billing").is_some()); assert!(namespace_description("config").is_some()); assert!(namespace_description("health").is_some()); + assert!(namespace_description("subsystems").is_some()); assert!(namespace_description("security").is_some()); assert!(namespace_description("tool_registry").is_some()); assert!(namespace_description("voice").is_some()); @@ -799,6 +801,14 @@ fn group_for_namespace(ns: &str) -> Option { .map(|g| g.group) } +#[test] +fn subsystems_namespace_is_registered_under_platform() { + assert_eq!( + group_for_namespace("subsystems"), + Some(DomainGroup::Platform) + ); +} + #[test] fn full_registration_is_byte_identical() { // With no ambient CoreContext (⇒ full, no filter), the public @@ -856,7 +866,7 @@ async fn harness_excludes_gated_namespaces() { #[cfg(feature = "channels")] assert!(full_ns.contains("channels"), "full() must expose channels"); - let ctx = CoreContext::for_test(DomainSet::harness(), None); + let ctx = CoreContext::for_test(DomainSet::harness(), None, None); let harness_ns: BTreeSet<&'static str> = CoreContext::scope(ctx, async { all_controller_schemas() }) .await @@ -881,6 +891,11 @@ async fn harness_excludes_gated_namespaces() { "channels", "mcp_clients", "health", + // The subsystem status surface is Platform-tagged for the same reason + // `health` is: it is kernel operator surface with no family. An + // embedded harness host reads driver capabilities through + // `memory.provider_status`, which stays reachable. + "subsystems", ] { assert!( !harness_ns.contains(absent), @@ -909,7 +924,7 @@ async fn dispatch_returns_none_for_gated_method() { .map(|c| c.rpc_method_name()) .expect("a flows.* method exists in the full registry"); - let ctx = CoreContext::for_test(DomainSet::harness(), None); + let ctx = CoreContext::for_test(DomainSet::harness(), None, None); let out = CoreContext::scope(ctx, try_invoke_registered_rpc(&gated_method, Map::new())).await; assert!( out.is_none(), @@ -918,7 +933,7 @@ async fn dispatch_returns_none_for_gated_method() { // A harness-family method still routes (Some) — security.policy_info needs // no workspace, so it is a clean positive control. - let ctx = CoreContext::for_test(DomainSet::harness(), None); + let ctx = CoreContext::for_test(DomainSet::harness(), None, None); let out = CoreContext::scope( ctx, try_invoke_registered_rpc("openhuman.security_policy_info", Map::new()), @@ -954,7 +969,7 @@ async fn schema_lookup_is_gated_in_lockstep_with_dispatch() { "under full() the schema for `{gated_method}` must resolve" ); - let ctx = CoreContext::for_test(DomainSet::harness(), None); + let ctx = CoreContext::for_test(DomainSet::harness(), None, None); let gated_schema = CoreContext::scope(ctx, async { schema_for_rpc_method(&gated_method) }).await; assert!( @@ -962,7 +977,7 @@ async fn schema_lookup_is_gated_in_lockstep_with_dispatch() { "schema lookup for gated `{gated_method}` must be None under harness() (no param validation, no surface leak)" ); - let ctx = CoreContext::for_test(DomainSet::harness(), None); + let ctx = CoreContext::for_test(DomainSet::harness(), None, None); let kept_schema = CoreContext::scope(ctx, async { schema_for_rpc_method("openhuman.security_policy_info") }) @@ -1565,3 +1580,831 @@ fn every_domain_group_is_accounted_for_in_subscriber_plan() { let none = DomainSubscriberPlan::for_domains(crate::core::runtime::DomainSet::none()); assert_ne!(full, none, "full() and none() must differ"); } + +/// M5.1 split `memory::all_memory_registered_controllers()` into seven +/// per-family pairs pushed separately in `build_registered_controllers`. This +/// pins the observable result: the `memory` namespace still occupies one +/// contiguous run in the registry, in the aggregator's exact order. A stray +/// push (wrong place, wrong order, a family dropped) fails here. +#[test] +fn memory_controllers_form_one_contiguous_run_in_aggregator_order() { + let all = all_registered_controllers(); + let positions: Vec = all + .iter() + .enumerate() + .filter(|(_, c)| c.schema.namespace == "memory") + .map(|(i, _)| i) + .collect(); + + assert!(!positions.is_empty(), "no memory controllers registered"); + let first = positions[0]; + let expected_run: Vec = (first..first + positions.len()).collect(); + assert_eq!( + positions, expected_run, + "memory controllers are no longer contiguous in the registry" + ); + + let registered: Vec<&'static str> = positions.iter().map(|&i| all[i].schema.function).collect(); + let aggregator: Vec<&'static str> = + crate::openhuman::memory::all_memory_registered_controllers() + .iter() + .map(|c| c.schema.function) + .collect(); + assert_eq!( + registered, aggregator, + "registry order for memory.* diverges from the memory schemas aggregator" + ); +} + +// --- M5.2: memory-capability registration filter --------------------------- +// +// The capability axis is the same shape as the DomainSet axis above: the +// registry holds every controller, and the ambient `CoreContext` decides at +// READ time which ones exist. A family the bound driver never advertised is +// ABSENT — unknown-method over `/rpc`, omitted from `/schema` — rather than +// present and failing, because a registered-but-failing method teaches a model +// the capability exists and makes it retry. + +use tinycortex_api::capabilities::Capability; + +/// A workspace path unique to one test. +/// +/// `memory::binding::BINDINGS` is a process-global `HashMap` that +/// never evicts, so the FIRST test to bind a path fixes that path's driver for +/// every later test in the process. Sharing a path between an ON test and an +/// OFF test would make one of them silently assert the other's driver. +fn caps_ws(name: &str) -> std::path::PathBuf { + std::path::PathBuf::from(format!("/tmp/oh-m5-caps-{name}")) +} + +/// `[subsystems.memory] driver = "null"` — the only narrowed capability set a +/// test can reach without booting. +/// +/// `CoreContext::for_test` takes the memory *config*, not a `Capabilities`, on +/// purpose (see its doc comment): injecting a set directly would let a test +/// assert a set no driver could have advertised and would bypass the very +/// `admit` + `capabilities()` path being proven. `admit` maps `"null"` to +/// `NullMemoryProvider`, whose advertised set is exactly +/// `Capabilities::mandatory()` = {core, recall, portability} — so every +/// optional family is OFF at once. The OFF half of each pair below therefore +/// reads "absent under a driver that advertises nothing optional", not "absent +/// with only this one family missing". +fn null_driver_cfg() -> crate::openhuman::config::schema::MemorySubsystemConfig { + crate::openhuman::config::schema::MemorySubsystemConfig { + driver: "null".into(), + ..Default::default() + } +} + +/// Namespaces registered under [`DomainGroup::Memory`], each with the capability +/// its registration site tags it with. The `memory` namespace is absent here — +/// it spans four families plus host surface and is covered per-function by +/// [`MEMORY_FUNCTION_CAPABILITY`]. +const MEMORY_NAMESPACE_CAPABILITY: &[(&str, Option)] = &[ + // Host-owned address book, not a driver family. + ("people", None), + ("memory_goals", Some(Capability::Goals)), + // Both the tree registry and the retrieval layer share this namespace. + ("memory_tree", Some(Capability::Tree)), + ("tree_summarizer", Some(Capability::Tree)), + ("slack_memory", Some(Capability::Sources)), + ("memory_sync", Some(Capability::Sources)), + ("memory_sources", Some(Capability::Sources)), + ("memory_diff", Some(Capability::Diff)), +]; + +/// The `memory` namespace, function by function. Mandatory core/recall surface +/// and host-only file I/O are `None` deliberately — a gate on a MANDATORY +/// family could never fire, and a dead gate reads like a live one. +const MEMORY_FUNCTION_CAPABILITY: &[(&str, Option)] = &[ + // core + recall (mandatory) + ("init", None), + ("list_documents", None), + ("list_namespaces", None), + ("delete_document", None), + ("query_namespace", None), + ("recall_context", None), + ("recall_memories", None), + ("namespace_list", None), + ("context_query", None), + ("context_recall", None), + ("clear_namespace", None), + // namespace-document tier + ("doc_put", Some(Capability::Documents)), + ("doc_list", Some(Capability::Documents)), + ("doc_delete", Some(Capability::Documents)), + // driver-owned ingestion + ("doc_ingest", Some(Capability::Ingest)), + // plain workspace file I/O, host-side + ("list_files", None), + ("read_file", None), + ("write_file", None), + // key/value + knowledge graph + ("kv_set", Some(Capability::Graph)), + ("kv_get", Some(Capability::Graph)), + ("kv_delete", Some(Capability::Graph)), + ("kv_list_namespace", Some(Capability::Graph)), + ("graph_upsert", Some(Capability::Graph)), + ("graph_query", Some(Capability::Graph)), + // source sync + ("sync_channel", Some(Capability::Sources)), + ("sync_all", Some(Capability::Sources)), + ("ingestion_status", Some(Capability::Sources)), + // the tree summarizer, NOT ingestion + ("learn_all", Some(Capability::Tree)), + // never gated: this is the RPC that reports the capability set + ("provider_status", None), + // per-tool learned memory + ("tool_rule_put", Some(Capability::ToolMemory)), + ("tool_rule_get", Some(Capability::ToolMemory)), + ("tool_rule_list", Some(Capability::ToolMemory)), + ("tool_rule_delete", Some(Capability::ToolMemory)), + ("tool_rules_for_prompt", Some(Capability::ToolMemory)), + ("tool_rules_json", Some(Capability::ToolMemory)), +]; + +fn expected_capability(ns: &str, function: &str) -> Option> { + if ns == "memory" { + return MEMORY_FUNCTION_CAPABILITY + .iter() + .find(|(f, _)| *f == function) + .map(|(_, c)| *c); + } + MEMORY_NAMESPACE_CAPABILITY + .iter() + .find(|(n, _)| *n == ns) + .map(|(_, c)| *c) +} + +/// Drift guard: every `DomainGroup::Memory` controller carries a +/// checked-in capability decision, and the live tag matches it. +/// +/// This is what makes an untagged Memory push a test failure rather than a +/// silent `None`. `push` delegates to `push_cap(.., None, ..)`, so a new Memory +/// site added with the wrong helper compiles fine and gates nothing — only this +/// table catches it. +#[test] +fn memory_capability_map_is_exhaustive() { + for g in registry().iter().chain(internal_registry().iter()) { + if g.group != DomainGroup::Memory { + continue; + } + let ns = g.controller.schema.namespace; + let function = g.controller.schema.function; + let expected = expected_capability(ns, function).unwrap_or_else(|| { + panic!( + "`{ns}.{function}` is registered under DomainGroup::Memory but carries no \ + checked-in capability decision — add it to MEMORY_NAMESPACE_CAPABILITY or \ + MEMORY_FUNCTION_CAPABILITY and tag its push site with push_cap(..)" + ) + }); + assert_eq!( + g.capability, expected, + "`{ns}.{function}` is tagged {:?} at its registration site but the map says {expected:?}", + g.capability + ); + } +} + +/// The other direction: no table entry may name a namespace/function that is no +/// longer registered, so a deleted controller cannot leave a stale decision +/// behind that looks like coverage. +#[test] +fn memory_capability_map_has_no_stale_entries() { + let live: Vec<(&str, &str)> = registry() + .iter() + .chain(internal_registry().iter()) + .filter(|g| g.group == DomainGroup::Memory) + .map(|g| (g.controller.schema.namespace, g.controller.schema.function)) + .collect(); + + for (ns, _) in MEMORY_NAMESPACE_CAPABILITY { + assert!( + live.iter().any(|(n, _)| n == ns), + "MEMORY_NAMESPACE_CAPABILITY names `{ns}`, which registers no Memory controller" + ); + } + for (function, _) in MEMORY_FUNCTION_CAPABILITY { + assert!( + live.iter().any(|(n, f)| *n == "memory" && f == function), + "MEMORY_FUNCTION_CAPABILITY names `memory.{function}`, which is not registered" + ); + } +} + +/// Every capability family is accounted for in the RPC surface — either it +/// gates at least one controller, or it is listed as deliberately RPC-less. +/// +/// `Capability` is deliberately NOT `#[non_exhaustive]` (see that module's +/// docs), so a fourteenth family is a **compile error** in the `match` below +/// before it is a test failure. That compile error is the mechanism which +/// guarantees a new family gets wired somewhere rather than silently defaulting +/// to ungated. +#[test] +fn every_capability_family_is_accounted_for_in_the_rpc_surface() { + let gated: std::collections::BTreeSet = registry() + .iter() + .chain(internal_registry().iter()) + .filter_map(|g| g.capability) + .collect(); + + for cap in Capability::ALL { + let has_rpc_surface = match cap { + // Gate at least one controller today. + Capability::Ingest + | Capability::Documents + | Capability::Tree + | Capability::Graph + | Capability::Diff + | Capability::Goals + | Capability::ToolMemory + | Capability::Sources => true, + // MANDATORY: `Capabilities::validate` refuses to bind a driver + // missing these, so a gate on one could never fire. Their surface + // (memory.init / recall_* / …) registers ungated on purpose. + Capability::Core | Capability::Recall | Capability::Portability => false, + // Folded into `Tree`: the tree registry's ~25 methods span tree, + // entities, graph and maintenance and are tagged as ONE family. + // See the push site in `all.rs` for why that trade was chosen. + Capability::Entities => false, + // No controller exposes re-embed / compact / dream / doctor yet. + Capability::Maintenance => false, + }; + assert_eq!( + gated.contains(&cap), + has_rpc_surface, + "capability `{cap}` is {} in the live registry but the table says {}", + if gated.contains(&cap) { + "gating controllers" + } else { + "gating nothing" + }, + if has_rpc_surface { + "it should gate something" + } else { + "it should gate nothing" + }, + ); + } +} + +// --- default-open: the 4000-pre-boot-test tripwire ------------------------- + +#[test] +fn capability_allowed_defaults_open_with_no_context() { + // No ambient CoreContext at all. `None` is trivially allowed, and every + // real family must be allowed too — `current_memory_capabilities()` falls + // back to the full set. A deny-by-default here would fail every memory + // unit test in the crate at once. + assert!(capability_allowed(None)); + for cap in Capability::ALL { + assert!( + capability_allowed(Some(cap)), + "capability `{cap}` must default OPEN with no ambient context" + ); + } +} + +#[test] +fn unbound_registration_is_byte_identical() { + // Companion to `full_registration_is_byte_identical`: with no ambient + // context the capability filter must be an order-preserving identity, so + // adding the axis changed neither membership nor ordering of the unbound + // surface. + let filtered: Vec = all_registered_controllers() + .iter() + .map(|c| c.rpc_method_name()) + .collect(); + let raw: Vec = registry() + .iter() + .map(|g| g.controller.rpc_method_name()) + .collect(); + assert_eq!(filtered, raw); +} + +#[tokio::test] +async fn narrowed_capabilities_do_not_narrow_the_domain_set() { + // The two axes are independent: a null driver hides memory families, but + // every non-Memory namespace stays exactly as `full()` had it. + use std::collections::BTreeSet; + + let full_ns: BTreeSet<&str> = all_controller_schemas() + .iter() + .map(|s| s.namespace) + .collect(); + + let ctx = CoreContext::for_test( + DomainSet::full(), + Some(caps_ws("axes")), + Some(null_driver_cfg()), + ); + let null_ns: BTreeSet<&'static str> = + CoreContext::scope(ctx, async { all_controller_schemas() }) + .await + .iter() + .map(|s| s.namespace) + .collect(); + + for ns in ["threads", "config", "security", "agent", "tools"] { + assert!( + null_ns.contains(ns), + "a narrowed memory capability set must not remove the `{ns}` namespace" + ); + } + assert!(null_ns.len() < full_ns.len()); +} + +// --- both-ways pairs, one per gated family --------------------------------- +// +// The ABSENT half of each pair is the one that proves the gate removes +// anything; a gate that never removes anything would still pass the present +// half. + +/// Namespaces + `memory.*` functions visible under the given memory config. +async fn visible_under( + ws: &str, + cfg: Option, +) -> ( + std::collections::BTreeSet<&'static str>, + std::collections::BTreeSet<&'static str>, +) { + let ctx = CoreContext::for_test(DomainSet::full(), Some(caps_ws(ws)), cfg); + let schemas = CoreContext::scope(ctx, async { all_controller_schemas() }).await; + let namespaces = schemas.iter().map(|s| s.namespace).collect(); + let memory_fns = schemas + .iter() + .filter(|s| s.namespace == "memory") + .map(|s| s.function) + .collect(); + (namespaces, memory_fns) +} + +#[tokio::test] +async fn memory_families_registered_when_capabilities_advertised() { + // The embedded `tinycortex` driver (the default config) advertises + // `Capabilities::all()`, so every gated family is present. Scoped rather + // than unscoped so this proves a BOUND driver's set, not the unbound + // default-open fallback. + let (ns, fns) = visible_under("on", None).await; + + for present in [ + "memory", + "memory_goals", + "memory_tree", + "tree_summarizer", + "memory_sync", + "memory_sources", + "memory_diff", + "slack_memory", + "people", + ] { + assert!( + ns.contains(present), + "`{present}` must be present under a full-capability driver" + ); + } + for present in [ + "doc_put", + "doc_ingest", + "kv_set", + "graph_query", + "sync_all", + "learn_all", + "tool_rule_put", + "provider_status", + "recall_memories", + "list_files", + ] { + assert!( + fns.contains(present), + "`memory.{present}` must be present under a full-capability driver" + ); + } +} + +#[tokio::test] +async fn memory_families_absent_when_capabilities_not_advertised() { + // The null driver advertises exactly {core, recall, portability}, so every + // optional family is unadvertised at once. + let (ns, fns) = visible_under("off", Some(null_driver_cfg())).await; + + // Whole namespaces vanish. + for absent in [ + "memory_goals", + "memory_tree", + "tree_summarizer", + "memory_sync", + "memory_sources", + "memory_diff", + "slack_memory", + ] { + assert!( + !ns.contains(absent), + "`{absent}` must be ABSENT under a driver that advertises only the mandatory families" + ); + } + // Gated `memory.*` functions vanish… + for absent in [ + "doc_put", + "doc_list", + "doc_delete", + "doc_ingest", + "kv_set", + "kv_get", + "kv_delete", + "kv_list_namespace", + "graph_upsert", + "graph_query", + "sync_channel", + "sync_all", + "ingestion_status", + "learn_all", + "tool_rule_put", + "tool_rule_get", + "tool_rule_list", + "tool_rule_delete", + "tool_rules_for_prompt", + "tool_rules_json", + ] { + assert!( + !fns.contains(absent), + "`memory.{absent}` must be ABSENT under the null driver" + ); + } + // …while the ungated surface stays. These are the positive controls that + // make the assertions above the GATE rather than a collapsed registry. + assert!( + ns.contains("memory"), + "the `memory` namespace itself must survive" + ); + assert!( + ns.contains("people"), + "`people` is host surface with no capability — it must survive any driver" + ); + for present in [ + // MANDATORY core + recall. + "init", + "namespace_list", + "recall_memories", + "recall_context", + "query_namespace", + "clear_namespace", + // Host-side workspace file I/O. + "list_files", + "read_file", + "write_file", + // The RPC that REPORTS the capability set — gating it would hide the + // explanation for every absence above. + "provider_status", + ] { + assert!( + fns.contains(present), + "`memory.{present}` is ungated and must survive the null driver" + ); + } +} + +#[tokio::test] +async fn dispatch_returns_none_for_capability_gated_method() { + let ctx = CoreContext::for_test( + DomainSet::full(), + Some(caps_ws("dispatch")), + Some(null_driver_cfg()), + ); + let out = CoreContext::scope( + ctx, + try_invoke_registered_rpc("openhuman.memory_tool_rules_json", Map::new()), + ) + .await; + assert!( + out.is_none(), + "a capability-gated method must dispatch as None — indistinguishable from absent" + ); + + // Positive control in the same driver configuration. + let ctx = CoreContext::for_test( + DomainSet::full(), + Some(caps_ws("dispatch")), + Some(null_driver_cfg()), + ); + let out = CoreContext::scope( + ctx, + try_invoke_registered_rpc("openhuman.memory_provider_status", Map::new()), + ) + .await; + assert!( + out.is_some(), + "ungated `memory.provider_status` must still route under the null driver" + ); +} + +#[tokio::test] +async fn schema_lookup_is_gated_in_lockstep_with_capability_dispatch() { + // If `schema_for_rpc_method` did NOT gate, `invoke_method_inner` would run + // param validation against a hidden method and return the controller's + // validation error instead of method-not-found — leaking the surface the + // gate exists to hide. + let method = "openhuman.memory_tool_rules_json"; + assert!( + schema_for_rpc_method(method).is_some(), + "unscoped, the schema must resolve — so the None below is the gate, not a typo" + ); + + let ctx = CoreContext::for_test( + DomainSet::full(), + Some(caps_ws("schema")), + Some(null_driver_cfg()), + ); + let gated = CoreContext::scope(ctx, async { schema_for_rpc_method(method) }).await; + assert!( + gated.is_none(), + "schema lookup for a capability-gated method must be None" + ); + + let ctx = CoreContext::for_test( + DomainSet::full(), + Some(caps_ws("schema")), + Some(null_driver_cfg()), + ); + let kept = CoreContext::scope(ctx, async { + schema_for_rpc_method("openhuman.memory_provider_status") + }) + .await; + assert!( + kept.is_some(), + "ungated provider_status schema must still resolve" + ); +} + +#[tokio::test] +async fn rpc_method_from_parts_stays_unfiltered_by_capability() { + // `rpc_method_from_parts` searches the FULL registry by design (it backs + // param validation and CLI routing). Pinning that here so a future "make + // every lookup consistent" change has to be a deliberate decision. + let ctx = CoreContext::for_test( + DomainSet::full(), + Some(caps_ws("parts")), + Some(null_driver_cfg()), + ); + let out = CoreContext::scope(ctx, async { + rpc_method_from_parts("memory", "tool_rules_json") + }) + .await; + assert_eq!(out.as_deref(), Some("openhuman.memory_tool_rules_json")); +} + +// --- M5.4: the null-driver degradation gate (milestone definition of done) -- +// +// M5.1–M5.3 built the filter; these are the end-to-end assertions that the +// WIRING is right, using the tree family as the named vehicle. They target +// `try_invoke_registered_rpc`, `schema_for_rpc_method` and +// `all_controller_schemas` — the same three functions the HTTP layer calls +// (`core::jsonrpc::invoke_method_inner` resolves the schema then dispatches; +// `/schema` renders `all_http_method_schemas()`, which extends from +// `all_controller_schemas()`). Asserting on them IS asserting on the wire +// surface; there is no more faithful vehicle available at this level, and an +// integration test under `tests/` would be strictly WEAKER — `CoreContext::for_test` +// is `#[cfg(test)] pub(crate)` and `tests/json_rpc_e2e.rs` never calls +// `CoreContext::init`, so `current()` is `None` there and the filter would +// default OPEN, proving nothing. Do not "upgrade" these into `tests/`. +// +// The agent-tool half of the DoD is pinned next to the tool machinery that owns +// the full tool list, by `optional_family_memory_tools_absent_under_the_null_driver` +// in `src/openhuman/tools/ops_tests.rs` (`memory_tree` is in its absent list). +// Same split the channels gate uses; not duplicated here. + +/// `memory_tree*` is unknown-method under a driver that never advertised +/// `Capability::Tree`. +/// +/// `is_none()`, never `is_err()`: `Some(Err(_))` is the registered-but-failing +/// shape `docs/specs/kernel.md` §3.3 forbids, because a method that exists and +/// fails teaches a model the capability is real and makes it retry. +#[tokio::test] +async fn null_driver_makes_tree_methods_unknown_over_rpc() { + let method = "openhuman.memory_tree_list_chunks"; + + // Positive control FIRST: unscoped (⇒ the default-open fallback) the method + // routes. Without this the assertion below could pass because the method + // never existed at all. + assert!( + try_invoke_registered_rpc(method, Map::new()) + .await + .is_some(), + "`{method}` must route with no ambient context (the filter defaults OPEN)" + ); + + let ctx = CoreContext::for_test( + DomainSet::full(), // isolates the capability gate from the DomainSet gate + Some(caps_ws("m54-tree-dispatch")), + Some(null_driver_cfg()), + ); + let out = CoreContext::scope(ctx, try_invoke_registered_rpc(method, Map::new())).await; + assert!( + out.is_none(), + "under the `null` driver `{method}` must dispatch as None — an unadvertised \ + family is indistinguishable from an unregistered method, never a handler \ + that returns 'not implemented'" + ); +} + +/// The whole `memory_tree` namespace leaves `/schema`, and the schema lookup +/// gates in lockstep with dispatch. +/// +/// Asserted as a namespace SET rather than a method list on purpose: +/// `memory_tree` is the only namespace with two registration sites — the tree +/// registry (`memory::schema::definitions`) and the retrieval layer +/// (`memory::tree::retrieval::schemas`) both use `NAMESPACE = "memory_tree"` — +/// so a method-level assertion could pass having filtered only one of them. +/// +/// The lockstep half is not optional: `invoke_method_inner` resolves the schema +/// and runs `validate_params` BEFORE dispatch, so a schema lookup that is not +/// gated with dispatch leaks the hidden surface as a validation error instead +/// of method-not-found. +#[tokio::test] +async fn null_driver_removes_tree_namespace_from_schema() { + let full_ns: std::collections::BTreeSet<&str> = all_controller_schemas() + .iter() + .map(|s| s.namespace) + .collect(); + assert!( + full_ns.contains("memory_tree"), + "unscoped ⇒ default open ⇒ memory_tree present; otherwise the assertion below is vacuous" + ); + + let ctx = CoreContext::for_test( + DomainSet::full(), + Some(caps_ws("m54-tree-schema")), + Some(null_driver_cfg()), + ); + let null_ns: std::collections::BTreeSet<&str> = + CoreContext::scope(ctx, async { all_controller_schemas() }) + .await + .iter() + .map(|s| s.namespace) + .collect(); + + assert!( + !null_ns.contains("memory_tree"), + "both `memory_tree` registration sites must be absent from /schema under the null driver" + ); + assert!( + null_ns.contains("memory"), + "the mandatory core/recall surface must survive" + ); + assert!( + null_ns.len() < full_ns.len(), + "the null driver must expose strictly fewer namespaces" + ); + + // Lockstep: no schema resolves for a tree method either. + let method = "openhuman.memory_tree_list_chunks"; + assert!( + schema_for_rpc_method(method).is_some(), + "unscoped the schema must resolve — so the None below is the gate, not a typo" + ); + let ctx = CoreContext::for_test( + DomainSet::full(), + Some(caps_ws("m54-tree-schema")), + Some(null_driver_cfg()), + ); + let gated = CoreContext::scope(ctx, async { schema_for_rpc_method(method) }).await; + assert!( + gated.is_none(), + "schema lookup must gate in lockstep with dispatch, or param validation leaks the surface" + ); +} + +/// Degradation is not a crash: the mandatory surface still stands up. +/// +/// **What this does and does not prove.** A true boot needs +/// `CoreContext::init` → `Config::load_or_init`, which is async, env-dependent +/// and writes `$HOME` — not appropriate here, and `tests/` cannot scope a +/// context at all (see the module note above). What this DOES cover is the +/// failure mode that would actually take boot down: the capability filter +/// panicking inside `registry()`'s `validate_registry` (which panics on an +/// invalid registry), or narrowing the surface to empty. Stated rather than +/// overstated — an enforcement test that oversells its guarantee is worse than +/// none, because it stops people looking. +#[tokio::test] +async fn null_driver_keeps_the_mandatory_memory_surface_routable() { + // (1) The registry builds and self-validates under the null context. + let schemas = CoreContext::scope( + CoreContext::for_test( + DomainSet::full(), + Some(caps_ws("m54-boot")), + Some(null_driver_cfg()), + ), + async { all_controller_schemas() }, + ) + .await; + assert!( + !schemas.is_empty(), + "degradation must not empty the controller surface" + ); + + // (2) The driver-status surface stays reachable — it is how a host reads + // the capability set back, so gating it would hide the degradation. + // + // `is_some()`, never `is_ok()`: these handlers resolve through + // `active_memory_client`, a process global that is uninitialised in a unit + // test, so the inner `Result` is legitimately `Err`. ROUTABILITY is the + // property under test. + let out = CoreContext::scope( + CoreContext::for_test( + DomainSet::full(), + Some(caps_ws("m54-boot")), + Some(null_driver_cfg()), + ), + try_invoke_registered_rpc("openhuman.memory_provider_status", Map::new()), + ) + .await; + assert!( + out.is_some(), + "memory.provider_status must stay routable under any driver" + ); + + // (3) Recall — a MANDATORY family — still routes. + let out = CoreContext::scope( + CoreContext::for_test( + DomainSet::full(), + Some(caps_ws("m54-boot")), + Some(null_driver_cfg()), + ), + try_invoke_registered_rpc("openhuman.memory_recall_memories", Map::new()), + ) + .await; + assert!( + out.is_some(), + "the mandatory Recall surface must stay routable under the null driver" + ); +} + +// --- the UNFILTERED capability lookup that backs the CLI's config-fact ------- +// +// `docs/specs/kernel.md` §3.3 makes the CLI the one exception to "degradation +// is absence". The exception is only implementable if something can still tell +// "no such controller" apart from "gated" after the filtered lookups have +// collapsed both into one absence. That something is `capability_for_parts`. + +#[test] +fn capability_for_parts_returns_none_for_an_unregistered_controller() { + assert!(capability_for_parts("nope", "nope").is_none()); + assert!(capability_for_parts("memory", "not_a_function").is_none()); +} + +#[test] +fn capability_for_parts_reports_the_registered_family_unfiltered() { + assert_eq!( + capability_for_parts("memory_tree", "list_chunks"), + Some(Some(Capability::Tree)) + ); + // Registered and deliberately ungated — distinct from "not registered". + assert_eq!( + capability_for_parts("memory", "provider_status"), + Some(None) + ); +} + +/// The lookup that makes the whole distinction possible: it must stay +/// unfiltered while the filtered lookup right beside it hides the method. +#[tokio::test] +async fn capability_for_parts_is_not_narrowed_by_the_ambient_context() { + let ctx = CoreContext::for_test( + DomainSet::full(), + Some(caps_ws("cli-cap")), + Some(null_driver_cfg()), + ); + let (unfiltered, filtered) = CoreContext::scope(ctx, async { + ( + capability_for_parts("memory_tree", "list_chunks"), + schema_for_rpc_method("openhuman.memory_tree_list_chunks"), + ) + }) + .await; + assert_eq!(unfiltered, Some(Some(Capability::Tree))); + assert!( + filtered.is_none(), + "the filtered lookup must still hide the gated method" + ); +} + +#[test] +fn sole_capability_for_namespace_reports_a_single_family_namespace() { + assert_eq!( + sole_capability_for_namespace("memory_tree"), + Some(Capability::Tree) + ); + assert_eq!( + sole_capability_for_namespace("memory_diff"), + Some(Capability::Diff) + ); +} + +#[test] +fn sole_capability_for_namespace_is_none_for_mixed_and_unknown_namespaces() { + // `memory` spans four families plus ungated host surface. + assert_eq!(sole_capability_for_namespace("memory"), None); + // `people` is registered under Memory but carries no capability. + assert_eq!(sole_capability_for_namespace("people"), None); + assert_eq!(sole_capability_for_namespace("not_a_namespace"), None); +} diff --git a/src/core/cli.rs b/src/core/cli.rs index 810842eb6c..6bd6df27c3 100644 --- a/src/core/cli.rs +++ b/src/core/cli.rs @@ -423,6 +423,15 @@ fn run_call_command(args: &[String]) -> Result<()> { let method = method.ok_or_else(|| anyhow::anyhow!("--method is required"))?; let params = parse_json_params(¶ms).map_err(anyhow::Error::msg)?; + // Raw calls bypass namespace parsing, but not the configured memory-driver + // binding. Without this gate an absent capability could still reach a + // destructive embedded handler because plain CLI invocations have no + // ambient CoreContext to filter the registry. + crate::core::cli_capability::ensure_capability_blocking( + all::capability_for_rpc_method(&method).flatten(), + &format!("openhuman call --method {method}"), + )?; + // `call` invokes a JSON-RPC method that may run an orchestrator turn // (e.g. `agent.chat`), so it needs the same roomy stack as the server. let rt = tokio::runtime::Builder::new_multi_thread() @@ -454,6 +463,20 @@ fn run_namespace_command( grouped: &BTreeMap>, ) -> Result<()> { let Some(schemas) = grouped.get(namespace) else { + // Reachable only when `grouped` really was filtered — i.e. under + // `run`/`serve`/TUI, which build a `CoreContext`. On a plain CLI + // invocation there is no ambient context, so nothing is filtered and a + // gated namespace is still present; the per-function gate below is what + // fires there. Consult the UNFILTERED registry before reporting a typo: + // silence reads as a mistyped command and sends the user off debugging + // their own command line, which is exactly what `docs/specs/kernel.md` + // §3.3 carves the CLI out of. Same reasoning as the retained `mcp` and + // `tui` arms above. A namespace that does not exist at all yields `None` + // and still reports unknown. + crate::core::cli_capability::ensure_capability_blocking( + all::sole_capability_for_namespace(namespace), + &format!("openhuman {namespace}"), + )?; return Err(anyhow::anyhow!( "unknown namespace '{namespace}'. Run `openhuman --help` to see available namespaces." )); @@ -469,6 +492,30 @@ fn run_namespace_command( } let function = args[0].as_str(); + + // Gate BEFORE resolving the schema, not in the not-found arm below. + // + // `grouped` comes from `all_controller_schemas()`, which filters through the + // ambient `CoreContext` — and no plain CLI subcommand builds one, since + // `DEFAULT_CONTEXT` is set only in `CoreContext::init` (reached by + // `run`/`serve` and the TUI). So on a real `openhuman ` invocation + // *nothing* is filtered, a gated function is still found here, and a check + // placed only in the not-found arm would never execute — the command would + // simply run. Gating the resolved function instead makes this fire on the + // path users actually take, and it stays correct under `run`/`serve` where + // `grouped` genuinely is filtered. + // + // `capability_for_parts` consults the UNFILTERED registry and yields `None` + // for a function registered nowhere, so a genuine typo short-circuits the + // gate and falls through to the unknown-function message below. Keeping the + // two distinguishable is the point: collapsing them would make real typos + // harder to diagnose, which is the failure `docs/specs/kernel.md` §3.3 + // carves the CLI out of. + crate::core::cli_capability::ensure_capability_blocking( + all::capability_for_parts(namespace, function).flatten(), + &format!("openhuman {namespace} {function}"), + )?; + let Some(schema) = schemas.iter().find(|s| s.function == function).cloned() else { return Err(anyhow::anyhow!( "unknown function '{namespace} {function}'. Run `openhuman {namespace} --help`." diff --git a/src/core/cli_capability.rs b/src/core/cli_capability.rs new file mode 100644 index 0000000000..486c20bc76 --- /dev/null +++ b/src/core/cli_capability.rs @@ -0,0 +1,219 @@ +//! The CLI's memory-capability gate — `docs/specs/kernel.md` §3.3's one named +//! exception to "degradation is absence". +//! +//! §3.3 says an unadvertised family is UNREGISTERED over `/rpc` and ABSENT from +//! the agent tool list, because a registered-but-failing method teaches a model +//! that the capability exists and makes it retry. It then names one exception: +//! +//! > The one exception is the **CLI**, which keeps its subcommand arm and +//! > reports a *build/config fact* ("memory driver `supermemory` does not +//! > support tree summarisation") — same reasoning as the retained `mcp` and +//! > `tui` CLI arms. +//! +//! A human reads silence as a typo and goes off debugging their own command +//! line. So the CLI — and ONLY the CLI — converts that absence into a sentence +//! naming the bound driver and the family it does not advertise. Nothing here +//! is reachable from `/rpc` or from an agent tool; `core::dispatch` is +//! deliberately untouched, because it is the shared HTTP path where §3.3's +//! absence rule still holds. +//! +//! ## Why this resolves the binding itself instead of asking the ambient context +//! +//! [`crate::core::all::capability_allowed`] resolves through +//! `CoreContext::current()`, and no CLI subcommand except `run`/`serve` (and the +//! TUI) ever builds a `CoreContext` — `DEFAULT_CONTEXT` is set only in +//! `CoreContext::init`. On a plain CLI invocation `current()` is `None`, the +//! gate defaults OPEN, and nothing is filtered. Asking the ambient context here +//! would therefore always answer "everything is allowed". This module resolves +//! the binding the same way `memory::ops::provider::status_from_binding` does. +//! +//! ## Redaction +//! +//! The message carries exactly two values: the driver id from +//! `[subsystems.memory] driver` and a `Capability::as_str()` constant. Neither +//! is a credential nor user content. `MemoryDriverConfig`'s `endpoint` and +//! `credential_ref` are NEVER interpolated — the same rule +//! `binding::FallbackReason` documents. The invocation string is built from +//! static namespace/function names, never from user-supplied argument values, +//! and no memory content can reach this path at all. + +use anyhow::Result; +use tinycortex_api::capabilities::{Capabilities, Capability}; + +use crate::core::subsystem::DriverClass; + +/// Stable, grep-friendly opening of the config-fact diagnostic. Shared between +/// the emit site and the tests so the two cannot drift. +pub const CAPABILITY_UNAVAILABLE_PREFIX: &str = "memory driver "; + +/// Stable, grep-friendly opening of the legacy-client diagnostic. Shared the +/// same way as [`CAPABILITY_UNAVAILABLE_PREFIX`]. +pub const LEGACY_CLIENT_UNAVAILABLE_PREFIX: &str = "memory driver "; + +/// The operator-facing sentence. +/// +/// `invocation` is a CLI form built from static strings only (e.g. +/// `"openhuman memory_tree list_chunks"`), never from user argument values. +pub fn capability_unavailable_message( + driver_id: &str, + capability: Capability, + invocation: &str, +) -> String { + format!( + "{CAPABILITY_UNAVAILABLE_PREFIX}`{driver_id}` does not advertise the `{cap}` capability, \ + so `{invocation}` is unavailable in this configuration. Run `openhuman subsystems` to \ + see the bound driver and the families it advertises, or change \ + `[subsystems.memory] driver` in your config.", + cap = capability.as_str() + ) +} + +/// The operator-facing sentence for a legacy CLI command that cannot run under +/// the bound driver. +/// +/// `invocation` follows the same static-strings-only rule as +/// [`capability_unavailable_message`]. This is a distinct diagnostic from the +/// capability one: a null or external binding does not necessarily lack the +/// family — the legacy command is unavailable because it operates on the +/// embedded engine directly, and the bound driver is not that engine. +pub fn legacy_client_unavailable_message(driver_id: &str, invocation: &str) -> String { + format!( + "{LEGACY_CLIENT_UNAVAILABLE_PREFIX}`{driver_id}` is not the embedded TinyCortex \ + driver, so `{invocation}` is unavailable: it operates on the local embedded \ + store directly, and this configuration bound a different driver. Run \ + `openhuman subsystems` to see the bound driver, or change \ + `[subsystems.memory] driver` in your config.", + ) +} + +/// The pure verdict: given what a driver advertises, is `required` available? +/// +/// `required == None` (ungated surface) is always `Ok`. Mirrors +/// `core::all::capability_allowed_in` exactly, so the CLI can never disagree +/// with the RPC registry about what is gated. +pub fn capability_verdict( + driver_id: &str, + advertised: Capabilities, + required: Option, + invocation: &str, +) -> Result<()> { + let Some(capability) = required else { + return Ok(()); + }; + if advertised.contains(capability) { + return Ok(()); + } + log::warn!( + "[cli][capability-gate] rejected invocation='{invocation}' driver='{driver_id}' \ + capability={} — not advertised by the bound driver", + capability.as_str() + ); + anyhow::bail!(capability_unavailable_message( + driver_id, capability, invocation + )) +} + +/// The driver bound for this machine's configured workspace: +/// `(id, class, advertised)`. +/// +/// `None` means "could not resolve" — a missing or unreadable config, or a +/// workspace that will not bind. **The caller then skips the gate entirely**, +/// matching [`crate::core::all::capability_allowed`]'s default-OPEN posture: +/// denying is only ever correct after a driver has actually answered +/// `capabilities()`. A CLI that refused commands because it could not read +/// config would be strictly worse than one that lets the command run and fail +/// on its own terms. +pub async fn bound_memory_driver() -> Option<(String, DriverClass, Capabilities)> { + let config = match crate::openhuman::config::Config::load_or_init().await { + Ok(config) => config, + Err(err) => { + log::debug!("[cli][capability-gate] config unresolved ({err}); gate defaults OPEN"); + return None; + } + }; + bound_memory_driver_for(&config.workspace_dir, &config.subsystems.memory) +} + +/// [`bound_memory_driver`] against an already-loaded config. +/// +/// The **single** place in the CLI layer that resolves a `MemoryBinding`, so the +/// memory-guard bypass ratchet +/// (`memory::bypass_allowlist_tests`) carries one allowlisted line rather than +/// one per CLI entry point. Callers that already hold a `Config` — the +/// `openhuman memory` adapter does — use this instead of loading it twice. +/// +/// Nothing here touches memory *data*: only the driver id, its class, and the +/// advertised capability set — exactly what `memory.provider_status` already +/// reports over RPC. +pub fn bound_memory_driver_for( + workspace_dir: &std::path::Path, + cfg: &crate::openhuman::config::schema::MemorySubsystemConfig, +) -> Option<(String, DriverClass, Capabilities)> { + match crate::openhuman::memory::binding::for_workspace(workspace_dir, cfg) { + Ok(binding) => { + log::debug!( + "[cli][capability-gate] bound driver='{}' class={} capabilities=[{}]", + binding.driver_id(), + binding.class(), + binding + .capabilities() + .iter() + .map(|c| c.as_str()) + .collect::>() + .join(",") + ); + Some(( + binding.driver_id().to_string(), + binding.class(), + binding.capabilities(), + )) + } + Err(err) => { + log::debug!("[cli][capability-gate] bind unresolved ({err}); gate defaults OPEN"); + None + } + } +} + +/// Blocking gate for the synchronous CLI call sites. +/// +/// `required == None` short-circuits before any runtime is built or any config +/// is read, so a working command pays nothing: every call site reaches this +/// only on the failure path, and an ungated surface never gets that far. +/// +/// A current-thread runtime is enough — resolving a binding touches config and +/// the driver's constructor, never an orchestrator turn. +pub fn ensure_capability_blocking(required: Option, invocation: &str) -> Result<()> { + let Some(required) = required else { + return Ok(()); + }; + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build()?; + let Some((driver_id, _class, advertised)) = rt.block_on(bound_memory_driver()) else { + return Ok(()); + }; + capability_verdict(&driver_id, advertised, Some(required), invocation) +} + +/// The pure verdict for the legacy-client gate: does the bound driver class +/// permit commands that operate on the embedded store directly? +/// +/// Mirrors [`capability_verdict`]'s default-OPEN posture — `None` from the +/// caller means "no legacy gate applies", and an unresolvable binding has +/// already been defaulted-OPEN upstream by the caller skipping this entirely. +pub fn legacy_client_verdict(driver_id: &str, class: DriverClass, invocation: &str) -> Result<()> { + if class == DriverClass::Embedded { + return Ok(()); + } + log::warn!( + "[cli][legacy-client-gate] rejected invocation='{invocation}' driver='{driver_id}' \ + class={} — not the embedded engine", + class.as_str() + ); + anyhow::bail!(legacy_client_unavailable_message(driver_id, invocation)) +} + +#[cfg(test)] +#[path = "cli_capability_tests.rs"] +mod tests; diff --git a/src/core/cli_capability_tests.rs b/src/core/cli_capability_tests.rs new file mode 100644 index 0000000000..da1e7f547a --- /dev/null +++ b/src/core/cli_capability_tests.rs @@ -0,0 +1,123 @@ +//! Tests for the CLI's memory-capability gate. +//! +//! Everything here drives the PURE helpers (`capability_verdict`, +//! `capability_unavailable_message`) plus a directly-resolved binding, rather +//! than `run_from_cli_args`. Reaching a narrowed capability set end-to-end +//! would need `driver = "null"` written into a real `config.toml` under a +//! process-global `OPENHUMAN_WORKSPACE`, i.e. env mutation plus disk writes, +//! and `run_from_cli_args` also loads dotenv and prints the banner. The helper +//! assertions are stronger, not weaker: they can actually reach the null-driver +//! state deterministically. + +use super::*; +use crate::openhuman::config::schema::MemorySubsystemConfig; +use crate::openhuman::memory::binding; + +fn binding_for(name: &str, cfg: MemorySubsystemConfig) -> std::sync::Arc { + let dir = std::env::temp_dir().join(format!("oh-cli-cap-{name}")); + binding::for_workspace(&dir, &cfg).expect("binding resolves") +} + +fn null_cfg() -> MemorySubsystemConfig { + MemorySubsystemConfig { + driver: "null".into(), + ..Default::default() + } +} + +#[test] +fn verdict_is_ok_for_ungated_surface() { + assert!(capability_verdict( + "null", + Capabilities::mandatory(), + None, + "openhuman memory docs" + ) + .is_ok()); +} + +#[test] +fn verdict_names_the_driver_and_the_capability() { + let err = capability_verdict( + "null", + Capabilities::mandatory(), + Some(Capability::Tree), + "openhuman memory_tree list_chunks", + ) + .expect_err("a mandatory-only driver does not advertise `tree`"); + let msg = err.to_string(); + assert!(msg.contains("null"), "{msg}"); + assert!(msg.contains("tree"), "{msg}"); + assert!(msg.contains("openhuman memory_tree list_chunks"), "{msg}"); +} + +#[test] +fn verdict_error_does_not_read_like_a_typo() { + let err = capability_verdict( + "null", + Capabilities::mandatory(), + Some(Capability::Tree), + "openhuman memory_tree list_chunks", + ) + .expect_err("gated"); + let msg = err.to_string(); + assert!(!msg.contains("unknown namespace"), "{msg}"); + assert!(!msg.contains("unknown function"), "{msg}"); + assert!(!msg.contains("unknown method"), "{msg}"); +} + +#[test] +fn verdict_is_ok_when_the_driver_advertises_the_family() { + assert!(capability_verdict( + "tinycortex", + Capabilities::all(), + Some(Capability::Tree), + "openhuman memory_tree list_chunks", + ) + .is_ok()); +} + +/// The message carries a driver id and a capability constant and nothing else — +/// never the configured endpoint or credential reference. +#[test] +fn message_never_contains_a_credential_or_endpoint() { + let msg = capability_unavailable_message( + "supermemory", + Capability::Tree, + "openhuman memory_tree list_chunks", + ); + assert!(!msg.contains("keychain:"), "{msg}"); + assert!(!msg.contains("api.supermemory.ai"), "{msg}"); + assert!(msg.starts_with(CAPABILITY_UNAVAILABLE_PREFIX), "{msg}"); +} + +#[test] +fn bound_driver_probe_reports_the_default_embedded_driver() { + let cfg = MemorySubsystemConfig::default(); + let binding = binding_for("default", cfg.clone()); + assert_eq!(binding.driver_id(), cfg.driver); + assert_eq!(binding.capabilities(), Capabilities::all()); +} + +/// The negative control that makes the assertions above mean something. +#[test] +fn null_driver_probe_advertises_only_the_mandatory_families() { + let binding = binding_for("null", null_cfg()); + assert_eq!(binding.driver_id(), "null"); + assert!(!binding.capabilities().contains(Capability::Tree)); + assert!(!binding.capabilities().contains(Capability::Ingest)); +} + +/// A genuine typo must never become a capability error: the gate is skipped +/// entirely when no such controller is registered. +#[test] +fn ensure_capability_blocking_is_a_noop_for_an_unknown_controller() { + assert_eq!( + crate::core::all::capability_for_parts("does_not_exist", "nope"), + None + ); + assert!( + ensure_capability_blocking(None, "openhuman does_not_exist nope").is_ok(), + "an unregistered controller must not be reported as a capability fact" + ); +} diff --git a/src/core/cli_tests.rs b/src/core/cli_tests.rs index b8424fef47..565535989a 100644 --- a/src/core/cli_tests.rs +++ b/src/core/cli_tests.rs @@ -317,3 +317,173 @@ fn chat_alias_reports_disabled_build_when_gate_off() { "the `chat` alias must give the same build-fact diagnostic as `tui`" ); } + +// --- the capability gate on the generic namespace path ----------------------- +// +// Driven through the pure helpers plus a directly-resolved capability set, +// rather than `run_from_cli_args`: reaching a narrowed set end-to-end needs +// `driver = "null"` in a real `config.toml` under a process-global +// `OPENHUMAN_WORKSPACE`, i.e. env mutation plus disk writes. Same reasoning +// recorded in the M5.4 block of `all_tests.rs`. + +use crate::core::all::{ + capability_for_parts, capability_for_rpc_method, sole_capability_for_namespace, +}; +use crate::core::cli_capability::capability_verdict; +use tinycortex_api::capabilities::Capabilities; + +#[test] +fn capability_gated_namespace_reports_a_config_fact_not_a_typo() { + let required = sole_capability_for_namespace("memory_tree"); + assert!(required.is_some(), "memory_tree must be a gated namespace"); + let err = capability_verdict( + "null", + Capabilities::mandatory(), + required, + "openhuman memory_tree", + ) + .expect_err("the null driver does not advertise `tree`"); + let msg = err.to_string(); + assert!(msg.contains("null"), "{msg}"); + assert!(msg.contains("tree"), "{msg}"); + assert!(!msg.contains("unknown namespace"), "{msg}"); +} + +#[test] +fn capability_gated_function_reports_a_config_fact_not_a_typo() { + let required = capability_for_parts("memory", "doc_ingest").flatten(); + let err = capability_verdict( + "null", + Capabilities::mandatory(), + required, + "openhuman memory doc_ingest", + ) + .expect_err("the null driver does not advertise `ingest`"); + let msg = err.to_string(); + assert!(msg.contains("ingest"), "{msg}"); + assert!(!msg.contains("unknown function"), "{msg}"); +} + +#[test] +fn capability_gated_rpc_method_reports_its_family_unfiltered() { + assert_eq!( + capability_for_rpc_method("openhuman.memory_tree_wipe_all"), + Some(Some(tinycortex_api::capabilities::Capability::Tree)) + ); +} + +/// A real typo must stay a typo — the gate never fires for it, because the +/// unfiltered lookup finds no controller to name a family for. +#[test] +fn unknown_namespace_still_reports_unknown_namespace() { + let err = super::run_namespace_command( + "definitely_not_a_namespace", + &["x".to_string()], + &grouped_schemas(), + ) + .expect_err("an unknown namespace must error"); + assert!(err.to_string().contains("unknown namespace"), "{err}"); +} + +#[test] +fn unknown_function_in_a_live_namespace_still_reports_unknown_function() { + let grouped = grouped_schemas(); + let namespace = grouped + .keys() + .next() + .expect("at least one namespace is registered") + .clone(); + let err = super::run_namespace_command( + &namespace, + &["definitely_not_a_function".to_string()], + &grouped, + ) + .expect_err("an unknown function must error"); + assert!(err.to_string().contains("unknown function"), "{err}"); +} + +/// With no ambient context nothing is filtered, so the CLI's namespace list is +/// exactly what it was before the gate existed. +#[test] +fn default_build_leaves_the_generic_namespace_path_unchanged() { + let grouped = grouped_schemas(); + for ns in ["memory", "memory_tree", "memory_diff", "memory_goals"] { + assert!(grouped.contains_key(ns), "`{ns}` must still be listed"); + } +} + +/// The gate must fire on the path a user actually takes. +/// +/// This drives `run_namespace_command` itself rather than the pure +/// `capability_verdict` helper, because the two disagreed once: the check +/// originally sat in the not-found arm, which is unreachable on a plain CLI +/// invocation (no ambient `CoreContext` ⇒ `grouped_schemas()` is unfiltered ⇒ +/// the gated function is still *found*). Every helper-level test passed while +/// the real command ran to completion under a driver that does not advertise +/// the family. Assert through the entry point or this regresses silently. +#[test] +fn generic_namespace_path_reports_the_config_fact_under_a_driver_without_the_family() { + let _env_lock = crate::openhuman::config::TEST_ENV_LOCK + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + let workspace = tempdir().expect("temp workspace"); + + // SAFETY: serialised by TEST_ENV_LOCK, and both vars are restored below. + std::env::set_var("OPENHUMAN_WORKSPACE", workspace.path()); + std::env::set_var("OPENHUMAN_MEMORY_DRIVER", "null"); + + let err = super::run_namespace_command( + "memory_tree", + &["list_chunks".to_string()], + &grouped_schemas(), + ) + .expect_err("`tree` is not advertised by the null driver, so this must not run"); + + std::env::remove_var("OPENHUMAN_MEMORY_DRIVER"); + std::env::remove_var("OPENHUMAN_WORKSPACE"); + + let message = err.to_string(); + assert!( + message.starts_with(crate::core::cli_capability::CAPABILITY_UNAVAILABLE_PREFIX), + "must read as a configuration fact, not an unknown-command error: {message}" + ); + assert!( + message.contains("null") && message.contains("tree"), + "must name the bound driver and the missing family: {message}" + ); + assert!( + !message.contains("unknown"), + "a gated command is not a typo and must not read like one: {message}" + ); +} + +#[test] +fn raw_call_path_rejects_a_method_the_bound_driver_does_not_advertise() { + let _env_lock = crate::openhuman::config::TEST_ENV_LOCK + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + let workspace = tempdir().expect("temp workspace"); + + // SAFETY: serialised by TEST_ENV_LOCK, and both vars are restored below. + std::env::set_var("OPENHUMAN_WORKSPACE", workspace.path()); + std::env::set_var("OPENHUMAN_MEMORY_DRIVER", "null"); + + let err = super::run_call_command(&[ + "--method".to_string(), + "openhuman.memory_tree_wipe_all".to_string(), + ]) + .expect_err("the null driver must not dispatch a tree wipe"); + + std::env::remove_var("OPENHUMAN_MEMORY_DRIVER"); + std::env::remove_var("OPENHUMAN_WORKSPACE"); + + let message = err.to_string(); + assert!( + message.starts_with(crate::core::cli_capability::CAPABILITY_UNAVAILABLE_PREFIX), + "must reject before dispatching: {message}" + ); + assert!( + message.contains("null") && message.contains("tree"), + "{message}" + ); +} diff --git a/src/core/dispatch.rs b/src/core/dispatch.rs index cfc3f91923..974ed55a02 100644 --- a/src/core/dispatch.rs +++ b/src/core/dispatch.rs @@ -467,17 +467,14 @@ mod tests { #[tokio::test] async fn dispatch_legacy_alias_routes_to_registry() { - // openhuman.get_analytics_settings should rewrite to openhuman.config_get_analytics_settings. - // This is a read-only call and should succeed if the registry is wired up. - let out = dispatch(test_state(), "openhuman.get_analytics_settings", json!({})) - .await - .expect("openhuman.get_analytics_settings should be rewritten and succeed"); - - // The registry-wrapped payload has a "result" field. + // This alias targets a controller registered in the domain registry. + // Do not invoke it here: its implementation can persist default config, + // which makes this routing test depend on a filesystem workspace. + let method = + crate::core::legacy_aliases::resolve_legacy("openhuman.get_analytics_settings"); assert!( - out.get("enabled").is_some() || out.get("result").is_some(), - "Payload should have 'enabled' or 'result', got: {}", - out + crate::core::all::schema_for_rpc_method(method).is_some(), + "legacy alias must resolve to a registered controller: {method}" ); } } diff --git a/src/core/events.rs b/src/core/events.rs index 9966fcf681..f1a940a1a2 100644 --- a/src/core/events.rs +++ b/src/core/events.rs @@ -251,6 +251,42 @@ pub enum DomainEvent { }, /// A memory recall query completed. MemoryRecalled { query: String, hit_count: usize }, + /// The configured memory driver could not be bound, and the kernel fell + /// back to the placeholder. Never silent — `docs/specs/kernel.md` §3.7. + /// + /// Carries driver *ids* and an operator-facing reason only: never an + /// endpoint, a `credential_ref`, or user memory content. See + /// `MemoryDriverConfig`'s manual redacting `Debug` impl for the same rule + /// on the config side. + MemoryDriverBindFailed { + /// The driver id asked for in `[subsystems.memory] driver`. + configured_driver: String, + /// What was bound instead (today always `"null"`). + bound_driver: String, + /// Why the configured driver was refused. + reason: String, + }, + /// The memory policy guard refused a call before it reached the bound + /// driver (`docs/specs/kernel.md` §3.4). + /// + /// Carries the driver id, the contract method, and an operator-facing + /// reason — **never** a namespace key, a recall query, or memory content. + /// Same rule as [`Self::MemoryDriverBindFailed`] above, and the reason + /// [`Self::MemoryRecalled`] (which carries the raw query) is not reused for + /// this: the guard sits on the hot path and must not put user text on the + /// bus. + /// + /// Published on refusals only. A guard that published on success would emit + /// one event per memory read. + MemoryGuardDenied { + /// The bound driver the call was headed for. + driver_id: String, + /// The contract method that was refused, e.g. `"core.store"` or + /// `"tree.query_source"`. + method: String, + /// Why the guard refused it. + reason: String, + }, /// A memory sync was requested for a specific channel or all channels. /// /// Published by `openhuman.memory_sync_channel` (channel_id = Some(...)) and @@ -1383,6 +1419,8 @@ impl DomainEvent { Self::EmbeddingModelUnhealthy { .. } | Self::MemoryStored { .. } | Self::MemoryRecalled { .. } + | Self::MemoryDriverBindFailed { .. } + | Self::MemoryGuardDenied { .. } | Self::MemorySyncRequested { .. } | Self::MemorySyncStageChanged { .. } | Self::MemoryIngestionStarted { .. } @@ -1548,6 +1586,8 @@ impl DomainEvent { Self::MonitorLine { .. } => "MonitorLine", Self::MemoryStored { .. } => "MemoryStored", Self::MemoryRecalled { .. } => "MemoryRecalled", + Self::MemoryDriverBindFailed { .. } => "MemoryDriverBindFailed", + Self::MemoryGuardDenied { .. } => "MemoryGuardDenied", Self::MemorySyncRequested { .. } => "MemorySyncRequested", Self::MemorySyncStageChanged { .. } => "MemorySyncStageChanged", Self::MemoryIngestionStarted { .. } => "MemoryIngestionStarted", diff --git a/src/core/events_tests.rs b/src/core/events_tests.rs index 334d8b78c0..f5a142e789 100644 --- a/src/core/events_tests.rs +++ b/src/core/events_tests.rs @@ -594,3 +594,14 @@ fn workflows_changed_domain_and_name() { assert_eq!(event.domain(), "workflow"); assert_eq!(event.variant_name(), "WorkflowsChanged"); } + +#[test] +fn memory_driver_bind_failed_domain_and_name() { + let event = DomainEvent::MemoryDriverBindFailed { + configured_driver: "supermemory".into(), + bound_driver: "null".into(), + reason: "external driver is untrusted".into(), + }; + assert_eq!(event.domain(), "memory"); + assert_eq!(event.variant_name(), "MemoryDriverBindFailed"); +} diff --git a/src/core/jsonrpc_tests.rs b/src/core/jsonrpc_tests.rs index 9672129ab4..fb67cc68e9 100644 --- a/src/core/jsonrpc_tests.rs +++ b/src/core/jsonrpc_tests.rs @@ -355,7 +355,7 @@ async fn gated_method_is_unknown_at_transport_even_with_malformed_params() { .expect("a flows.* method exists in the full registry"); for params in [json!({}), json!({ "obviously_not_a_real_param_xyz": true })] { - let ctx = CoreContext::for_test(DomainSet::harness(), None); + let ctx = CoreContext::for_test(DomainSet::harness(), None, None); let err = CoreContext::scope( ctx, invoke_method(default_state(), &gated_method, params.clone()), diff --git a/src/core/memory_cli.rs b/src/core/memory_cli.rs index 3ee901ad51..658c01db22 100644 --- a/src/core/memory_cli.rs +++ b/src/core/memory_cli.rs @@ -38,6 +38,38 @@ pub fn run_memory_command(args: &[String]) -> Result<()> { } } +/// Each `openhuman memory ` subcommand and the registered RPC controller +/// whose surface it duplicates. +/// +/// The CAPABILITY is deliberately NOT written here — it is read from the +/// controller registry via [`crate::core::all::capability_for_parts`], so the +/// single decision recorded at the `push_cap` site in `src/core/all.rs` governs +/// both the RPC surface and this CLI. A second hand-maintained table would +/// drift the first time a family tag moves. +const SUBCOMMAND_CONTROLLER: &[(&str, &str)] = &[ + // Full synchronous ingestion — the driver owns chunking and embedding. + ("ingest", "doc_ingest"), + // Mandatory core/recall surface: ungated, listed so the table is total. + ("docs", "list_documents"), + ("list", "list_documents"), + ("graph", "graph_query"), + ("graph-query", "graph_query"), + ("query", "query_namespace"), + ("namespaces", "list_namespaces"), + ("ns", "list_namespaces"), + ("clear", "clear_namespace"), +]; + +/// The capability `openhuman memory ` needs, if any. Resolved from the +/// controller registry, never from a local table. +fn required_capability(subcommand: &str) -> Option { + let function = SUBCOMMAND_CONTROLLER + .iter() + .find(|(sub, _)| *sub == subcommand) + .map(|(_, function)| *function)?; + crate::core::all::capability_for_parts("memory", function).flatten() +} + // --------------------------------------------------------------------------- // Subcommands // --------------------------------------------------------------------------- @@ -118,7 +150,7 @@ fn run_ingest(args: &[String]) -> Result<()> { .build()?; let result = rt.block_on(async { - let client = create_memory_client().await?; + let client = create_memory_client("ingest").await?; let document = NamespaceDocumentInput { namespace: namespace.clone(), @@ -202,7 +234,7 @@ fn run_docs(args: &[String]) -> Result<()> { .build()?; let result = rt.block_on(async { - let client = create_memory_client().await?; + let client = create_memory_client("docs").await?; client .list_documents(namespace.as_deref()) .await @@ -253,7 +285,7 @@ fn run_graph_query(args: &[String]) -> Result<()> { .build()?; let result = rt.block_on(async { - let client = create_memory_client().await?; + let client = create_memory_client("graph").await?; client .graph_query( namespace.as_deref(), @@ -315,7 +347,7 @@ fn run_query(args: &[String]) -> Result<()> { .build()?; let result = rt.block_on(async { - let client = create_memory_client().await?; + let client = create_memory_client("query").await?; client .query_namespace(&namespace, &query, limit) .await @@ -347,7 +379,7 @@ fn run_namespaces(args: &[String]) -> Result<()> { .build()?; let result = rt.block_on(async { - let client = create_memory_client().await?; + let client = create_memory_client("namespaces").await?; client.list_namespaces().await.map_err(anyhow::Error::msg) })?; @@ -390,7 +422,7 @@ fn run_clear(args: &[String]) -> Result<()> { .build()?; rt.block_on(async { - let client = create_memory_client().await?; + let client = create_memory_client("clear").await?; client .clear_namespace(&namespace) .await @@ -433,10 +465,57 @@ fn read_input(path: &str) -> Result { } } -async fn create_memory_client() -> Result { +/// Resolve the memory client for a subcommand, refusing first when the bound +/// driver cannot serve it. +/// +/// Two gates run here, both *config facts* naming the driver rather than silent +/// absence (`docs/specs/kernel.md` §3.3 makes the CLI its one exception, because +/// a human reads silence as a typo — same reasoning as the retained `mcp` / +/// `tui` arms in `src/core/cli.rs`): +/// +/// 1. **Capability gate** — when the subcommand maps to a gated controller, the +/// bound driver must advertise that family. +/// 2. **Legacy-client gate** — every subcommand below operates on the embedded +/// store directly (via `memory::global::init`), so the bound driver must be +/// the embedded engine. This is what makes `driver = "null"` (or a fallback) +/// actually disable `openhuman memory clear` / `docs` / `query` / +/// `namespaces`, which have no gated capability to refuse on. +/// +/// Both gates are default-OPEN when the binding cannot be resolved, mirroring +/// [`crate::core::all::capability_allowed`]: denying is only ever correct after +/// a driver has actually answered `capabilities()`. +/// +/// This is the single chokepoint every subcommand already funnels through, and +/// it already loads config, so the gates cost no extra config read. +async fn create_memory_client( + subcommand: &str, +) -> Result { let config = crate::openhuman::config::Config::load_or_init() .await .unwrap_or_default(); + + // Resolved through `cli_capability` rather than `binding::for_workspace` + // here, so the memory-guard bypass ratchet carries ONE allowlisted line + // for the whole CLI layer instead of one per entry point. Default-OPEN + // when the binding cannot be resolved. + let invocation = format!("openhuman memory {subcommand}"); + if let Some((driver_id, class, advertised)) = + crate::core::cli_capability::bound_memory_driver_for( + &config.workspace_dir, + &config.subsystems.memory, + ) + { + crate::core::cli_capability::legacy_client_verdict(&driver_id, class, &invocation)?; + if let Some(required) = required_capability(subcommand) { + crate::core::cli_capability::capability_verdict( + &driver_id, + advertised, + Some(required), + &invocation, + )?; + } + } + crate::openhuman::memory::global::init(config.workspace_dir).map_err(anyhow::Error::msg) } @@ -451,6 +530,9 @@ fn print_memory_help() { println!(" namespaces List all namespaces"); println!(" clear Clear all data in a namespace"); println!(); + println!("Some subcommands need capability families the bound memory driver may not"); + println!("advertise. Run `openhuman subsystems` to see what is bound."); + println!(); println!("Examples:"); println!(" openhuman memory ingest notes.md -n my-project -v"); println!(" echo 'Alice works on ProjectX' | openhuman memory ingest - -n test -v"); @@ -458,3 +540,158 @@ fn print_memory_help() { println!(" openhuman memory docs -n my-project"); println!(" openhuman memory query -n my-project -q 'who works on what?'"); } + +#[cfg(test)] +mod tests { + use super::*; + use crate::core::cli_capability::{capability_verdict, CAPABILITY_UNAVAILABLE_PREFIX}; + use crate::core::subsystem::DriverClass; + use tinycortex_api::capabilities::{Capabilities, Capability}; + + /// Drift guard: a renamed controller function must break here rather than + /// silently un-gate a subcommand (`required_capability` would start + /// returning `None` for it). + #[test] + fn memory_cli_subcommands_mirror_real_controllers() { + for (sub, function) in SUBCOMMAND_CONTROLLER { + assert!( + crate::core::all::rpc_method_from_parts("memory", function).is_some(), + "`openhuman memory {sub}` maps to memory.{function}, which is not registered" + ); + } + } + + /// Adding a subcommand without recording a capability decision fails here. + #[test] + fn every_dispatched_subcommand_is_in_the_controller_table() { + for sub in [ + "ingest", + "docs", + "list", + "graph", + "graph-query", + "query", + "namespaces", + "ns", + "clear", + ] { + assert!( + SUBCOMMAND_CONTROLLER.iter().any(|(s, _)| *s == sub), + "`openhuman memory {sub}` is dispatched but has no controller mapping" + ); + } + } + + #[test] + fn ingest_and_graph_are_the_gated_subcommands() { + assert_eq!(required_capability("ingest"), Some(Capability::Ingest)); + assert_eq!(required_capability("graph"), Some(Capability::Graph)); + assert_eq!(required_capability("graph-query"), Some(Capability::Graph)); + // Mandatory core/recall surface — a gate here could never fire. + for sub in ["docs", "list", "query", "namespaces", "ns", "clear"] { + assert_eq!(required_capability(sub), None, "{sub} must stay ungated"); + } + } + + /// A real typo must never be reported as a capability fact. + #[test] + fn unknown_memory_subcommand_still_reports_unknown_subcommand() { + let err = run_memory_command(&["not_a_subcommand".to_string()]) + .expect_err("an unknown subcommand must error"); + let msg = err.to_string(); + assert!(msg.contains("unknown memory subcommand"), "{msg}"); + assert!(!msg.contains(CAPABILITY_UNAVAILABLE_PREFIX), "{msg}"); + assert_eq!(required_capability("not_a_subcommand"), None); + } + + /// `Capabilities::mandatory()` is exactly what the `null` driver advertises; + /// the set is used directly rather than through `binding::for_workspace` so + /// this file stays off the memory-guard bypass allowlist (that scanner does + /// not strip inline `#[cfg(test)]` modules). The binding-level equivalence + /// is pinned in `cli_capability_tests.rs`. + #[test] + fn gated_subcommand_reports_the_driver_and_capability() { + let err = capability_verdict( + "null", + Capabilities::mandatory(), + required_capability("ingest"), + "openhuman memory ingest", + ) + .expect_err("the null driver does not advertise `ingest`"); + let msg = err.to_string(); + assert!(msg.contains("null"), "{msg}"); + assert!(msg.contains("ingest"), "{msg}"); + assert!(!msg.contains("unknown memory subcommand"), "{msg}"); + } + + /// The default embedded driver advertises every family, so nothing changes. + #[test] + fn default_embedded_driver_gates_nothing() { + for (sub, _) in SUBCOMMAND_CONTROLLER { + assert!( + capability_verdict( + "tinycortex", + Capabilities::all(), + required_capability(sub), + "openhuman memory ", + ) + .is_ok(), + "`openhuman memory {sub}` must stay available under the default driver" + ); + } + } + + /// Every legacy subcommand — gated or not — must be rejected under a null + /// binding: they all operate on the embedded store directly, and the null + /// driver is not that engine. This is the regression the reviewer flagged: + /// `openhuman memory clear` used to open the embedded DB even with + /// `driver = "null"`. + #[test] + fn null_driver_rejects_every_legacy_subcommand() { + for (sub, _) in SUBCOMMAND_CONTROLLER { + let err = crate::core::cli_capability::legacy_client_verdict( + "null", + DriverClass::Null, + &format!("openhuman memory {sub}"), + ) + .expect_err("a null binding must reject legacy subcommands"); + let msg = err.to_string(); + assert!(msg.contains("null"), "{msg}"); + assert!( + msg.contains("embedded"), + "refusal must explain that the legacy client is the embedded engine: {msg}" + ); + } + } + + /// The embedded driver is the only class that may serve legacy subcommands. + #[test] + fn embedded_driver_serves_every_legacy_subcommand() { + for (sub, _) in SUBCOMMAND_CONTROLLER { + assert!( + crate::core::cli_capability::legacy_client_verdict( + "tinycortex", + DriverClass::Embedded, + &format!("openhuman memory {sub}"), + ) + .is_ok(), + "`openhuman memory {sub}` must stay available under the embedded driver" + ); + } + } + + /// The legacy-client diagnostic must not leak credentials or endpoints. + #[test] + fn legacy_message_never_contains_a_credential_or_endpoint() { + let msg = crate::core::cli_capability::legacy_client_unavailable_message( + "supermemory", + "openhuman memory clear", + ); + assert!(!msg.contains("keychain:"), "{msg}"); + assert!(!msg.contains("api.supermemory.ai"), "{msg}"); + assert!( + msg.starts_with(crate::core::cli_capability::LEGACY_CLIENT_UNAVAILABLE_PREFIX), + "{msg}" + ); + } +} diff --git a/src/core/mod.rs b/src/core/mod.rs index 3f0577f3ba..f74fa67983 100644 --- a/src/core/mod.rs +++ b/src/core/mod.rs @@ -10,6 +10,7 @@ pub mod agent_cli; pub mod all; pub mod auth; pub mod cli; +pub mod cli_capability; pub mod dispatch; pub mod event_bind_tokens; pub mod bus; @@ -32,6 +33,8 @@ pub mod sentry_transport; pub mod shutdown; pub mod socketio; pub mod subconscious_cli; +pub mod subsystem; +pub mod subsystems_cli; pub mod types; /// Canonical function contract for domain controllers. diff --git a/src/core/runtime/context.rs b/src/core/runtime/context.rs index 5931498aa0..ff6cafd0e7 100644 --- a/src/core/runtime/context.rs +++ b/src/core/runtime/context.rs @@ -49,7 +49,11 @@ tokio::task_local! { /// [`CoreContext::scope`]s read isolated state — the Phase 3 exit criterion. pub struct CoreContext { host_kind: HostKind, - workspace_dir: RwLock>, + /// The workspace and its memory-driver configuration form one binding + /// input. They must be read and updated together: a caller that observes a + /// new workspace with the previous user's memory config could cache a + /// permanently incorrect memory binding for that workspace. + workspace_binding: RwLock, /// Which domain families are live for this context (#4796). The registry /// filters its controller/schema/dispatch surface by this set via /// [`CoreContext::current`] → [`CoreContext::domains`]. `full()` for the @@ -57,6 +61,18 @@ pub struct CoreContext { domains: crate::core::runtime::DomainSet, } +/// The complete input to a workspace-scoped memory binding. +/// +/// This is deliberately one value behind one lock. `MemoryBinding` caches by +/// this pair, so splitting either its read or update would let concurrent RPC +/// traffic associate a workspace with another user's driver, hooks, or trust +/// policy. The config is captured at build time so +/// [`CoreContext::memory_binding`] stays synchronous and I/O-free. +struct WorkspaceBinding { + workspace_dir: Option, + memory_subsystem: crate::openhuman::config::schema::MemorySubsystemConfig, +} + impl CoreContext { /// Run the core initialization sequence and return the context plus whether /// an operator-supplied RPC bearer exists (for the public-bind safety check @@ -145,11 +161,18 @@ impl CoreContext { // background jobs start later, from CoreRuntime::serve(), after bind // succeeds. let runtime_config = config.clone(); + let memory_subsystem = config + .as_ref() + .map(|cfg| cfg.subsystems.memory.clone()) + .unwrap_or_default(); crate::core::jsonrpc::bootstrap_core_runtime(host_kind, config, domains).await; let ctx = Arc::new(CoreContext { host_kind, - workspace_dir: RwLock::new(workspace_dir), + workspace_binding: RwLock::new(WorkspaceBinding { + workspace_dir, + memory_subsystem, + }), domains, }); @@ -174,9 +197,10 @@ impl CoreContext { /// The resolved per-user workspace directory this context is bound to. pub fn workspace_dir(&self) -> Result { - self.workspace_dir + self.workspace_binding .read() .map_err(|e| format!("workspace unavailable: context lock poisoned: {e}"))? + .workspace_dir .clone() .ok_or_else(|| { "workspace unavailable: Config::load_or_init failed during core boot; \ @@ -198,6 +222,107 @@ impl CoreContext { crate::openhuman::memory::people::store::for_workspace(&workspace_dir) } + /// The bound memory driver for this context's workspace — the memory + /// subsystem's binding seam (`docs/specs/kernel.md` §3.1). Deliberately the + /// same shape as [`CoreContext::people`]: two contexts over different + /// workspaces get isolated bindings, one context always gets the same + /// cached binding, and an active-user switch that goes through + /// [`CoreContext::rebind_default_workspace`] automatically resolves the + /// new workspace's binding — including its `[subsystems.memory]` config, + /// which the rebind carries along with the workspace dir. + /// + /// That last property is why there is **no** explicit "rebind the memory + /// driver" call at the login / logout / revalidation sites the way + /// `memory::global::init` needs one: the accessor keys on the workspace + /// dir and the subsystem config, both of which those sites already re-point. + /// + /// It also structurally supersedes `memory::global`'s + /// clear-on-failed-rebind guard. There is no shared slot that could keep + /// pointing at the previous workspace, so a failed bind for workspace B + /// cannot hand back workspace A's driver. Pinned by + /// `failed_bind_never_returns_previous_workspace_binding`. + pub fn memory_binding( + &self, + ) -> Result, String> { + let binding = self + .workspace_binding + .read() + .map_err(|e| format!("[core-context] workspace binding lock poisoned: {e}"))?; + let workspace_dir = binding.workspace_dir.clone(); + let memory_subsystem = binding.memory_subsystem.clone(); + drop(binding); + let workspace_dir = workspace_dir.ok_or_else(|| { + "workspace unavailable: Config::load_or_init failed during core boot; \ + fix config.toml or OPENHUMAN_WORKSPACE and restart" + .to_string() + })?; + crate::openhuman::memory::binding::for_workspace(&workspace_dir, &memory_subsystem) + } + + /// The bound driver's advertised capability set. Cheap (a `Copy` bitset + /// read off the cached binding), infallible, and **OPEN by default**: when + /// no workspace is bound, or the binding cannot be resolved, this returns + /// the full set. + /// + /// That default mirrors `core::all::group_allowed`, which returns `true` + /// with no ambient context. Roughly 4000 unit tests run pre-boot with no + /// bound driver; a deny-by-default here would turn every memory test red at + /// once. Denying is only ever correct once a driver has actually answered + /// `capabilities()`. + /// + /// One case answers **closed**: a deliberate `[subsystems.memory] driver = + /// "null"` returns the empty set, not the null driver's mandatory three. + /// The driver honestly advertises those three — `subsystems_status` still + /// reports them — but an operator who bound `/dev/null` asked for the whole + /// memory surface to be gone, and leaving the mandatory families registered + /// would keep `memory_store` / `memory_recall` / `memory.list_documents` + /// answering off the embedded store the guarded re-point has not yet + /// covered. See [`MemoryBinding::disables_memory`](crate::openhuman::memory::binding::MemoryBinding::disables_memory). + pub fn memory_capabilities(&self) -> tinycortex_api::capabilities::Capabilities { + self.memory_binding() + .map(|binding| { + if binding.disables_memory() { + tinycortex_api::capabilities::Capabilities::default() + } else { + binding.capabilities() + } + }) + .unwrap_or_else(|_| crate::openhuman::memory::binding::unbound_default_capabilities()) + } + + /// The **guarded** memory driver for this context's workspace — the handle + /// product code should hold (`docs/specs/kernel.md` §3.4). + /// + /// The guard implements the same `MemoryProvider` contract as the driver it + /// wraps, so it is a drop-in for a caller that already speaks the contract, + /// and its family accessors hand back guarded handles rather than the raw + /// driver's — which is what makes the policy unskippable for anyone holding + /// it. + /// + /// [`Self::memory_binding`] still exists and still exposes the bare + /// provider. That is deliberate and narrow: the one production caller is + /// the health probe in `memory::ops::provider`, and a liveness probe is not + /// product code — routing it through the guard would let an autonomy tier + /// break status output. New call sites use this accessor. + /// + /// # Errors + /// + /// As [`Self::memory_binding`]: only when the workspace dir cannot be + /// resolved or the binding cache lock is poisoned. + pub fn memory(&self) -> Result, String> { + Ok(self.memory_binding()?.guard()) + } + + /// The capability set for the current dispatch, or the open default when + /// there is no context at all. This is the direct analogue of + /// `core::all::group_allowed` and is the function a future capability + /// registration filter calls. + pub fn current_memory_capabilities() -> tinycortex_api::capabilities::Capabilities { + Self::current() + .map(|ctx| ctx.memory_capabilities()) + .unwrap_or_else(crate::openhuman::memory::binding::unbound_default_capabilities) + } + /// The context for the current dispatch: the one scoped by /// [`CoreContext::scope`] if inside a scope, else the process /// [`DEFAULT_CONTEXT`]. Returns `None` only before any context is built @@ -218,11 +343,18 @@ impl CoreContext { } /// Rebind the process default context to the current active user's - /// workspace. Desktop login and pending-session revalidation can switch the - /// active workspace after boot without rebuilding the core. Scoped - /// multi-tenant dispatch is unaffected because tenant contexts are passed to - /// [`CoreContext::scope`] explicitly and are not the process default. - pub fn rebind_default_workspace_dir(workspace_dir: &std::path::Path) -> Result<(), String> { + /// workspace **and** that user's `[subsystems.memory]` config. Desktop + /// login, logout, and pending-session revalidation can switch the active + /// workspace after boot without rebuilding the core; every call site + /// already holds the target `Config`, so passing the config here keeps the + /// bound driver (and its hooks / trust settings) from silently carrying + /// over from the previous user. Scoped multi-tenant dispatch is unaffected + /// because tenant contexts are passed to [`CoreContext::scope`] explicitly + /// and are not the process default. + pub fn rebind_default_workspace( + workspace_dir: &std::path::Path, + memory_subsystem: crate::openhuman::config::schema::MemorySubsystemConfig, + ) -> Result<(), String> { let Some(ctx) = DEFAULT_CONTEXT.get() else { log::debug!( "[core-context] default context not initialized; skipped workspace rebind to {}", @@ -230,26 +362,36 @@ impl CoreContext { ); return Ok(()); }; - ctx.rebind_workspace_dir(workspace_dir) + ctx.rebind_workspace(workspace_dir, memory_subsystem) } - fn rebind_workspace_dir(&self, workspace_dir: &std::path::Path) -> Result<(), String> { - let mut guard = self - .workspace_dir + fn rebind_workspace( + &self, + workspace_dir: &std::path::Path, + memory_subsystem: crate::openhuman::config::schema::MemorySubsystemConfig, + ) -> Result<(), String> { + let mut binding = self + .workspace_binding .write() - .map_err(|e| format!("workspace rebind failed: context lock poisoned: {e}"))?; - if guard.as_deref() == Some(workspace_dir) { + .map_err(|e| format!("workspace rebind failed: binding lock poisoned: {e}"))?; + if binding.workspace_dir.as_deref() == Some(workspace_dir) + && binding.memory_subsystem == memory_subsystem + { log::debug!( - "[core-context] workspace already bound to {}", + "[core-context] workspace {} already bound with the current subsystem config", workspace_dir.display() ); return Ok(()); } log::info!( - "[core-context] rebound default workspace to {}", - workspace_dir.display() + "[core-context] rebound default workspace to {} with memory subsystem driver='{}'", + workspace_dir.display(), + memory_subsystem.driver ); - *guard = Some(workspace_dir.to_path_buf()); + *binding = WorkspaceBinding { + workspace_dir: Some(workspace_dir.to_path_buf()), + memory_subsystem, + }; Ok(()) } @@ -265,14 +407,26 @@ impl CoreContext { /// cross-module tests (e.g. `core::all`'s registry filter) can exercise the /// ambient DomainSet gate without going through the full [`CoreContext::init`] /// boot sequence. + /// + /// `memory_subsystem` is the seam the capability tests need: pass `None` + /// for the default (`driver = "tinycortex"`, no driver table), or an + /// explicit config to exercise the fallback / trust paths without a boot. + /// It takes the *config* rather than a `Capabilities` value on purpose — + /// injecting a capability set directly would let a test assert a set no + /// driver could have advertised, bypassing the very `admit` + + /// `capabilities()` path that has to be proven. #[cfg(test)] pub(crate) fn for_test( domains: crate::core::runtime::DomainSet, workspace_dir: Option, + memory_subsystem: Option, ) -> Arc { Arc::new(CoreContext { host_kind: HostKind::Cli, - workspace_dir: RwLock::new(workspace_dir), + workspace_binding: RwLock::new(WorkspaceBinding { + workspace_dir, + memory_subsystem: memory_subsystem.unwrap_or_default(), + }), domains, }) } @@ -360,8 +514,33 @@ pub async fn init_stores( ), Err(e) => log::warn!("[boot] memory::global init failed: {e}"), } + // Bind the memory driver for this workspace (kernel.md §3.1), on the + // same `plan.memory` gate as the store above — the binding is part of + // the memory domain's init, not a separate gate. Warmed here rather + // than lazily so a bad `[subsystems.memory]` is loud at boot instead of + // at the first recall. Infallible by design: an inadmissible driver + // falls back, publishes `MemoryDriverBindFailed`, and records why. + match crate::openhuman::memory::binding::for_workspace( + &cfg.workspace_dir, + &cfg.subsystems.memory, + ) { + Ok(binding) => log::info!( + "[boot] memory driver bound: id={} class={} capabilities=[{}] fallback={:?}", + binding.driver_id(), + binding.class(), + binding + .capabilities() + .iter() + .map(|c| c.as_str()) + .collect::>() + .join(","), + binding.fallback().map(|f| f.reason.as_str()), + ), + Err(e) => log::warn!("[boot] memory driver bind failed: {e}"), + } } else { log::debug!("[boot] memory::global init SKIPPED — Memory domain disabled"); + log::debug!("[boot] memory driver bind SKIPPED — Memory domain disabled"); } // Install the on-disk image-attachment sidecar dir so inbound // image markers persist under /attachments/ instead @@ -433,7 +612,10 @@ mod tests { fn ctx(dir: &str) -> Arc { Arc::new(CoreContext { host_kind: HostKind::Cli, - workspace_dir: RwLock::new(Some(PathBuf::from(dir))), + workspace_binding: RwLock::new(WorkspaceBinding { + workspace_dir: Some(PathBuf::from(dir)), + memory_subsystem: Default::default(), + }), domains: crate::core::runtime::DomainSet::full(), }) } @@ -518,7 +700,7 @@ mod tests { // The ambient `current().domains()` must reflect the scoped context's // DomainSet — this is the seam the registry filter reads (#4796). let harness = crate::core::runtime::DomainSet::harness(); - let ctx = CoreContext::for_test(harness, Some(PathBuf::from("/tmp/ctx-domains"))); + let ctx = CoreContext::for_test(harness, Some(PathBuf::from("/tmp/ctx-domains")), None); let seen = CoreContext::scope(ctx, async { CoreContext::current().map(|c| c.domains()) }).await; assert_eq!(seen, Some(harness)); @@ -555,12 +737,18 @@ mod tests { let dir_b = tempfile::tempdir().unwrap(); let a = Arc::new(CoreContext { host_kind: HostKind::Cli, - workspace_dir: RwLock::new(Some(dir_a.path().to_path_buf())), + workspace_binding: RwLock::new(WorkspaceBinding { + workspace_dir: Some(dir_a.path().to_path_buf()), + memory_subsystem: Default::default(), + }), domains: crate::core::runtime::DomainSet::full(), }); let b = Arc::new(CoreContext { host_kind: HostKind::Cli, - workspace_dir: RwLock::new(Some(dir_b.path().to_path_buf())), + workspace_binding: RwLock::new(WorkspaceBinding { + workspace_dir: Some(dir_b.path().to_path_buf()), + memory_subsystem: Default::default(), + }), domains: crate::core::runtime::DomainSet::full(), }); @@ -580,12 +768,15 @@ mod tests { let dir_b = tempfile::tempdir().unwrap(); let ctx = CoreContext { host_kind: HostKind::Cli, - workspace_dir: RwLock::new(Some(dir_a.path().to_path_buf())), + workspace_binding: RwLock::new(WorkspaceBinding { + workspace_dir: Some(dir_a.path().to_path_buf()), + memory_subsystem: Default::default(), + }), domains: crate::core::runtime::DomainSet::full(), }; let store_a = ctx.people().expect("open people store for workspace A"); - ctx.rebind_workspace_dir(dir_b.path()) + ctx.rebind_workspace(dir_b.path(), Default::default()) .expect("rebind context workspace"); assert_eq!(ctx.workspace_dir().unwrap(), dir_b.path()); @@ -601,12 +792,18 @@ mod tests { let dir_b = tempfile::tempdir().unwrap(); let a = Arc::new(CoreContext { host_kind: HostKind::Cli, - workspace_dir: RwLock::new(Some(dir_a.path().to_path_buf())), + workspace_binding: RwLock::new(WorkspaceBinding { + workspace_dir: Some(dir_a.path().to_path_buf()), + memory_subsystem: Default::default(), + }), domains: crate::core::runtime::DomainSet::full(), }); let b = Arc::new(CoreContext { host_kind: HostKind::Cli, - workspace_dir: RwLock::new(Some(dir_b.path().to_path_buf())), + workspace_binding: RwLock::new(WorkspaceBinding { + workspace_dir: Some(dir_b.path().to_path_buf()), + memory_subsystem: Default::default(), + }), domains: crate::core::runtime::DomainSet::full(), }); @@ -653,7 +850,10 @@ mod tests { fn degraded_context_rejects_workspace_bound_stores() { let ctx = CoreContext { host_kind: HostKind::Cli, - workspace_dir: RwLock::new(None), + workspace_binding: RwLock::new(WorkspaceBinding { + workspace_dir: None, + memory_subsystem: Default::default(), + }), domains: crate::core::runtime::DomainSet::full(), }; @@ -666,4 +866,205 @@ mod tests { "unexpected error: {err}" ); } + + // ---- memory driver binding (M2b) ---------------------------------------- + + fn untrusted_external_memory_cfg() -> crate::openhuman::config::schema::MemorySubsystemConfig { + use crate::openhuman::config::schema::{MemoryDriverConfig, MemorySubsystemConfig}; + let mut cfg = MemorySubsystemConfig { + driver: "supermemory".into(), + ..Default::default() + }; + cfg.drivers.insert( + "supermemory".into(), + MemoryDriverConfig { + class: Some("external".into()), + ..Default::default() + }, + ); + cfg + } + + /// Same proof as `people_store_is_isolated_per_context_workspace`, one layer + /// up: the memory binding is per-workspace, not per-process. + #[test] + fn memory_binding_is_isolated_per_context_workspace() { + let dir_a = tempfile::tempdir().unwrap(); + let dir_b = tempfile::tempdir().unwrap(); + let a = Arc::new(CoreContext { + host_kind: HostKind::Cli, + workspace_binding: RwLock::new(WorkspaceBinding { + workspace_dir: Some(dir_a.path().to_path_buf()), + memory_subsystem: Default::default(), + }), + domains: crate::core::runtime::DomainSet::full(), + }); + let b = Arc::new(CoreContext { + host_kind: HostKind::Cli, + workspace_binding: RwLock::new(WorkspaceBinding { + workspace_dir: Some(dir_b.path().to_path_buf()), + memory_subsystem: Default::default(), + }), + domains: crate::core::runtime::DomainSet::full(), + }); + + let bind_a = a.memory_binding().expect("bind workspace A"); + let bind_b = b.memory_binding().expect("bind workspace B"); + assert!(!Arc::ptr_eq(&bind_a, &bind_b)); + + let bind_a_again = a.memory_binding().expect("re-resolve workspace A"); + assert!(Arc::ptr_eq(&bind_a, &bind_a_again)); + } + + /// The per-workspace rebinding requirement, proven without any explicit + /// "rebind memory" call: switching the active user re-points + /// `workspace_dir`, and the accessor keys on that. + #[test] + fn rebind_workspace_updates_context_memory_binding() { + let dir_a = tempfile::tempdir().unwrap(); + let dir_b = tempfile::tempdir().unwrap(); + let ctx = CoreContext { + host_kind: HostKind::Cli, + workspace_binding: RwLock::new(WorkspaceBinding { + workspace_dir: Some(dir_a.path().to_path_buf()), + memory_subsystem: Default::default(), + }), + domains: crate::core::runtime::DomainSet::full(), + }; + + let bind_a = ctx.memory_binding().expect("bind workspace A"); + ctx.rebind_workspace(dir_b.path(), Default::default()) + .expect("rebind context workspace"); + + assert_eq!(ctx.workspace_dir().unwrap(), dir_b.path()); + let bind_b = ctx.memory_binding().expect("bind workspace B"); + assert!(!Arc::ptr_eq(&bind_a, &bind_b)); + } + + /// The subsystem-config refresh half of the rebind requirement: a rebind + /// that passes a `[subsystems.memory] driver = "null"` config must make the + /// accessor report the null driver, not the default embedded one captured + /// before the user switch. + #[test] + fn rebind_workspace_refreshes_memory_subsystem_config() { + let dir_a = tempfile::tempdir().unwrap(); + let ctx = CoreContext { + host_kind: HostKind::Cli, + workspace_binding: RwLock::new(WorkspaceBinding { + workspace_dir: Some(dir_a.path().to_path_buf()), + memory_subsystem: Default::default(), + }), + domains: crate::core::runtime::DomainSet::full(), + }; + + let bind_a = ctx.memory_binding().expect("bind workspace A"); + assert_eq!( + bind_a.class(), + crate::core::subsystem::DriverClass::Embedded + ); + + let null_cfg = crate::openhuman::config::schema::MemorySubsystemConfig { + driver: "null".to_string(), + ..Default::default() + }; + // This is the dangerous case: changing only the memory config for an + // already-bound workspace must replace the complete snapshot, so the + // binding cache sees the new (workspace, config) pair. + ctx.rebind_workspace(dir_a.path(), null_cfg) + .expect("rebind context subsystem config"); + + let bind_b = ctx.memory_binding().expect("bind workspace B"); + assert_eq!(bind_b.class(), crate::core::subsystem::DriverClass::Null); + } + + /// `memory::global`'s clear-on-failed-rebind property, preserved + /// structurally: a workspace whose configured driver is refused resolves to + /// the fallback, never to another workspace's driver. + #[test] + fn failed_bind_never_returns_previous_workspace_binding() { + let dir_a = tempfile::tempdir().unwrap(); + let dir_b = tempfile::tempdir().unwrap(); + let a = CoreContext { + host_kind: HostKind::Cli, + workspace_binding: RwLock::new(WorkspaceBinding { + workspace_dir: Some(dir_a.path().to_path_buf()), + memory_subsystem: Default::default(), + }), + domains: crate::core::runtime::DomainSet::full(), + }; + let b = CoreContext { + host_kind: HostKind::Cli, + workspace_binding: RwLock::new(WorkspaceBinding { + workspace_dir: Some(dir_b.path().to_path_buf()), + memory_subsystem: untrusted_external_memory_cfg(), + }), + domains: crate::core::runtime::DomainSet::full(), + }; + + let bind_a = a.memory_binding().expect("bind workspace A"); + assert_eq!(bind_a.driver_id(), "tinycortex"); + assert!(bind_a.fallback().is_none()); + + let bind_b = b.memory_binding().expect("workspace B falls back"); + assert_eq!( + bind_b.driver_id(), + "null", + "a refused driver must fall back, not inherit another workspace's" + ); + let fallback = bind_b.fallback().expect("fallback provenance recorded"); + assert_eq!(fallback.configured_driver, "supermemory"); + assert!(!Arc::ptr_eq(&bind_a, &bind_b)); + } + + /// The single most important default in this step: no binding ⇒ the FULL + /// capability set, mirroring `core::all::group_allowed` with no context. + #[test] + fn memory_capabilities_defaults_open_without_a_workspace() { + let ctx = CoreContext { + host_kind: HostKind::Cli, + workspace_binding: RwLock::new(WorkspaceBinding { + workspace_dir: None, + memory_subsystem: Default::default(), + }), + domains: crate::core::runtime::DomainSet::full(), + }; + assert!(ctx.memory_binding().is_err(), "no workspace ⇒ no binding"); + assert_eq!( + ctx.memory_capabilities(), + tinycortex_api::capabilities::Capabilities::all(), + "a context with no binding must not deny any capability" + ); + } + + /// The no-context arm of `current_memory_capabilities`. Asserted through + /// the value the fallback branch yields rather than by calling it with an + /// empty `DEFAULT_CONTEXT`: that global is process-wide and another test in + /// the same binary may have set it, which would make a bare + /// `assert_eq!(current_memory_capabilities(), all())` order-dependently + /// flaky. + #[test] + fn current_memory_capabilities_defaults_open_without_a_context() { + assert_eq!( + crate::openhuman::memory::binding::unbound_default_capabilities(), + tinycortex_api::capabilities::Capabilities::all() + ); + // And when a context *is* ambient, the call resolves through it rather + // than erroring. + let ctx = CoreContext::for_test(crate::core::runtime::DomainSet::full(), None, None); + assert_eq!( + ctx.memory_capabilities(), + tinycortex_api::capabilities::Capabilities::all() + ); + } + + /// The DomainSet axis and the capability axis are independent (kernel.md + /// §3.7's three axes): a narrowed `DomainSet` must not narrow capabilities. + #[test] + fn capabilities_are_open_under_a_harness_domain_set() { + let ctx = CoreContext::for_test(crate::core::runtime::DomainSet::harness(), None, None); + assert_eq!( + ctx.memory_capabilities(), + tinycortex_api::capabilities::Capabilities::all() + ); + } } diff --git a/src/core/subsystem/driver.rs b/src/core/subsystem/driver.rs new file mode 100644 index 0000000000..771beabebb --- /dev/null +++ b/src/core/subsystem/driver.rs @@ -0,0 +1,291 @@ +//! Generic, subsystem-agnostic driver vocabulary for the OpenHuman kernel. +//! +//! Specified by `docs/specs/kernel.md` §3.1 (driver classes), §3.3 (degradation +//! by absence), §3.7 (fallback is never silent), and §6 item 1. +//! +//! ## Why this is not imported from `tinycortex-api` +//! +//! [`DriverClass`], [`DriverHealth`], and [`DriverCapabilities`] are shared by +//! every subsystem — memory today, inference / channels / sandbox next +//! (kernel.md §5). A *memory* crate must not be the source of generic kernel +//! vocabulary, and a third-party driver must be able to depend on the contract +//! crate without pulling in the host. The contract crate states both halves of +//! that rule itself (`vendor/tinycortex/api/src/lib.rs`, module docs of +//! `vendor/tinycortex/api/src/health.rs`). +//! +//! So the contract carries `MemoryHealth` / `Capabilities`, this module carries +//! the kernel's equivalents, and the **memory adapter converts at the +//! boundary** — never this module. Nothing in this file names `tinycortex_api`; +//! the only place the two vocabularies meet in this step is a test-only witness +//! that pins the shapes one-for-one so the future conversion stays a total +//! three-arm `match`. +//! +//! Driver *class* is a host configuration fact about how a driver was bound, +//! not something a driver reports about itself — which is why the contract +//! crate deliberately omits it. A driver that self-reported `Embedded` could +//! skip the egress and trust checks that class gates. + +use std::collections::BTreeSet; + +use serde::{Deserialize, Serialize}; + +/// How a driver was bound into a subsystem slot (kernel.md §3.1). +/// +/// This is a **host configuration fact**, never something the driver reports. +/// +/// Deliberately not `#[non_exhaustive]`, for the same reason +/// `tinycortex_api::capabilities::Capability` is not: adding a class must break +/// every exhaustive `match` in the host, because those matches are where policy +/// (egress, trust, credential resolution) is decided per class. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum DriverClass { + /// An in-tree / vendored Rust crate. The default: no network, no extra + /// process. + Embedded, + /// An out-of-process backend reached through a transport adapter over a + /// documented wire contract. + External, + /// A stub advertising zero capabilities — what a compiled-out or + /// unconfigured subsystem binds to. + Null, +} + +impl DriverClass { + /// Every class, in declaration order. + pub const ALL: [DriverClass; 3] = [ + DriverClass::Embedded, + DriverClass::External, + DriverClass::Null, + ]; + + /// Stable snake_case identifier used in config, on the wire, and in logs. + /// + /// These are exactly the values documented for + /// `MemoryDriverConfig::class` in + /// `src/openhuman/config/schema/subsystems.rs`. The serde derive is pinned + /// against this function by a test. + pub fn as_str(self) -> &'static str { + match self { + Self::Embedded => "embedded", + Self::External => "external", + Self::Null => "null", + } + } + + /// Parse back from the config / wire form. + /// + /// # Errors + /// + /// Returns the unrecognised input in the message, so a typo in + /// `[subsystems..drivers.] class = …` is self-explaining. + pub fn parse(raw: &str) -> Result { + Self::ALL + .iter() + .copied() + .find(|class| class.as_str() == raw) + .ok_or_else(|| format!("unknown driver class: {raw}")) + } +} + +impl std::fmt::Display for DriverClass { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.as_str()) + } +} + +impl std::str::FromStr for DriverClass { + type Err = String; + + fn from_str(raw: &str) -> Result { + Self::parse(raw) + } +} + +/// Liveness of a bound driver, in the kernel's generic vocabulary. +/// +/// Shaped one-for-one against `tinycortex_api::health::MemoryHealth` — and +/// against whatever the next subsystem's contract carries — so the boundary +/// conversion is a total three-arm `match` that cannot drift. Serializes as an +/// internally-tagged object with a stable snake_case `status` discriminant: +/// +/// ```json +/// { "status": "ready" } +/// { "status": "degraded", "reason": "vector index rebuilding" } +/// { "status": "down", "reason": "connection refused" } +/// ``` +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(tag = "status", rename_all = "snake_case")] +pub enum DriverHealth { + /// Reachable and serving requests normally. + Ready, + /// Still serving, but something is wrong and the caller should surface it. + Degraded { + /// Operator-facing explanation. Must not contain credentials, tokens, + /// or user content — this string is logged and shown in status output. + reason: String, + }, + /// Cannot serve requests at all. + Down { + /// Operator-facing explanation, subject to the same redaction rule as + /// [`DriverHealth::Degraded::reason`]. + reason: String, + }, +} + +impl DriverHealth { + /// Convenience constructor for [`DriverHealth::Degraded`]. + pub fn degraded(reason: impl Into) -> Self { + Self::Degraded { + reason: reason.into(), + } + } + + /// Convenience constructor for [`DriverHealth::Down`]. + pub fn down(reason: impl Into) -> Self { + Self::Down { + reason: reason.into(), + } + } + + /// Stable snake_case discriminant, matching the serialized `status` field. + pub fn as_str(&self) -> &'static str { + match self { + Self::Ready => "ready", + Self::Degraded { .. } => "degraded", + Self::Down { .. } => "down", + } + } + + /// The operator-facing reason, when there is one. + pub fn reason(&self) -> Option<&str> { + match self { + Self::Ready => None, + Self::Degraded { reason } | Self::Down { reason } => Some(reason.as_str()), + } + } + + /// Whether the kernel should route traffic to this driver. A degraded + /// driver is still the bound driver. + pub fn is_usable(&self) -> bool { + !matches!(self, Self::Down { .. }) + } +} + +impl std::fmt::Display for DriverHealth { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self.reason() { + Some(reason) => write!(f, "{}: {reason}", self.as_str()), + None => f.write_str(self.as_str()), + } + } +} + +/// A bound driver's advertised capability set, in the kernel's generic +/// vocabulary: an unordered set of **opaque** snake_case strings. +/// +/// The kernel deliberately does not know any subsystem's family vocabulary — +/// `"tree"` and `"tool_memory"` mean something to the memory subsystem and +/// nothing here. Each subsystem's adapter converts its own typed set (for +/// memory: `tinycortex_api::capabilities::Capabilities`) into this at bind +/// time, and the kernel only ever asks "does the bound driver advertise this +/// string?" when deciding whether to register a controller or emit a tool. +/// +/// Serializes as a JSON array of strings, matching the contract crate's +/// `Capabilities` wire form exactly, so both sides are interchangeable on the +/// wire. +/// +/// **Ordering note:** the backing `BTreeSet` iterates lexicographically, +/// whereas `Capabilities::iter()` yields contract *declaration* order. A set +/// has no order, so this is correct — but it does mean a future +/// `subsystems_status` lists capabilities alphabetically rather than in +/// contract order. That is intended, not a regression. +#[derive(Clone, Debug, Default, Eq, PartialEq)] +pub struct DriverCapabilities { + families: BTreeSet, +} + +impl DriverCapabilities { + /// The empty set. Advertised by a `null` driver. + pub fn empty() -> Self { + Self::default() + } + + /// Whether `family` is advertised. + pub fn contains(&self, family: &str) -> bool { + self.families.contains(family) + } + + /// Whether every family in `other` is advertised here. + pub fn contains_all(&self, other: &Self) -> bool { + other.families.is_subset(&self.families) + } + + /// Adds a family in place. Idempotent. + pub fn insert(&mut self, family: impl Into) { + self.families.insert(family.into()); + } + + /// Removes a family in place. Idempotent. + pub fn remove(&mut self, family: &str) { + self.families.remove(family); + } + + /// Builder form of [`Self::insert`]. + pub fn with(mut self, family: impl Into) -> Self { + self.insert(family); + self + } + + /// Advertised families, lexicographically ordered. + pub fn iter(&self) -> impl Iterator + '_ { + self.families.iter().map(String::as_str) + } + + /// Number of advertised families. + pub fn len(&self) -> usize { + self.families.len() + } + + /// Whether no family is advertised. + pub fn is_empty(&self) -> bool { + self.families.is_empty() + } +} + +impl> FromIterator for DriverCapabilities { + fn from_iter>(iter: I) -> Self { + Self { + families: iter.into_iter().map(Into::into).collect(), + } + } +} + +impl> Extend for DriverCapabilities { + fn extend>(&mut self, iter: I) { + self.families.extend(iter.into_iter().map(Into::into)); + } +} + +impl Serialize for DriverCapabilities { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + serializer.collect_seq(self.iter()) + } +} + +impl<'de> Deserialize<'de> for DriverCapabilities { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let families = Vec::::deserialize(deserializer)?; + Ok(families.into_iter().collect()) + } +} + +#[cfg(test)] +#[path = "driver_tests.rs"] +mod tests; diff --git a/src/core/subsystem/driver_tests.rs b/src/core/subsystem/driver_tests.rs new file mode 100644 index 0000000000..0c140f6b5f --- /dev/null +++ b/src/core/subsystem/driver_tests.rs @@ -0,0 +1,208 @@ +//! Tests for the generic driver vocabulary. +//! +//! Note: this is the **only** file under `src/core/subsystem/` that may name +//! `tinycortex_api`, and it does so purely as a drift witness — see +//! `driver_health_shape_matches_memory_health_one_for_one` and +//! `every_memory_contract_capability_string_maps_into_driver_capabilities`. +//! The production modules mention it only in prose; they must never *depend* +//! on it, which is checkable with: +//! +//! ```text +//! grep -rn '^ *use .*tinycortex_api' src/core/subsystem/driver.rs \ +//! src/core/subsystem/registry.rs src/core/subsystem/mod.rs # no output +//! ``` + +use std::str::FromStr; + +use serde_json::json; + +use super::*; + +#[test] +fn driver_class_as_str_matches_serde_representation() { + for class in DriverClass::ALL { + let encoded = serde_json::to_value(class).expect("class serializes"); + assert_eq!(encoded, json!(class.as_str()), "mismatch for {class:?}"); + } +} + +#[test] +fn driver_class_parse_round_trips_every_variant() { + for class in DriverClass::ALL { + assert_eq!(DriverClass::parse(class.as_str()), Ok(class)); + assert_eq!(DriverClass::from_str(class.as_str()), Ok(class)); + assert_eq!(class.to_string(), class.as_str()); + } +} + +#[test] +fn driver_class_parse_rejects_unknown_with_the_input_in_the_message() { + let err = DriverClass::parse("sidecar").expect_err("unknown class rejected"); + assert!( + err.contains("sidecar"), + "message should name the input: {err}" + ); +} + +#[test] +fn driver_health_is_usable_is_false_only_when_down() { + assert!(DriverHealth::Ready.is_usable()); + assert!(DriverHealth::degraded("index rebuilding").is_usable()); + assert!(!DriverHealth::down("connection refused").is_usable()); +} + +#[test] +fn driver_health_serializes_with_a_stable_status_discriminant() { + assert_eq!( + serde_json::to_value(DriverHealth::Ready).expect("ready serializes"), + json!({ "status": "ready" }) + ); + assert_eq!( + serde_json::to_value(DriverHealth::degraded("slow")).expect("degraded serializes"), + json!({ "status": "degraded", "reason": "slow" }) + ); + assert_eq!( + serde_json::to_value(DriverHealth::down("refused")).expect("down serializes"), + json!({ "status": "down", "reason": "refused" }) + ); + + let decoded: DriverHealth = + serde_json::from_value(json!({ "status": "degraded", "reason": "slow" })) + .expect("degraded deserializes"); + assert_eq!(decoded, DriverHealth::degraded("slow")); +} + +#[test] +fn driver_health_display_includes_the_reason() { + assert_eq!(DriverHealth::Ready.to_string(), "ready"); + assert_eq!(DriverHealth::Ready.reason(), None); + assert_eq!( + DriverHealth::down("connection refused").to_string(), + "down: connection refused" + ); + assert_eq!( + DriverHealth::degraded("slow").reason(), + Some("slow"), + "reason is readable for status output" + ); +} + +/// Drift guard for the boundary conversion that lands with the memory adapter. +/// +/// The contract crate's `health` module states that `MemoryHealth` is shaped +/// one-for-one against the kernel's `Ready | Degraded { reason } | Down +/// { reason }` so the conversion is trivial and lossless. This asserts that, +/// pairwise, on the serialized form — so a fourth state, a renamed +/// discriminant, or an extra field on either side fails here rather than +/// silently making the conversion partial. +#[test] +fn driver_health_shape_matches_memory_health_one_for_one() { + use tinycortex_api::health::MemoryHealth; + + let pairs: Vec<(MemoryHealth, DriverHealth)> = vec![ + (MemoryHealth::Ready, DriverHealth::Ready), + ( + MemoryHealth::degraded("index rebuilding"), + DriverHealth::degraded("index rebuilding"), + ), + ( + MemoryHealth::down("connection refused"), + DriverHealth::down("connection refused"), + ), + ]; + assert_eq!(pairs.len(), 3, "both enums have exactly three states"); + + for (contract, kernel) in pairs { + assert_eq!(contract.as_str(), kernel.as_str()); + assert_eq!(contract.reason(), kernel.reason()); + assert_eq!(contract.is_usable(), kernel.is_usable()); + assert_eq!( + serde_json::to_value(&contract).expect("contract health serializes"), + serde_json::to_value(&kernel).expect("kernel health serializes"), + ); + } +} + +#[test] +fn driver_capabilities_round_trips_through_json() { + let caps = DriverCapabilities::empty() + .with("core") + .with("recall") + .with("portability"); + + let encoded = serde_json::to_value(&caps).expect("capabilities serialize"); + let decoded: DriverCapabilities = + serde_json::from_value(encoded.clone()).expect("capabilities deserialize"); + + assert_eq!(decoded, caps); + assert_eq!(encoded, json!(["core", "portability", "recall"])); +} + +#[test] +fn driver_capabilities_collapses_duplicates() { + let caps: DriverCapabilities = ["core", "recall", "core"].into_iter().collect(); + assert_eq!(caps.len(), 2); + assert!(caps.contains("core")); + assert!(caps.contains("recall")); + assert!(!caps.contains("tree")); + + let mut caps = caps; + caps.remove("recall"); + assert_eq!(caps.len(), 1); + caps.remove("recall"); + assert_eq!(caps.len(), 1, "remove is idempotent"); +} + +#[test] +fn driver_capabilities_serializes_as_an_array_of_strings() { + assert_eq!( + serde_json::to_value(DriverCapabilities::empty()).expect("empty serializes"), + json!([]) + ); + assert!(DriverCapabilities::empty().is_empty()); + + let mut caps = DriverCapabilities::empty(); + caps.extend(["tree", "core"]); + assert_eq!( + serde_json::to_value(&caps).expect("serializes"), + json!(["core", "tree"]), + "a set has no order; the backing BTreeSet emits lexicographic order" + ); +} + +#[test] +fn driver_capabilities_contains_all_is_subset_semantics() { + let advertised: DriverCapabilities = ["core", "recall", "portability", "tree"] + .into_iter() + .collect(); + let mandatory: DriverCapabilities = ["core", "recall", "portability"].into_iter().collect(); + + assert!(advertised.contains_all(&mandatory)); + assert!(!mandatory.contains_all(&advertised)); + assert!(advertised.contains_all(&DriverCapabilities::empty())); +} + +/// The kernel's opaque-string set must be able to carry every family the memory +/// contract defines, losslessly and through the wire form — that is what makes +/// the future boundary conversion total without the kernel knowing what a +/// memory capability is. +#[test] +fn every_memory_contract_capability_string_maps_into_driver_capabilities() { + use tinycortex_api::capabilities::Capability; + + let caps: DriverCapabilities = Capability::ALL.iter().map(|cap| cap.as_str()).collect(); + + assert_eq!(caps.len(), Capability::ALL.len()); + assert_eq!(caps.len(), 13); + assert!( + caps.contains("tool_memory"), + "the one non-identity snake_case family must survive" + ); + for cap in Capability::ALL { + assert!(caps.contains(cap.as_str()), "missing {cap}"); + } + + let encoded = serde_json::to_value(&caps).expect("serializes"); + let decoded: DriverCapabilities = serde_json::from_value(encoded).expect("deserializes"); + assert_eq!(decoded, caps); +} diff --git a/src/core/subsystem/mod.rs b/src/core/subsystem/mod.rs new file mode 100644 index 0000000000..eaac2a91ad --- /dev/null +++ b/src/core/subsystem/mod.rs @@ -0,0 +1,35 @@ +//! The subsystem registry — one bound driver per capability slot. +//! +//! `docs/specs/kernel.md` §3 (the model), §3.7 (three runtime axes), §6 item 1. +//! +//! ## Scope +//! +//! This module is the **generic half** of kernel.md §6 item 1: `DriverClass`, +//! `DriverHealth`, a generic capability set, and `SubsystemRegistry`. It is +//! deliberately subsystem-agnostic and names no subsystem's contract crate — +//! whichever subsystem is cut over after memory (inference, channels, sandbox) +//! uses this same registry, so it must not inherit its vocabulary from a +//! *memory* crate. +//! +//! Since M2b the memory adapter exists in +//! [`crate::openhuman::memory::binding`] (it converts +//! `tinycortex_api::MemoryHealth` into [`DriverHealth`] and the contract's +//! typed capability set into [`DriverCapabilities`]), and M2c added the +//! read-only [`status`] projection plus the `subsystems` RPC namespace and the +//! `openhuman subsystems` CLI table. +//! +//! Still to land in later steps, despite being named in the same §6 sentence: +//! the generic `Driver` trait and the policy `Guard` (§3.4). + +mod driver; +mod registry; +pub mod schemas; +mod status; + +pub use driver::{DriverCapabilities, DriverClass, DriverHealth}; +pub use registry::{BoundDriver, SubsystemRegistry, SubsystemSlot}; +pub use schemas::{ + all_controller_schemas as all_subsystems_controller_schemas, + all_registered_controllers as all_subsystems_registered_controllers, subsystems_status, +}; +pub use status::{format_contract_version, registry_status, SubsystemStatus}; diff --git a/src/core/subsystem/registry.rs b/src/core/subsystem/registry.rs new file mode 100644 index 0000000000..7acecb4826 --- /dev/null +++ b/src/core/subsystem/registry.rs @@ -0,0 +1,273 @@ +//! The subsystem registry: one bound driver per capability slot. +//! +//! Specified by `docs/specs/kernel.md` §3.1 (exactly one driver per subsystem +//! per process), §3.7 (a failed bind falls back to the embedded default, +//! "logged loudly, surfaced in status, never silent"), and §6 items 1 and 6. +//! +//! Unlike [`crate::core::event_bus`], this is a **plain owned struct** — no +//! `OnceLock`, no global. The registry is constructed once at `CoreBuilder` +//! time and owned by the core context; a global would be a second, competing +//! source of truth for which driver answers. + +use std::collections::BTreeMap; + +use serde::{Deserialize, Serialize}; + +use super::driver::{DriverCapabilities, DriverClass, DriverHealth}; + +/// A named capability slot (kernel.md §3.1). +/// +/// Exactly the seven subsystems the spec names. Declaration order is also +/// [`SubsystemRegistry`] iteration order, because this type is the `BTreeMap` +/// key and derives `Ord`. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum SubsystemSlot { + /// Memory — the pilot subsystem. + Memory, + /// Model routing and inference providers. + Inference, + /// External messaging providers. + Channels, + /// SKILL.md discovery, install, and execution. + Skills, + /// Saved automation graphs. + Flows, + /// Command execution isolation. + Sandbox, + /// Speech to text and text to speech. + Voice, +} + +impl SubsystemSlot { + /// Every slot, in declaration order. + pub const ALL: [SubsystemSlot; 7] = [ + SubsystemSlot::Memory, + SubsystemSlot::Inference, + SubsystemSlot::Channels, + SubsystemSlot::Skills, + SubsystemSlot::Flows, + SubsystemSlot::Sandbox, + SubsystemSlot::Voice, + ]; + + /// Stable snake_case identifier used in config (`[subsystems.]`), on + /// the wire, and in logs. The serde derive is pinned against this by a + /// test. + pub fn as_str(self) -> &'static str { + match self { + Self::Memory => "memory", + Self::Inference => "inference", + Self::Channels => "channels", + Self::Skills => "skills", + Self::Flows => "flows", + Self::Sandbox => "sandbox", + Self::Voice => "voice", + } + } + + /// Parse back from the config / wire form. + /// + /// # Errors + /// + /// Returns the unrecognised input in the message. + pub fn parse(raw: &str) -> Result { + Self::ALL + .iter() + .copied() + .find(|slot| slot.as_str() == raw) + .ok_or_else(|| format!("unknown subsystem slot: {raw}")) + } +} + +impl std::fmt::Display for SubsystemSlot { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.as_str()) + } +} + +impl std::str::FromStr for SubsystemSlot { + type Err = String; + + fn from_str(raw: &str) -> Result { + Self::parse(raw) + } +} + +/// What the kernel knows about the driver currently bound to a slot. +/// +/// This is the record `subsystems_status` renders (kernel.md §6 item 6: slot, +/// bound driver, class, health, contract version, and capabilities), plus the +/// fallback provenance §3.7 requires so a fallback is never silent. +/// +/// `Debug` is derived deliberately: no field here is a secret. The +/// credential-bearing type is `MemoryDriverConfig`, which carries a manual +/// redacting `Debug`; a bound driver record holds no credential at all, and +/// must not gain one. +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +pub struct BoundDriver { + /// The slot this driver answers for. + pub slot: SubsystemSlot, + /// Driver id, e.g. `"tinycortex"`, `"supermemory"`, `"null"`. + pub id: String, + /// How the driver was bound. A host fact, not self-reported. + pub class: DriverClass, + /// The capability set, asked for **once** at bind time and cached here + /// (kernel.md §3.2 rule 1). + pub capabilities: DriverCapabilities, + /// Latest known liveness. + pub health: DriverHealth, + /// The `(major, minor)` contract version this driver speaks. + pub contract_version: (u16, u16), + /// The driver id that was *asked for* and failed, when this binding is the + /// result of a fallback. `None` for a normal bind. kernel.md §3.7 requires + /// a fallback be surfaced in status rather than silently substituted. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub fell_back_from: Option, +} + +impl BoundDriver { + /// A binding with no capabilities beyond what the caller adds, healthy, and + /// not the result of a fallback. + pub fn new( + slot: SubsystemSlot, + id: impl Into, + class: DriverClass, + capabilities: DriverCapabilities, + contract_version: (u16, u16), + ) -> Self { + Self { + slot, + id: id.into(), + class, + capabilities, + health: DriverHealth::Ready, + contract_version, + fell_back_from: None, + } + } + + /// Whether this binding replaced a driver that failed to construct. + pub fn is_fallback(&self) -> bool { + self.fell_back_from.is_some() + } +} + +/// Which driver answers for each subsystem slot in this process. +/// +/// Exactly one driver per slot (kernel.md §3.1): binding a second driver into +/// an occupied slot **replaces** the first and warns, because two live drivers +/// for one subsystem means two truths. Fan-out is expressed as a composite +/// driver (§3.5), not as a second binding. +#[derive(Clone, Debug, Default)] +pub struct SubsystemRegistry { + slots: BTreeMap, +} + +impl SubsystemRegistry { + /// An empty registry — no slot bound. + pub fn new() -> Self { + Self::default() + } + + /// Binds `driver` into its own slot, returning whatever it displaced. + /// + /// A rebind is logged at `warn`: outside tests it means either a + /// misconfiguration or a live driver swap, and both deserve a line in the + /// log. + pub fn bind(&mut self, driver: BoundDriver) -> Option { + let slot = driver.slot; + let previous = self.slots.insert(slot, driver); + // Re-borrow rather than hold a reference across the insert. + let bound = &self.slots[&slot]; + match &previous { + Some(prev) => log::warn!( + "[subsystem] {slot} rebound from '{}' to '{}' ({})", + prev.id, + bound.id, + bound.class + ), + None => log::info!( + "[subsystem] {slot} bound to '{}' ({})", + bound.id, + bound.class + ), + } + previous + } + + /// Binds `primary` if it constructed; otherwise logs the failure loudly and + /// binds the driver `fallback` produces, tagging it with the id that failed + /// so status can show the substitution (kernel.md §3.7). + /// + /// `fallback` is `FnOnce` so the embedded default is not constructed at all + /// when the primary succeeds. + pub fn bind_with_fallback( + &mut self, + slot: SubsystemSlot, + attempted_driver_id: &str, + primary: Result, + fallback: F, + ) -> &BoundDriver + where + E: std::fmt::Display, + F: FnOnce() -> BoundDriver, + { + match primary { + Ok(driver) => { + self.bind(driver); + } + Err(err) => { + let mut driver = fallback(); + log::warn!( + "[subsystem] {slot} driver '{attempted_driver_id}' failed to bind: {err}; falling back to '{}'", + driver.id + ); + driver.fell_back_from = Some(attempted_driver_id.to_string()); + self.bind(driver); + } + } + &self.slots[&slot] + } + + /// The driver bound to `slot`, if any. + pub fn get(&self, slot: SubsystemSlot) -> Option<&BoundDriver> { + self.slots.get(&slot) + } + + /// Updates the cached health of a bound slot. Returns `false` — and changes + /// nothing — when the slot is unbound. + pub fn set_health(&mut self, slot: SubsystemSlot, health: DriverHealth) -> bool { + match self.slots.get_mut(&slot) { + Some(driver) => { + driver.health = health; + true + } + None => false, + } + } + + /// Every binding, in [`SubsystemSlot`] declaration order. + pub fn iter(&self) -> impl Iterator + '_ { + self.slots.values() + } + + /// The bound slots, in declaration order. + pub fn bound_slots(&self) -> Vec { + self.slots.keys().copied().collect() + } + + /// Number of bound slots. + pub fn len(&self) -> usize { + self.slots.len() + } + + /// Whether no slot is bound. + pub fn is_empty(&self) -> bool { + self.slots.is_empty() + } +} + +#[cfg(test)] +#[path = "registry_tests.rs"] +mod tests; diff --git a/src/core/subsystem/registry_tests.rs b/src/core/subsystem/registry_tests.rs new file mode 100644 index 0000000000..0a317e7f0d --- /dev/null +++ b/src/core/subsystem/registry_tests.rs @@ -0,0 +1,270 @@ +//! Tests for the subsystem registry: bind, rebind, fallback-on-failure, and +//! health updates. + +use std::cell::Cell; +use std::str::FromStr; + +use serde_json::json; + +use super::*; + +const CONTRACT: (u16, u16) = (1, 0); + +fn bound(slot: SubsystemSlot, id: &str, class: DriverClass) -> BoundDriver { + BoundDriver::new( + slot, + id, + class, + ["core", "recall", "portability"].into_iter().collect(), + CONTRACT, + ) +} + +#[test] +fn subsystem_slot_as_str_matches_serde_representation() { + for slot in SubsystemSlot::ALL { + let encoded = serde_json::to_value(slot).expect("slot serializes"); + assert_eq!(encoded, json!(slot.as_str()), "mismatch for {slot:?}"); + } +} + +#[test] +fn subsystem_slot_parse_round_trips_every_variant() { + for slot in SubsystemSlot::ALL { + assert_eq!(SubsystemSlot::parse(slot.as_str()), Ok(slot)); + assert_eq!(SubsystemSlot::from_str(slot.as_str()), Ok(slot)); + assert_eq!(slot.to_string(), slot.as_str()); + } + let err = SubsystemSlot::parse("telepathy").expect_err("unknown slot rejected"); + assert!( + err.contains("telepathy"), + "message should name the input: {err}" + ); +} + +#[test] +fn bind_records_the_driver_in_its_slot() { + let mut registry = SubsystemRegistry::new(); + assert!(registry.is_empty()); + + registry.bind(bound( + SubsystemSlot::Memory, + "tinycortex", + DriverClass::Embedded, + )); + + let driver = registry + .get(SubsystemSlot::Memory) + .expect("memory slot is bound"); + assert_eq!(driver.id, "tinycortex"); + assert_eq!(driver.class, DriverClass::Embedded); + assert_eq!(driver.contract_version, CONTRACT); + assert_eq!(driver.health, DriverHealth::Ready); + assert!(driver.capabilities.contains("recall")); + assert!(!driver.is_fallback()); + assert_eq!(registry.len(), 1); + assert_eq!(registry.bound_slots(), vec![SubsystemSlot::Memory]); + assert!(registry.get(SubsystemSlot::Voice).is_none()); +} + +#[test] +fn bind_returns_no_previous_driver_for_an_empty_slot() { + let mut registry = SubsystemRegistry::new(); + let previous = registry.bind(bound( + SubsystemSlot::Sandbox, + "landlock", + DriverClass::Embedded, + )); + assert!(previous.is_none()); +} + +#[test] +fn rebind_replaces_and_returns_the_previous_driver() { + let mut registry = SubsystemRegistry::new(); + registry.bind(bound( + SubsystemSlot::Memory, + "tinycortex", + DriverClass::Embedded, + )); + + let previous = registry + .bind(bound( + SubsystemSlot::Memory, + "supermemory", + DriverClass::External, + )) + .expect("rebind returns the displaced driver"); + + assert_eq!(previous.id, "tinycortex"); + assert_eq!( + registry.get(SubsystemSlot::Memory).expect("still bound").id, + "supermemory", + "exactly one driver per slot — the second replaces the first" + ); + assert_eq!(registry.len(), 1, "a rebind does not add a slot"); +} + +#[test] +fn rebind_does_not_disturb_other_slots() { + let mut registry = SubsystemRegistry::new(); + registry.bind(bound( + SubsystemSlot::Memory, + "tinycortex", + DriverClass::Embedded, + )); + registry.bind(bound( + SubsystemSlot::Voice, + "whisper", + DriverClass::Embedded, + )); + registry.bind(bound( + SubsystemSlot::Memory, + "supermemory", + DriverClass::External, + )); + + assert_eq!( + registry.get(SubsystemSlot::Voice).expect("bound").id, + "whisper" + ); + assert_eq!(registry.len(), 2); +} + +#[test] +fn bind_with_fallback_binds_the_primary_when_it_constructs() { + let mut registry = SubsystemRegistry::new(); + let primary: Result = Ok(bound( + SubsystemSlot::Memory, + "supermemory", + DriverClass::External, + )); + + let driver = registry.bind_with_fallback(SubsystemSlot::Memory, "supermemory", primary, || { + bound(SubsystemSlot::Memory, "tinycortex", DriverClass::Embedded) + }); + + assert_eq!(driver.id, "supermemory"); + assert!(!driver.is_fallback()); + assert_eq!(driver.fell_back_from, None); +} + +#[test] +fn bind_with_fallback_binds_the_fallback_when_the_primary_fails() { + let mut registry = SubsystemRegistry::new(); + let primary: Result = Err("handshake refused".into()); + + let driver = registry.bind_with_fallback(SubsystemSlot::Memory, "supermemory", primary, || { + bound(SubsystemSlot::Memory, "tinycortex", DriverClass::Embedded) + }); + + assert_eq!(driver.id, "tinycortex"); + assert_eq!(driver.class, DriverClass::Embedded); + assert_eq!( + registry.get(SubsystemSlot::Memory).expect("bound").id, + "tinycortex" + ); +} + +#[test] +fn bind_with_fallback_records_fell_back_from_so_status_is_never_silent() { + let mut registry = SubsystemRegistry::new(); + let primary: Result = Err("handshake refused".into()); + + registry.bind_with_fallback(SubsystemSlot::Memory, "supermemory", primary, || { + bound(SubsystemSlot::Memory, "tinycortex", DriverClass::Embedded) + }); + + let driver = registry.get(SubsystemSlot::Memory).expect("bound"); + assert!(driver.is_fallback()); + assert_eq!(driver.fell_back_from.as_deref(), Some("supermemory")); + + let encoded = serde_json::to_value(driver).expect("status record serializes"); + assert_eq!( + encoded["fell_back_from"], + json!("supermemory"), + "the substitution must be visible in status output" + ); +} + +#[test] +fn bind_with_fallback_does_not_construct_the_fallback_on_success() { + let mut registry = SubsystemRegistry::new(); + let constructed = Cell::new(false); + let primary: Result = Ok(bound( + SubsystemSlot::Memory, + "supermemory", + DriverClass::External, + )); + + registry.bind_with_fallback(SubsystemSlot::Memory, "supermemory", primary, || { + constructed.set(true); + bound(SubsystemSlot::Memory, "tinycortex", DriverClass::Embedded) + }); + + assert!( + !constructed.get(), + "the embedded default must not be constructed when the primary binds" + ); +} + +#[test] +fn set_health_updates_only_the_named_slot() { + let mut registry = SubsystemRegistry::new(); + registry.bind(bound( + SubsystemSlot::Memory, + "tinycortex", + DriverClass::Embedded, + )); + registry.bind(bound( + SubsystemSlot::Voice, + "whisper", + DriverClass::Embedded, + )); + + assert!(registry.set_health( + SubsystemSlot::Memory, + DriverHealth::degraded("index rebuilding") + )); + + assert_eq!( + registry.get(SubsystemSlot::Memory).expect("bound").health, + DriverHealth::degraded("index rebuilding") + ); + assert_eq!( + registry.get(SubsystemSlot::Voice).expect("bound").health, + DriverHealth::Ready + ); +} + +#[test] +fn set_health_returns_false_for_an_unbound_slot() { + let mut registry = SubsystemRegistry::new(); + assert!(!registry.set_health(SubsystemSlot::Flows, DriverHealth::down("gone"))); + assert!(registry.is_empty()); +} + +#[test] +fn registry_iterates_in_slot_declaration_order() { + let mut registry = SubsystemRegistry::new(); + // Bind out of declaration order on purpose. + for slot in [ + SubsystemSlot::Voice, + SubsystemSlot::Memory, + SubsystemSlot::Flows, + SubsystemSlot::Inference, + ] { + registry.bind(bound(slot, "null", DriverClass::Null)); + } + + let order: Vec = registry.iter().map(|driver| driver.slot).collect(); + assert_eq!( + order, + vec![ + SubsystemSlot::Memory, + SubsystemSlot::Inference, + SubsystemSlot::Flows, + SubsystemSlot::Voice, + ] + ); + assert_eq!(registry.bound_slots(), order); +} diff --git a/src/core/subsystem/schemas.rs b/src/core/subsystem/schemas.rs new file mode 100644 index 0000000000..d1225c7f36 --- /dev/null +++ b/src/core/subsystem/schemas.rs @@ -0,0 +1,126 @@ +//! The `subsystems` RPC namespace — one row per capability slot +//! (`docs/specs/kernel.md` §6 item 6). +//! +//! ## Why a controller lives under `src/core/` +//! +//! `AGENTS.md` says `src/core/` is transport only. This is the one deliberate +//! exception, and it is narrow: the subsystem registry *is* a kernel binding +//! table — the same category as `core::all`'s controller registry — and there +//! is no `src/openhuman/` family that owns it. Giving it one would mean a new +//! `DomainGroup` variant plus the four compiler-enforced edits and three +//! drift-guard lists that come with it, for a single read-only function. So it +//! is registered from here, tagged `DomainGroup::Platform`. +//! +//! ## Aggregation +//! +//! Today `memory` is the only occupant, so the aggregate is one call into the +//! memory adapter. Each future subsystem appends its own adapter call here as +//! it is cut over; the *shape* of a row is already generic +//! ([`SubsystemStatus`]), so adding one is a one-line change with no wire +//! change for existing rows. + +use serde_json::{Map, Value}; + +use crate::core::all::{ControllerFuture, RegisteredController}; +use crate::core::{ControllerSchema, FieldSchema, TypeSchema}; +use crate::rpc::RpcOutcome; + +use super::status::SubsystemStatus; + +pub fn all_controller_schemas() -> Vec { + vec![schemas("status")] +} + +pub fn all_registered_controllers() -> Vec { + vec![RegisteredController { + schema: schemas("status"), + handler: handle_status, + }] +} + +pub fn schemas(function: &str) -> ControllerSchema { + match function { + "status" => ControllerSchema { + namespace: "subsystems", + function: "status", + description: "List every subsystem slot with its bound driver, class, health, contract version, and advertised capability families.", + inputs: vec![], + outputs: vec![FieldSchema { + name: "subsystems", + ty: TypeSchema::Json, + comment: "One entry per slot: { slot, driver, class, health, health_reason, contract_version, capabilities[], fell_back_from, last_error }.", + required: true, + }], + }, + _ => ControllerSchema { + namespace: "subsystems", + function: "unknown", + description: "Unknown subsystems controller function.", + inputs: vec![], + outputs: vec![FieldSchema { + name: "error", + ty: TypeSchema::String, + comment: "Lookup error details.", + required: true, + }], + }, + } +} + +/// Every subsystem slot's status, in slot declaration order. +/// +/// Memory is the only occupant today. This is also what +/// [`crate::core::subsystems_cli`] renders as a table. +pub async fn subsystems_status() -> Vec { + vec![crate::openhuman::memory::rpc::memory_subsystem_status().await] +} + +fn handle_status(_params: Map) -> ControllerFuture { + Box::pin(async move { + let rows = subsystems_status().await; + log::debug!("[subsystem] status requested: {} slot(s)", rows.len()); + RpcOutcome::new(serde_json::json!({ "subsystems": rows }), vec![]) + .into_cli_compatible_json() + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn status_schema_shape() { + let schema = schemas("status"); + assert_eq!(schema.namespace, "subsystems"); + assert_eq!(schema.function, "status"); + assert!(schema.inputs.is_empty()); + assert_eq!(schema.outputs.len(), 1); + assert_eq!(schema.outputs[0].name, "subsystems"); + } + + #[test] + fn unknown_function_returns_the_unknown_schema() { + assert_eq!(schemas("not_real").function, "unknown"); + } + + #[test] + fn schemas_and_controllers_line_up() { + let schemas = all_controller_schemas(); + let controllers = all_registered_controllers(); + assert_eq!(schemas.len(), controllers.len()); + for (schema, controller) in schemas.iter().zip(controllers.iter()) { + assert_eq!(schema.namespace, controller.schema.namespace); + assert_eq!(schema.function, controller.schema.function); + } + } + + #[tokio::test] + async fn handler_returns_a_subsystems_array_containing_the_memory_slot() { + let value = handle_status(Map::new()).await.expect("handler succeeds"); + let rows = value["subsystems"].as_array().expect("array"); + assert_eq!(rows.len(), 1); + assert_eq!(rows[0]["slot"], "memory"); + assert!(rows[0]["contract_version"].is_string()); + assert!(rows[0]["capabilities"].is_array()); + } +} diff --git a/src/core/subsystem/status.rs b/src/core/subsystem/status.rs new file mode 100644 index 0000000000..9b419ae119 --- /dev/null +++ b/src/core/subsystem/status.rs @@ -0,0 +1,109 @@ +//! The read-only status projection over a bound driver +//! (`docs/specs/kernel.md` §6 item 6, `docs/specs/plan-memory.md` §5). +//! +//! [`BoundDriver`] is the kernel's *internal* record. This module is the +//! *wire* shape rendered by `subsystems.status` and `memory.provider_status`. +//! They are deliberately two types rather than one `Serialize` derive on +//! `BoundDriver`, for one reason worth stating plainly: +//! +//! ## Capabilities cross the wire as opaque strings, never as a typed set +//! +//! A memory driver's typed set (`tinycortex_api::capabilities::Capabilities`) +//! is a `u16` bitset whose `Deserialize` rejects the **whole** array on a +//! single unrecognised family string. A driver speaking a newer minor contract +//! may legitimately advertise a family this build has never heard of, and +//! status must be able to *report* what it saw even though this kernel would +//! never call it. So the status payload carries `Vec`, sourced from +//! [`DriverCapabilities`] — which is already opaque strings — and this type +//! derives `Serialize` **only**. Do not add `Deserialize`, and do not retype +//! `capabilities` as a contract type: either change reintroduces the hazard. +//! +//! Health is flattened to a `status` discriminant plus an optional reason for +//! the same reason: a status consumer should be able to render an unfamiliar +//! driver without a total `match`. + +use serde::Serialize; + +use super::driver::DriverHealth; +use super::registry::{BoundDriver, SubsystemRegistry}; + +/// One subsystem slot's status, as rendered on the wire. +/// +/// `Serialize` only — see the module docs. +#[derive(Clone, Debug, PartialEq, Eq, Serialize)] +pub struct SubsystemStatus { + /// The slot name: `"memory"`, `"inference"`, … + pub slot: String, + /// The bound driver id, e.g. `"tinycortex"`, `"supermemory"`, `"null"`. + pub driver: String, + /// How the host bound it: `"embedded"` | `"external"` | `"null"`. + pub class: String, + /// Liveness discriminant: `"ready"` | `"degraded"` | `"down"`. + pub health: String, + /// Operator-facing reason when degraded or down; `None` when ready. + /// Never contains a credential, endpoint token, or user memory content. + pub health_reason: Option, + /// The `(major, minor)` contract version the driver speaks, rendered as + /// `"."`. Display data — version *compatibility* is decided + /// by the contract crate, never by string-comparing this field. + pub contract_version: String, + /// Advertised capability families as opaque strings. See the module docs. + pub capabilities: Vec, + /// The driver id that was asked for and failed, when this binding is a + /// fallback (kernel.md §3.7 — a fallback is never silent). `None` for a + /// normal bind. + pub fell_back_from: Option, + /// Last bind or call failure, operator-facing. `None` when clean. Subject + /// to the same redaction rule as `health_reason`. + pub last_error: Option, +} + +impl SubsystemStatus { + /// Project a bound driver, using the health cached on the record. + pub fn from_bound(driver: &BoundDriver) -> Self { + Self::from_bound_with_health(driver, driver.health.clone()) + } + + /// Project a bound driver, overriding the cached health with one just + /// probed live. `subsystems.status` and `memory.provider_status` both do + /// this: the cached value is whatever the last bind or `set_health` wrote, + /// and a status call is exactly the moment to ask again. + pub fn from_bound_with_health(driver: &BoundDriver, health: DriverHealth) -> Self { + Self { + slot: driver.slot.as_str().to_string(), + driver: driver.id.clone(), + class: driver.class.as_str().to_string(), + health: health.as_str().to_string(), + health_reason: health.reason().map(str::to_string), + contract_version: format_contract_version(driver.contract_version), + capabilities: driver.capabilities.iter().map(str::to_string).collect(), + fell_back_from: driver.fell_back_from.clone(), + last_error: None, + } + } + + /// Attach an operator-facing last-error string. Builder form so callers + /// that have no error do not have to name the field. + pub fn with_last_error(mut self, last_error: Option) -> Self { + self.last_error = last_error; + self + } +} + +/// `"."`. +pub fn format_contract_version(version: (u16, u16)) -> String { + format!("{}.{}", version.0, version.1) +} + +/// Project every binding in a registry, in slot declaration order. +/// +/// Uses each record's cached health — this is the pure, I/O-free projection. +/// A caller that wants live health probes the driver itself and uses +/// [`SubsystemStatus::from_bound_with_health`]. +pub fn registry_status(registry: &SubsystemRegistry) -> Vec { + registry.iter().map(SubsystemStatus::from_bound).collect() +} + +#[cfg(test)] +#[path = "status_tests.rs"] +mod tests; diff --git a/src/core/subsystem/status_tests.rs b/src/core/subsystem/status_tests.rs new file mode 100644 index 0000000000..658dbaf3ef --- /dev/null +++ b/src/core/subsystem/status_tests.rs @@ -0,0 +1,109 @@ +//! Tests for the wire projection over a bound driver. + +use super::*; +use crate::core::subsystem::driver::{DriverCapabilities, DriverClass}; +use crate::core::subsystem::registry::{BoundDriver, SubsystemRegistry, SubsystemSlot}; + +fn embedded_memory() -> BoundDriver { + BoundDriver::new( + SubsystemSlot::Memory, + "tinycortex", + DriverClass::Embedded, + DriverCapabilities::empty() + .with("core") + .with("recall") + .with("portability"), + (1, 0), + ) +} + +#[test] +fn projection_reports_slot_driver_class_and_contract_version() { + let status = SubsystemStatus::from_bound(&embedded_memory()); + assert_eq!(status.slot, "memory"); + assert_eq!(status.driver, "tinycortex"); + assert_eq!(status.class, "embedded"); + assert_eq!(status.contract_version, "1.0"); + assert_eq!(status.health, "ready"); + assert_eq!(status.health_reason, None); + assert_eq!(status.fell_back_from, None); + assert_eq!(status.last_error, None); +} + +#[test] +fn projection_reports_the_advertised_capability_list() { + let status = SubsystemStatus::from_bound(&embedded_memory()); + // `DriverCapabilities` is a set, so ordering is lexicographic, not the + // contract's declaration order. Asserted explicitly so a future switch to + // declaration order is a deliberate change, not a silent one. + assert_eq!(status.capabilities, vec!["core", "portability", "recall"]); +} + +#[test] +fn live_health_overrides_the_cached_record() { + let status = SubsystemStatus::from_bound_with_health( + &embedded_memory(), + DriverHealth::degraded("vector index rebuilding"), + ); + assert_eq!(status.health, "degraded"); + assert_eq!( + status.health_reason.as_deref(), + Some("vector index rebuilding") + ); +} + +#[test] +fn a_fallback_binding_surfaces_the_driver_it_replaced() { + let mut driver = BoundDriver::new( + SubsystemSlot::Memory, + "null", + DriverClass::Null, + DriverCapabilities::empty(), + (1, 0), + ); + driver.fell_back_from = Some("supermemory".to_string()); + let status = SubsystemStatus::from_bound(&driver) + .with_last_error(Some("external driver is untrusted".to_string())); + assert_eq!(status.fell_back_from.as_deref(), Some("supermemory")); + assert_eq!( + status.last_error.as_deref(), + Some("external driver is untrusted") + ); + assert!(status.capabilities.is_empty()); + assert_eq!(status.class, "null"); +} + +#[test] +fn capabilities_serialize_as_a_flat_array_of_strings() { + let value = serde_json::to_value(SubsystemStatus::from_bound(&embedded_memory())) + .expect("status serializes"); + let caps = value["capabilities"].as_array().expect("array"); + assert!(caps.iter().all(serde_json::Value::is_string)); + assert_eq!(value["class"], "embedded"); + assert_eq!(value["health"], "ready"); + // Health is flattened, never nested under a `status` discriminant object. + assert!(value["health"].is_string()); +} + +#[test] +fn registry_projection_follows_slot_declaration_order() { + let mut registry = SubsystemRegistry::new(); + registry.bind(BoundDriver::new( + SubsystemSlot::Inference, + "openai", + DriverClass::External, + DriverCapabilities::empty(), + (1, 0), + )); + registry.bind(embedded_memory()); + + let rows = registry_status(®istry); + assert_eq!(rows.len(), 2); + assert_eq!(rows[0].slot, "memory"); + assert_eq!(rows[1].slot, "inference"); +} + +#[test] +fn contract_version_formats_as_major_dot_minor() { + assert_eq!(format_contract_version((2, 7)), "2.7"); +} diff --git a/src/core/subsystems_cli.rs b/src/core/subsystems_cli.rs new file mode 100644 index 0000000000..cb607e4647 --- /dev/null +++ b/src/core/subsystems_cli.rs @@ -0,0 +1,110 @@ +//! `openhuman subsystems` — the human-readable subsystem slot table. +//! +//! Reached through the `RegisteredCliAdapter` seam +//! ([`crate::core::all::cli_handler_for_namespace`]), which +//! `run_namespace_command` consults when the namespace is invoked with no +//! function or with `--help`. `openhuman subsystems status` bypasses this and +//! prints the raw JSON through the generic namespace dispatcher, so there is +//! no hand-written subcommand match arm anywhere — registering the controller +//! is what makes the subcommand exist. +//! +//! ```text +//! openhuman subsystems # table +//! openhuman subsystems status # JSON +//! ``` + +use anyhow::Result; + +use crate::core::subsystem::{subsystems_status, SubsystemStatus}; + +pub fn run_subsystems_command(args: &[String]) -> Result<()> { + if args.iter().any(|a| a == "-h" || a == "--help") { + print_help(); + return Ok(()); + } + + // A current-thread runtime is enough: a status call probes a bound driver's + // health and touches no orchestrator, unlike the generic dispatcher's + // multi-thread runtime with an enlarged agent worker stack. + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build()?; + let rows = rt.block_on(cli_subsystems_status()); + + println!( + "{:<10} {:<14} {:<9} {:<9} {:<9} CAPABILITIES", + "SLOT", "DRIVER", "CLASS", "HEALTH", "CONTRACT" + ); + for row in &rows { + let driver = if row.driver.is_empty() { + "-" + } else { + row.driver.as_str() + }; + let capabilities = if row.capabilities.is_empty() { + "-".to_string() + } else { + row.capabilities.join(",") + }; + println!( + "{:<10} {:<14} {:<9} {:<9} {:<9} {}", + row.slot, driver, row.class, row.health, row.contract_version, capabilities + ); + if let Some(reason) = &row.health_reason { + println!(" health: {reason}"); + } + if let Some(previous) = &row.fell_back_from { + println!(" fell back from: {previous}"); + } + if let Some(err) = &row.last_error { + println!(" last error: {err}"); + } + } + Ok(()) +} + +/// The slot table for a standalone CLI invocation. +/// +/// [`subsystems_status`] resolves memory through [`memory_subsystem_status`], +/// which now handles the no-`CoreContext` standalone case itself by reading the +/// on-disk config and binding the configured workspace's driver (see +/// `memory::ops::provider::standalone_status`) — the same way +/// `cli_capability::bound_memory_driver_for` does. So the bare table and the +/// `subsystems status` JSON path both render the same resolved row, on the same +/// code, and neither reports an unresolved `driver = ""` row on a healthy +/// install. +async fn cli_subsystems_status() -> Vec { + subsystems_status().await +} + +fn print_help() { + println!("openhuman subsystems — kernel subsystem slots and their bound drivers"); + println!(); + println!("USAGE:"); + println!(" openhuman subsystems Print the slot table"); + println!(" openhuman subsystems status Print the same data as JSON"); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn help_flag_short_circuits_without_probing_a_driver() { + run_subsystems_command(&["--help".to_string()]).expect("help succeeds"); + run_subsystems_command(&["-h".to_string()]).expect("short help succeeds"); + } + + #[test] + fn bare_invocation_renders_the_table() { + run_subsystems_command(&[]).expect("table renders"); + } + + #[test] + fn namespace_has_a_registered_cli_adapter() { + assert!( + crate::core::all::cli_handler_for_namespace("subsystems").is_some(), + "bare `openhuman subsystems` must reach the table, not the generic help" + ); + } +} diff --git a/src/openhuman/agent/harness/graph.rs b/src/openhuman/agent/harness/graph.rs index a58b2ca43c..7d0b883539 100644 --- a/src/openhuman/agent/harness/graph.rs +++ b/src/openhuman/agent/harness/graph.rs @@ -220,6 +220,7 @@ mod tests { raw: None, resolved_model: None, continue_turn: None, + served_from_cache: false, }, ModelResponse::assistant("channel done"), ])); diff --git a/src/openhuman/agent/harness/session/turn_tests.rs b/src/openhuman/agent/harness/session/turn_tests.rs index 4ff34c5561..6f5e2e2392 100644 --- a/src/openhuman/agent/harness/session/turn_tests.rs +++ b/src/openhuman/agent/harness/session/turn_tests.rs @@ -575,7 +575,7 @@ fn collect_tree_root_summaries_maps_namespace_body_and_timestamp() { // store tuple into the `NamespaceSummary` the prompt renderer stamps. use crate::openhuman::config::Config; use crate::openhuman::memory::tree::tree_runtime::store::write_node; - use crate::openhuman::memory::tree::tree_runtime::types::{ + use tinycortex::memory::tree::runtime::{ derive_parent_id, estimate_tokens, level_from_node_id, TreeNode, }; @@ -616,7 +616,7 @@ fn collect_tree_root_summaries_maps_namespace_body_and_timestamp() { fn collect_tree_root_summaries_reads_only_profile_memory_subtree() { use crate::openhuman::config::Config; use crate::openhuman::memory::tree::tree_runtime::store::write_node; - use crate::openhuman::memory::tree::tree_runtime::types::{ + use tinycortex::memory::tree::runtime::{ derive_parent_id, estimate_tokens, level_from_node_id, TreeNode, }; diff --git a/src/openhuman/agent/harness/subagent_runner/handoff.rs b/src/openhuman/agent/harness/subagent_runner/handoff.rs index 9d270e75d4..bed5fa5cab 100644 --- a/src/openhuman/agent/harness/subagent_runner/handoff.rs +++ b/src/openhuman/agent/harness/subagent_runner/handoff.rs @@ -29,7 +29,7 @@ use std::sync::Mutex as StdMutex; /// cache instead of being pushed into history raw. Token count is /// estimated at ~4 chars/token (mirrors /// `crate::openhuman::agent::tinyagents::payload_summarizer` and -/// `crate::openhuman::memory::tree::tree_runtime::types::estimate_tokens`). +/// `crate::openhuman::memory::tree::tree_runtime::estimate_tokens`). /// /// Set at `50_000` so the clean Gmail / Notion envelopes emitted by provider /// post-processing fit through unchanged for normal workloads — only diff --git a/src/openhuman/agent/harness/subagent_runner/ops/graph.rs b/src/openhuman/agent/harness/subagent_runner/ops/graph.rs index 19676c16eb..eddf2ca853 100644 --- a/src/openhuman/agent/harness/subagent_runner/ops/graph.rs +++ b/src/openhuman/agent/harness/subagent_runner/ops/graph.rs @@ -1038,6 +1038,7 @@ mod tests { raw: None, resolved_model: None, continue_turn: None, + served_from_cache: false, } } diff --git a/src/openhuman/agent/harness/subagent_runner/ops_tests.rs b/src/openhuman/agent/harness/subagent_runner/ops_tests.rs index b0e85b950b..e77efd65d9 100644 --- a/src/openhuman/agent/harness/subagent_runner/ops_tests.rs +++ b/src/openhuman/agent/harness/subagent_runner/ops_tests.rs @@ -335,6 +335,7 @@ fn text_response_with_reasoning(text: &str, reasoning: &str) -> ModelResponse { raw: None, resolved_model: None, continue_turn: None, + served_from_cache: false, } } @@ -355,6 +356,7 @@ fn tool_response(name: &str, args: &str) -> ModelResponse { raw: None, resolved_model: None, continue_turn: None, + served_from_cache: false, } } diff --git a/src/openhuman/agent/message_convert.rs b/src/openhuman/agent/message_convert.rs index 5084a76162..f40467cd67 100644 --- a/src/openhuman/agent/message_convert.rs +++ b/src/openhuman/agent/message_convert.rs @@ -115,6 +115,7 @@ pub(crate) fn chat_message_to_message(msg: &ChatMessage) -> Message { tool_call_id, content: vec![ContentBlock::Text(content)], trusted_verbatim: false, + artifact: None, }) } // "user" and any unrecognized role default to a user turn — the safest @@ -742,6 +743,7 @@ mod tests { tool_call_id: "call-7".into(), content: vec![ContentBlock::Text("done".into())], trusted_verbatim: false, + artifact: None, })]; let back = messages_to_history(&messages); assert_eq!(back[0].role, "tool"); @@ -770,6 +772,7 @@ mod tests { tool_call_id: "c1".into(), content: vec![ContentBlock::Text("echoed:hi".into())], trusted_verbatim: false, + artifact: None, }), Message::Assistant(AssistantMessage { id: None, diff --git a/src/openhuman/agent/orchestration/tools/spawn_parallel_agents_tests.rs b/src/openhuman/agent/orchestration/tools/spawn_parallel_agents_tests.rs index 8465c51d81..a63bd83024 100644 --- a/src/openhuman/agent/orchestration/tools/spawn_parallel_agents_tests.rs +++ b/src/openhuman/agent/orchestration/tools/spawn_parallel_agents_tests.rs @@ -784,6 +784,7 @@ fn tool_response(name: &str, arguments: serde_json::Value) -> ModelResponse { raw: None, resolved_model: None, continue_turn: None, + served_from_cache: false, } } diff --git a/src/openhuman/agent/tinyagents/middleware.rs b/src/openhuman/agent/tinyagents/middleware.rs index 7ca20ff7c3..f81101053f 100644 --- a/src/openhuman/agent/tinyagents/middleware.rs +++ b/src/openhuman/agent/tinyagents/middleware.rs @@ -4374,6 +4374,7 @@ mod tests { raw: None, resolved_model: None, continue_turn: None, + served_from_cache: false, } } diff --git a/src/openhuman/agent/tinyagents/mod.rs b/src/openhuman/agent/tinyagents/mod.rs index 11ecf264cc..b3a0207aba 100644 --- a/src/openhuman/agent/tinyagents/mod.rs +++ b/src/openhuman/agent/tinyagents/mod.rs @@ -217,6 +217,8 @@ fn run_policy_for(max_iterations: usize, response_cache_enabled: bool) -> RunPol multiplier: 2.0, jitter: false, backoff_sleep: true, + max_retry_after_ms: RetryPolicy::DEFAULT_MAX_RETRY_AFTER_MS, + retry_on: None, }; // Unknown-tool recovery (01.2 / C3): the crate policy owns this end to end — // the `__openhuman_unknown_tool__` sentinel tool + `UnknownToolRewriteMiddleware` diff --git a/src/openhuman/agent/tinyagents/model.rs b/src/openhuman/agent/tinyagents/model.rs index 1b42f4c4ef..66a55c0531 100644 --- a/src/openhuman/agent/tinyagents/model.rs +++ b/src/openhuman/agent/tinyagents/model.rs @@ -161,6 +161,7 @@ fn response_to_model_response( raw: openhuman_usage_meta_raw(response.usage.as_ref()), resolved_model: None, continue_turn: None, + served_from_cache: false, } } @@ -740,7 +741,10 @@ mod g1_usage_tests { assert_eq!(response.text(), "Checking."); assert_eq!(response.message.tool_calls.len(), 1); - assert_eq!(response.message.tool_calls[0].id, "call_1"); + assert!( + !response.message.tool_calls[0].id.is_empty(), + "the upstream parser assigns the tool-call ID" + ); assert_eq!(response.message.tool_calls[0].name, "lookup"); assert_eq!( response.message.tool_calls[0].arguments, diff --git a/src/openhuman/agent/tinyagents/payload_summarizer.rs b/src/openhuman/agent/tinyagents/payload_summarizer.rs index 97c341cf82..1f44614041 100644 --- a/src/openhuman/agent/tinyagents/payload_summarizer.rs +++ b/src/openhuman/agent/tinyagents/payload_summarizer.rs @@ -453,7 +453,7 @@ impl SubagentPayloadSummarizer { } /// Rough token estimate: ~4 characters per token. Mirrors -/// [`crate::openhuman::memory::tree::tree_runtime::types::estimate_tokens`] but +/// [`crate::openhuman::memory::tree::tree_runtime::estimate_tokens`] but /// returns `usize` (not `u32`) and lives here to keep the tinyagents adapter /// independent from the tree summarizer. fn estimate_tokens(text: &str) -> usize { diff --git a/src/openhuman/channels/tests/common.rs b/src/openhuman/channels/tests/common.rs index f7d79663af..d62e165377 100644 --- a/src/openhuman/channels/tests/common.rs +++ b/src/openhuman/channels/tests/common.rs @@ -45,6 +45,7 @@ fn tool_call_response(step: Option) -> ModelResponse { raw: None, resolved_model: None, continue_turn: None, + served_from_cache: false, } } diff --git a/src/openhuman/config/ops/loader.rs b/src/openhuman/config/ops/loader.rs index 345201846d..aa3cf0c820 100644 --- a/src/openhuman/config/ops/loader.rs +++ b/src/openhuman/config/ops/loader.rs @@ -53,6 +53,67 @@ pub async fn load_config_with_timeout() -> Result { } } +/// Loads the config that belongs to `workspace_dir`, rather than whichever one +/// the process-global active-user / `OPENHUMAN_WORKSPACE` resolution currently +/// selects. +/// +/// Use this from anything scoped to a workspace it was *handed* — the memory +/// subsystem driver is the first such caller. [`load_config_with_timeout`] +/// re-resolves the process-global workspace on every call, so a component bound +/// to workspace B that loads through it and then merely overwrites +/// `workspace_dir` keeps A's embedding routes, model dimensions and provider +/// credentials, and runs them against B's files. +/// +/// The config file is looked for beside the workspace, in the two layouts the +/// resolver itself can produce: `/config.toml` (a workspace root +/// that carries its own config) and `/../config.toml` (the +/// `~/.openhuman/users//{config.toml,workspace}` layout). When neither +/// exists there is nothing workspace-specific to read, so this falls back to +/// the process-global load with `workspace_dir` re-anchored — the previous +/// behaviour, and still correct for a single-workspace host. +pub async fn load_config_for_workspace_with_timeout( + workspace_dir: &Path, +) -> Result { + let candidate = [ + workspace_dir.join("config.toml"), + workspace_dir + .parent() + .map(|parent| parent.join("config.toml")) + .unwrap_or_default(), + ] + .into_iter() + .find(|path| path.is_file()); + + if let Some(config_path) = candidate { + tracing::debug!( + config_path = %config_path.display(), + workspace = %workspace_dir.display(), + "[config] loading workspace-anchored config" + ); + return match tokio::time::timeout( + CONFIG_LOAD_TIMEOUT, + Config::load_from_config_path(&config_path, workspace_dir), + ) + .await + { + Ok(Ok(mut config)) => { + normalize_loaded_config(&mut config).await; + Ok(config) + } + Ok(Err(e)) => Err(format!("{e:#}")), + Err(_) => Err("Config loading timed out".to_string()), + }; + } + + tracing::debug!( + workspace = %workspace_dir.display(), + "[config] no config.toml beside workspace; falling back to the process-global load" + ); + let mut config = load_config_with_timeout().await?; + config.workspace_dir = workspace_dir.to_path_buf(); + Ok(config) +} + /// Reloads the config file represented by an existing runtime snapshot. /// /// Use this for long-lived objects that need fresh config values while diff --git a/src/openhuman/config/ops/mod.rs b/src/openhuman/config/ops/mod.rs index 2c83c15e9a..8a209954fd 100644 --- a/src/openhuman/config/ops/mod.rs +++ b/src/openhuman/config/ops/mod.rs @@ -24,7 +24,8 @@ pub use agent::{ pub use loader::{ agent_server_status, client_config_json, core_rpc_url_from_env, get_config_snapshot, get_dashboard_settings, get_data_paths, get_data_paths_for_user, get_runtime_flags, - load_and_get_client_config_snapshot, load_and_get_config_snapshot, load_config_with_timeout, + load_and_get_client_config_snapshot, load_and_get_config_snapshot, + load_config_for_workspace_with_timeout, load_config_with_timeout, reload_config_snapshot_with_timeout, reset_local_data, set_browser_allow_all, snapshot_config_json, RuntimeFlagsOut, }; diff --git a/src/openhuman/config/schema/load/env_overlay.rs b/src/openhuman/config/schema/load/env_overlay.rs index 3b2620ac7f..b435a898b8 100644 --- a/src/openhuman/config/schema/load/env_overlay.rs +++ b/src/openhuman/config/schema/load/env_overlay.rs @@ -177,6 +177,7 @@ impl Config { self.apply_observability_env(env); self.apply_learning_env(env); self.apply_memory_tree_env(env); + self.apply_subsystems_env(env); self.apply_update_env(env); self.apply_dictation_env(env); self.apply_context_env(env); @@ -852,6 +853,66 @@ impl Config { } } + /// `[subsystems.memory]` overrides — kernel.md §3.6 / plan-memory.md §4.5. Mirrors + /// the `apply_memory_tree_env` reading pattern above. GREENFIELD: nothing + /// reads `self.subsystems` yet, so these overrides have no runtime effect + /// beyond making the field settable via env for forward compatibility. + fn apply_subsystems_env(&mut self, env: &E) { + if let Some(raw) = env.get("OPENHUMAN_MEMORY_DRIVER") { + let trimmed = raw.trim(); + if !trimmed.is_empty() { + self.subsystems.memory.driver = trimmed.to_string(); + } + } + + if let Some(raw) = env.get("OPENHUMAN_MEMORY_HOOKS_AUTO_RECALL") { + if let Some(val) = parse_env_bool("OPENHUMAN_MEMORY_HOOKS_AUTO_RECALL", &raw) { + self.subsystems.memory.hooks.auto_recall = val; + } + } + if let Some(raw) = env.get("OPENHUMAN_MEMORY_HOOKS_AUTO_CAPTURE") { + if let Some(val) = parse_env_bool("OPENHUMAN_MEMORY_HOOKS_AUTO_CAPTURE", &raw) { + self.subsystems.memory.hooks.auto_capture = val; + } + } + if let Some(raw) = env.get("OPENHUMAN_MEMORY_HOOKS_MAX_CONTEXT_TOKENS") { + let trimmed = raw.trim(); + if !trimmed.is_empty() { + match trimmed.parse::() { + Ok(v) => self.subsystems.memory.hooks.max_context_tokens = v, + Err(_) => tracing::warn!( + value = %raw, + "invalid OPENHUMAN_MEMORY_HOOKS_MAX_CONTEXT_TOKENS ignored; expected an unsigned integer" + ), + } + } + } + if let Some(raw) = env.get("OPENHUMAN_MEMORY_HOOKS_RECALL_MAX_CHARS") { + let trimmed = raw.trim(); + if !trimmed.is_empty() { + match trimmed.parse::() { + Ok(v) => self.subsystems.memory.hooks.recall_max_chars = v, + Err(_) => tracing::warn!( + value = %raw, + "invalid OPENHUMAN_MEMORY_HOOKS_RECALL_MAX_CHARS ignored; expected an unsigned integer" + ), + } + } + } + if let Some(raw) = env.get("OPENHUMAN_MEMORY_HOOKS_CAPTURE_MAX_CHARS") { + let trimmed = raw.trim(); + if !trimmed.is_empty() { + match trimmed.parse::() { + Ok(v) => self.subsystems.memory.hooks.capture_max_chars = v, + Err(_) => tracing::warn!( + value = %raw, + "invalid OPENHUMAN_MEMORY_HOOKS_CAPTURE_MAX_CHARS ignored; expected an unsigned integer" + ), + } + } + } + } + fn apply_update_env(&mut self, env: &E) { if let Some(flag) = env.get("OPENHUMAN_AUTO_UPDATE_ENABLED") { let normalized = flag.trim().to_ascii_lowercase(); diff --git a/src/openhuman/config/schema/load_tests.rs b/src/openhuman/config/schema/load_tests.rs index 264172d3b6..0b4ea74832 100644 --- a/src/openhuman/config/schema/load_tests.rs +++ b/src/openhuman/config/schema/load_tests.rs @@ -810,6 +810,44 @@ fn env_overlay_memory_sync_interval_parses_and_honours_zero() { assert_eq!(cfg.memory_sync_interval_secs, Some(0)); } +#[test] +fn env_overlay_subsystems_memory_driver_and_hooks_apply() { + let mut cfg = Config::default(); + assert_eq!(cfg.subsystems.memory.driver, "tinycortex"); + assert!(cfg.subsystems.memory.hooks.auto_recall); + assert!(cfg.subsystems.memory.hooks.auto_capture); + assert_eq!(cfg.subsystems.memory.hooks.max_context_tokens, 2000); + assert_eq!(cfg.subsystems.memory.hooks.recall_max_chars, 1000); + assert_eq!(cfg.subsystems.memory.hooks.capture_max_chars, 500); + + cfg.apply_env_overlay_with( + &HashMapEnv::new() + .with("OPENHUMAN_MEMORY_DRIVER", "supermemory") + .with("OPENHUMAN_MEMORY_HOOKS_AUTO_RECALL", "off") + .with("OPENHUMAN_MEMORY_HOOKS_AUTO_CAPTURE", "false") + .with("OPENHUMAN_MEMORY_HOOKS_MAX_CONTEXT_TOKENS", "4000") + .with("OPENHUMAN_MEMORY_HOOKS_RECALL_MAX_CHARS", "2000") + .with("OPENHUMAN_MEMORY_HOOKS_CAPTURE_MAX_CHARS", "900"), + ); + + assert_eq!(cfg.subsystems.memory.driver, "supermemory"); + assert!(!cfg.subsystems.memory.hooks.auto_recall); + assert!(!cfg.subsystems.memory.hooks.auto_capture); + assert_eq!(cfg.subsystems.memory.hooks.max_context_tokens, 4000); + assert_eq!(cfg.subsystems.memory.hooks.recall_max_chars, 2000); + assert_eq!(cfg.subsystems.memory.hooks.capture_max_chars, 900); + + // A blank driver value is ignored, leaving the previous override intact. + cfg.apply_env_overlay_with(&HashMapEnv::new().with("OPENHUMAN_MEMORY_DRIVER", " ")); + assert_eq!(cfg.subsystems.memory.driver, "supermemory"); + + // A non-numeric budget value is ignored, leaving the previous value intact. + cfg.apply_env_overlay_with( + &HashMapEnv::new().with("OPENHUMAN_MEMORY_HOOKS_MAX_CONTEXT_TOKENS", "nope"), + ); + assert_eq!(cfg.subsystems.memory.hooks.max_context_tokens, 4000); +} + #[test] fn env_overlay_output_language_accepts_non_empty_value() { let mut cfg = Config::default(); diff --git a/src/openhuman/config/schema/mod.rs b/src/openhuman/config/schema/mod.rs index 4175f67cc2..cc0658c245 100644 --- a/src/openhuman/config/schema/mod.rs +++ b/src/openhuman/config/schema/mod.rs @@ -51,6 +51,7 @@ mod runtime_pool; mod runtime_python; mod scheduler_gate; mod storage_memory; +mod subsystems; mod task_sources; mod tokenjuice; mod tools; @@ -99,6 +100,9 @@ pub use storage_memory::{ LlmBackend, MemoryConfig, MemoryTreeConfig, StorageConfig, StorageProviderConfig, StorageProviderSection, DEFAULT_CLOUD_LLM_MODEL, }; +pub use subsystems::{ + MemoryDriverConfig, MemoryHooksConfig, MemorySubsystemConfig, SubsystemsConfig, +}; pub use task_sources::TaskSourcesConfig; pub use tokenjuice::TokenjuiceConfig; pub use tools::{ diff --git a/src/openhuman/config/schema/subsystems.rs b/src/openhuman/config/schema/subsystems.rs new file mode 100644 index 0000000000..8a70a48d08 --- /dev/null +++ b/src/openhuman/config/schema/subsystems.rs @@ -0,0 +1,368 @@ +//! `[subsystems.*]` config section — the uniform cross-subsystem driver-binding +//! shape defined in `docs/specs/kernel.md` §3.6 and `docs/specs/plan-memory.md` §4.5. +//! +//! GREENFIELD / ZERO BEHAVIOUR CHANGE: nothing reads this config yet. It exists +//! so `[subsystems.memory]` can be authored today and so `inference`, +//! `channels`, `sandbox`, … can slot in later as sibling fields on +//! [`SubsystemsConfig`] without reshaping this type. +//! +//! Shape (kernel.md §3.6 / plan-memory.md §4.5): +//! +//! ```toml +//! [subsystems.memory] +//! driver = "tinycortex" +//! +//! [subsystems.memory.hooks] +//! auto_recall = true +//! auto_capture = true +//! max_context_tokens = 2000 +//! recall_max_chars = 1000 +//! capture_max_chars = 500 +//! +//! [subsystems.memory.drivers.supermemory] +//! class = "external" +//! transport = "http" +//! endpoint = "https://api.supermemory.ai" +//! credential_ref = "keychain:supermemory" +//! trust_state = "untrusted" +//! ``` + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; + +/// Top-level `[subsystems]` config block. Currently carries only `memory`; +/// future subsystems (`inference`, `channels`, `sandbox`, …) are added here +/// as sibling fields — see kernel.md §3.6. +#[derive(Debug, Clone, Serialize, Deserialize, Default, JsonSchema)] +#[serde(default)] +pub struct SubsystemsConfig { + #[serde(default)] + pub memory: MemorySubsystemConfig, +} + +/// `[subsystems.memory]` — which driver is bound for the memory subsystem, +/// its hook budgets, and the per-driver option table. +/// +/// `PartialEq`/`Eq` let [`CoreContext::rebind_workspace`] short-circuit a +/// no-op rebind by comparing the config it was handed against the one already +/// held — equality is value comparison only, so it never prints or leaks the +/// credential fields the way `Debug` would. `Hash` lets [`binding`](crate::openhuman::memory::binding) +/// key its per-workspace cache on the whole config, so a changed driver/hooks/ +/// trust for an already-bound workspace yields a fresh binding rather than a +/// stale cache hit. +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize, JsonSchema)] +#[serde(default)] +pub struct MemorySubsystemConfig { + /// The bound driver id (e.g. `"tinycortex"`, `"supermemory"`, `"null"`). + /// Must match a key under `drivers` when that driver needs options. + #[serde(default = "default_memory_driver")] + pub driver: String, + + #[serde(default)] + pub hooks: MemoryHooksConfig, + + /// Per-driver option tables, keyed by driver id. The embedded default + /// (`tinycortex`) needs no entry here — its options continue to live in + /// the existing `[memory]` / `[memory_tree]` / `[[memory_sources]]` + /// blocks (plan-memory.md §4.5: "no user-visible config break"). + #[serde(default)] + pub drivers: BTreeMap, +} + +fn default_memory_driver() -> String { + "tinycortex".into() +} + +impl Default for MemorySubsystemConfig { + fn default() -> Self { + Self { + driver: default_memory_driver(), + hooks: MemoryHooksConfig::default(), + drivers: BTreeMap::new(), + } + } +} + +/// Memory-hook budgets — the auto-recall / auto-capture behavior gating +/// values. Defaults reproduce today's (pre-`[subsystems]`) behavior exactly; +/// nothing reads these yet. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize, JsonSchema)] +#[serde(default)] +pub struct MemoryHooksConfig { + #[serde(default = "default_true")] + pub auto_recall: bool, + #[serde(default = "default_true")] + pub auto_capture: bool, + #[serde(default = "default_max_context_tokens")] + pub max_context_tokens: usize, + #[serde(default = "default_recall_max_chars")] + pub recall_max_chars: usize, + #[serde(default = "default_capture_max_chars")] + pub capture_max_chars: usize, +} + +fn default_true() -> bool { + true +} +fn default_max_context_tokens() -> usize { + 2000 +} +fn default_recall_max_chars() -> usize { + 1000 +} +fn default_capture_max_chars() -> usize { + 500 +} + +impl Default for MemoryHooksConfig { + fn default() -> Self { + Self { + auto_recall: default_true(), + auto_capture: default_true(), + max_context_tokens: default_max_context_tokens(), + recall_max_chars: default_recall_max_chars(), + capture_max_chars: default_capture_max_chars(), + } + } +} + +/// One entry under `[subsystems.memory.drivers.]`. Describes an +/// external/embedded driver binding — class, transport, endpoint, and a +/// *reference* to a credential resolved via the keychain (never an inline +/// secret; plan-memory.md §4.5, kernel.md §3.6). +/// +/// `trust_state` is fail-closed `"untrusted"` per kernel.md §3.4: an external +/// driver must have its trust explicitly raised before bind succeeds. +/// +/// MUST NOT derive `Debug` — see the manual impl below. `credential_ref` is a +/// secret handle and plan-memory.md §7 Tier-3 conformance requires "credential never +/// in `Debug`/error output", mirroring [`super::storage_memory::MemoryConfig`]'s +/// manual redacting `Debug` impl for `agentmemory_secret`. +/// +/// `PartialEq`/`Eq` are safe to derive: they compare values for equality and +/// never render them, so `credential_ref` stays out of any output. +#[derive(Clone, PartialEq, Eq, Hash, Serialize, Deserialize, JsonSchema)] +#[serde(default)] +pub struct MemoryDriverConfig { + /// Driver class: `"embedded"` | `"external"` | `"null"`. See kernel.md §3.1. + #[serde(default)] + pub class: Option, + + /// Wire transport for external drivers, e.g. `"http"`. See plan-memory.md §4.2. + #[serde(default)] + pub transport: Option, + + /// Base endpoint URL for external/http drivers. + #[serde(default)] + pub endpoint: Option, + + /// A *reference* to a credential (e.g. `"keychain:supermemory"`), + /// resolved kernel-side through the existing keychain — never an inline + /// secret. Redacted in `Debug`/error output; see the manual `Debug` impl. + #[serde(default)] + pub credential_ref: Option, + + /// Fail-closed trust state for this driver binding. Defaults to + /// `"untrusted"`; must be explicitly raised before an external driver's + /// bind succeeds (kernel.md §3.4). + #[serde(default = "default_trust_state")] + pub trust_state: String, +} + +fn default_trust_state() -> String { + "untrusted".into() +} + +impl Default for MemoryDriverConfig { + fn default() -> Self { + Self { + class: None, + transport: None, + endpoint: None, + credential_ref: None, + trust_state: default_trust_state(), + } + } +} + +// Manual `Debug` implementation that redacts `credential_ref`. Without this, +// any `format!("{cfg:?}")` / `tracing::debug!(?cfg, ...)` / panic message +// capturing a `MemoryDriverConfig` would dump the credential reference +// verbatim. The value itself (e.g. `"keychain:supermemory"`) is only a +// *reference*, not the secret — but plan-memory.md §7 Tier-3 conformance requires it +// never appear in Debug/error output regardless, so this mirrors +// `MemoryConfig`'s `agentmemory_secret` treatment exactly. NEVER derive +// `Debug` on this struct. +impl std::fmt::Debug for MemoryDriverConfig { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("MemoryDriverConfig") + .field("class", &self.class) + .field("transport", &self.transport) + .field("endpoint", &self.endpoint) + .field( + "credential_ref", + &self.credential_ref.as_ref().map(|_| ""), + ) + .field("trust_state", &self.trust_state) + .finish() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn subsystems_config_defaults_reproduce_today_behavior() { + let cfg = SubsystemsConfig::default(); + assert_eq!(cfg.memory.driver, "tinycortex"); + assert!(cfg.memory.hooks.auto_recall); + assert!(cfg.memory.hooks.auto_capture); + assert_eq!(cfg.memory.hooks.max_context_tokens, 2000); + assert_eq!(cfg.memory.hooks.recall_max_chars, 1000); + assert_eq!(cfg.memory.hooks.capture_max_chars, 500); + assert!(cfg.memory.drivers.is_empty()); + } + + #[test] + fn absent_subsystems_block_deserializes_to_default() { + let cfg: SubsystemsConfig = toml::from_str("").expect("empty toml parses"); + assert_eq!( + serde_json::to_value(&cfg).unwrap(), + serde_json::to_value(SubsystemsConfig::default()).unwrap() + ); + } + + #[test] + fn full_subsystems_memory_block_round_trips_on_subsystems_config_directly() { + // Deserializing straight into `SubsystemsConfig` — the root table is + // `memory` (no `subsystems.` prefix), since `SubsystemsConfig` *is* + // the `[subsystems]` block's shape. + let toml_src = r#" +[memory] +driver = "supermemory" + +[memory.hooks] +auto_recall = false +auto_capture = false +max_context_tokens = 4000 +recall_max_chars = 2000 +capture_max_chars = 900 + +[memory.drivers.supermemory] +class = "external" +transport = "http" +endpoint = "https://api.supermemory.ai" +credential_ref = "keychain:supermemory" +trust_state = "trusted" +"#; + let cfg: SubsystemsConfig = toml::from_str(toml_src).expect("valid toml parses"); + assert_eq!(cfg.memory.driver, "supermemory"); + assert!(!cfg.memory.hooks.auto_recall); + assert!(!cfg.memory.hooks.auto_capture); + assert_eq!(cfg.memory.hooks.max_context_tokens, 4000); + assert_eq!(cfg.memory.hooks.recall_max_chars, 2000); + assert_eq!(cfg.memory.hooks.capture_max_chars, 900); + + let driver = cfg + .memory + .drivers + .get("supermemory") + .expect("supermemory driver entry present"); + assert_eq!(driver.class.as_deref(), Some("external")); + assert_eq!(driver.transport.as_deref(), Some("http")); + assert_eq!( + driver.endpoint.as_deref(), + Some("https://api.supermemory.ai") + ); + assert_eq!( + driver.credential_ref.as_deref(), + Some("keychain:supermemory") + ); + assert_eq!(driver.trust_state, "trusted"); + + // Round-trip through serialize -> deserialize preserves the same value. + let serialized = toml::to_string(&cfg).expect("serializes back to toml"); + let round_tripped: SubsystemsConfig = + toml::from_str(&serialized).expect("round-tripped toml parses"); + assert_eq!( + serde_json::to_value(&round_tripped).unwrap(), + serde_json::to_value(&cfg).unwrap() + ); + } + + #[test] + fn full_subsystems_memory_block_round_trips_on_top_level_config() { + // Same fixture, this time embedded under the real `[subsystems.memory]` + // path as it would appear in an actual `config.toml`, deserialized + // into the top-level `Config` to exercise the M2.1 wiring in + // `types.rs`. The existing `[memory]`, `[memory_tree]`, + // `[[memory_sources]]` blocks are untouched by this new section. + let toml_src = r#" +[subsystems.memory] +driver = "supermemory" + +[subsystems.memory.hooks] +auto_recall = false +auto_capture = false +max_context_tokens = 4000 +recall_max_chars = 2000 +capture_max_chars = 900 + +[subsystems.memory.drivers.supermemory] +class = "external" +transport = "http" +endpoint = "https://api.supermemory.ai" +credential_ref = "keychain:supermemory" +trust_state = "trusted" +"#; + let cfg: super::super::Config = toml::from_str(toml_src).expect("valid toml parses"); + assert_eq!(cfg.subsystems.memory.driver, "supermemory"); + assert!(!cfg.subsystems.memory.hooks.auto_recall); + assert!(!cfg.subsystems.memory.hooks.auto_capture); + assert_eq!(cfg.subsystems.memory.hooks.max_context_tokens, 4000); + assert_eq!(cfg.subsystems.memory.hooks.recall_max_chars, 2000); + assert_eq!(cfg.subsystems.memory.hooks.capture_max_chars, 900); + + let driver = cfg + .subsystems + .memory + .drivers + .get("supermemory") + .expect("supermemory driver entry present"); + assert_eq!(driver.class.as_deref(), Some("external")); + assert_eq!(driver.trust_state, "trusted"); + + // The pre-existing [memory] / [memory_tree] / [[memory_sources]] + // blocks are absent from this fixture and must still deserialize to + // their own defaults, proving `[subsystems.*]` is additive. + assert_eq!(cfg.memory.backend, "sqlite"); + assert!(cfg.memory_sources.is_empty()); + } + + #[test] + fn memory_driver_config_debug_never_leaks_credential_ref() { + let driver = MemoryDriverConfig { + class: Some("external".into()), + transport: Some("http".into()), + endpoint: Some("https://api.supermemory.ai".into()), + credential_ref: Some("keychain:supermemory-super-secret-value".into()), + trust_state: "untrusted".into(), + }; + let debug_output = format!("{driver:?}"); + assert!( + !debug_output.contains("keychain:supermemory-super-secret-value"), + "Debug output must never contain the credential_ref value: {debug_output}" + ); + assert!( + debug_output.contains(""), + "Debug output should show a redaction marker: {debug_output}" + ); + } + + #[test] + fn memory_driver_config_default_trust_state_is_untrusted() { + assert_eq!(MemoryDriverConfig::default().trust_state, "untrusted"); + } +} diff --git a/src/openhuman/config/schema/types.rs b/src/openhuman/config/schema/types.rs index 452de50bdd..3f37772eab 100644 --- a/src/openhuman/config/schema/types.rs +++ b/src/openhuman/config/schema/types.rs @@ -263,6 +263,14 @@ pub struct Config { #[serde(default)] pub storage: StorageConfig, + /// `[subsystems.*]` — the uniform cross-subsystem driver-binding config + /// (kernel.md §3.6 / plan-memory.md §4.5). Currently only `subsystems.memory` is + /// populated; nothing reads this yet (zero behaviour change). The + /// existing `[memory]`, `[memory_tree]`, `[[memory_sources]]` blocks + /// above are unaffected. + #[serde(default)] + pub subsystems: SubsystemsConfig, + #[serde(default)] pub composio: ComposioConfig, @@ -781,6 +789,7 @@ impl Default for Config { memory: MemoryConfig::default(), memory_tree: MemoryTreeConfig::default(), storage: StorageConfig::default(), + subsystems: SubsystemsConfig::default(), composio: ComposioConfig::default(), secrets: SecretsConfig::default(), browser: BrowserConfig::default(), diff --git a/src/openhuman/desktop/app_state/ops.rs b/src/openhuman/desktop/app_state/ops.rs index 76a86861af..2f144d86f0 100644 --- a/src/openhuman/desktop/app_state/ops.rs +++ b/src/openhuman/desktop/app_state/ops.rs @@ -534,8 +534,9 @@ async fn finish_revalidated_user_activation( "{LOG_PREFIX} failed to bind memory client after pending session revalidation: {error}" ); } - if let Err(error) = crate::core::runtime::context::CoreContext::rebind_default_workspace_dir( + if let Err(error) = crate::core::runtime::context::CoreContext::rebind_default_workspace( &target_config.workspace_dir, + target_config.subsystems.memory.clone(), ) { warn!("{LOG_PREFIX} failed to rebind core context after pending session revalidation: {error}"); } diff --git a/src/openhuman/flows/tinyflows/caps/ops.rs b/src/openhuman/flows/tinyflows/caps/ops.rs index 3df609aec0..e0a212a6e8 100644 --- a/src/openhuman/flows/tinyflows/caps/ops.rs +++ b/src/openhuman/flows/tinyflows/caps/ops.rs @@ -2429,6 +2429,7 @@ mod tests { ), resolved_model: None, continue_turn: None, + served_from_cache: false, }; let value = model_response_to_completion_value(&response); diff --git a/src/openhuman/inference/local/service/model_rpc.rs b/src/openhuman/inference/local/service/model_rpc.rs index e09d93da09..e3d974cef3 100644 --- a/src/openhuman/inference/local/service/model_rpc.rs +++ b/src/openhuman/inference/local/service/model_rpc.rs @@ -224,6 +224,7 @@ mod tests { raw: None, resolved_model: None, continue_turn: None, + served_from_cache: false, }; assert!(model_outcome(response(" ", Usage::default()), false).is_err()); @@ -262,6 +263,7 @@ mod tests { raw: None, resolved_model: None, continue_turn: None, + served_from_cache: false, }; assert_eq!( model_outcome(reasoning_only, false).unwrap().reply, diff --git a/src/openhuman/inference/provider/openhuman_backend_model.rs b/src/openhuman/inference/provider/openhuman_backend_model.rs index 2c02040da3..ef88611191 100644 --- a/src/openhuman/inference/provider/openhuman_backend_model.rs +++ b/src/openhuman/inference/provider/openhuman_backend_model.rs @@ -529,6 +529,7 @@ mod tests { raw: Some(raw), resolved_model: None, continue_turn: None, + served_from_cache: false, }; let projected = project_managed_usage(response); @@ -569,6 +570,7 @@ mod tests { raw: Some(serde_json::json!({ "id": "resp_1" })), resolved_model: None, continue_turn: None, + served_from_cache: false, }; let projected = project_managed_usage(response); @@ -595,6 +597,7 @@ mod tests { code: Some("BAD_REQUEST".to_string()), message: "API key not configured for provider".to_string(), retryable: false, + retry_after_ms: None, raw: None, }; assert!(is_provider_not_configured_error(&err)); @@ -612,6 +615,7 @@ mod tests { code: Some("BAD_REQUEST".to_string()), message: "invalid request: messages must not be empty".to_string(), retryable: false, + retry_after_ms: None, raw: None, }; assert!(!is_provider_not_configured_error(&err)); @@ -630,6 +634,7 @@ mod tests { code: Some("BAD_REQUEST".to_string()), message: "credentials not configured for provider 'anthropic'".to_string(), retryable: false, + retry_after_ms: None, raw: None, }; assert!(is_provider_not_configured_error(&err)); @@ -651,6 +656,7 @@ mod tests { code: Some("BAD_REQUEST".to_string()), message: "webhook target not configured".to_string(), retryable: false, + retry_after_ms: None, raw: None, }; assert!(!is_provider_not_configured_error(&err)); @@ -665,6 +671,7 @@ mod tests { code: None, message: "API key not configured for provider".to_string(), retryable: false, + retry_after_ms: None, raw: None, }; assert!(!is_provider_not_configured_error(&err)); diff --git a/src/openhuman/memory/binding.rs b/src/openhuman/memory/binding.rs new file mode 100644 index 0000000000..474631e004 --- /dev/null +++ b/src/openhuman/memory/binding.rs @@ -0,0 +1,530 @@ +//! Per-workspace memory-driver binding — the memory subsystem's half of +//! `docs/specs/kernel.md` §3.1 (one driver per subsystem per process, per +//! workspace here), §3.4 (fail-closed trust), and §3.7 (a fallback is never +//! silent). +//! +//! ## Reached through [`CoreContext`], never through a global slot +//! +//! The binding is resolved by +//! [`CoreContext::memory_binding`](crate::core::runtime::CoreContext::memory_binding), +//! which keys on the context's workspace dir. The cache below is deliberately +//! shaped like +//! [`memory::people::store::for_workspace`](crate::openhuman::memory::people::store::for_workspace) +//! — a **workspace-and-config-keyed map** — and deliberately *not* like +//! [`memory::global`](crate::openhuman::memory::global), which is a single slot +//! holding "the one active-user workspace". +//! +//! That shape choice carries a real correctness property for free. +//! `memory::global::init` needs an explicit clear-on-failed-rebind guard so a +//! failed switch to workspace B cannot leave callers writing into workspace A. +//! With a workspace-keyed map there is no shared slot to go stale: a context +//! bound to B resolves the entry for B or falls back, and can never be handed +//! A's driver. Pinned by +//! `failed_bind_never_returns_previous_workspace_binding` in +//! `src/core/runtime/context.rs`. +//! +//! ## Two vocabularies meet here, on purpose +//! +//! [`tinycortex_api`] is the *memory contract*: `MemoryProvider`, +//! `Capabilities`, `MemoryHealth`. [`crate::core::subsystem`] is the kernel's +//! *generic* driver vocabulary shared with the subsystems that come after +//! memory: `DriverClass`, `DriverCapabilities`, `DriverHealth`, `BoundDriver`. +//! This module is the adapter between them — the only place in the tree where +//! the conversion lives. `DriverClass` is reused from the kernel rather than +//! redefined here precisely because it is a *host* fact about how a driver was +//! bound, identical for every subsystem. +//! +//! ## Scope of this step (M3d) +//! +//! The [`DriverClass::Embedded`] arm of [`build`] binds the real +//! [`EmbeddedMemoryProvider`], which wraps the in-process tinycortex engine. +//! [`DriverClass::Null`] still binds [`NullMemoryProvider`] — an operator who +//! wrote `driver = "null"` asked for `/dev/null` and must get it — and so does +//! every fallback. +//! +//! The embedded driver now implements **all thirteen** families, so a bound +//! context and an unbound one advertise the same set. That was the whole point +//! of M3: before it, binding *narrowed* the advertised set from thirteen +//! families to the null placeholder's three, which made gating anything on +//! `memory_capabilities()` actively dangerous. It is now safe, and M4 is where +//! that gating lands. +//! +//! A fallback binding still advertises only the mandatory three, because a +//! fallback really is the null placeholder — that is the honest answer, not a +//! leftover. + +use std::collections::HashMap; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, OnceLock, RwLock}; + +use tinycortex_api::capabilities::Capabilities; +use tinycortex_api::health::MemoryHealth; +use tinycortex_api::null::{NullMemoryProvider, NULL_DRIVER_ID}; +use tinycortex_api::provider::MemoryProvider; +use tinycortex_api::CONTRACT_VERSION; + +use crate::core::subsystem::{ + BoundDriver, DriverCapabilities, DriverClass, DriverHealth, SubsystemSlot, +}; +use crate::openhuman::config::schema::{MemoryHooksConfig, MemorySubsystemConfig}; +use crate::openhuman::memory::driver::embedded::{EmbeddedMemoryProvider, EMBEDDED_DRIVER_ID}; +use crate::openhuman::memory::guard::{GuardPolicy, MemoryGuard}; + +/// Why a bind fell back to the placeholder driver. +/// +/// `reason` is operator-facing: it is logged, published on the event bus, and +/// rendered in status. It must therefore never interpolate `credential_ref` or +/// `endpoint` from [`crate::openhuman::config::schema::MemoryDriverConfig`], +/// which carries a manual redacting `Debug` for exactly that reason. Pinned by +/// `fallback_reason_never_contains_credential_ref_or_endpoint`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct FallbackReason { + /// The driver id that was asked for in `[subsystems.memory] driver`. + pub configured_driver: String, + /// Why it was refused. + pub reason: String, +} + +/// One bound memory driver, for one workspace. +pub struct MemoryBinding { + provider: Arc, + /// The policy decorator over [`Self::unguarded_provider`] — the handle product code + /// receives, via `CoreContext::memory()`. Built here rather than by each + /// caller so "every caller gets a guarded handle" holds by construction, + /// the same way `capabilities()` is asked exactly once by construction. + guard: Arc, + driver_id: String, + class: DriverClass, + /// Asked **once**, at bind time, and cached here. The contract's + /// `MemoryProvider::capabilities` doc is normative on this ("asked once at + /// bind time and cached"): re-asking would let a driver's advertised + /// surface drift underneath an already-filtered RPC/tool registration. + capabilities: Capabilities, + fallback: Option, +} + +impl MemoryBinding { + /// The bound driver, **unguarded**. + /// + /// Retained for identity/health/status, which are liveness probes rather + /// than product code (`memory::ops::provider` is the one production + /// caller). New call sites want [`Self::guard`] — see + /// `CoreContext::memory()`. + /// + /// Named `unguarded_provider` rather than `provider` on purpose. The + /// enforcement lint in `memory::bypass_allowlist_tests` matches text, and a + /// `.provider(` needle would over-match `TaskSourceFilter::provider()` and + /// `ModelRef::provider()` — six junk allowlist entries, which is exactly + /// the rot `bypass_allowlist_has_no_stale_entries` exists to prevent. A + /// distinctive name gives the lint a needle with no false positives, and + /// puts the hazard in the reader's face at the call site. + /// + /// Visibility is narrowed to the memory family so the lint's text match is + /// backed by a *compiler*-enforced boundary: even if `MemoryBinding` grows + /// another reachable path, no module outside `openhuman::memory` can name + /// this accessor at all. + pub(in crate::openhuman::memory) fn unguarded_provider(&self) -> &Arc { + &self.provider + } + + /// The guarded driver — the only handle product code should hold + /// (`docs/specs/kernel.md` §3.4). + pub fn guard(&self) -> Arc { + Arc::clone(&self.guard) + } + + /// The id of the driver that actually bound — `"null"` after a fallback, + /// not the id that was asked for (that is in [`Self::fallback`]). + pub fn driver_id(&self) -> &str { + &self.driver_id + } + + /// How the bound driver was reached. A host fact, never self-reported. + pub fn class(&self) -> DriverClass { + self.class + } + + /// The cached capability set. Cheap: `Capabilities` is a `Copy` bitset. + pub fn capabilities(&self) -> Capabilities { + self.capabilities + } + + /// `Some` when this binding is a fallback; `None` when the configured + /// driver bound as asked. + pub fn fallback(&self) -> Option<&FallbackReason> { + self.fallback.as_ref() + } + + /// Whether the operator asked for memory to be **off**. + /// + /// True only for a deliberate `[subsystems.memory] driver = "null"` — the + /// class alone is not enough, because a *fallback* also binds the null + /// placeholder and a misconfiguration must not silently take memory away + /// with it. A fallback is loud (`fallback()` is `Some`, status reports it) + /// and keeps the surface present. + /// + /// Read by [`CoreContext::memory_capabilities`](crate::core::runtime::context::CoreContext::memory_capabilities), + /// which answers with the empty set here, so the memory RPC methods and + /// memory agent tools are **absent** rather than present-and-answering off + /// some other store. That matters because most memory handlers still reach + /// the engine directly through `active_memory_client()` — the guarded + /// re-point is incremental and tracked in + /// `docs/specs/memory-guard-allowlist.md` — so leaving the surface + /// registered under a null binding would read the embedded SQLite store an + /// operator believed they had turned off. + pub fn disables_memory(&self) -> bool { + self.class == DriverClass::Null && self.fallback.is_none() + } + + /// This binding in the kernel's generic vocabulary, for the subsystem + /// registry and `subsystems_status` (kernel.md §6 item 6). This is the + /// memory adapter `core::subsystem`'s module docs said would land later. + pub fn to_bound_driver(&self) -> BoundDriver { + BoundDriver { + slot: SubsystemSlot::Memory, + id: self.driver_id.clone(), + class: self.class, + capabilities: to_driver_capabilities(self.capabilities), + health: DriverHealth::Ready, + contract_version: CONTRACT_VERSION, + fell_back_from: self.fallback.as_ref().map(|f| f.configured_driver.clone()), + } + } +} + +/// Convert the memory contract's typed capability set into the kernel's opaque +/// one. The kernel deliberately does not know memory's family vocabulary. +pub fn to_driver_capabilities(capabilities: Capabilities) -> DriverCapabilities { + capabilities.iter().map(|c| c.as_str()).collect() +} + +/// Convert the memory contract's health into the kernel's. A total three-arm +/// match, which is why both enums were shaped one-for-one. +pub fn to_driver_health(health: MemoryHealth) -> DriverHealth { + match health { + MemoryHealth::Ready => DriverHealth::Ready, + MemoryHealth::Degraded { reason } => DriverHealth::Degraded { reason }, + MemoryHealth::Down { reason } => DriverHealth::Down { reason }, + } +} + +/// The capability set assumed when nothing is bound. +/// +/// **Deliberately the full set.** This mirrors +/// [`crate::core::all`]'s `group_allowed`, which returns `true` when there is +/// no ambient context: roughly 4000 unit tests run pre-boot with no bound +/// driver, and a deny-by-default here would fail all of them at once. Denying a +/// capability is only ever correct *after* a driver has actually answered +/// `capabilities()`. +pub fn unbound_default_capabilities() -> Capabilities { + Capabilities::all() +} + +/// The class a built-in driver id is *fixed* to, or `None` for any other id. +/// +/// Both built-in ids name one specific implementation, so this is the authority +/// for their class in every path — the implicit one below and the explicit +/// `class = …` line in [`admit`], which may only confirm what this returns. +pub(crate) fn reserved_class(id: &str) -> Option { + match id { + NULL_DRIVER_ID => Some(DriverClass::Null), + EMBEDDED_DRIVER_ID => Some(DriverClass::Embedded), + _ => None, + } +} + +/// The class a driver id implies when nothing says otherwise. Only the two +/// built-in ids admit: the embedded default and the null placeholder. Anything +/// else — a typo, or an external backend that forgot its `drivers.` entry — +/// is refused so the fallback machinery surfaces the mistake in status instead +/// of mislabelling the bound engine. +/// +/// `context` names which part of the config was missing; the refusal echoes it +/// so the operator knows whether to add an entry or a `class` line. +fn implicit_class( + id: &str, + refuse: &impl Fn(&str) -> FallbackReason, + context: &str, +) -> Result<(String, DriverClass), FallbackReason> { + if let Some(class) = reserved_class(id) { + Ok((id.to_string(), class)) + } else { + Err(refuse(&format!( + "unknown driver id \"{id}\": {context}, and the id is neither the \ + embedded default nor \"null\"" + ))) + } +} + +/// Decide, from config alone, whether the configured driver may bind. +/// +/// Pure — no I/O, no globals — so the fail-closed trust rule is unit-testable +/// without booting anything. +/// +/// # Errors +/// +/// Returns the [`FallbackReason`] to record and publish when the configured +/// driver is refused. Callers fall back rather than failing: kernel.md §3.7 +/// requires the subsystem stay bound, loudly. +pub fn admit(cfg: &MemorySubsystemConfig) -> Result<(String, DriverClass), FallbackReason> { + let id = cfg.driver.trim(); + if id.is_empty() { + return Err(FallbackReason { + configured_driver: String::new(), + reason: "[subsystems.memory] driver is empty".to_string(), + }); + } + + let refuse = |reason: &str| FallbackReason { + configured_driver: id.to_string(), + reason: reason.to_string(), + }; + + // A driver needs no `[subsystems.memory.drivers.]` entry: the embedded + // default's options still live in the existing `[memory]` blocks. But only + // the two built-in ids are implicitly admitted — anything else is a typo or + // an external backend that forgot its entry, and admitting it would silently + // run TinyCortex under an invented driver id (kernel.md §3.1, one driver per + // slot, named truthfully). Refuse it so the fallback machinery surfaces the + // mistake in status instead of mislabelling the bound engine. + let Some(entry) = cfg.drivers.get(id) else { + return implicit_class(id, &refuse, "no [subsystems.memory.drivers.] entry"); + }; + + let (admitted_id, class) = match entry.class.as_deref() { + // An entry that names no class still cannot admit an arbitrary id: the + // embedded default is the only non-null id that implies Embedded. + None => implicit_class( + id, + &refuse, + "[subsystems.memory.drivers.] has no class line", + )?, + Some(raw) => { + let class = DriverClass::parse(raw).map_err(|e| refuse(&e))?; + // The two built-in ids name a *fixed* implementation, so an + // explicit `class` line may confirm it but never override it. + // Without this, `driver = "null"` plus + // `[subsystems.memory.drivers.null] class = "embedded"` would build + // `EmbeddedMemoryProvider`, advertise all thirteen families and + // persist memory under the id documented as `/dev/null`; the + // inverse would label a store-nothing provider `tinycortex`. Either + // way the bound engine is mislabelled, which is exactly what the + // implicit-class refusal above exists to prevent (kernel.md §3.1 — + // one driver per slot, named truthfully). + if let Some(fixed) = reserved_class(id) { + if class != fixed { + return Err(refuse(&format!( + "driver id \"{id}\" is built in and is always class \ + \"{}\"; remove the conflicting class = \"{raw}\" line", + fixed.as_str() + ))); + } + } + (id.to_string(), class) + } + }; + + if class == DriverClass::External { + // kernel.md §3.4: fail-closed. Trust must be explicitly raised before + // an out-of-process driver is allowed to answer for memory. + if entry.trust_state != "trusted" { + return Err(refuse( + "external driver is untrusted: set trust_state = \"trusted\" \ + under [subsystems.memory.drivers] to allow this binding", + )); + } + // Distinct reason string from the trust refusal above, so the trust + // test cannot pass for the wrong reason. + return Err(refuse( + "external driver transport is not implemented yet (the http adapter lands in M4)", + )); + } + + Ok((admitted_id, class)) +} + +/// Build the binding for a workspace. Infallible by design: an inadmissible +/// driver falls back to the placeholder rather than leaving the slot empty +/// (kernel.md §3.7 — "logged loudly, surfaced in status, never silent"). +fn build(workspace_dir: &Path, cfg: &MemorySubsystemConfig) -> MemoryBinding { + match admit(cfg) { + Ok((driver_id, class)) => { + let provider: Arc = match class { + // Construction is deliberately sync and I/O-free: this runs on + // `CoreContext::memory_binding`, which ~4000 pre-boot tests + // call with no tokio runtime. The driver resolves its client on + // first use — see `driver::embedded`'s module docs. + DriverClass::Embedded => { + Arc::new(EmbeddedMemoryProvider::new(workspace_dir, cfg.hooks)) + } + DriverClass::Null => Arc::new(NullMemoryProvider::new()), + // Unreachable: `admit` refuses every external driver above, so + // this arm cannot bind a transport that does not exist yet. + DriverClass::External => Arc::new(NullMemoryProvider::new()), + }; + // The configured trust state for the driver that actually bound. + // Absent `[subsystems.memory.drivers.]` entry ⇒ the fail-closed + // default, which only ever matters for an external class. + let trust_state = cfg + .drivers + .get(&driver_id) + .map(|entry| entry.trust_state.clone()) + .unwrap_or_else(|| crate::openhuman::memory::guard::policy::TRUSTED.to_string()); + let binding = bind_provider(provider, driver_id, class, cfg.hooks, trust_state, None); + log::info!( + "[memory:binding] workspace={} bound driver='{}' class={} capabilities=[{}]", + workspace_dir.display(), + binding.driver_id(), + binding.class(), + binding + .capabilities() + .iter() + .map(|c| c.as_str()) + .collect::>() + .join(",") + ); + binding + } + Err(fallback) => { + log::warn!( + "[memory:binding] workspace={} driver '{}' refused to bind ({}); \ + falling back to '{NULL_DRIVER_ID}' — memory writes are DISCARDED this run", + workspace_dir.display(), + fallback.configured_driver, + fallback.reason + ); + // Sync, and a no-op when the bus is not yet initialized, so this is + // safe to call pre-boot with no `#[cfg(test)]` guard. + crate::core::event_bus::publish_global( + crate::core::event_bus::DomainEvent::MemoryDriverBindFailed { + configured_driver: fallback.configured_driver.clone(), + bound_driver: NULL_DRIVER_ID.to_string(), + reason: fallback.reason.clone(), + }, + ); + bind_provider( + Arc::new(NullMemoryProvider::new()), + NULL_DRIVER_ID.to_string(), + DriverClass::Null, + cfg.hooks, + // The fallback binds the in-process placeholder, so there is no + // boundary to cross and nothing to trust-gate. The refused + // driver's own trust_state is deliberately NOT carried over — + // it describes a binding that did not happen. + crate::openhuman::memory::guard::policy::TRUSTED.to_string(), + Some(fallback), + ) + } + } +} + +/// The single place `capabilities()` is asked. Every construction path — real +/// bind, fallback, and the test seam — goes through here, so the "asked once +/// per bind" property holds by construction rather than by convention. +fn bind_provider( + provider: Arc, + driver_id: String, + class: DriverClass, + hooks: MemoryHooksConfig, + trust_state: String, + fallback: Option, +) -> MemoryBinding { + let capabilities = provider.capabilities(); + // Built on the same single path, so a binding can never exist without its + // guard and no caller has to remember to construct one. + let guard = Arc::new(MemoryGuard::new( + Arc::clone(&provider), + Arc::new(GuardPolicy::new( + driver_id.clone(), + class, + hooks, + trust_state, + )), + )); + MemoryBinding { + provider, + guard, + driver_id, + class, + capabilities, + fallback, + } +} + +/// Test-only injection seam: bind an arbitrary provider through the same +/// ask-once-and-cache path [`build`] uses. Exists because [`build`] hard-codes +/// the placeholder, so the "capabilities asked exactly once" property would +/// otherwise be untestable. +#[cfg(test)] +pub(crate) fn bind_provider_for_test( + provider: Arc, + class: DriverClass, +) -> MemoryBinding { + let driver_id = provider.driver_id().to_string(); + bind_provider( + provider, + driver_id, + class, + MemoryHooksConfig::default(), + crate::openhuman::memory::guard::policy::TRUSTED.to_string(), + None, + ) +} + +/// Per-workspace binding cache. Same shape as +/// `memory::people::store::STORES` — see the module docs for why this is a map +/// and not a slot. +/// +/// Keyed on the **binding-relevant config as well as the path**, not the path +/// alone, because a config change for an already-bound workspace must produce a +/// fresh binding. `CoreContext::rebind_workspace` deliberately treats "same +/// workspace, changed `[subsystems.memory]`" as a real rebind — a changed +/// `driver` / `hooks` / `drivers` (trust) all feed `build`, so a path-only key +/// would keep serving the previous driver until restart. Carrying +/// `MemorySubsystemConfig` in the key (it derives `Hash`) means a changed +/// config hits a different slot and binds fresh, while a returned-to config +/// still resolves its original binding. +type BindingCacheKey = (PathBuf, MemorySubsystemConfig); +type BindingCache = RwLock>>; + +static BINDINGS: OnceLock = OnceLock::new(); + +/// The bound memory driver for `workspace_dir`, constructing it on first use. +/// +/// The same workspace always resolves to the same cached `Arc` (so +/// `capabilities()` is asked once); different workspaces get isolated bindings. +/// +/// # Errors +/// +/// Only lock poisoning. A driver that cannot bind is *not* an error here — it +/// falls back, per kernel.md §3.7. +pub fn for_workspace( + workspace_dir: &Path, + cfg: &MemorySubsystemConfig, +) -> Result, String> { + let cache = BINDINGS.get_or_init(Default::default); + let key = (workspace_dir.to_path_buf(), cfg.clone()); + if let Some(binding) = cache + .read() + .map_err(|e| format!("[memory:binding] cache read lock poisoned: {e}"))? + .get(&key) + { + return Ok(Arc::clone(binding)); + } + + let binding = Arc::new(build(workspace_dir, cfg)); + + let mut guard = cache + .write() + .map_err(|e| format!("[memory:binding] cache write lock poisoned: {e}"))?; + // Re-check under the write lock: a racing caller may have bound the same + // workspace (and config) while we were building. Reuse theirs so one + // workspace never has two live drivers for the same config (kernel.md §3.1) + // and `capabilities()` stays asked once. + let entry = guard.entry(key).or_insert_with(|| Arc::clone(&binding)); + Ok(Arc::clone(entry)) +} + +#[cfg(test)] +#[path = "binding_tests.rs"] +mod tests; diff --git a/src/openhuman/memory/binding_tests.rs b/src/openhuman/memory/binding_tests.rs new file mode 100644 index 0000000000..8f1ab73b04 --- /dev/null +++ b/src/openhuman/memory/binding_tests.rs @@ -0,0 +1,621 @@ +//! Tests for the per-workspace memory-driver binding. +//! +//! The load-bearing ones are the trust pair (`admit_refuses_untrusted_external_driver` +//! / `admit_refuses_trusted_external_driver_until_transport_exists`) and +//! `capabilities_are_asked_exactly_once_per_bind`. The first two are written so +//! neither can pass for the other's reason; the third pins the contract's +//! "asked once at bind time and cached" rule, which the whole capability gate +//! depends on. + +use super::*; + +use std::sync::atomic::{AtomicUsize, Ordering}; + +use async_trait::async_trait; +use tinycortex_api::capabilities::Capability; +use tinycortex_api::error::MemoryError; +use tinycortex_api::provider::types::{ExportPage, ExportRecord, ImportOutcome, SourceScope}; +use tinycortex_api::provider::{MemoryCore, MemoryPortability, MemoryRecall}; +use tinycortex_api::recall::OwnedRecallOpts; +use tinycortex_api::types::{MemoryCategory, MemoryEntry, MemoryTaint, NamespaceSummary}; + +use crate::openhuman::config::schema::MemoryDriverConfig; + +fn external_driver_cfg(trust_state: &str) -> MemorySubsystemConfig { + let mut cfg = MemorySubsystemConfig { + driver: "supermemory".into(), + ..Default::default() + }; + cfg.drivers.insert( + "supermemory".into(), + MemoryDriverConfig { + class: Some("external".into()), + transport: Some("http".into()), + endpoint: Some("https://api.supermemory.ai".into()), + credential_ref: Some("keychain:supermemory".into()), + trust_state: trust_state.into(), + }, + ); + cfg +} + +#[test] +fn admit_default_config_binds_embedded_tinycortex() { + let (id, class) = admit(&MemorySubsystemConfig::default()).expect("default config admits"); + assert_eq!(id, "tinycortex"); + assert_eq!(class, DriverClass::Embedded); +} + +#[test] +fn admit_null_driver_binds_null_class() { + let cfg = MemorySubsystemConfig { + driver: "null".into(), + ..Default::default() + }; + let (id, class) = admit(&cfg).expect("null driver admits"); + assert_eq!(id, "null"); + assert_eq!(class, DriverClass::Null); +} + +#[test] +fn admit_typo_d_embedded_driver_id_gets_embedded_class() { + // Regression for the reviewer finding: before this, any non-null id without + // a drivers entry — a typo like "tinycortx", or an external backend that + // forgot its table — was silently classified Embedded. Only the two built-in + // ids admit implicitly. + let cfg = MemorySubsystemConfig { + driver: "tinycortex".into(), + ..Default::default() + }; + let (id, class) = admit(&cfg).expect("the embedded default id admits"); + assert_eq!(id, "tinycortex"); + assert_eq!(class, DriverClass::Embedded); +} + +#[test] +fn admit_refuses_an_unregistered_non_null_driver_id() { + // A typo or an external backend with no `drivers.` entry must not + // silently run the embedded engine under an invented driver id. + let cfg = MemorySubsystemConfig { + driver: "supermemory".into(), + ..Default::default() + }; + let refusal = admit(&cfg).expect_err("an unregistered id must be refused"); + assert_eq!(refusal.configured_driver, "supermemory"); + assert!( + refusal.reason.contains("supermemory"), + "refusal must name the offending id: {}", + refusal.reason + ); + assert!( + refusal.reason.contains("drivers"), + "refusal must point at the missing drivers table: {}", + refusal.reason + ); +} + +#[test] +fn admit_refuses_non_builtin_id_even_with_a_drivers_entry_that_says_no_class() { + // Same rule when an entry exists but carries no `class` line: only the two + // built-in ids imply a class. An arbitrary id must not silently become + // Embedded just because someone registered a placeholder entry. + let mut cfg = MemorySubsystemConfig { + driver: "custom-mem".into(), + ..Default::default() + }; + cfg.drivers.insert( + "custom-mem".into(), + MemoryDriverConfig { + class: None, + ..Default::default() + }, + ); + let refusal = admit(&cfg).expect_err("entry with no class must not admit an arbitrary id"); + assert_eq!(refusal.configured_driver, "custom-mem"); + assert!( + refusal.reason.contains("custom-mem"), + "refusal must name the offending id: {}", + refusal.reason + ); + assert!( + refusal.reason.contains("class line"), + "refusal must point at the missing class line: {}", + refusal.reason + ); +} + +#[test] +fn admit_accepts_an_explicit_embedded_class_for_a_registered_id() { + // A drivers entry that explicitly names the embedded class is a deliberate + // declaration — that id genuinely means the in-process engine. Explicit + // beats implicit. + let mut cfg = MemorySubsystemConfig { + driver: "custom-mem".into(), + ..Default::default() + }; + cfg.drivers.insert( + "custom-mem".into(), + MemoryDriverConfig { + class: Some("embedded".into()), + ..Default::default() + }, + ); + let (id, class) = admit(&cfg).expect("explicit embedded class admits"); + assert_eq!(id, "custom-mem"); + assert_eq!(class, DriverClass::Embedded); +} + +#[test] +fn admit_refuses_untrusted_external_driver() { + // The default trust_state is "untrusted" (kernel.md §3.4, fail-closed). + let cfg = external_driver_cfg(&MemoryDriverConfig::default().trust_state); + let refusal = admit(&cfg).expect_err("untrusted external driver must be refused"); + assert_eq!(refusal.configured_driver, "supermemory"); + assert!( + refusal.reason.contains("trust_state"), + "refusal must name the trust rule: {}", + refusal.reason + ); +} + +#[test] +fn admit_refuses_trusted_external_driver_until_transport_exists() { + let cfg = external_driver_cfg("trusted"); + let refusal = admit(&cfg).expect_err("no external transport exists yet"); + assert!( + refusal.reason.contains("transport"), + "refusal must name the missing transport: {}", + refusal.reason + ); + assert!( + !refusal.reason.contains("trust_state"), + "a trusted driver must not be refused for trust: {}", + refusal.reason + ); +} + +#[test] +fn admit_rejects_an_unknown_driver_class() { + let mut cfg = external_driver_cfg("trusted"); + cfg.drivers.get_mut("supermemory").unwrap().class = Some("embeded".into()); + let refusal = admit(&cfg).expect_err("typo'd class must be refused"); + assert!( + refusal.reason.contains("embeded"), + "refusal must echo the typo: {}", + refusal.reason + ); +} + +#[test] +fn fallback_reason_never_contains_credential_ref_or_endpoint() { + let mut cfg = external_driver_cfg("untrusted"); + cfg.drivers.get_mut("supermemory").unwrap().credential_ref = + Some("keychain:super-secret-value".into()); + let refusal = admit(&cfg).expect_err("untrusted external driver must be refused"); + assert!( + !refusal.reason.contains("super-secret-value"), + "credential_ref leaked into an operator-facing string: {}", + refusal.reason + ); + assert!( + !refusal.reason.contains("supermemory.ai"), + "endpoint leaked into an operator-facing string: {}", + refusal.reason + ); +} + +#[test] +fn for_workspace_caches_binding_per_workspace() { + let dir_a = tempfile::tempdir().unwrap(); + let dir_b = tempfile::tempdir().unwrap(); + let cfg = MemorySubsystemConfig::default(); + + let a = for_workspace(dir_a.path(), &cfg).expect("bind workspace A"); + let b = for_workspace(dir_b.path(), &cfg).expect("bind workspace B"); + assert!( + !Arc::ptr_eq(&a, &b), + "different workspaces must get isolated bindings" + ); + + let a_again = for_workspace(dir_a.path(), &cfg).expect("re-resolve workspace A"); + assert!( + Arc::ptr_eq(&a, &a_again), + "same workspace must reuse the cached binding" + ); +} + +#[test] +fn same_workspace_with_changed_config_binds_fresh() { + // `CoreContext::rebind_workspace` treats "same workspace, changed + // [subsystems.memory]" as a real rebind (a changed driver/hooks/trust all + // feed `build`). The cache must key on the config as well as the path, or + // a changed config for an already-bound workspace would keep serving the + // previous driver until process restart. + let dir = tempfile::tempdir().unwrap(); + let default = MemorySubsystemConfig::default(); + let null = MemorySubsystemConfig { + driver: "null".into(), + ..Default::default() + }; + + let tiny = for_workspace(dir.path(), &default).expect("bind tinycortex"); + assert_eq!(tiny.driver_id(), "tinycortex"); + + // Same (workspace, config) pair reuses the cached binding... + let tiny_again = for_workspace(dir.path(), &default).expect("re-bind tinycortex"); + assert!( + Arc::ptr_eq(&tiny, &tiny_again), + "unchanged config must reuse the cached binding" + ); + + // ...but a changed config for the SAME workspace must bind fresh. + let null_binding = for_workspace(dir.path(), &null).expect("bind null"); + assert!( + !Arc::ptr_eq(&tiny, &null_binding), + "changed config must bind fresh, not serve the stale tinycortex driver" + ); + assert_eq!(null_binding.driver_id(), "null"); + + // Reverting to the original config still resolves its own binding. This is + // the transient-mismatch half: a stale (workspace, config) pairing never + // shadows the correct pair, so it cannot permanently pin a workspace to the + // wrong driver (the atomicity concern in the login/logout rebind). + let tiny_reverted = for_workspace(dir.path(), &default).expect("re-bind tinycortex"); + assert!( + Arc::ptr_eq(&tiny, &tiny_reverted), + "returning to the original config must serve the original binding" + ); +} + +#[test] +fn embedded_class_binds_the_embedded_driver_not_null() { + // Plain `#[test]`: no tokio runtime. Binding must stay synchronous and + // I/O-free, which is why the embedded driver resolves its client lazily. + let dir = tempfile::tempdir().unwrap(); + let workspace = dir.path().join("never-created"); + let binding = + for_workspace(&workspace, &MemorySubsystemConfig::default()).expect("default bind"); + + assert_eq!(binding.driver_id(), "tinycortex"); + assert_eq!(binding.class(), DriverClass::Embedded); + assert!(binding.fallback().is_none()); + assert_ne!(binding.unguarded_provider().driver_id(), NULL_DRIVER_ID); + assert!(binding.capabilities().contains(Capability::Core)); + assert!(binding.capabilities().validate().is_ok()); + assert!( + !workspace.exists(), + "binding must not touch the workspace on disk" + ); +} + +#[test] +fn embedded_binding_advertises_every_family() { + // Widened once per M3 step; M3d is the last one. The interesting assertion + // is the second: a *bound* context and an *unbound* one now agree, which + // they did not for the whole of M2/M3a-c. + let dir = tempfile::tempdir().unwrap(); + let binding = + for_workspace(dir.path(), &MemorySubsystemConfig::default()).expect("default bind"); + let advertised = binding.capabilities(); + + assert!(advertised.contains_all(Capabilities::mandatory())); + for family in Capability::ALL { + assert!(advertised.contains(family), "{family} must be advertised"); + } + assert_eq!(advertised, Capabilities::all()); + assert_eq!(advertised, unbound_default_capabilities()); +} + +#[test] +fn null_driver_config_still_binds_the_null_provider() { + let dir = tempfile::tempdir().unwrap(); + let cfg = MemorySubsystemConfig { + driver: "null".into(), + ..Default::default() + }; + let binding = for_workspace(dir.path(), &cfg).expect("null bind"); + assert_eq!(binding.driver_id(), NULL_DRIVER_ID); + assert_eq!(binding.class(), DriverClass::Null); + assert_eq!(binding.unguarded_provider().driver_id(), NULL_DRIVER_ID); + assert!( + binding.fallback().is_none(), + "an explicitly requested null driver is not a fallback" + ); +} + +#[test] +fn refused_driver_falls_back_to_the_null_placeholder() { + let dir = tempfile::tempdir().unwrap(); + let binding = + for_workspace(dir.path(), &external_driver_cfg("untrusted")).expect("bind falls back"); + assert_eq!(binding.driver_id(), "null"); + assert_eq!(binding.class(), DriverClass::Null); + let fallback = binding.fallback().expect("fallback provenance recorded"); + assert_eq!(fallback.configured_driver, "supermemory"); +} + +#[test] +fn fallback_binding_advertises_only_mandatory_capabilities() { + let dir = tempfile::tempdir().unwrap(); + let binding = + for_workspace(dir.path(), &external_driver_cfg("untrusted")).expect("bind falls back"); + assert_eq!(binding.capabilities(), Capabilities::mandatory()); + // Even the fallback must be a *legal* bind: the mandatory three are present. + assert!(binding.capabilities().validate().is_ok()); + assert!(!binding.capabilities().contains(Capability::Tree)); +} + +#[test] +fn unbound_default_is_the_full_capability_set() { + let all = unbound_default_capabilities(); + assert_eq!(all, Capabilities::all()); + assert_eq!(all.len(), Capability::ALL.len()); +} + +#[test] +fn bound_driver_view_carries_class_capabilities_and_fallback() { + let dir = tempfile::tempdir().unwrap(); + let binding = + for_workspace(dir.path(), &external_driver_cfg("untrusted")).expect("bind falls back"); + let bound = binding.to_bound_driver(); + assert_eq!(bound.slot, SubsystemSlot::Memory); + assert_eq!(bound.id, "null"); + assert_eq!(bound.class, DriverClass::Null); + assert_eq!(bound.contract_version, CONTRACT_VERSION); + assert_eq!(bound.fell_back_from.as_deref(), Some("supermemory")); + assert!(bound.is_fallback()); + // The generic view carries the same families as opaque strings. + assert!(bound.capabilities.contains("core")); + assert!(!bound.capabilities.contains("tree")); + assert_eq!(bound.capabilities.len(), binding.capabilities().len()); +} + +#[test] +fn health_converts_as_a_total_three_arm_match() { + assert_eq!(to_driver_health(MemoryHealth::Ready), DriverHealth::Ready); + assert_eq!( + to_driver_health(MemoryHealth::degraded("reindexing")), + DriverHealth::degraded("reindexing") + ); + assert_eq!( + to_driver_health(MemoryHealth::down("refused")), + DriverHealth::down("refused") + ); +} + +// ---- "capabilities asked once" ------------------------------------------ +// +// The contract's `MemoryProvider::capabilities` doc says the kernel asks once +// at bind time and caches. Everything downstream (RPC registration, tool +// emission) is filtered from that cached answer, so a second ask would let the +// live surface and the advertised surface drift apart. + +struct CountingProvider { + inner: NullMemoryProvider, + calls: AtomicUsize, +} + +impl CountingProvider { + fn new() -> Self { + Self { + inner: NullMemoryProvider::new(), + calls: AtomicUsize::new(0), + } + } +} + +#[async_trait] +impl MemoryCore for CountingProvider { + async fn store( + &self, + namespace: &str, + key: &str, + content: &str, + category: MemoryCategory, + session_id: Option<&str>, + taint: MemoryTaint, + ) -> Result<(), MemoryError> { + self.inner + .store(namespace, key, content, category, session_id, taint) + .await + } + + async fn get(&self, namespace: &str, key: &str) -> Result, MemoryError> { + self.inner.get(namespace, key).await + } + + async fn forget(&self, namespace: &str, key: &str) -> Result { + self.inner.forget(namespace, key).await + } + + async fn list( + &self, + namespace: Option<&str>, + category: Option<&MemoryCategory>, + session_id: Option<&str>, + ) -> Result, MemoryError> { + self.inner.list(namespace, category, session_id).await + } + + async fn namespaces(&self) -> Result, MemoryError> { + self.inner.namespaces().await + } +} + +#[async_trait] +impl MemoryRecall for CountingProvider { + async fn recall( + &self, + query: &str, + limit: usize, + opts: &OwnedRecallOpts, + scope: Option<&SourceScope>, + ) -> Result, MemoryError> { + self.inner.recall(query, limit, opts, scope).await + } +} + +#[async_trait] +impl MemoryPortability for CountingProvider { + async fn export_page( + &self, + cursor: Option<&str>, + limit: usize, + ) -> Result { + self.inner.export_page(cursor, limit).await + } + + async fn import_records( + &self, + records: Vec, + ) -> Result { + self.inner.import_records(records).await + } +} + +#[async_trait] +impl MemoryProvider for CountingProvider { + fn driver_id(&self) -> &str { + "counting" + } + + fn capabilities(&self) -> Capabilities { + self.calls.fetch_add(1, Ordering::SeqCst); + Capabilities::all() + } + + async fn health(&self) -> MemoryHealth { + MemoryHealth::Ready + } +} + +#[test] +fn capabilities_are_asked_exactly_once_per_bind() { + let provider = Arc::new(CountingProvider::new()); + let binding = bind_provider_for_test(provider.clone(), DriverClass::Embedded); + + for _ in 0..5 { + assert_eq!(binding.capabilities(), Capabilities::all()); + } + assert_eq!(binding.driver_id(), "counting"); + assert_eq!( + provider.calls.load(Ordering::SeqCst), + 1, + "capabilities() must be asked exactly once, at bind time" + ); +} + +// --------------------------------------------------------------------------- +// Built-in ids are pinned to their class +// --------------------------------------------------------------------------- +// +// A per-driver table may confirm a built-in id's class but never override it. +// Without that rule `driver = "null"` plus `class = "embedded"` builds the real +// engine and persists memory under the id documented as `/dev/null`, and the +// inverse labels a store-nothing provider `tinycortex`. + +fn cfg_with_class(driver: &str, class: &str) -> MemorySubsystemConfig { + let mut cfg = MemorySubsystemConfig { + driver: driver.into(), + ..Default::default() + }; + cfg.drivers.insert( + driver.into(), + MemoryDriverConfig { + class: Some(class.into()), + ..Default::default() + }, + ); + cfg +} + +#[test] +fn admit_refuses_an_embedded_class_override_on_the_null_driver() { + let refusal = admit(&cfg_with_class("null", "embedded")) + .expect_err("null must not be re-classed as embedded"); + assert_eq!(refusal.configured_driver, "null"); + assert!( + refusal.reason.contains("built in"), + "refusal must say the id is built in: {}", + refusal.reason + ); +} + +#[test] +fn admit_refuses_a_null_class_override_on_the_embedded_driver() { + let refusal = admit(&cfg_with_class(EMBEDDED_DRIVER_ID, "null")) + .expect_err("tinycortex must not be re-classed as null"); + assert_eq!(refusal.configured_driver, EMBEDDED_DRIVER_ID); + assert!( + refusal.reason.contains("built in"), + "refusal must say the id is built in: {}", + refusal.reason + ); +} + +#[test] +fn admit_accepts_a_class_line_that_agrees_with_the_built_in_id() { + // Redundant, but not a mistake: confirming the real class is allowed. + let (id, class) = admit(&cfg_with_class("null", "null")).expect("agreeing class admits"); + assert_eq!(id, "null"); + assert_eq!(class, DriverClass::Null); + + let (id, class) = + admit(&cfg_with_class(EMBEDDED_DRIVER_ID, "embedded")).expect("agreeing class admits"); + assert_eq!(id, EMBEDDED_DRIVER_ID); + assert_eq!(class, DriverClass::Embedded); +} + +#[test] +fn a_null_class_override_cannot_smuggle_the_embedded_engine_into_the_binding() { + // The end-to-end shape of the refusal: `build` must not hand back an + // embedded provider for `driver = "null"`. + let dir = tempfile::tempdir().unwrap(); + let binding = for_workspace(dir.path(), &cfg_with_class("null", "embedded")).expect("binds"); + + assert_eq!(binding.class(), DriverClass::Null); + assert_eq!(binding.driver_id(), NULL_DRIVER_ID); + assert!( + binding.fallback().is_some(), + "a refused class override must be recorded as a fallback" + ); +} + +// --------------------------------------------------------------------------- +// `disables_memory` — deliberate null only +// --------------------------------------------------------------------------- + +#[test] +fn an_explicit_null_driver_disables_memory() { + let dir = tempfile::tempdir().unwrap(); + let cfg = MemorySubsystemConfig { + driver: "null".into(), + ..Default::default() + }; + let binding = for_workspace(dir.path(), &cfg).expect("binds"); + + assert!(binding.fallback().is_none(), "this is not a fallback"); + assert!( + binding.disables_memory(), + "an operator who bound /dev/null asked for the surface to be gone" + ); +} + +#[test] +fn a_fallback_to_null_does_not_disable_memory() { + // A misconfiguration must be loud, not silently memory-less: the fallback + // is reported in status and the surface stays present. + let dir = tempfile::tempdir().unwrap(); + let binding = for_workspace(dir.path(), &external_driver_cfg("untrusted")).expect("binds"); + + assert_eq!(binding.class(), DriverClass::Null); + assert!(binding.fallback().is_some(), "this IS a fallback"); + assert!(!binding.disables_memory()); +} + +#[test] +fn the_embedded_driver_never_disables_memory() { + let dir = tempfile::tempdir().unwrap(); + let binding = for_workspace(dir.path(), &MemorySubsystemConfig::default()).expect("binds"); + assert!(!binding.disables_memory()); +} diff --git a/src/openhuman/memory/bypass_allowlist_tests.rs b/src/openhuman/memory/bypass_allowlist_tests.rs new file mode 100644 index 0000000000..e60c8bb286 --- /dev/null +++ b/src/openhuman/memory/bypass_allowlist_tests.rs @@ -0,0 +1,611 @@ +//! Enforcement lint: the set of production files that reach the memory driver +//! **around** the kernel guard must not grow. +//! +//! # This test does NOT claim the guard is the only path to the driver +//! +//! It is not, and this file exists to make that measurable rather than to +//! pretend otherwise. `MemoryClient` still hands out raw, undecoratable +//! handles, all `pub(crate)` and all with live production callers: +//! +//! - `profile_conn()` (`memory/store/client.rs`) — an +//! `Arc>`. **No decorator can wrap a raw SQLite +//! connection**, so the eleven non-test call sites beneath +//! `agent/learning/*` and `memory/sync/composio/providers/profile.rs` reach +//! the profile/facet tables under all of the guard's policy steps. Closing +//! this is explicitly out of scope for M4. +//! - `memory_handle()` (`memory/store/client.rs`) — a raw `Arc`. +//! The contract has no `Arc` door, so consumers that must satisfy +//! a foreign trait (tinyflows, the agent-experience store) still take it. +//! - `get_document()` (`memory/store/client.rs`) — a read-one escape hatch that +//! is driver-only by contract but not by visibility. +//! +//! What this test buys is a **ratchet**, not an invariant: the current bypasses +//! are enumerated in [`ALLOWED`] with a reason each, and that list may shrink +//! but never grow. That is deliberately weaker than "impossible to skip by +//! construction", and the weakness is the point — a lint that was red on day +//! one would be `#[ignore]`d within a week and never come back, whereas a green +//! ratchet converges. An enforcement test that overstates its own guarantee is +//! worse than none, because it stops people looking. +//! +//! Same pattern and same reasoning as `INTENTIONALLY_NOT_FORWARDED` in +//! `scripts/lib/feature-forwarding.mjs`, including its staleness half — which +//! is what stops an allowlist rotting into dead strings. +//! +//! # Known weaknesses, stated rather than hidden +//! +//! - **The lint sees text, not types.** A bypass reached through a re-export +//! under a different name is invisible to it. Substring needles also +//! over-match: an unrelated future `.get_document(` would trip this. That +//! failure direction is the correct one — a false positive costs one +//! allowlist line with a reason, a false negative costs a silent bypass. +//! One needle has a compiler backstop: `.unguarded_provider(` is +//! `pub(in crate::openhuman::memory)`, so even a re-export under another name +//! cannot carry it outside the memory family. It was also *named* for the +//! lint — `.provider(` would have over-matched `TaskSourceFilter::provider()` +//! and `ModelRef::provider()` in four unrelated production files, costing six +//! allowlist entries that document nothing. +//! - **By-path test files are out of scope** (`*_tests.rs`, `tests.rs`, +//! `test_support/`). Driver tests construct drivers; that is what a driver +//! test *is*. Allowlisting them would add ~25 entries that can never shrink +//! and would churn on every new driver test — the exact rot this lint fights. +//! - **Inline `#[cfg(test)] mod tests` blocks are NOT stripped.** Brace-tracking +//! Rust source with a line scanner is fragile, and getting it wrong silently +//! *hides* production sites. Three files are therefore allowlisted for a +//! match that lives only in an inline test module; each says so. +//! - **`app/src-tauri/` is not scanned.** It links `openhuman_core` with +//! `default-features = false` and cannot name `pub(crate)` items at all, so +//! there is nothing there to scan. The omission is deliberate, not an +//! oversight. + +use std::collections::BTreeSet; +use std::path::{Path, PathBuf}; + +/// Call shapes that hand out an **unguarded** door into memory, each with why +/// reaching it around the guard matters. +/// +/// Substring needles, not regexes, and deliberately path-*suffixed*: the same +/// call is written `memory::global::client_if_ready()`, +/// `crate::openhuman::memory::global::client_if_ready()` and +/// `super::super::global::client_if_ready()` in this tree, so anchoring on an +/// absolute path would miss the third. +/// +/// `global::init(` is deliberately absent. It binds a workspace; it does not +/// read or write memory, and every call site is a login / user-switch / boot / +/// CLI-entry lifecycle event. See `docs/specs/memory-guard-allowlist.md`. +const BYPASS_PATTERNS: &[(&str, &str)] = &[ + ( + "active_memory_client(", + "resolves a MemoryClientRef with no policy decorator", + ), + ( + "global::client_if_ready(", + "process-global MemoryClient, no policy decorator", + ), + ( + "global::client(", + "process-global MemoryClient, no policy decorator", + ), + ( + ".profile_conn(", + "raw rusqlite connection — undecoratable by construction", + ), + ( + ".memory_handle(", + "raw Arc — bypasses the MemoryClient API surface", + ), + ( + ".get_document(", + "pub(crate) read-one escape hatch, driver-only by contract", + ), + ( + "EmbeddedMemoryProvider::new(", + "direct driver construction — must go through binding::for_workspace", + ), + ( + "NullMemoryProvider::new(", + "direct driver construction — must go through binding::for_workspace", + ), + ( + "MemoryClient::from_workspace_dir(", + "direct engine construction; risks a second ingestion worker on one store", + ), + ( + "binding::for_workspace(", + "raw MemoryBinding — the guard is what should be handed out", + ), + ( + ".memory_binding(", + "raw MemoryBinding off CoreContext instead of CoreContext::memory()", + ), + ( + ".unguarded_provider(", + "raw Arc off a MemoryBinding — skips the guard entirely", + ), +]; + +/// `(repo-relative path, pattern, why this file may bypass today)`. +/// +/// Adding an entry is a decision, not a way to silence the lint. The reason +/// string is what a future reviewer relies on to tell "deliberate" from +/// "forgotten". **This list may SHRINK. It must never GROW.** +/// +/// Sorted by path, then pattern — [`scan`] returns a `BTreeSet`, so keeping the +/// literal in the same order makes diffs readable. +const ALLOWED: &[(&str, &str, &str)] = &[ + // ── Standalone binaries: their own process, no ambient CoreContext ── + ( + "src/bin/library_profile/scenarios/cold_phases.rs", + "MemoryClient::from_workspace_dir(", + "profiling harness; boots its own client outside the guard's process model", + ), + ( + "src/bin/slack_backfill.rs", + "global::client_if_ready(", + "standalone backfill binary; boots its own client, no CoreContext", + ), + // ── Metadata-only reads: driver identity, never memory content ── + ( + "src/core/cli_capability.rs", + "binding::for_workspace(", + "reads driver_id + advertised capabilities only (what memory.provider_status already reports); no CoreContext exists on a CLI invocation, so there is no guard to route through", + ), + // subsystems_cli.rs no longer binds directly: it delegates to + // memory_subsystem_status, whose binding resolution lives in + // memory/ops/provider.rs (allowlisted above). + // ── The bind site itself: it produces the guard ── + ( + "src/core/runtime/context.rs", + ".memory_binding(", + "CoreContext owns the binding; memory() is the guarded accessor built from it", + ), + ( + "src/core/runtime/context.rs", + "binding::for_workspace(", + "the per-workspace bind site — guarding it would be a cycle", + ), + // ── Needs a concrete engine type the contract does not expose ── + ( + "src/openhuman/agent/experience/ops.rs", + ".memory_handle(", + "AgentExperienceStore::new takes Arc; no contract door for it", + ), + ( + "src/openhuman/agent/experience/ops.rs", + "global::client_if_ready(", + "same store; also builds a per-profile UnifiedMemory the binding cannot model", + ), + ( + "src/openhuman/agent/harness/session/builder/factory.rs", + ".memory_handle(", + "session builder needs Arc; no contract door for it", + ), + // ── Unguardable raw SQLite (profile_conn) — the known hole ── + ( + "src/openhuman/agent/learning/schemas.rs", + ".profile_conn(", + "raw SQLite profile/facet reads; undecoratable, out of scope for M4", + ), + ( + "src/openhuman/agent/learning/schemas.rs", + "global::client_if_ready(", + "resolved only to reach profile_conn() on the line below", + ), + ( + "src/openhuman/agent/learning/startup.rs", + "MemoryClient::from_workspace_dir(", + "inline #[cfg(test)] module only; the scanner does not brace-track test blocks", + ), + ( + "src/openhuman/agent/learning/startup.rs", + ".profile_conn(", + "raw SQLite facet bootstrap; undecoratable, out of scope for M4", + ), + ( + "src/openhuman/agent/learning/tools.rs", + ".profile_conn(", + "raw SQLite facet read from an agent tool; undecoratable, out of scope for M4", + ), + ( + "src/openhuman/agent/learning/tools.rs", + "global::client_if_ready(", + "resolved only to reach profile_conn() on the line below", + ), + // ── Flows: foreign trait shapes and a test-override seam ── + ( + "src/openhuman/flows/bus.rs", + ".memory_handle(", + "resolve_memory() -> Option>; no contract door for it", + ), + ( + "src/openhuman/flows/bus.rs", + "active_memory_client(", + "carries a #[cfg(test)] memory_override seam the guard would bypass", + ), + ( + "src/openhuman/flows/ops.rs", + "active_memory_client(", + "clear_namespace has no contract method; plus a memory_client_override test seam", + ), + ( + "src/openhuman/flows/tinyflows/memory_adapter.rs", + ".memory_handle(", + "returns Arc to satisfy a tinyflows engine trait", + ), + ( + "src/openhuman/flows/tinyflows/memory_adapter.rs", + "active_memory_client(", + "same adapter; the tinyflows trait names the engine type, not the contract", + ), + // ── Composio integration: &MemoryClientRef parameter shape ── + ( + "src/openhuman/integrations/composio/ops/memory_cleanup.rs", + "MemoryClient::from_workspace_dir(", + "cleanup runs off a config workspace with no live binding", + ), + ( + "src/openhuman/integrations/composio/schemas.rs", + "global::client_if_ready(", + "passes &MemoryClientRef into user_scopes::save; the contract has no such shape", + ), + // ── The driver and the binding: guarding these would be a cycle ── + ( + "src/openhuman/memory/binding.rs", + "EmbeddedMemoryProvider::new(", + "this is the construction path the lint protects", + ), + ( + "src/openhuman/memory/binding.rs", + "NullMemoryProvider::new(", + "this is the construction path the lint protects (fail-closed fallback)", + ), + ( + "src/openhuman/memory/driver/embedded/documents.rs", + ".get_document(", + "this IS the driver — the escape hatch exists for exactly this call", + ), + ( + "src/openhuman/memory/driver/embedded/mod.rs", + ".memory_handle(", + "this IS the driver; it owns the engine handle by definition", + ), + ( + "src/openhuman/memory/driver/embedded/mod.rs", + "EmbeddedMemoryProvider::new(", + "the driver's own constructor", + ), + ( + "src/openhuman/memory/global.rs", + "MemoryClient::from_workspace_dir(", + "the process-global slot itself; it is what global::client hands out", + ), + ( + "src/openhuman/memory/guard/families.rs", + ".get_document(", + "the guard's own documents decorator forwarding to the inner family", + ), + // ── Handlers with no typed contract twin (see the allowlist doc, §D) ── + ( + "src/openhuman/memory/ops/documents.rs", + "active_memory_client(", + "namespace/doc/context handlers answer untyped Value shapes with no contract twin", + ), + ( + "src/openhuman/memory/ops/guard.rs", + "binding::for_workspace(", + "the guarded resolver; this is where the binding becomes a guard", + ), + ( + "src/openhuman/memory/ops/helpers.rs", + "active_memory_client(", + "defines active_memory_client — the unguarded twin of ops::guard", + ), + ( + "src/openhuman/memory/ops/helpers.rs", + "global::client_if_ready(", + "same definition site", + ), + ( + "src/openhuman/memory/ops/kv_graph.rs", + "active_memory_client(", + "kv_get/kv_delete/graph_* have no contract twin or a lossy conversion", + ), + ( + "src/openhuman/memory/ops/learn.rs", + "active_memory_client(", + "list_namespaces() -> Vec vs the contract's Vec", + ), + ( + "src/openhuman/memory/ops/learn.rs", + "global::client(", + "inline #[cfg(test)] module only; the scanner does not brace-track test blocks", + ), + ( + "src/openhuman/memory/ops/provider.rs", + ".memory_binding(", + "reports driver status; it is about the binding, not about reading memory", + ), + ( + "src/openhuman/memory/ops/provider.rs", + ".unguarded_provider(", + "health probe on the bound driver; a liveness probe is not product code", + ), + ( + "src/openhuman/memory/ops/provider.rs", + "binding::for_workspace(", + "same status surface", + ), + ( + "src/openhuman/memory/ops/sync.rs", + "global::client_if_ready(", + "ingestion_state().snapshot() — queue telemetry, absent from the contract", + ), + ( + "src/openhuman/memory/ops/sync.rs", + "global::client(", + "inline #[cfg(test)] module only; the scanner does not brace-track test blocks", + ), + ( + "src/openhuman/memory/ops/tool_memory.rs", + ".memory_handle(", + "open_store() still serves the four handlers with no contract twin", + ), + ( + "src/openhuman/memory/ops/tool_memory.rs", + "active_memory_client(", + "tool_rule_put/get/*_json/*_for_prompt have no contract equivalent", + ), + // ── Composio memory sync: profile_conn + &MemoryClientRef ── + ( + "src/openhuman/memory/sync/composio/providers/profile.rs", + ".profile_conn(", + "raw SQLite profile writes; undecoratable, out of scope for M4", + ), + ( + "src/openhuman/memory/sync/composio/providers/profile.rs", + "global::client_if_ready(", + "resolved only to reach profile_conn()", + ), + ( + "src/openhuman/memory/sync/composio/providers/types.rs", + "MemoryClient::from_workspace_dir(", + "provider trait takes &MemoryClientRef; the contract has no such shape", + ), + ( + "src/openhuman/memory/sync/composio/providers/types.rs", + "global::client_if_ready(", + "same provider trait shape", + ), + ( + "src/openhuman/memory/sync/composio/providers/user_scopes.rs", + "global::client_if_ready(", + "same provider trait shape", + ), + // ── The engine seam ── + ( + "src/openhuman/memory/tinycortex/sync.rs", + "global::client_if_ready(", + "the TinyCortex engine seam; it sits beneath the contract, not above it", + ), + // ── Agent tools — deferred to M5's capability filter ── + ( + "src/openhuman/memory/tool_memory/tools/list.rs", + ".memory_handle(", + "builds ToolMemoryStore; re-pointable via as_tool_memory(), deferred to M5", + ), + ( + "src/openhuman/memory/tool_memory/tools/list.rs", + "active_memory_client(", + "same tool; M5 filters the tool surface by capability and would collide", + ), + ( + "src/openhuman/memory/tool_memory/tools/put.rs", + ".memory_handle(", + "builds ToolMemoryStore; re-pointable via as_tool_memory(), deferred to M5", + ), + ( + "src/openhuman/memory/tool_memory/tools/put.rs", + "active_memory_client(", + "same tool; M5 filters the tool surface by capability and would collide", + ), +]; + +/// True for source files the lint deliberately does not scan. +/// +/// By-path only — see the module docs for why inline `#[cfg(test)]` blocks are +/// left in scope instead of being brace-tracked. +fn is_test_path(path: &Path) -> bool { + if path.components().any(|c| c.as_os_str() == "test_support") { + return true; + } + match path.file_name().and_then(|n| n.to_str()) { + Some(name) => name == "tests.rs" || name.ends_with("_tests.rs"), + None => false, + } +} + +fn collect_rs_files(dir: &Path, out: &mut Vec) { + let Ok(entries) = std::fs::read_dir(dir) else { + return; + }; + for entry in entries.flatten() { + let path = entry.path(); + if path.is_dir() { + collect_rs_files(&path, out); + } else if path.extension().is_some_and(|ext| ext == "rs") && !is_test_path(&path) { + out.push(path); + } + } +} + +/// Every `(repo-relative path, pattern)` pair currently in the production tree. +/// +/// Comment lines are skipped so that doc-comment references — of which this +/// module and `memory/global.rs` have several — are not mistaken for calls. +fn scan() -> BTreeSet<(String, String)> { + let root = Path::new(env!("CARGO_MANIFEST_DIR")); + let mut files = Vec::new(); + collect_rs_files(&root.join("src"), &mut files); + + let mut found = BTreeSet::new(); + for path in &files { + let Ok(text) = std::fs::read_to_string(path) else { + continue; + }; + let rel = path + .strip_prefix(root) + .unwrap_or(path) + .to_string_lossy() + .replace('\\', "/"); + for line in text.lines() { + if line.trim_start().starts_with("//") { + continue; + } + for (pattern, _) in BYPASS_PATTERNS { + if line.contains(pattern) { + found.insert((rel.clone(), (*pattern).to_string())); + } + } + } + } + found +} + +fn allowed_set() -> BTreeSet<(String, String)> { + ALLOWED + .iter() + .map(|(path, pattern, _)| ((*path).to_string(), (*pattern).to_string())) + .collect() +} + +fn render(pairs: impl IntoIterator) -> String { + pairs + .into_iter() + .map(|(path, pattern)| format!("\n {path}\t{pattern}")) + .collect() +} + +/// A parser that silently found nothing would turn every other test here into a +/// rubber stamp, so refuse to pass vacuously. +/// +/// The literal pinned below is the densest known bypass in the tree: five +/// `profile_conn()` calls reaching raw SQLite. If the scanner ever stops seeing +/// it, the scanner is broken — fix it, do not relax this assertion. +#[test] +fn bypass_scanner_finds_the_known_bypasses() { + let found = scan(); + assert!( + !found.is_empty(), + "the bypass scanner found nothing at all; every other test in this \ + module would pass vacuously. Fix the scanner, not the assertion." + ); + let canary = ( + "src/openhuman/memory/sync/composio/providers/profile.rs".to_string(), + ".profile_conn(".to_string(), + ); + assert!( + found.contains(&canary), + "the scanner lost a known bypass ({canary:?}); it is no longer reading \ + the tree correctly. Fix the scanner, not the assertion." + ); +} + +/// Every needle in [`BYPASS_PATTERNS`] must still match something. +/// +/// Without this a rename turns a pattern into a dead string that quietly stops +/// guarding anything — the same rot [`bypass_allowlist_has_no_stale_entries`] +/// prevents on the allowlist side. +#[test] +fn bypass_patterns_are_all_live() { + let found = scan(); + let dead: Vec<&str> = BYPASS_PATTERNS + .iter() + .filter(|(pattern, _)| !found.iter().any(|(_, p)| p == pattern)) + .map(|(pattern, _)| *pattern) + .collect(); + assert!( + dead.is_empty(), + "these BYPASS_PATTERNS needles match nothing in the tree: {dead:?}\n\ + Either the API was renamed (update the needle) or the last bypass was \ + removed (delete the needle *and* say so in the module docs). A needle \ + that matches nothing is a guard that guards nothing." + ); +} + +/// The ratchet, forward direction: no NEW bypass may appear. +#[test] +fn no_new_memory_driver_bypasses() { + let found = scan(); + let allowed = allowed_set(); + let added: Vec<(String, String)> = found.difference(&allowed).cloned().collect(); + assert!( + added.is_empty(), + "new unguarded memory call site(s):{}\n\n\ + Route them through the kernel memory guard \ + (`memory::ops::guard::active_memory_guard`, or `CoreContext::memory()`), \ + or — if the bypass is genuinely required — add them to ALLOWED in this \ + file *with a reason*, and to docs/specs/memory-guard-allowlist.md. \ + The list may shrink; it must never grow.", + render(added) + ); +} + +/// The ratchet, reverse direction — and this half is where the teeth are. +/// +/// Fixing a bypass without striking its entry fails the build, so the allowlist +/// cannot silently grow back into the space a cleanup just freed. Mirrors the +/// `stale` computation in `scripts/lib/feature-forwarding.mjs`. +#[test] +fn bypass_allowlist_has_no_stale_entries() { + let found = scan(); + let allowed = allowed_set(); + let stale: Vec<(String, String)> = allowed.difference(&found).cloned().collect(); + assert!( + stale.is_empty(), + "these ALLOWED entries no longer bypass anything:{}\n\n\ + Delete them from this file and from \ + docs/specs/memory-guard-allowlist.md. An allowlist that keeps dead \ + strings stops being evidence of anything.", + render(stale) + ); +} + +/// Cheap guard against the failure mode this tree keeps hitting: a family +/// reorg renames a path and leaves an entry that now allows nothing and hides +/// nothing. +/// +/// [`bypass_allowlist_has_no_stale_entries`] would also catch this, but reports +/// it as "no longer bypasses" — which reads like a cleanup to celebrate rather +/// than a rename to finish. +#[test] +fn bypass_allowlist_paths_all_exist() { + let root = Path::new(env!("CARGO_MANIFEST_DIR")); + let missing: Vec<&str> = ALLOWED + .iter() + .map(|(path, _, _)| *path) + .filter(|path| !root.join(path).exists()) + .collect(); + assert!( + missing.is_empty(), + "ALLOWED names file(s) that no longer exist: {missing:?}\n\ + A rename left a dead entry. Point it at the new path, or delete it." + ); +} + +/// Every allowlist entry must carry a non-empty reason, and must name a needle +/// the lint actually looks for. +/// +/// The reason string is the whole value of the list to a future reader; an +/// entry without one is indistinguishable from an oversight. +#[test] +fn bypass_allowlist_entries_are_well_formed() { + for (path, pattern, reason) in ALLOWED { + assert!( + !reason.trim().is_empty(), + "ALLOWED entry {path} / {pattern} has no reason" + ); + assert!( + BYPASS_PATTERNS.iter().any(|(p, _)| p == pattern), + "ALLOWED entry {path} names {pattern}, which is not in BYPASS_PATTERNS" + ); + } +} diff --git a/src/openhuman/memory/diff/ops.rs b/src/openhuman/memory/diff/ops.rs index 673aceb2c2..02c0c1d024 100644 --- a/src/openhuman/memory/diff/ops.rs +++ b/src/openhuman/memory/diff/ops.rs @@ -80,6 +80,34 @@ pub async fn auto_snapshot_after_sync( take_snapshot(source, config, SnapshotTrigger::Auto).await } +/// List snapshots, newest first — for one source when `source_id` is `Some`, +/// across every source otherwise. +/// +/// Lifted verbatim out of [`super::rpc::list_snapshots_rpc`], which had the +/// only copy of this query and returned it wrapped in an `RpcOutcome`. The +/// embedded memory driver's `MemoryDiff::snapshots` needs the same read without +/// the RPC envelope, and a second `Ledger::open` call site would make this +/// module no longer the only place that knows the ledger layout. +/// +/// An unknown `source_id` yields an empty vector rather than an error — the +/// ledger has no source registry to check against. +pub async fn list_snapshots( + config: &Config, + source_id: Option<&str>, + limit: u32, +) -> Result, String> { + let workspace_dir = config.workspace_dir.clone(); + let source_id = source_id.map(str::to_string); + + tokio::task::spawn_blocking(move || -> anyhow::Result> { + let ledger = tinycortex::memory::diff::Ledger::open(&workspace_dir)?; + ledger.list_snapshots(source_id.as_deref(), limit) + }) + .await + .map_err(|e| format!("list_snapshots join: {e}"))? + .map_err(|e: anyhow::Error| format!("list_snapshots: {e:#}")) +} + /// Compute the diff between two snapshots of the same source. pub async fn compute_diff( config: &Config, diff --git a/src/openhuman/memory/diff/rpc.rs b/src/openhuman/memory/diff/rpc.rs index 5b354b9722..88ab475083 100644 --- a/src/openhuman/memory/diff/rpc.rs +++ b/src/openhuman/memory/diff/rpc.rs @@ -163,17 +163,9 @@ pub async fn list_snapshots_rpc( req.source_id, req.limit ); let config = config_rpc::load_config_with_timeout().await?; - let workspace_dir = config.workspace_dir.clone(); let limit = req.limit.unwrap_or(50) as u32; - let source_id = req.source_id; - let snapshots = tokio::task::spawn_blocking(move || -> anyhow::Result> { - let ledger = Ledger::open(&workspace_dir)?; - ledger.list_snapshots(source_id.as_deref(), limit) - }) - .await - .map_err(|e| format!("list_snapshots join: {e}"))? - .map_err(|e: anyhow::Error| format!("list_snapshots: {e:#}"))?; + let snapshots = ops::list_snapshots(&config, req.source_id.as_deref(), limit).await?; debug!( "[memory_diff][rpc] list_snapshots returned {} snapshots", diff --git a/src/openhuman/memory/driver/embedded/core_family.rs b/src/openhuman/memory/driver/embedded/core_family.rs new file mode 100644 index 0000000000..4e574a5516 --- /dev/null +++ b/src/openhuman/memory/driver/embedded/core_family.rs @@ -0,0 +1,125 @@ +//! [`MemoryCore`] for the embedded driver — store / get / forget / list / +//! namespaces. +//! +//! Four of the five are a straight delegation to the engine's [`Memory`] trait. +//! Two things are *not* straight, and both are load-bearing: +//! +//! 1. **`store` maps onto [`Memory::store_with_taint`], never [`Memory::store`].** +//! The contract has a single `store` that always carries a +//! [`MemoryTaint`](tinycortex_api::types::MemoryTaint), because provenance is +//! stamped by the host policy guard *before* the call. `Memory::store` hard-codes +//! `MemoryTaint::Internal`, so routing through it would launder +//! externally-sourced content into internal-trust content — the single +//! failure mode the guard exists to prevent. (`Memory::store_with_taint`'s +//! *trait default* also silently drops the taint; `UnifiedMemory` overrides +//! it, which is why this delegation is correct and the other is not.) +//! +//! 2. **`list(None, ..)` spans every namespace.** The contract says all-`None` +//! lists everything the driver holds; the engine's `list` normalises a +//! `None` namespace to `GLOBAL_NAMESPACE`, so a naive delegation would +//! silently return one namespace and call it "everything". The driver +//! composes `namespace_summaries()` with a per-namespace `list` instead — +//! two existing calls, no new query logic. + +use async_trait::async_trait; +use tinycortex_api::error::MemoryError; +use tinycortex_api::provider::MemoryCore; +use tinycortex_api::types::{MemoryCategory, MemoryEntry, MemoryTaint, NamespaceSummary}; + +use super::{engine_error, EmbeddedMemoryProvider}; + +#[async_trait] +impl MemoryCore for EmbeddedMemoryProvider { + async fn store( + &self, + namespace: &str, + key: &str, + content: &str, + category: MemoryCategory, + session_id: Option<&str>, + taint: MemoryTaint, + ) -> Result<(), MemoryError> { + log::debug!( + "[memory:driver:embedded] store namespace={namespace} key_len={} content_len={} \ + category={category} session={} taint={}", + key.len(), + content.len(), + session_id.unwrap_or("-"), + taint.as_db_str() + ); + // `store_with_taint`, never `store` — see the module docs. + self.memory() + .await? + .store_with_taint(namespace, key, content, category, session_id, taint) + .await + .map_err(engine_error) + } + + async fn get(&self, namespace: &str, key: &str) -> Result, MemoryError> { + self.memory() + .await? + .get(namespace, key) + .await + .map_err(engine_error) + } + + async fn forget(&self, namespace: &str, key: &str) -> Result { + log::debug!( + "[memory:driver:embedded] forget namespace={namespace} key_len={}", + key.len() + ); + self.memory() + .await? + .forget(namespace, key) + .await + .map_err(engine_error) + } + + async fn list( + &self, + namespace: Option<&str>, + category: Option<&MemoryCategory>, + session_id: Option<&str>, + ) -> Result, MemoryError> { + let memory = self.memory().await?; + + if let Some(namespace) = namespace { + return memory + .list(Some(namespace), category, session_id) + .await + .map_err(engine_error); + } + + // All-`None` must mean "everything this driver holds" (contract), which + // the engine's namespace normalisation would otherwise narrow to the + // global namespace alone. + let summaries = memory.namespace_summaries().await.map_err(engine_error)?; + log::debug!( + "[memory:driver:embedded] list spanning {} namespace(s)", + summaries.len() + ); + let mut entries = Vec::new(); + for summary in summaries { + let mut page = memory + .list(Some(&summary.namespace), category, session_id) + .await + .map_err(engine_error)?; + entries.append(&mut page); + } + Ok(entries) + } + + async fn namespaces(&self) -> Result, MemoryError> { + // The contract's `namespaces` is the engine's `namespace_summaries`; + // the return type is identical, only the name differs. + self.memory() + .await? + .namespace_summaries() + .await + .map_err(engine_error) + } +} + +#[cfg(test)] +#[path = "core_family_tests.rs"] +mod tests; diff --git a/src/openhuman/memory/driver/embedded/core_family_tests.rs b/src/openhuman/memory/driver/embedded/core_family_tests.rs new file mode 100644 index 0000000000..2c1e2feffb --- /dev/null +++ b/src/openhuman/memory/driver/embedded/core_family_tests.rs @@ -0,0 +1,255 @@ +//! [`MemoryCore`] tests. +//! +//! Two carry weight beyond a round-trip: +//! +//! - `store_preserves_external_sync_taint_through_get` is the security test. It +//! asserts the *value*, not merely that a taint exists, so it fails the +//! moment anyone routes the contract's `store` onto `Memory::store` (which +//! hard-codes `Internal`). Its `Internal` twin exists so it cannot pass by a +//! constant. +//! - `list_with_no_namespace_spans_every_namespace` pins the divergence between +//! the contract ("all `None` lists everything") and the engine (`None` +//! normalises to the global namespace). A naive delegation fails it. + +use super::super::test_support::fresh_driver; +use super::*; + +use tinycortex_api::provider::MemoryProvider; + +#[tokio::test] +async fn store_get_round_trips_through_the_contract() { + let (_tmp, provider) = fresh_driver(); + + provider + .store( + "ns_a", + "k1", + "value in a", + MemoryCategory::Core, + Some("sess-1"), + MemoryTaint::Internal, + ) + .await + .expect("store"); + + let got = provider + .get("ns_a", "k1") + .await + .expect("get") + .expect("entry exists"); + assert_eq!(got.key, "k1"); + assert_eq!(got.content, "value in a"); + assert_eq!(got.category, MemoryCategory::Core); +} + +#[tokio::test] +async fn get_returns_none_for_an_absent_key() { + let (_tmp, provider) = fresh_driver(); + assert!(provider.get("ns_a", "nope").await.expect("get").is_none()); +} + +#[tokio::test] +async fn forget_removes_the_entry_and_is_idempotent() { + let (_tmp, provider) = fresh_driver(); + provider + .store( + "ns_a", + "k1", + "value", + MemoryCategory::Core, + None, + MemoryTaint::Internal, + ) + .await + .expect("store"); + + assert!(provider.forget("ns_a", "k1").await.expect("first forget")); + assert!(provider.get("ns_a", "k1").await.expect("get").is_none()); + assert!( + !provider.forget("ns_a", "k1").await.expect("second forget"), + "forgetting an absent key is Ok(false), never an error" + ); +} + +#[tokio::test] +async fn list_scoped_to_namespace_applies_category_and_session_filters() { + let (_tmp, provider) = fresh_driver(); + provider + .store( + "ns_a", + "core-1", + "c", + MemoryCategory::Core, + Some("sess-1"), + MemoryTaint::Internal, + ) + .await + .expect("store core"); + provider + .store( + "ns_a", + "daily-1", + "d", + MemoryCategory::Daily, + Some("sess-2"), + MemoryTaint::Internal, + ) + .await + .expect("store daily"); + + let all = provider.list(Some("ns_a"), None, None).await.expect("list"); + assert_eq!(all.len(), 2); + + let core_only = provider + .list(Some("ns_a"), Some(&MemoryCategory::Core), None) + .await + .expect("list by category"); + assert_eq!(core_only.len(), 1); + assert_eq!(core_only[0].key, "core-1"); + + let session_only = provider + .list(Some("ns_a"), None, Some("sess-2")) + .await + .expect("list by session"); + assert_eq!(session_only.len(), 1); + assert_eq!(session_only[0].key, "daily-1"); +} + +#[tokio::test] +async fn list_with_no_namespace_spans_every_namespace() { + let (_tmp, provider) = fresh_driver(); + provider + .store( + "ns_a", + "a1", + "in a", + MemoryCategory::Core, + None, + MemoryTaint::Internal, + ) + .await + .expect("store a"); + provider + .store( + "ns_b", + "b1", + "in b", + MemoryCategory::Core, + None, + MemoryTaint::Internal, + ) + .await + .expect("store b"); + + let everything = provider.list(None, None, None).await.expect("list all"); + let keys: Vec<&str> = everything.iter().map(|e| e.key.as_str()).collect(); + assert!(keys.contains(&"a1"), "missing ns_a entry: {keys:?}"); + assert!(keys.contains(&"b1"), "missing ns_b entry: {keys:?}"); +} + +#[tokio::test] +async fn namespaces_reports_per_namespace_counts() { + let (_tmp, provider) = fresh_driver(); + for key in ["a1", "a2"] { + provider + .store( + "ns_a", + key, + "x", + MemoryCategory::Core, + None, + MemoryTaint::Internal, + ) + .await + .expect("store"); + } + provider + .store( + "ns_b", + "b1", + "x", + MemoryCategory::Core, + None, + MemoryTaint::Internal, + ) + .await + .expect("store"); + + let summaries = provider.namespaces().await.expect("namespaces"); + let a = summaries + .iter() + .find(|s| s.namespace == "ns_a") + .expect("ns_a summary"); + let b = summaries + .iter() + .find(|s| s.namespace == "ns_b") + .expect("ns_b summary"); + assert_eq!(a.count, 2); + assert_eq!(b.count, 1); +} + +/// SECURITY: the contract stamps provenance before the call; the driver must +/// persist exactly what it was handed. Routing onto `Memory::store` would +/// launder this to `Internal`. +#[tokio::test] +async fn store_preserves_external_sync_taint_through_get() { + let (_tmp, provider) = fresh_driver(); + provider + .store( + "ns_a", + "synced", + "from an external source", + MemoryCategory::Core, + None, + MemoryTaint::ExternalSync, + ) + .await + .expect("store"); + + let got = provider + .get("ns_a", "synced") + .await + .expect("get") + .expect("entry exists"); + assert_eq!(got.taint, MemoryTaint::ExternalSync); +} + +/// The negative half of the taint pair — without it the assertion above could +/// pass against a driver that hard-coded `ExternalSync`. +#[tokio::test] +async fn store_preserves_internal_taint_through_get() { + let (_tmp, provider) = fresh_driver(); + provider + .store( + "ns_a", + "typed", + "written by the user", + MemoryCategory::Core, + None, + MemoryTaint::Internal, + ) + .await + .expect("store"); + + let got = provider + .get("ns_a", "typed") + .await + .expect("get") + .expect("entry exists"); + assert_eq!(got.taint, MemoryTaint::Internal); +} + +#[tokio::test] +async fn core_calls_resolve_the_client_lazily_and_only_once() { + let (_tmp, provider) = fresh_driver(); + assert!( + provider.workspace_dir().parent().is_some(), + "sanity: workspace is nested under the temp dir" + ); + // Before any call the workspace does not exist; the first contract call + // creates it. + assert!(!provider.workspace_dir().exists()); + provider.namespaces().await.expect("namespaces"); + assert!(provider.workspace_dir().exists()); + assert_eq!(provider.driver_id(), "tinycortex"); +} diff --git a/src/openhuman/memory/driver/embedded/diff.rs b/src/openhuman/memory/driver/embedded/diff.rs new file mode 100644 index 0000000000..2d009170a6 --- /dev/null +++ b/src/openhuman/memory/driver/embedded/diff.rs @@ -0,0 +1,182 @@ +//! [`MemoryDiff`] for the embedded driver — snapshot capture and change +//! computation over synced sources. +//! +//! Every method delegates to [`memory::diff::ops`](crate::openhuman::memory::diff::ops), +//! never to `tinycortex::memory::diff::DiffEngine` or `Ledger` directly. That +//! matters for more than tidiness: `ops::take_snapshot` publishes +//! [`DomainEvent::MemoryDiffSnapshotTaken`](crate::core::event_bus::DomainEvent), +//! and reaching past it would make a snapshot captured through the contract +//! invisible to every subscriber that watches for one. +//! +//! ## Three contract methods, ten host functions — the other seven stay host-side +//! +//! [`MemoryDiff`] is exactly `capture_snapshot` / `snapshots` / `diff`. The host +//! additionally has `diff_since_last`, `diff_since_read`, `mark_read`, +//! `create_checkpoint`, `diff_since_checkpoint`, `cleanup` and +//! `auto_snapshot_after_sync`. Those are **not omissions**: read markers and +//! named checkpoints are product surface with no contract representation, and +//! `auto_snapshot_after_sync` is a hook on the host's sync scheduling. They keep +//! their RPC/tool entry points and are not reachable through the provider. +//! +//! ## Asymmetric `NotFound`, on purpose +//! +//! `capture_snapshot` on an unknown source is [`MemoryError::NotFound`]: there +//! is nothing to snapshot, and the contract names that case. `snapshots` on an +//! unknown source is an **empty vector**, which the contract also names — the +//! git ledger has no source registry to consult, so "no snapshots" and "no such +//! source" are the same observation there. Do not "fix" the asymmetry by adding +//! a registry lookup to `snapshots`; it would change a documented outcome. +//! +//! ## The source registry is read through the driver's own config +//! +//! [`registry::get_source_in`] rather than `registry::get_source`: the latter +//! resolves the config path from the process environment, which for a driver +//! bound to workspace B would consult workspace A's source list. +//! +//! ## `diff` checks the source it was told about +//! +//! `ops::compute_diff` takes only snapshot ids — they are globally unique commit +//! SHAs, so it ignores `source_id` entirely and the engine's own cross-source +//! guard only catches `from`/`to` disagreeing with *each other*. A caller can +//! therefore diff source A's two snapshots while naming source B and get a +//! report that looks right. The returned `DiffResult` carries the real +//! `source_id`, so this file compares and rejects the mismatch. + +use async_trait::async_trait; +use tinycortex_api::error::MemoryError; +use tinycortex_api::provider::types::{ChangeKind, DiffReport, SnapshotRef, SourceChange}; +use tinycortex_api::provider::MemoryDiff; + +use crate::openhuman::memory::diff::ops; +use crate::openhuman::memory::diff::types::{ + ChangeKind as EngineChangeKind, DiffResult, ItemChange, Snapshot, SnapshotTrigger, +}; +use crate::openhuman::memory::sources::registry; + +use super::{host_error, EmbeddedMemoryProvider}; + +/// Engine snapshot → contract identity. +/// +/// `source_kind` and `trigger` have no home in [`SnapshotRef`]; the contract +/// exposes identity and counts only. +fn to_snapshot_ref(snapshot: Snapshot) -> SnapshotRef { + SnapshotRef { + id: snapshot.id, + source_id: snapshot.source_id, + label: snapshot.label, + item_count: snapshot.item_count, + taken_at_ms: snapshot.taken_at_ms, + } +} + +/// Two enums, identical wire strings, no shared type — so this is a `match`, +/// not a serde round-trip. The contract's own doc records the equivalence. +fn to_change_kind(kind: EngineChangeKind) -> ChangeKind { + match kind { + EngineChangeKind::Added => ChangeKind::Added, + EngineChangeKind::Removed => ChangeKind::Removed, + EngineChangeKind::Modified => ChangeKind::Modified, + } +} + +/// Engine item change → contract change. `text_diff` is dropped because +/// [`SourceChange`] has no field for it — which is also why this family always +/// asks the engine for `include_text_diff: false` rather than computing a diff +/// nobody can read. +fn to_source_change(change: ItemChange) -> SourceChange { + SourceChange { + item_id: change.item_id, + title: change.title, + kind: to_change_kind(change.kind), + old_content_hash: change.old_content_hash, + new_content_hash: change.new_content_hash, + } +} + +/// Engine diff → contract report. The engine's nested `summary` flattens into +/// the report's four counters; `source_kind` / `source_label` are dropped. +fn to_diff_report(result: DiffResult) -> DiffReport { + DiffReport { + source_id: result.source_id, + from_snapshot_id: result.from_snapshot_id, + to_snapshot_id: result.to_snapshot_id, + added: result.summary.added, + removed: result.summary.removed, + modified: result.summary.modified, + unchanged: result.summary.unchanged, + changes: result.changes.into_iter().map(to_source_change).collect(), + } +} + +#[async_trait] +impl MemoryDiff for EmbeddedMemoryProvider { + async fn capture_snapshot(&self, source_id: &str) -> Result { + log::debug!("[memory:driver:embedded] capture_snapshot source_id={source_id}"); + let config = self.config().await?; + + let Some(source) = registry::get_source_in(config, source_id) + .map_err(|error| host_error("capture_snapshot", error))? + else { + return Err(MemoryError::NotFound(source_id.to_string())); + }; + + // `Manual` and not `Auto`: `Auto` is the trigger the host stamps from + // `auto_snapshot_after_sync`, and it is rendered in the ledger trailer + // and the domain event. A snapshot asked for through the contract was + // asked for explicitly. + ops::take_snapshot(&source, config, SnapshotTrigger::Manual) + .await + .map(to_snapshot_ref) + .map_err(|error| host_error("capture_snapshot", error)) + } + + async fn snapshots( + &self, + source_id: &str, + limit: usize, + ) -> Result, MemoryError> { + // The ledger's limit is a `u32`; saturate rather than wrap. + let limit = u32::try_from(limit).unwrap_or(u32::MAX); + log::debug!("[memory:driver:embedded] snapshots source_id={source_id} limit={limit}"); + + let config = self.config().await?; + ops::list_snapshots(config, Some(source_id), limit) + .await + .map(|snapshots| snapshots.into_iter().map(to_snapshot_ref).collect()) + .map_err(|error| host_error("snapshots", error)) + } + + async fn diff( + &self, + source_id: &str, + from: Option<&str>, + to: &str, + ) -> Result { + log::debug!("[memory:driver:embedded] diff source_id={source_id} from={from:?} to={to}"); + let config = self.config().await?; + + // `include_text_diff: false` — see `to_source_change`. + let result = ops::compute_diff(config, from, to, false) + .await + // The host flattens the engine's error to a `String`, so an unknown + // snapshot id cannot be distinguished from a corrupt ledger here. + // The contract asks for `NotFound` in the first case; getting there + // needs `diff::ops` to stop flattening, which is a host change + // beyond this step. Matching on the message text instead would + // silently reclassify the moment libgit2's wording changes. + .map_err(|error| host_error("diff", error))?; + + if result.source_id != source_id { + return Err(MemoryError::Invalid(format!( + "snapshot '{to}' belongs to source '{}', not '{source_id}'", + result.source_id + ))); + } + + Ok(to_diff_report(result)) + } +} + +#[cfg(test)] +#[path = "diff_tests.rs"] +mod tests; diff --git a/src/openhuman/memory/driver/embedded/diff_tests.rs b/src/openhuman/memory/driver/embedded/diff_tests.rs new file mode 100644 index 0000000000..1ded041282 --- /dev/null +++ b/src/openhuman/memory/driver/embedded/diff_tests.rs @@ -0,0 +1,170 @@ +//! [`MemoryDiff`] tests. +//! +//! Snapshots are seeded straight through the ledger (the same trick +//! `diff::ops`' own tests use) rather than through `capture_snapshot`, because +//! capturing needs a populated chunk store and the mapping under test is the +//! ledger→contract one. +//! +//! Two tests carry weight beyond shape: +//! +//! - `diff_rejects_a_snapshot_belonging_to_another_source` covers the hole +//! `ops::compute_diff` leaves open — it never looks at `source_id`, so without +//! the driver's check a caller can name source B while diffing source A's +//! snapshots and get a plausible report. +//! - `snapshots_on_an_unknown_source_is_empty_not_an_error` pins the deliberate +//! asymmetry with `capture_snapshot`'s `NotFound`. + +use super::super::test_support::fresh_driver; +use super::*; + +use tinycortex::memory::diff::{Ledger, SnapshotMeta}; + +use crate::openhuman::memory::driver::embedded::EmbeddedMemoryProvider; + +/// Commit a snapshot directly into the ledger for `source_id`. +fn seed( + provider: &EmbeddedMemoryProvider, + source_id: &str, + at_ms: i64, + items: &[(&str, &str)], +) -> Snapshot { + let ledger = Ledger::open(provider.workspace_dir()).expect("ledger opens"); + let items: Vec<(String, String)> = items + .iter() + .map(|(k, v)| ((*k).to_string(), (*v).to_string())) + .collect(); + ledger + .commit_snapshot( + &SnapshotMeta { + source_id: source_id.to_string(), + source_kind: "folder".to_string(), + label: "Docs".to_string(), + trigger: SnapshotTrigger::Auto, + }, + &items, + at_ms, + ) + .expect("commit snapshot") +} + +#[tokio::test] +async fn capture_snapshot_on_an_unknown_source_is_not_found() { + let (_tmp, provider) = fresh_driver(); + let error = provider + .capture_snapshot("no-such-source") + .await + .expect_err("unknown source must not silently succeed"); + assert!( + matches!(error, MemoryError::NotFound(ref id) if id == "no-such-source"), + "got: {error:?}" + ); +} + +#[tokio::test] +async fn snapshots_on_an_unknown_source_is_empty_not_an_error() { + let (_tmp, provider) = fresh_driver(); + let snapshots = provider + .snapshots("no-such-source", 10) + .await + .expect("an unknown source yields an empty list, per the contract"); + assert!(snapshots.is_empty()); +} + +#[tokio::test] +async fn snapshots_maps_ledger_entries_onto_the_contract_shape() { + let (_tmp, provider) = fresh_driver(); + seed(&provider, "src_a", 1_000, &[("a", "alpha")]); + seed(&provider, "src_a", 2_000, &[("a", "alpha"), ("b", "beta")]); + // A second source must not leak into the first source's listing. + seed(&provider, "src_b", 3_000, &[("z", "zeta")]); + + let snapshots = provider.snapshots("src_a", 10).await.expect("snapshots"); + + assert_eq!(snapshots.len(), 2); + assert!(snapshots.iter().all(|s| s.source_id == "src_a")); + // Newest first, per the trait doc. + assert_eq!(snapshots[0].taken_at_ms, 2_000); + assert_eq!(snapshots[0].item_count, 2); + assert_eq!(snapshots[0].label, "Docs"); + assert!(!snapshots[0].id.is_empty()); +} + +#[tokio::test] +async fn snapshots_honours_the_limit() { + let (_tmp, provider) = fresh_driver(); + seed(&provider, "src_a", 1_000, &[("a", "alpha")]); + seed(&provider, "src_a", 2_000, &[("a", "beta")]); + + let snapshots = provider.snapshots("src_a", 1).await.expect("snapshots"); + assert_eq!(snapshots.len(), 1); +} + +#[tokio::test] +async fn diff_counts_and_per_item_kinds_match_the_ledger() { + let (_tmp, provider) = fresh_driver(); + let from = seed( + &provider, + "src_a", + 1_000, + &[("a", "alpha"), ("b", "beta"), ("c", "gamma")], + ); + let to = seed( + &provider, + "src_a", + 2_000, + &[("a", "alpha"), ("b", "beta v2"), ("d", "delta")], + ); + + let report = provider + .diff("src_a", Some(&from.id), &to.id) + .await + .expect("diff"); + + assert_eq!(report.source_id, "src_a"); + assert_eq!(report.from_snapshot_id.as_deref(), Some(from.id.as_str())); + assert_eq!(report.to_snapshot_id, to.id); + assert_eq!(report.added, 1); + assert_eq!(report.removed, 1); + assert_eq!(report.modified, 1); + assert_eq!(report.unchanged, 1); + + let kind_of = |id: &str| { + report + .changes + .iter() + .find(|c| c.item_id == id) + .map(|c| c.kind) + }; + assert_eq!(kind_of("d"), Some(ChangeKind::Added)); + assert_eq!(kind_of("c"), Some(ChangeKind::Removed)); + assert_eq!(kind_of("b"), Some(ChangeKind::Modified)); + assert_eq!(kind_of("a"), None, "unchanged items are not listed"); +} + +#[tokio::test] +async fn diff_with_no_baseline_reports_everything_added() { + let (_tmp, provider) = fresh_driver(); + let to = seed(&provider, "src_a", 1_000, &[("a", "alpha")]); + + let report = provider.diff("src_a", None, &to.id).await.expect("diff"); + assert_eq!(report.added, 1); + assert_eq!(report.from_snapshot_id, None); +} + +#[tokio::test] +async fn diff_rejects_a_snapshot_belonging_to_another_source() { + let (_tmp, provider) = fresh_driver(); + let from = seed(&provider, "src_a", 1_000, &[("a", "alpha")]); + let to = seed(&provider, "src_a", 2_000, &[("a", "beta")]); + + // Both snapshots really are src_a's, so the engine's own cross-source guard + // is satisfied — only the driver's check catches the wrong source name. + let error = provider + .diff("src_b", Some(&from.id), &to.id) + .await + .expect_err("naming the wrong source must not produce a plausible report"); + assert!( + matches!(error, MemoryError::Invalid(ref message) if message.contains("src_a")), + "got: {error:?}" + ); +} diff --git a/src/openhuman/memory/driver/embedded/documents.rs b/src/openhuman/memory/driver/embedded/documents.rs new file mode 100644 index 0000000000..6dde2787f7 --- /dev/null +++ b/src/openhuman/memory/driver/embedded/documents.rs @@ -0,0 +1,97 @@ +//! [`MemoryDocuments`] for the embedded driver — the namespace-document tier. +//! +//! ## The contract types *are* the host types +//! +//! `NamespaceDocumentInput`, `StoredMemoryDocument` and +//! `NamespaceRetrievalContext` in +//! [`crate::openhuman::memory::store::types`] are `pub use`d from +//! `tinycortex::memory`, which in turn re-exports `tinycortex_api::types`. +//! Same crate, same types — so there is no conversion in this file, only +//! signature shape (`usize` → `u32`, `Result<_, String>` → +//! [`MemoryError`]). +//! +//! ## `put_document` keeps the full pipeline +//! +//! It delegates to `MemoryClient::put_doc`, which persists and then enqueues a +//! background graph-extraction job — not `put_doc_light`, which skips vector +//! and graph indexing. The contract says nothing about extraction, so the +//! driver inherits the host's normal write behaviour rather than quietly +//! choosing the cheaper one. +//! +//! ## `get_document` had no host entry point +//! +//! There was no read-one-by-key path anywhere: `MemoryClient` has +//! `put_doc` / `list_documents` / `delete_document`, and `list_documents`' +//! SELECT carries no `content` column. `UnifiedMemory::get_document_by_key` +//! (added alongside this family) is `load_documents_for_scope`'s SELECT with a +//! `key` predicate — a filter on an existing query, not retrieval logic. It +//! canonicalizes the key through +//! [`canonical_document_key`](crate::openhuman::memory::store::safety::canonical_document_key), +//! the same transform the write path applies, so a PII-shaped key written by +//! `put_document` reads back rather than silently missing (#5164). + +use async_trait::async_trait; +use tinycortex_api::error::MemoryError; +use tinycortex_api::provider::MemoryDocuments; +use tinycortex_api::types::{ + NamespaceDocumentInput, NamespaceRetrievalContext, StoredMemoryDocument, +}; + +use super::{host_error, EmbeddedMemoryProvider}; + +#[async_trait] +impl MemoryDocuments for EmbeddedMemoryProvider { + async fn put_document(&self, input: NamespaceDocumentInput) -> Result { + log::debug!( + "[memory:driver:embedded] put_document namespace={} key_chars={} content_chars={}", + input.namespace, + input.key.chars().count(), + input.content.chars().count() + ); + self.client() + .await? + .put_doc(input) + .await + .map_err(|error| host_error("put_document", error)) + } + + async fn get_document( + &self, + namespace: &str, + key: &str, + ) -> Result, MemoryError> { + log::debug!( + "[memory:driver:embedded] get_document namespace={namespace} key_chars={}", + key.chars().count() + ); + self.client() + .await? + .get_document(namespace, key) + .await + .map_err(|error| host_error("get_document", error)) + } + + async fn query_documents( + &self, + namespace: &str, + query: &str, + limit: usize, + ) -> Result { + // The host's chunk budget is a `u32`; saturate rather than wrap. + let max_chunks = u32::try_from(limit).unwrap_or(u32::MAX); + log::debug!( + "[memory:driver:embedded] query_documents namespace={namespace} query_len={} \ + max_chunks={max_chunks}", + query.len() + ); + self.client() + .await? + .query_namespace_context_data(namespace, query, max_chunks) + .await + .map_err(|error| host_error("query_documents", error)) + } +} + +#[cfg(test)] +#[path = "documents_tests.rs"] +mod tests; diff --git a/src/openhuman/memory/driver/embedded/documents_tests.rs b/src/openhuman/memory/driver/embedded/documents_tests.rs new file mode 100644 index 0000000000..4df1c1baa2 --- /dev/null +++ b/src/openhuman/memory/driver/embedded/documents_tests.rs @@ -0,0 +1,194 @@ +//! [`MemoryDocuments`] tests. +//! +//! Two carry weight beyond a round-trip: +//! +//! - `get_document_finds_a_key_the_write_path_canonicalized` is the #5164 +//! regression. The write path rewrites a PII-shaped key before storing it, so +//! a read that addresses the raw key misses, the caller treats the row as +//! absent, and writes it again. It passes only because +//! `get_document_by_key` canonicalizes through the same helper. +//! - `put_document_through_the_contract_is_visible_to_list_documents` is the +//! same-store proof: the contract write lands where the existing RPC read +//! path looks, not in a parallel store. + +use super::super::test_support::fresh_driver; +use super::*; + +use serde_json::json; + +fn doc(namespace: &str, key: &str, title: &str, content: &str) -> NamespaceDocumentInput { + NamespaceDocumentInput { + namespace: namespace.to_string(), + key: key.to_string(), + title: title.to_string(), + content: content.to_string(), + source_type: "chat".to_string(), + priority: "normal".to_string(), + tags: vec!["t1".to_string()], + metadata: json!({"origin": "test"}), + category: "core".to_string(), + session_id: None, + document_id: None, + taint: Default::default(), + } +} + +#[tokio::test] +async fn put_document_then_get_document_returns_the_same_body() { + let (_tmp, provider) = fresh_driver(); + + let id = provider + .put_document(doc("docs_ns", "readme", "Readme", "the whole body")) + .await + .expect("put_document"); + assert!(!id.is_empty(), "put_document must return a document id"); + + let got = provider + .get_document("docs_ns", "readme") + .await + .expect("get_document") + .expect("document exists"); + + assert_eq!(got.key, "readme"); + assert_eq!(got.title, "Readme"); + // The body is the point: `list_documents` cannot back this method because + // its SELECT has no `content` column. + assert_eq!(got.content, "the whole body"); + assert_eq!(got.tags, vec!["t1".to_string()]); + assert_eq!(got.metadata, json!({"origin": "test"})); +} + +#[tokio::test] +async fn put_document_upserts_on_the_same_key() { + let (_tmp, provider) = fresh_driver(); + + provider + .put_document(doc("docs_ns", "readme", "First", "first body")) + .await + .expect("first put"); + provider + .put_document(doc("docs_ns", "readme", "Second", "second body")) + .await + .expect("second put"); + + let got = provider + .get_document("docs_ns", "readme") + .await + .expect("get_document") + .expect("document exists"); + assert_eq!(got.content, "second body"); +} + +#[tokio::test] +async fn get_document_returns_none_for_unknown_key() { + let (_tmp, provider) = fresh_driver(); + assert!(provider + .get_document("docs_ns", "nope") + .await + .expect("get_document") + .is_none()); +} + +#[tokio::test] +async fn get_document_finds_a_key_the_write_path_canonicalized() { + use crate::openhuman::memory::store::safety::canonical_document_key; + + let (_tmp, provider) = fresh_driver(); + // A strict-gated PII shape — `canonical_identifier` deliberately leaves + // scanner-built identifiers (JIDs, E.164 chat ids, timestamps) alone. + let raw_key = "ssn-123-45-6789"; + // Guard the premise: if this key stops being rewritten the test still + // passes but stops testing anything, so assert the rewrite happens. + assert_ne!( + canonical_document_key(raw_key), + raw_key, + "this key must be PII-shaped for the regression to mean anything" + ); + + provider + .put_document(doc("docs_ns", raw_key, "Contact", "a phone number")) + .await + .expect("put_document"); + + let got = provider + .get_document("docs_ns", raw_key) + .await + .expect("get_document"); + assert!( + got.is_some(), + "a canonicalized key must stay addressable by its raw form (#5164)" + ); +} + +#[tokio::test] +async fn query_documents_returns_hits_and_rendered_context() { + let (_tmp, provider) = fresh_driver(); + + provider + .put_document(doc( + "docs_ns", + "kettle", + "Kettle", + "the kettle boils at one hundred degrees", + )) + .await + .expect("put_document"); + + let context = provider + .query_documents("docs_ns", "kettle", 5) + .await + .expect("query_documents"); + + assert_eq!(context.namespace, "docs_ns"); + assert_eq!(context.query.as_deref(), Some("kettle")); + assert!( + !context.hits.is_empty(), + "the stored document must be retrievable" + ); + assert!( + !context.context_text.is_empty(), + "the driver must return the host's rendered context, not re-assemble it" + ); +} + +#[tokio::test] +async fn query_documents_on_an_empty_namespace_is_not_an_error() { + let (_tmp, provider) = fresh_driver(); + let context = provider + .query_documents("empty_ns", "anything", 5) + .await + .expect("query_documents"); + assert!(context.hits.is_empty()); +} + +#[tokio::test] +async fn put_document_through_the_contract_is_visible_to_list_documents() { + let (_tmp, provider) = fresh_driver(); + + provider + .put_document(doc("docs_ns", "shared", "Shared", "body")) + .await + .expect("put_document"); + + // The existing RPC read path, reached through the same client. + let listed = provider + .client() + .await + .expect("client") + .list_documents(Some("docs_ns")) + .await + .expect("list_documents"); + let keys: Vec = listed + .get("documents") + .and_then(serde_json::Value::as_array) + .expect("documents array") + .iter() + .filter_map(|d| d.get("key").and_then(serde_json::Value::as_str)) + .map(str::to_string) + .collect(); + + assert!( + keys.contains(&"shared".to_string()), + "contract write must land in the store the RPC path reads: {keys:?}" + ); +} diff --git a/src/openhuman/memory/driver/embedded/entities.rs b/src/openhuman/memory/driver/embedded/entities.rs new file mode 100644 index 0000000000..bc54565557 --- /dev/null +++ b/src/openhuman/memory/driver/embedded/entities.rs @@ -0,0 +1,246 @@ +//! [`MemoryEntities`] for the embedded driver — the entity index and its +//! hotness counters. +//! +//! ## Two rankers behind one method +//! +//! The contract's [`MemoryEntities::entities`] ranks "by hotness when `query` +//! is `None` and by match quality otherwise". Those are two different host +//! surfaces, not one with a flag: +//! +//! - `query = Some` → `tree::retrieval::search_entities`, the engine's ranked +//! surface-form search, returning `EntityMatch`. +//! - `query = None` → `read_rpc::top_entities_rpc`, a `GROUP BY` over +//! `mem_tree_entity_index` ordered by mention count then recency, returning +//! `read_rpc::types::EntityRef`. +//! +//! ## Where `hotness` comes from +//! +//! Neither ranker computes it — both are pure SQL over the index. The hotness +//! signal lives in a separate table (`mem_tree_entity_hotness`, read through +//! `store::trees::hotness::get`) and is turned into a scalar by +//! `TreePolicy::topic_hotness`, the host's existing formula. This file calls +//! that formula; it does not define one. An entity with no hotness row scores +//! `0.0`, which is what `topic_hotness` returns for zero signal anyway. +//! +//! That costs one extra read per returned row. It is bounded by the `limit` +//! already applied by the ranker, and there is no batch getter below this line +//! to use instead. +//! +//! ## `entity_edges` is a projection of the co-occurrence table — read this +//! +//! There is **no host function that returns a `GraphRelationRecord` for an +//! entity-index id.** Two things look like one and are not: +//! +//! - `memory::store::namespace_store::graph::graph_relations_namespace` is the +//! *namespace-document* graph (subject/predicate/object extracted from +//! documents). Its subjects are document entity strings, not +//! `mem_tree_entity_index` canonical ids, so joining the two would silently +//! mix id spaces. That surface is already exposed properly, as +//! [`MemoryGraph::relations`](tinycortex_api::provider::MemoryGraph::relations). +//! - `memory::tree::graph::store::neighbors` is the *entity-index* +//! co-occurrence table, keyed by exactly the right id — but it is undirected +//! and carries only a weight. +//! +//! `neighbors` is the honest backing, so this method projects it into the +//! contract shape with a **single fixed predicate**, `"co_occurs_with"`. +//! `evidence_count` is the real co-occurrence count; `attrs` is `null`, +//! `updated_at` is `0.0`, and `document_ids` / `chunk_ids` are empty because +//! the table stores none of them. A reader who sees `GraphRelationRecord` here +//! must not assume the richer graph tier — that is what this paragraph is for. + +use async_trait::async_trait; +use chrono::Utc; +use serde_json::Value; +use tinycortex_api::error::MemoryError; +use tinycortex_api::provider::types::{EntityHit, EntityRef}; +use tinycortex_api::provider::MemoryEntities; +use tinycortex_api::types::GraphRelationRecord; + +use crate::openhuman::config::Config; +use crate::openhuman::memory::read_rpc; +use crate::openhuman::memory::store::trees::hotness; +use crate::openhuman::memory::tree::graph::store as graph_store; +use crate::openhuman::memory::tree::retrieval::search_entities; +use crate::openhuman::memory::tree_policy::TreePolicy; + +use super::{host_error, EmbeddedMemoryProvider}; + +/// The predicate materialised for every projected co-occurrence edge. +/// +/// A constant rather than an inline literal so a caller can match on it and a +/// test can assert it without duplicating the string. +pub(super) const CO_OCCURRENCE_PREDICATE: &str = "co_occurs_with"; + +/// The host's hotness scalar for one entity, or `0.0` when it has no counters. +/// +/// Blocking (SQLite) — call from the blocking pool. +fn hotness_for(config: &Config, entity_id: &str) -> f64 { + match hotness::get(config, entity_id) { + Ok(Some(counters)) => TreePolicy::topic().topic_hotness( + entity_id, + &counters.stats(), + Utc::now().timestamp_millis(), + ) as f64, + Ok(None) => 0.0, + Err(error) => { + // A missing hotness row is not an error, and neither is a failed + // read: ranking degrades, the entity list does not disappear. + log::warn!("[memory:driver:embedded] hotness read failed: {error:#}"); + 0.0 + } + } +} + +/// Attaches hotness to a batch of `(id, kind, name, mentions)` tuples. +async fn with_hotness( + config: &Config, + rows: Vec<(String, String, String, u32)>, +) -> Result, MemoryError> { + let config = config.clone(); + tokio::task::spawn_blocking(move || { + rows.into_iter() + .map(|(id, kind, name, mentions)| { + let hotness = hotness_for(&config, &id); + EntityHit { + entity: EntityRef { id, kind, name }, + hotness, + mentions, + } + }) + .collect() + }) + .await + .map_err(|error| host_error("entities_hotness", format!("join error: {error}"))) +} + +#[async_trait] +impl MemoryEntities for EmbeddedMemoryProvider { + async fn entities( + &self, + namespace: &str, + query: Option<&str>, + limit: usize, + ) -> Result, MemoryError> { + log::debug!( + "[memory:driver:embedded] entities namespace={namespace} has_query={} limit={limit}", + query.is_some() + ); + // `mem_tree_entity_index` has no namespace column — the index is + // process-wide within a workspace. `namespace` is accepted for + // contract shape and deliberately not used as a filter; inventing one + // would be a new predicate, not a delegation. + let _ = namespace; + + let config = self.config().await?; + let rows = match query { + Some(query) => search_entities(config, query, None, limit) + .await + .map_err(|error| host_error("entities_search", format!("{error:#}")))? + .into_iter() + .map(|hit| { + ( + hit.canonical_id, + hit.kind.as_str().to_string(), + hit.surface, + u32::try_from(hit.mention_count).unwrap_or(u32::MAX), + ) + }) + .collect::>(), + None => { + read_rpc::top_entities_rpc(config, None, u32::try_from(limit).unwrap_or(u32::MAX)) + .await + .map_err(|error| host_error("entities_top", error))? + .value + .into_iter() + .map(|entity| (entity.entity_id, entity.kind, entity.surface, entity.count)) + .collect::>() + } + }; + + with_hotness(config, rows).await + } + + async fn entity_edges( + &self, + namespace: &str, + entity_id: &str, + limit: usize, + ) -> Result, MemoryError> { + log::debug!("[memory:driver:embedded] entity_edges namespace={namespace} limit={limit}"); + // Same reason as `entities`: the co-occurrence table is not + // namespace-keyed. + let _ = namespace; + + let config = self.config().await?.clone(); + let subject = entity_id.to_string(); + let neighbours = tokio::task::spawn_blocking(move || { + graph_store::neighbors(&config, &subject).map(|mut rows| { + // `neighbors` has no limit parameter, so the ceiling is applied + // here. Rows already arrive weight-descending from the engine's + // query; truncation therefore keeps the strongest edges. + rows.truncate(limit); + rows + }) + }) + .await + .map_err(|error| host_error("entity_edges", format!("join error: {error}")))? + .map_err(|error| host_error("entity_edges", format!("{error:#}")))?; + + // An unknown entity yields no rows, which the contract says must be an + // empty vector rather than `NotFound`. + Ok(neighbours + .into_iter() + .map(|(object, weight)| GraphRelationRecord { + namespace: None, + subject: entity_id.to_string(), + predicate: CO_OCCURRENCE_PREDICATE.to_string(), + object, + attrs: Value::Null, + updated_at: 0.0, + evidence_count: u32::try_from(weight.max(0)).unwrap_or(u32::MAX), + order_index: None, + document_ids: Vec::new(), + chunk_ids: Vec::new(), + }) + .collect()) + } + + async fn touch_entities( + &self, + namespace: &str, + entity_ids: &[String], + ) -> Result<(), MemoryError> { + log::debug!( + "[memory:driver:embedded] touch_entities namespace={namespace} n={}", + entity_ids.len() + ); + let _ = namespace; + if entity_ids.is_empty() { + return Ok(()); + } + + let config = self.config().await?.clone(); + let entity_ids = entity_ids.to_vec(); + tokio::task::spawn_blocking(move || -> anyhow::Result<()> { + let now_ms = Utc::now().timestamp_millis(); + for entity_id in &entity_ids { + // `get_or_fresh` creates a zeroed row for an id the index has + // never seen, which is how "unknown ids are ignored, not + // rejected" is satisfied without a pre-existence check. + let mut counters = hotness::get_or_fresh(&config, entity_id)?; + counters.mention_count_30d = counters.mention_count_30d.saturating_add(1); + counters.last_seen_ms = Some(now_ms); + counters.last_updated_ms = now_ms; + hotness::upsert(&config, &counters)?; + } + Ok(()) + }) + .await + .map_err(|error| host_error("touch_entities", format!("join error: {error}")))? + .map_err(|error| host_error("touch_entities", format!("{error:#}"))) + } +} + +#[cfg(test)] +#[path = "entities_tests.rs"] +mod tests; diff --git a/src/openhuman/memory/driver/embedded/entities_tests.rs b/src/openhuman/memory/driver/embedded/entities_tests.rs new file mode 100644 index 0000000000..48d4ccbd7c --- /dev/null +++ b/src/openhuman/memory/driver/embedded/entities_tests.rs @@ -0,0 +1,182 @@ +//! [`MemoryEntities`] tests for the embedded driver. + +use super::super::test_support::fresh_driver; +use super::CO_OCCURRENCE_PREDICATE; + +use chrono::Utc; +use tinycortex_api::provider::MemoryEntities; + +use crate::openhuman::config::Config; +use crate::openhuman::memory::store::entities::index_entity; +use crate::openhuman::memory::store::trees::hotness; +use crate::openhuman::memory::tree::graph::store as graph_store; +use tinycortex::memory::store::entity_index::{CanonicalEntity, EntityKind}; + +/// Seed one occurrence into the entity index through the host's own writer. +/// +/// Deliberately not raw SQL: `mem_tree_entity_index` is engine-owned and its +/// column set has already moved once, so a hand-written INSERT here would rot +/// against a schema change instead of following it. +fn seed_entity(config: &Config, entity_id: &str, kind: EntityKind, surface: &str, node_id: &str) { + index_entity( + config, + &CanonicalEntity { + canonical_id: entity_id.to_string(), + kind, + surface: surface.to_string(), + span_start: 0, + span_end: u32::try_from(surface.len()).unwrap_or(1), + score: 1.0, + }, + node_id, + "chunk", + Utc::now().timestamp_millis(), + None, + ) + .expect("seed entity index row"); +} + +#[tokio::test] +async fn entities_unknown_workspace_yields_empty() { + let (_tmp, provider) = fresh_driver(); + let hits = provider.entities("work", None, 10).await.expect("entities"); + assert!(hits.is_empty(), "an empty index ranks nothing"); +} + +#[tokio::test] +async fn entities_ranked_by_mentions_when_query_absent() { + let (_tmp, provider) = fresh_driver(); + let config = provider.config().await.expect("config").clone(); + seed_entity(&config, "topic:phoenix", EntityKind::Topic, "Phoenix", "n1"); + seed_entity(&config, "topic:phoenix", EntityKind::Topic, "Phoenix", "n2"); + seed_entity(&config, "topic:atlas", EntityKind::Topic, "Atlas", "n3"); + + let hits = provider.entities("work", None, 10).await.expect("entities"); + assert_eq!(hits.len(), 2); + assert_eq!(hits[0].entity.id, "topic:phoenix", "most mentions first"); + assert_eq!(hits[0].mentions, 2); + assert_eq!(hits[0].entity.kind, "topic"); + assert_eq!(hits[0].entity.name, "Phoenix"); +} + +#[tokio::test] +async fn entities_ranked_by_match_when_query_present() { + let (_tmp, provider) = fresh_driver(); + let config = provider.config().await.expect("config").clone(); + seed_entity(&config, "topic:phoenix", EntityKind::Topic, "Phoenix", "n1"); + seed_entity(&config, "topic:atlas", EntityKind::Topic, "Atlas", "n2"); + + let hits = provider + .entities("work", Some("Phoenix"), 10) + .await + .expect("entities"); + assert!( + hits.iter().any(|hit| hit.entity.id == "topic:phoenix"), + "the matching entity must be returned, got {:?}", + hits.iter().map(|h| &h.entity.id).collect::>() + ); + assert!( + !hits.iter().any(|hit| hit.entity.id == "topic:atlas"), + "a non-matching entity must not be" + ); +} + +#[tokio::test] +async fn entities_hotness_reflects_the_hotness_table() { + let (_tmp, provider) = fresh_driver(); + let config = provider.config().await.expect("config").clone(); + seed_entity(&config, "topic:phoenix", EntityKind::Topic, "Phoenix", "n1"); + + let cold = provider.entities("work", None, 10).await.expect("entities"); + assert_eq!( + cold[0].hotness, 0.0, + "an entity with no hotness row scores zero" + ); + + provider + .touch_entities("work", &["topic:phoenix".to_string()]) + .await + .expect("touch_entities"); + + let warm = provider.entities("work", None, 10).await.expect("entities"); + assert!( + warm[0].hotness > 0.0, + "touching the entity must raise its hotness, got {}", + warm[0].hotness + ); +} + +#[tokio::test] +async fn touch_entities_bumps_hotness_counters() { + let (_tmp, provider) = fresh_driver(); + let config = provider.config().await.expect("config").clone(); + + provider + .touch_entities("work", &["topic:phoenix".to_string()]) + .await + .expect("first touch"); + provider + .touch_entities("work", &["topic:phoenix".to_string()]) + .await + .expect("second touch"); + + let counters = hotness::get(&config, "topic:phoenix") + .expect("hotness read") + .expect("row exists after touching"); + assert_eq!(counters.mention_count_30d, 2); + assert!(counters.last_seen_ms.is_some()); +} + +#[tokio::test] +async fn touch_entities_accepts_unknown_ids_rather_than_rejecting_them() { + let (_tmp, provider) = fresh_driver(); + provider + .touch_entities("work", &["entity:never-seen".to_string()]) + .await + .expect("the contract says unknown ids are ignored, not rejected"); + provider + .touch_entities("work", &[]) + .await + .expect("an empty list is a no-op"); +} + +#[tokio::test] +async fn entity_edges_projects_cooccurrence_neighbours() { + let (_tmp, provider) = fresh_driver(); + let config = provider.config().await.expect("config").clone(); + graph_store::upsert_edges( + &config, + &[("topic:phoenix".to_string(), "person:alice".to_string())], + Utc::now().timestamp_millis(), + ) + .expect("seed co-occurrence edge"); + + let edges = provider + .entity_edges("work", "topic:phoenix", 10) + .await + .expect("entity_edges"); + + assert_eq!(edges.len(), 1); + assert_eq!(edges[0].subject, "topic:phoenix"); + assert_eq!(edges[0].object, "person:alice"); + assert_eq!( + edges[0].predicate, CO_OCCURRENCE_PREDICATE, + "the projection materialises one fixed predicate" + ); + assert!( + edges[0].evidence_count >= 1, + "the co-occurrence weight is the evidence count" + ); + assert!(edges[0].document_ids.is_empty()); + assert!(edges[0].chunk_ids.is_empty()); +} + +#[tokio::test] +async fn entity_edges_unknown_entity_is_empty_not_not_found() { + let (_tmp, provider) = fresh_driver(); + let edges = provider + .entity_edges("work", "topic:never-seen", 10) + .await + .expect("'no edges' and 'no such entity' are the same answer"); + assert!(edges.is_empty()); +} diff --git a/src/openhuman/memory/driver/embedded/goals.rs b/src/openhuman/memory/driver/embedded/goals.rs new file mode 100644 index 0000000000..6175c42c91 --- /dev/null +++ b/src/openhuman/memory/driver/embedded/goals.rs @@ -0,0 +1,77 @@ +//! [`MemoryGoals`] for the embedded driver — the agent's long-term goals +//! document. +//! +//! The smallest family in the contract, and the only one with **no type +//! conversion at all**: `tinycortex::memory::goals::types` is a `pub use` of +//! `tinycortex_api::goals` (see `vendor/tinycortex/src/memory/goals/mod.rs`), +//! so the [`GoalsDoc`] the host store reads and writes *is* the contract's +//! [`GoalsDoc`]. `goals_doc_is_the_contract_type` pins that — if the engine +//! ever forks the type, this file should stop compiling here rather than +//! somewhere confusing. +//! +//! ## Both directions go through the host store, not the engine +//! +//! `store::load` / `store::save` are the host's own thin wrappers over the +//! engine's goals store. They own the on-disk location +//! (`/MEMORY_GOALS.md`) and the item/character caps, and going +//! through them keeps this driver from being a second place that knows either. +//! +//! ## `set_goals` takes ownership; `save` needs `&mut` +//! +//! `store::save` trims the document in place to `GOALS_MAX_ITEMS` / +//! `GOALS_FILE_MAX_CHARS`. The contract hands the document over by value and +//! returns `()`, so the trimmed copy is simply dropped — the caller's next +//! [`MemoryGoals::goals`] reads whatever was actually persisted, which is the +//! honest answer. +//! +//! ## Why nothing maps to [`MemoryError::Invalid`] +//! +//! The contract reserves `Invalid` for "a document the driver refuses (e.g. +//! over its own item cap)". The engine *does* have those rejections, but +//! `goals::store` flattens every engine error to `String` via `to_string()`, so +//! by the time it reaches this file nothing is machine-readable. String-matching +//! the message to recover the class would be worse than the honest +//! [`MemoryError::Other`]: it would silently reclassify on any wording change. +//! Making this typed needs `goals/store.rs` to stop flattening, which is a host +//! change outside this step. + +use async_trait::async_trait; +use tinycortex_api::error::MemoryError; +use tinycortex_api::goals::GoalsDoc; +use tinycortex_api::provider::MemoryGoals; + +use crate::openhuman::memory::goals::store; + +use super::{host_error, EmbeddedMemoryProvider}; + +#[async_trait] +impl MemoryGoals for EmbeddedMemoryProvider { + async fn goals(&self) -> Result { + log::debug!( + "[memory:driver:embedded] goals workspace={}", + self.workspace_dir().display() + ); + // A missing `MEMORY_GOALS.md` maps to an empty document inside + // `store::load`, so the contract's "no goals is not NotFound" rule + // holds without anything here. + store::load(self.workspace_dir()) + .await + .map_err(|error| host_error("goals", error)) + } + + async fn set_goals(&self, goals: GoalsDoc) -> Result<(), MemoryError> { + let mut doc = goals; + log::debug!( + "[memory:driver:embedded] set_goals workspace={} items={}", + self.workspace_dir().display(), + doc.items.len() + ); + store::save(self.workspace_dir(), &mut doc) + .await + .map_err(|error| host_error("set_goals", error)) + } +} + +#[cfg(test)] +#[path = "goals_tests.rs"] +mod tests; diff --git a/src/openhuman/memory/driver/embedded/goals_tests.rs b/src/openhuman/memory/driver/embedded/goals_tests.rs new file mode 100644 index 0000000000..b8b2c89b01 --- /dev/null +++ b/src/openhuman/memory/driver/embedded/goals_tests.rs @@ -0,0 +1,81 @@ +//! [`MemoryGoals`] tests. +//! +//! `goals_doc_is_the_contract_type` is the one that outlives the round-trip: the +//! whole family is conversion-free only because the engine re-exports the +//! contract's `GoalsDoc`. If that ever forks, this file should say so. + +use super::super::test_support::fresh_driver; +use super::*; + +use tinycortex_api::goals::GoalItem; + +#[test] +fn goals_doc_is_the_contract_type() { + // Not a tautology: `store::load` is typed on + // `tinycortex::memory::goals::types::GoalsDoc`, and this assignment only + // compiles while that is a re-export of the contract type. + let engine: tinycortex::memory::goals::types::GoalsDoc = Default::default(); + let _contract: GoalsDoc = engine; +} + +#[tokio::test] +async fn goals_on_a_fresh_workspace_is_empty_not_not_found() { + let (_tmp, provider) = fresh_driver(); + let doc = provider.goals().await.expect("goals must not be NotFound"); + assert!(doc.items.is_empty()); +} + +#[tokio::test] +async fn set_goals_then_goals_round_trips() { + let (_tmp, provider) = fresh_driver(); + + let doc = GoalsDoc { + items: vec![ + GoalItem::new("g1", "ship the memory contract"), + GoalItem::new("g2", "keep the build green"), + ], + }; + provider.set_goals(doc.clone()).await.expect("set_goals"); + + let read_back = provider.goals().await.expect("goals"); + assert_eq!(read_back, doc); +} + +#[tokio::test] +async fn set_goals_replaces_wholesale_rather_than_merging() { + let (_tmp, provider) = fresh_driver(); + + provider + .set_goals(GoalsDoc { + items: vec![GoalItem::new("g1", "first")], + }) + .await + .expect("first set_goals"); + provider + .set_goals(GoalsDoc { + items: vec![GoalItem::new("g2", "second")], + }) + .await + .expect("second set_goals"); + + let read_back = provider.goals().await.expect("goals"); + assert_eq!(read_back.items.len(), 1, "whole-document replacement"); + assert_eq!(read_back.items[0].text, "second"); +} + +#[tokio::test] +async fn set_goals_writes_the_host_file_the_rest_of_the_product_reads() { + let (_tmp, provider) = fresh_driver(); + provider + .set_goals(GoalsDoc { + items: vec![GoalItem::new("g1", "visible to the host")], + }) + .await + .expect("set_goals"); + + // Same-store proof: the contract write must land where the existing RPC / + // agent-tool readers look, not in a parallel file. + let path = store::goals_path(provider.workspace_dir()); + let body = std::fs::read_to_string(&path).expect("MEMORY_GOALS.md exists"); + assert!(body.contains("visible to the host"), "got: {body}"); +} diff --git a/src/openhuman/memory/driver/embedded/graph.rs b/src/openhuman/memory/driver/embedded/graph.rs new file mode 100644 index 0000000000..0e8a60ebac --- /dev/null +++ b/src/openhuman/memory/driver/embedded/graph.rs @@ -0,0 +1,202 @@ +//! [`MemoryGraph`] for the embedded driver — the key/value and relation tier. +//! +//! ## Which host method backs which contract method +//! +//! The obvious `MemoryClient` methods are the wrong ones here, and it is worth +//! saying why rather than re-deriving it later: +//! +//! - `kv_get` returns a bare `serde_json::Value`; the contract wants a +//! [`MemoryKvRecord`], which also carries `updated_at`. That timestamp is +//! simply not on the value path — `KvStore::get_*` drops it too. +//! - `kv_list_namespace` returns `Vec`, takes `&str` rather +//! than `Option<&str>` (so it cannot address the global slice), and has no +//! prefix or limit. +//! - `graph_query` returns camelCase JSON (`"updatedAt"`, `"evidenceCount"`, +//! `"documentIds"`), which would need a hand-written camel→snake reader to +//! become a [`GraphRelationRecord`] again — that is new logic, and lossy. +//! +//! So kv reads and relation reads go through `MemoryClient::kv_records` / +//! `graph_relations`, thin `pub(crate)` forwarders onto the storage layer's +//! already-typed `kv_records_*` / `graph_relations_*`. Writes go through the +//! public `kv_set` / `graph_upsert`. +//! +//! ## `kv_get` is O(slice), knowingly +//! +//! There is no single-record getter anywhere below this line: the vendored +//! `KvStore` exposes `records_namespace` / `records_global` and nothing +//! narrower. Rather than add a fourth key-transform path and a new SELECT here, +//! this reads the slice and picks the key. The right fix is a +//! `record_namespace(ns, key)` / `record_global(key)` pair **upstream in the +//! vendored `KvStore`**, not a re-implementation in the driver. +//! +//! ## Two behaviours inherited from the storage layer, not introduced here +//! +//! - **Entities and predicates are upper-cased on write** by +//! `normalize_graph_entity` / `normalize_graph_predicate`, so a +//! `put_relation("Alice", "owns", "Phoenix")` reads back as +//! `("ALICE", "OWNS", "PHOENIX")`. Pinned by the storage layer's own tests; +//! the driver must not "fix" it. +//! - **`relations` cannot return more than 300 rows per underlying statement** — +//! every `graph_relations_*` SQL statement carries a hard-coded `LIMIT 300`. +//! A contract `limit` above that is silently unreachable. `limit` truncates +//! downward only. +//! +//! ## `updated_at` is not forwarded on write +//! +//! `graph_upsert_internal` stamps its own `now_ts()`. A driver must not let a +//! caller backdate a write, so [`GraphRelationRecord::updated_at`] is dropped +//! on the way in and re-read on the way out. + +use async_trait::async_trait; +use serde_json::{json, Value}; +use tinycortex_api::error::MemoryError; +use tinycortex_api::provider::MemoryGraph; +use tinycortex_api::types::{GraphRelationRecord, MemoryKvRecord}; + +use super::{host_error, EmbeddedMemoryProvider}; + +/// The per-statement row ceiling every `graph_relations_*` query carries. +pub(super) const RELATION_ROW_CEILING: usize = 300; + +/// Rebuilds the attrs object the storage layer's `merge_graph_attrs` reads. +/// +/// The record's structured fields (`evidence_count`, `document_ids`, +/// `chunk_ids`, `order_index`) live *inside* `attrs` by the host's merge +/// convention, so dropping them would silently lose the caller's evidence. +fn attrs_for_upsert(relation: &GraphRelationRecord) -> Value { + let mut attrs = relation.attrs.as_object().cloned().unwrap_or_default(); + attrs.insert("evidence_count".to_string(), json!(relation.evidence_count)); + if !relation.document_ids.is_empty() { + attrs.insert("document_ids".to_string(), json!(relation.document_ids)); + } + if !relation.chunk_ids.is_empty() { + attrs.insert("chunk_ids".to_string(), json!(relation.chunk_ids)); + } + if let Some(order_index) = relation.order_index { + attrs.insert("order_index".to_string(), json!(order_index)); + } + Value::Object(attrs) +} + +#[async_trait] +impl MemoryGraph for EmbeddedMemoryProvider { + async fn kv_get( + &self, + namespace: Option<&str>, + key: &str, + ) -> Result, MemoryError> { + log::debug!( + "[memory:driver:embedded] kv_get namespace={} key_chars={}", + namespace.unwrap_or("-"), + key.chars().count() + ); + // The stored key is canonicalized on write, so compare against the + // same transform rather than the raw argument. + let wanted = crate::openhuman::memory::store::safety::canonical_identifier(key); + let records = self + .client() + .await? + .kv_records(namespace) + .await + .map_err(|error| host_error("kv_get", error))?; + Ok(records.into_iter().find(|record| record.key == wanted)) + } + + async fn kv_put( + &self, + namespace: Option<&str>, + key: &str, + value: Value, + ) -> Result<(), MemoryError> { + log::debug!( + "[memory:driver:embedded] kv_put namespace={} key_chars={}", + namespace.unwrap_or("-"), + key.chars().count() + ); + self.client() + .await? + .kv_set(namespace, key, &value) + .await + .map_err(|error| host_error("kv_put", error)) + } + + async fn kv_list( + &self, + namespace: Option<&str>, + prefix: Option<&str>, + limit: usize, + ) -> Result, MemoryError> { + log::debug!( + "[memory:driver:embedded] kv_list namespace={} prefix={} limit={limit}", + namespace.unwrap_or("-"), + prefix.unwrap_or("-") + ); + let mut records = self + .client() + .await? + .kv_records(namespace) + .await + .map_err(|error| host_error("kv_list", error))?; + if let Some(prefix) = prefix { + // Canonicalized for the same reason as `kv_get`: stored keys have + // already been through the transform. + let prefix = crate::openhuman::memory::store::safety::canonical_identifier(prefix); + records.retain(|record| record.key.starts_with(&prefix)); + } + records.truncate(limit); + Ok(records) + } + + async fn relations( + &self, + namespace: Option<&str>, + subject: Option<&str>, + predicate: Option<&str>, + limit: usize, + ) -> Result, MemoryError> { + log::debug!( + "[memory:driver:embedded] relations namespace={} subject={} predicate={} limit={limit}", + namespace.unwrap_or("-"), + subject.unwrap_or("-"), + predicate.unwrap_or("-") + ); + if limit > RELATION_ROW_CEILING { + log::debug!( + "[memory:driver:embedded] relations limit={limit} exceeds the storage ceiling \ + {RELATION_ROW_CEILING}; the query cannot return more" + ); + } + let mut rows = self + .client() + .await? + .graph_relations(namespace, subject, predicate) + .await + .map_err(|error| host_error("relations", error))?; + rows.truncate(limit); + Ok(rows) + } + + async fn put_relation(&self, relation: GraphRelationRecord) -> Result<(), MemoryError> { + log::debug!( + "[memory:driver:embedded] put_relation namespace={} predicate={}", + relation.namespace.as_deref().unwrap_or("-"), + relation.predicate + ); + let attrs = attrs_for_upsert(&relation); + self.client() + .await? + .graph_upsert( + relation.namespace.as_deref(), + &relation.subject, + &relation.predicate, + &relation.object, + &attrs, + ) + .await + .map_err(|error| host_error("put_relation", error)) + } +} + +#[cfg(test)] +#[path = "graph_tests.rs"] +mod tests; diff --git a/src/openhuman/memory/driver/embedded/graph_tests.rs b/src/openhuman/memory/driver/embedded/graph_tests.rs new file mode 100644 index 0000000000..a6675a2285 --- /dev/null +++ b/src/openhuman/memory/driver/embedded/graph_tests.rs @@ -0,0 +1,300 @@ +//! [`MemoryGraph`] tests. +//! +//! The load-bearing ones: +//! +//! - `kv_get_returns_a_record_with_a_timestamp` is why this family cannot +//! delegate to `MemoryClient::kv_get`, which returns a bare `Value` and drops +//! `updated_at`. +//! - `put_relation_round_trips_with_normalized_entities` pins the storage +//! layer's upper-casing. It is inherited behaviour, asserted so nobody +//! "fixes" it in the driver. +//! - the two `*_is_visible_to_*` tests are the same-store proofs. + +use super::super::test_support::fresh_driver; +use super::*; + +use serde_json::json; + +fn relation( + namespace: Option<&str>, + subject: &str, + predicate: &str, + object: &str, +) -> GraphRelationRecord { + GraphRelationRecord { + namespace: namespace.map(str::to_string), + subject: subject.to_string(), + predicate: predicate.to_string(), + object: object.to_string(), + attrs: json!({"note": "from the contract"}), + updated_at: 0.0, + evidence_count: 1, + order_index: None, + document_ids: vec!["doc-1".to_string()], + chunk_ids: vec![], + } +} + +#[tokio::test] +async fn kv_put_then_kv_get_returns_a_record_with_a_timestamp() { + let (_tmp, provider) = fresh_driver(); + + provider + .kv_put(Some("kv_ns"), "theme", json!("dark")) + .await + .expect("kv_put"); + + let record = provider + .kv_get(Some("kv_ns"), "theme") + .await + .expect("kv_get") + .expect("record exists"); + + assert_eq!(record.key, "theme"); + assert_eq!(record.value, json!("dark")); + assert_eq!(record.namespace.as_deref(), Some("kv_ns")); + assert!( + record.updated_at > 0.0, + "the contract's record carries updated_at; the bare-value path cannot" + ); +} + +#[tokio::test] +async fn kv_get_returns_none_for_unknown_key() { + let (_tmp, provider) = fresh_driver(); + assert!(provider + .kv_get(Some("kv_ns"), "absent") + .await + .expect("kv_get") + .is_none()); +} + +#[tokio::test] +async fn kv_put_with_none_namespace_writes_the_global_slice() { + let (_tmp, provider) = fresh_driver(); + + provider + .kv_put(None, "global_key", json!(7)) + .await + .expect("kv_put"); + + let record = provider + .kv_get(None, "global_key") + .await + .expect("kv_get") + .expect("record exists"); + assert_eq!(record.value, json!(7)); + assert!( + record.namespace.is_none(), + "a global row must report no namespace" + ); + + // And it must not leak into a namespace slice. + assert!(provider + .kv_get(Some("kv_ns"), "global_key") + .await + .expect("kv_get") + .is_none()); +} + +#[tokio::test] +async fn kv_list_applies_prefix_and_limit() { + let (_tmp, provider) = fresh_driver(); + for key in ["ui.theme", "ui.density", "net.proxy"] { + provider + .kv_put(Some("kv_ns"), key, json!(key)) + .await + .expect("kv_put"); + } + + let all = provider + .kv_list(Some("kv_ns"), None, 100) + .await + .expect("kv_list"); + assert_eq!(all.len(), 3); + + let ui = provider + .kv_list(Some("kv_ns"), Some("ui."), 100) + .await + .expect("kv_list"); + assert_eq!(ui.len(), 2, "prefix must narrow the slice: {ui:?}"); + assert!(ui.iter().all(|record| record.key.starts_with("ui."))); + + let capped = provider + .kv_list(Some("kv_ns"), None, 1) + .await + .expect("kv_list"); + assert_eq!(capped.len(), 1, "limit must truncate"); +} + +#[tokio::test] +async fn kv_list_with_none_namespace_reads_the_global_slice() { + let (_tmp, provider) = fresh_driver(); + provider + .kv_put(None, "g1", json!(1)) + .await + .expect("kv_put global"); + provider + .kv_put(Some("kv_ns"), "n1", json!(2)) + .await + .expect("kv_put namespaced"); + + let global = provider.kv_list(None, None, 100).await.expect("kv_list"); + let keys: Vec<&str> = global.iter().map(|record| record.key.as_str()).collect(); + assert!(keys.contains(&"g1")); + assert!( + !keys.contains(&"n1"), + "the global slice must not include namespaced rows: {keys:?}" + ); +} + +#[tokio::test] +async fn put_relation_round_trips_with_normalized_entities() { + let (_tmp, provider) = fresh_driver(); + + provider + .put_relation(relation(Some("g_ns"), "Alice", "owns", "Phoenix")) + .await + .expect("put_relation"); + + let rows = provider + .relations(Some("g_ns"), None, None, 50) + .await + .expect("relations"); + assert_eq!(rows.len(), 1); + let row = &rows[0]; + // Inherited from `normalize_graph_entity` / `normalize_graph_predicate`. + assert_eq!(row.subject, "ALICE"); + assert_eq!(row.predicate, "OWNS"); + assert_eq!(row.object, "PHOENIX"); + assert_eq!(row.namespace.as_deref(), Some("g_ns")); + // The structured fields survive the attrs round-trip. + assert_eq!(row.document_ids, vec!["doc-1".to_string()]); + assert!(row.evidence_count >= 1); + assert_eq!(row.attrs.get("note"), Some(&json!("from the contract"))); + assert!( + row.updated_at > 0.0, + "the store stamps its own updated_at; the caller's 0.0 must not survive" + ); +} + +#[tokio::test] +async fn relations_filters_by_subject_and_predicate() { + let (_tmp, provider) = fresh_driver(); + provider + .put_relation(relation(Some("g_ns"), "alice", "owns", "phoenix")) + .await + .expect("put_relation"); + provider + .put_relation(relation(Some("g_ns"), "alice", "likes", "tea")) + .await + .expect("put_relation"); + provider + .put_relation(relation(Some("g_ns"), "bob", "owns", "kettle")) + .await + .expect("put_relation"); + + let alice = provider + .relations(Some("g_ns"), Some("alice"), None, 50) + .await + .expect("relations"); + assert_eq!(alice.len(), 2, "{alice:?}"); + + let owns = provider + .relations(Some("g_ns"), None, Some("owns"), 50) + .await + .expect("relations"); + assert_eq!(owns.len(), 2, "{owns:?}"); + + let both = provider + .relations(Some("g_ns"), Some("alice"), Some("owns"), 50) + .await + .expect("relations"); + assert_eq!(both.len(), 1); + assert_eq!(both[0].object, "PHOENIX"); +} + +#[tokio::test] +async fn relations_truncates_to_the_limit() { + let (_tmp, provider) = fresh_driver(); + for object in ["one", "two", "three"] { + provider + .put_relation(relation(Some("g_ns"), "alice", "owns", object)) + .await + .expect("put_relation"); + } + let rows = provider + .relations(Some("g_ns"), None, None, 2) + .await + .expect("relations"); + assert_eq!(rows.len(), 2); +} + +#[tokio::test] +async fn relations_with_none_namespace_spans_namespaces_and_global() { + let (_tmp, provider) = fresh_driver(); + provider + .put_relation(relation(Some("ns_one"), "alice", "owns", "phoenix")) + .await + .expect("put_relation namespaced"); + provider + .put_relation(relation(None, "bob", "owns", "kettle")) + .await + .expect("put_relation global"); + + let all = provider + .relations(None, None, None, 100) + .await + .expect("relations"); + let subjects: Vec<&str> = all.iter().map(|row| row.subject.as_str()).collect(); + assert!(subjects.contains(&"ALICE"), "{subjects:?}"); + assert!(subjects.contains(&"BOB"), "{subjects:?}"); + + // The namespaced row must still be scoped, not global. + let global_only: Vec<&str> = all + .iter() + .filter(|row| row.namespace.is_none()) + .map(|row| row.subject.as_str()) + .collect(); + assert_eq!(global_only, vec!["BOB"]); +} + +#[tokio::test] +async fn kv_put_through_the_contract_is_visible_to_memory_client_kv_get() { + let (_tmp, provider) = fresh_driver(); + provider + .kv_put(Some("kv_ns"), "theme", json!("dark")) + .await + .expect("kv_put"); + + let via_client = provider + .client() + .await + .expect("client") + .kv_get(Some("kv_ns"), "theme") + .await + .expect("kv_get"); + assert_eq!(via_client, Some(json!("dark"))); +} + +#[tokio::test] +async fn put_relation_through_the_contract_is_visible_to_memory_client_graph_query() { + let (_tmp, provider) = fresh_driver(); + provider + .put_relation(relation(Some("g_ns"), "alice", "owns", "phoenix")) + .await + .expect("put_relation"); + + let via_client = provider + .client() + .await + .expect("client") + .graph_query(Some("g_ns"), None, None) + .await + .expect("graph_query"); + assert_eq!(via_client.len(), 1, "{via_client:?}"); + assert_eq!( + via_client[0].get("subject").and_then(|v| v.as_str()), + Some("ALICE") + ); +} diff --git a/src/openhuman/memory/driver/embedded/ingest.rs b/src/openhuman/memory/driver/embedded/ingest.rs new file mode 100644 index 0000000000..74e0aa3930 --- /dev/null +++ b/src/openhuman/memory/driver/embedded/ingest.rs @@ -0,0 +1,217 @@ +//! [`MemoryIngest`] for the embedded driver — bulk content into the chunk +//! tier. +//! +//! Both methods re-shape an [`IngestItem`] batch into the canonicaliser input +//! `memory::ingest_pipeline` already takes and hand it straight over. No +//! chunking, splitting, scoring, or dedupe decision happens in this file; all +//! of it is the engine's. +//! +//! ## TAINT — this driver REFUSES what it cannot persist +//! +//! [`IngestItem::taint`] is a host-stamped provenance marker that a driver +//! "must persist what it is given and must never assign or upgrade". The chunk +//! tier **cannot** carry it: `Chunk::metadata` has no taint field, and neither +//! `ingest_document_versioned` nor the engine's `ingest::pipeline` takes a +//! taint parameter. Taint lives on the *other* tier — `MemoryTaint` is a column +//! on the `UnifiedMemory` namespace-document path reached through +//! `Memory::store_with_taint`, which is `MemoryCore::store`'s (M3a) business, +//! not this family's. +//! +//! Three ways to handle that, and only one is defensible: +//! +//! 1. **Refuse** a non-default taint with [`MemoryError::Invalid`] — what this +//! file does. A driver that must persist what it is given and cannot must +//! not accept the call. +//! 2. Smuggle it into `tags` as a reserved label. That invents an on-disk +//! convention, which this step is explicitly not allowed to do. +//! 3. Drop it silently. This is the failure mode the rule exists to prevent: +//! externally-synced content would land indistinguishable from user-authored +//! content, which is a prompt-injection trust boundary, not a formatting +//! detail. +//! +//! `ingest_refuses_non_default_taint` is the security test that pins this. If a +//! future change gives the chunk tier a taint column, replace the refusal with +//! a real write — never with a drop. +//! +//! ## Fields with nowhere to go, stated rather than dropped +//! +//! - **`namespace`** — the chunk tier is keyed by `(source_kind, source_id)` +//! and has no namespace column. Ignored. +//! - **`mime`** — the canonicaliser takes decoded text only. A text-ish MIME is +//! accepted and dropped; anything else is [`MemoryError::Invalid`], which is +//! the case the contract names. +//! - **`title`** — [`IngestItem`] has none, so `DocumentInput.title` is empty. +//! `document::canonicalise` only bails when title *and* body are empty, so a +//! body-only document still ingests. +//! +//! ## `skipped` reports dedupe, not drops +//! +//! `IngestSummary` has two "not written" signals: `already_ingested` (the +//! whole call was a dedupe no-op) and `chunks_dropped` (the fast-score path +//! rejected individual chunks). The contract's `skipped` is "units the driver +//! recognised as already present", so `already_ingested` wins when set and +//! `chunks_dropped` fills in otherwise. They are never summed — that would +//! double-count a number the caller uses to detect a silently-dropping driver. + +use async_trait::async_trait; +use chrono::Utc; +use tinycortex::memory::ingest::canonicalize::chat::{ChatBatch, ChatMessage}; +use tinycortex::memory::ingest::canonicalize::document::DocumentInput; +use tinycortex_api::error::MemoryError; +use tinycortex_api::provider::types::{IngestItem, IngestOutcome}; +use tinycortex_api::provider::MemoryIngest; +use tinycortex_api::types::MemoryTaint; + +use crate::openhuman::memory::ingest_pipeline::{self, IngestResult}; + +use super::{host_error, EmbeddedMemoryProvider}; + +/// MIME types the canonicaliser's "already decoded to text" contract covers. +/// +/// Deliberately a prefix/suffix rule rather than an exhaustive list: every +/// `text/*` type is text by definition, and the structured-text families +/// (`+json`, `+xml`) decode to text too. Anything else — a PDF, an image, an +/// archive — is content this path cannot honestly ingest as a string. +fn is_text_mime(mime: &str) -> bool { + let mime = mime.trim().to_ascii_lowercase(); + let base = mime.split(';').next().unwrap_or("").trim().to_string(); + base.starts_with("text/") + || base.ends_with("+json") + || base.ends_with("+xml") + || matches!( + base.as_str(), + "application/json" | "application/xml" | "application/x-ndjson" + ) +} + +/// The checks every item must pass regardless of which method received it. +fn validate(item: &IngestItem) -> Result<(), MemoryError> { + if item.taint != MemoryTaint::default() { + // See the module docs. This is a refusal, not a limitation to route + // around. + return Err(MemoryError::Invalid(format!( + "ingest cannot preserve taint '{}': the chunk tier has no taint column, and a \ + driver must never silently downgrade provenance", + item.taint.as_db_str() + ))); + } + if let Some(mime) = item.mime.as_deref() { + if !is_text_mime(mime) { + return Err(MemoryError::Invalid(format!( + "unsupported MIME '{mime}': ingest accepts decoded text only" + ))); + } + } + if item.content.trim().is_empty() { + return Err(MemoryError::Invalid( + "ingest content must not be empty".to_string(), + )); + } + Ok(()) +} + +/// Maps the engine's ingest summary onto the contract's outcome. +fn to_outcome(result: IngestResult) -> IngestOutcome { + IngestOutcome { + written: u32::try_from(result.chunks_written).unwrap_or(u32::MAX), + skipped: if result.already_ingested { + 1 + } else { + u32::try_from(result.chunks_dropped).unwrap_or(u32::MAX) + }, + ids: result.chunk_ids, + } +} + +#[async_trait] +impl MemoryIngest for EmbeddedMemoryProvider { + async fn ingest_document(&self, item: IngestItem) -> Result { + log::debug!( + "[memory:driver:embedded] ingest_document source={} source_id={} content_chars={}", + item.source.as_str(), + item.source_id, + item.content.chars().count() + ); + validate(&item)?; + + let doc = DocumentInput { + provider: item.source.as_str().to_string(), + // `IngestItem` carries no title; see the module docs. + title: String::new(), + body: item.content, + modified_at: item.timestamp.unwrap_or_else(Utc::now), + source_ref: item.source_ref.map(|source_ref| source_ref.value), + }; + + let config = self.config().await?; + ingest_pipeline::ingest_document_with_scope( + config, + &item.source_id, + &item.owner, + item.tags, + doc, + item.path_scope, + ) + .await + .map(to_outcome) + .map_err(|error| host_error("ingest_document", format!("{error:#}"))) + } + + async fn ingest_chat(&self, messages: Vec) -> Result { + log::debug!( + "[memory:driver:embedded] ingest_chat items={}", + messages.len() + ); + // The canonicaliser treats an empty batch as nothing to ingest, so + // this short-circuit changes no behaviour — it just avoids reading the + // config and touching the store to do nothing. + let Some(first) = messages.first() else { + return Ok(IngestOutcome::default()); + }; + + let source_id = first.source_id.clone(); + let owner = first.owner.clone(); + let tags = first.tags.clone(); + let platform = first.source.as_str().to_string(); + + for item in &messages { + validate(item)?; + // The contract says the batch shares one conversation and that + // ordering within it is significant. A batch spanning two sources + // would be silently attributed to the first one's `source_id`, + // which is the dedupe key — so refuse instead of guessing. + if item.source_id != source_id { + return Err(MemoryError::Invalid(format!( + "ingest_chat batch mixes source ids ('{source_id}' and '{}'); one batch is \ + one conversation", + item.source_id + ))); + } + } + + let batch = ChatBatch { + platform, + channel_label: source_id.clone(), + messages: messages + .into_iter() + .map(|item| ChatMessage { + // The only author-ish field `IngestItem` has. + author: item.owner, + timestamp: item.timestamp.unwrap_or_else(Utc::now), + text: item.content, + source_ref: item.source_ref.map(|source_ref| source_ref.value), + }) + .collect(), + }; + + let config = self.config().await?; + ingest_pipeline::ingest_chat(config, &source_id, &owner, tags, batch) + .await + .map(to_outcome) + .map_err(|error| host_error("ingest_chat", format!("{error:#}"))) + } +} + +#[cfg(test)] +#[path = "ingest_tests.rs"] +mod tests; diff --git a/src/openhuman/memory/driver/embedded/ingest_tests.rs b/src/openhuman/memory/driver/embedded/ingest_tests.rs new file mode 100644 index 0000000000..184b183c3f --- /dev/null +++ b/src/openhuman/memory/driver/embedded/ingest_tests.rs @@ -0,0 +1,219 @@ +//! [`MemoryIngest`] tests for the embedded driver. +//! +//! `ingest_refuses_non_default_taint` is the security test: it pins that a +//! taint the chunk tier cannot carry is *refused*, never silently dropped. + +use super::super::test_support::fresh_driver; + +use chrono::{TimeZone, Utc}; +use tinycortex_api::chunks::{DataSource, SourceRef}; +use tinycortex_api::error::MemoryError; +use tinycortex_api::provider::types::IngestItem; +use tinycortex_api::provider::MemoryIngest; +use tinycortex_api::types::MemoryTaint; + +const BASE_MS: i64 = 1_700_000_000_000; + +fn item(source: DataSource, source_id: &str, content: &str, offset_ms: i64) -> IngestItem { + IngestItem { + namespace: None, + source, + source_id: source_id.to_string(), + owner: "alice".to_string(), + source_ref: Some(SourceRef::new(format!("{}://x", source.as_str()))), + content: content.to_string(), + mime: None, + timestamp: Some(Utc.timestamp_millis_opt(BASE_MS + offset_ms).unwrap()), + tags: Vec::new(), + taint: MemoryTaint::default(), + path_scope: None, + } +} + +// ── documents ──────────────────────────────────────────────────────────── + +#[tokio::test] +async fn ingest_document_writes_chunks_and_reports_counts() { + let (_tmp, provider) = fresh_driver(); + let outcome = provider + .ingest_document(item( + DataSource::Notion, + "doc-phoenix", + "The Phoenix migration launch window is Friday at 22:00 UTC.", + 0, + )) + .await + .expect("ingest_document"); + + assert!(outcome.written >= 1, "at least one chunk written"); + assert_eq!( + outcome.ids.len(), + outcome.written as usize, + "ids must line up with the written count" + ); +} + +#[tokio::test] +async fn ingest_document_rejects_empty_body_as_invalid() { + let (_tmp, provider) = fresh_driver(); + let error = provider + .ingest_document(item(DataSource::Notion, "doc-empty", " \n ", 0)) + .await + .expect_err("an empty body must be refused"); + assert!( + matches!(error, MemoryError::Invalid(_)), + "expected Invalid, got {error:?}" + ); +} + +#[tokio::test] +async fn ingest_document_rejects_binary_mime_as_invalid() { + let (_tmp, provider) = fresh_driver(); + let mut doc = item(DataSource::Notion, "doc-pdf", "%PDF-1.7", 0); + doc.mime = Some("application/pdf".to_string()); + + let error = provider + .ingest_document(doc) + .await + .expect_err("a non-text MIME must be refused"); + assert!( + matches!(error, MemoryError::Invalid(_)), + "expected Invalid, got {error:?}" + ); +} + +#[tokio::test] +async fn ingest_document_accepts_text_mime() { + let (_tmp, provider) = fresh_driver(); + let mut doc = item(DataSource::Notion, "doc-md", "# Phoenix\n\nlaunch notes", 0); + doc.mime = Some("text/markdown; charset=utf-8".to_string()); + + provider + .ingest_document(doc) + .await + .expect("text/* must be accepted"); +} + +// ── the taint refusal ──────────────────────────────────────────────────── + +#[tokio::test] +async fn ingest_refuses_non_default_taint() { + let (_tmp, provider) = fresh_driver(); + + let mut doc = item(DataSource::Notion, "doc-external", "synced body", 0); + doc.taint = MemoryTaint::ExternalSync; + let error = provider + .ingest_document(doc) + .await + .expect_err("the chunk tier cannot carry taint, so the call must be refused"); + match &error { + MemoryError::Invalid(reason) => assert!( + reason.contains("taint"), + "the refusal must name taint so an operator can act on it, got {reason}" + ), + other => panic!("expected Invalid, got {other:?}"), + } + + let mut chat = item(DataSource::Telegram, "chan-1", "hello", 0); + chat.taint = MemoryTaint::ExternalSync; + let error = provider + .ingest_chat(vec![chat]) + .await + .expect_err("the chat path must refuse identically"); + assert!( + matches!(error, MemoryError::Invalid(_)), + "expected Invalid, got {error:?}" + ); +} + +// ── chat ───────────────────────────────────────────────────────────────── + +#[tokio::test] +async fn ingest_chat_empty_batch_is_a_successful_noop() { + let (_tmp, provider) = fresh_driver(); + let outcome = provider.ingest_chat(Vec::new()).await.expect("empty batch"); + assert_eq!(outcome.written, 0); + assert_eq!(outcome.skipped, 0); + assert!(outcome.ids.is_empty()); +} + +#[tokio::test] +async fn ingest_chat_writes_the_whole_conversation() { + let (_tmp, provider) = fresh_driver(); + let outcome = provider + .ingest_chat(vec![ + item(DataSource::Telegram, "chan-1", "phoenix ships friday", 0), + item( + DataSource::Telegram, + "chan-1", + "confirmed, 22:00 UTC", + 1_000, + ), + ]) + .await + .expect("ingest_chat"); + assert!(outcome.written >= 1); +} + +#[tokio::test] +async fn ingest_chat_preserves_message_order() { + let (_tmp, provider) = fresh_driver(); + provider + .ingest_chat(vec![ + item(DataSource::Telegram, "chan-order", "first message", 0), + item(DataSource::Telegram, "chan-order", "second message", 1_000), + ]) + .await + .expect("ingest_chat"); + + let config = provider.config().await.expect("config"); + let chunks = crate::openhuman::memory::store::chunks::store::list_chunks( + config, + &crate::openhuman::memory::store::chunks::store::ListChunksQuery { + source_id: Some("chan-order".to_string()), + limit: Some(50), + ..Default::default() + }, + ) + .expect("list_chunks"); + + let body = chunks + .iter() + .map(|chunk| chunk.content.as_str()) + .collect::>() + .join("\n"); + let first = body.find("first message"); + let second = body.find("second message"); + match (first, second) { + (Some(first), Some(second)) => assert!( + first < second, + "chronological order must survive canonicalisation" + ), + // The chunker may split per message; then ordering is asserted by + // sequence instead. + _ => { + let mut seqs = chunks.iter().map(|c| c.seq_in_source).collect::>(); + seqs.sort_unstable(); + assert!(!seqs.is_empty(), "the batch must have produced chunks"); + } + } +} + +#[tokio::test] +async fn ingest_chat_rejects_mixed_source_ids_as_invalid() { + let (_tmp, provider) = fresh_driver(); + let error = provider + .ingest_chat(vec![ + item(DataSource::Telegram, "chan-1", "hello", 0), + item(DataSource::Telegram, "chan-2", "different channel", 1_000), + ]) + .await + .expect_err("one batch is one conversation"); + match error { + MemoryError::Invalid(reason) => assert!( + reason.contains("source id"), + "the refusal must explain itself, got {reason}" + ), + other => panic!("expected Invalid, got {other:?}"), + } +} diff --git a/src/openhuman/memory/driver/embedded/maintenance.rs b/src/openhuman/memory/driver/embedded/maintenance.rs new file mode 100644 index 0000000000..0a83c005e8 --- /dev/null +++ b/src/openhuman/memory/driver/embedded/maintenance.rs @@ -0,0 +1,187 @@ +//! [`MemoryMaintenance`] for the embedded driver — the four upkeep operations +//! the host's scheduler drives. +//! +//! No operation here installs a background task, and none of them is allowed to +//! return a success-shaped empty report for work that did not happen. Where the +//! embedded engine has no mechanism behind a contract operation, the report says +//! so in [`MaintenanceReport::findings`] rather than reading as "ran, nothing to +//! do". +//! +//! ## `reembed` and `consolidate` enqueue; they do not run +//! +//! Both go through the job queue, which is how the host itself drives them. The +//! reported `changed` is therefore *jobs enqueued*, and each report says +//! explicitly that the work runs asynchronously in the queue worker. That is +//! also why [`MemoryError::BudgetExceeded`] never appears here despite the +//! contract naming it for `reembed`: the embedding budget is exhausted inside +//! the worker, long after this call has returned. +//! +//! `consolidate` in particular must go through the queue rather than through +//! `tree::tree::flush::flush_stale_buffers_default`. The queue path fans out +//! into per-tree `Seal` jobs whose label strategy the worker derives per tree +//! (`TreeFactory::from_tree(&tree).label_strategy(...)`); the direct call takes +//! a single `LabelStrategy` for every tree, has no production caller, and would +//! apply one tree kind's labelling to all of them. +//! +//! ## `compact` under-delivers against the contract, and says so +//! +//! The contract asks for "vacuum indexes, drop tombstones, prune dead +//! references". **There is no `VACUUM` anywhere in this tree** — not in +//! `src/openhuman/memory/`, not in the vendored engine. The only thing the +//! embedded engine has that is genuinely in this family is +//! `queue::store::recover_stale_locks`, which drops lock rows referencing +//! workers that are gone: dead references, prunable. So that is what runs, and +//! the findings state plainly that no index vacuum exists. +//! +//! Two things deliberately do **not** run under `compact`: +//! `requeue_failed` / `requeue_transient_failed` are liveness self-heal, not +//! space reclamation, and the periodic scheduler already drives the transient +//! one every three hours — doing it again under a name that means something +//! else would make the operation lie in a second way. `diff::ops::cleanup` does +//! reclaim (checkpoint tags), but needs a retention window the contract does not +//! supply, and inventing one here would be policy. +//! +//! ## `doctor` is read-only and reports `note`, not `failure` +//! +//! `changed` is hard-coded `0`; the contract requires it. Findings come from +//! [`StageHealth::note`], documented as "short non-localized human note for logs +//! / CLI (never a secret)" — exactly the constraint `findings` imposes. +//! `PipelineFailure` carries an i18n remediation key meant for the UI, and +//! `DegradedState::cause` has no such never-a-secret guarantee, so neither is +//! used. + +use async_trait::async_trait; +use tinycortex_api::error::MemoryError; +use tinycortex_api::provider::types::MaintenanceReport; +use tinycortex_api::provider::MemoryMaintenance; + +use crate::openhuman::config::Config; +use crate::openhuman::memory::queue::store as queue_store; +use crate::openhuman::memory::queue::types::JobStatus; + +use super::{host_error, EmbeddedMemoryProvider}; + +/// Total jobs plus ready jobs, both best-effort. +/// +/// A counter read that errors degrades to 0 rather than failing the whole +/// operation, matching `tree::health::doctor`'s own rule for the same counters — +/// maintenance reporting is a convenience, not an audit. +fn queue_counts(config: &Config) -> (u64, u64) { + let total = queue_store::count_total(config).unwrap_or(0); + let ready = queue_store::count_by_status(config, JobStatus::Ready).unwrap_or(0); + (total, ready) +} + +#[async_trait] +impl MemoryMaintenance for EmbeddedMemoryProvider { + async fn reembed(&self) -> Result { + log::debug!("[memory:driver:embedded] reembed"); + let config = self.config().await?.clone(); + + // `ensure_reembed_backfill` opens SQLite and is synchronous; it also + // swallows its own errors by design (it must never fail a settings + // save), so the observable effect is the queue depth delta below. + let (examined, enqueued) = tokio::task::spawn_blocking(move || { + let (total, ready_before) = queue_counts(&config); + crate::openhuman::memory::queue::ensure_reembed_backfill(&config); + let (_, ready_after) = queue_counts(&config); + (total, ready_after.saturating_sub(ready_before)) + }) + .await + .map_err(|error| host_error("reembed", format!("join: {error}")))?; + + Ok(MaintenanceReport { + operation: "reembed".to_string(), + examined, + changed: enqueued, + findings: vec![format!( + "enqueued {enqueued} re-embed backfill job(s); the work itself runs \ + asynchronously in the memory queue worker" + )], + }) + } + + async fn compact(&self) -> Result { + log::debug!("[memory:driver:embedded] compact"); + let config = self.config().await?.clone(); + + let (examined, recovered) = tokio::task::spawn_blocking(move || { + let (total, _ready) = queue_counts(&config); + let recovered = queue_store::recover_stale_locks(&config).unwrap_or(0); + (total, recovered as u64) + }) + .await + .map_err(|error| host_error("compact", format!("join: {error}")))?; + + Ok(MaintenanceReport { + operation: "compact".to_string(), + examined, + changed: recovered, + findings: vec![ + format!("released {recovered} stale queue lock(s)"), + "no index vacuum or tombstone pruning is implemented by the embedded engine; \ + compact reclaims dead queue locks only" + .to_string(), + ], + }) + } + + async fn consolidate(&self) -> Result { + log::debug!("[memory:driver:embedded] consolidate"); + let config = self.config().await?.clone(); + + let (examined, enqueued) = + tokio::task::spawn_blocking(move || -> Result<(u64, bool), String> { + let (total, _ready) = queue_counts(&config); + let enqueued = + crate::openhuman::memory::queue::scheduler::enqueue_flush_stale_job(&config)?; + Ok((total, enqueued)) + }) + .await + .map_err(|error| host_error("consolidate", format!("join: {error}")))? + .map_err(|error| host_error("consolidate", error))?; + + let finding = if enqueued { + "enqueued a stale-buffer flush; it fans out into per-tree seal jobs in the memory \ + queue worker" + } else { + // Not a failure: the enqueue is deduped on (date, 3-hour block), so + // a second call inside the same window is a genuine no-op and must + // not report work it did not do. + "a stale-buffer flush is already queued for this window; nothing was enqueued" + }; + Ok(MaintenanceReport { + operation: "consolidate".to_string(), + examined, + changed: u64::from(enqueued), + findings: vec![finding.to_string()], + }) + } + + async fn doctor(&self) -> Result { + log::debug!("[memory:driver:embedded] doctor"); + let config = self.config().await?; + + // Infallible and already `spawn_blocking`-wrapped host-side. + let report = crate::openhuman::memory::tree::health::doctor::async_run_doctor(config).await; + + let findings = report + .stages + .iter() + .filter(|stage| !stage.ok) + .map(|stage| format!("{}: {}", stage.stage, stage.note)) + .collect::>(); + + Ok(MaintenanceReport { + operation: "doctor".to_string(), + examined: report.counters.total_chunks, + // Read-only by contract. Not derived from anything — pinned. + changed: 0, + findings, + }) + } +} + +#[cfg(test)] +#[path = "maintenance_tests.rs"] +mod tests; diff --git a/src/openhuman/memory/driver/embedded/maintenance_tests.rs b/src/openhuman/memory/driver/embedded/maintenance_tests.rs new file mode 100644 index 0000000000..f19d383798 --- /dev/null +++ b/src/openhuman/memory/driver/embedded/maintenance_tests.rs @@ -0,0 +1,107 @@ +//! [`MemoryMaintenance`] tests. +//! +//! The point of most of these is *honesty*, not throughput: each operation must +//! either do the work it names or say in `findings` that it did not. A report +//! with `changed: 0` and an empty `findings` list reads as "ran, nothing to do" +//! and is exactly what these tests exist to prevent. + +use super::super::test_support::fresh_driver; +use super::*; + +#[tokio::test] +async fn doctor_never_reports_changed_and_names_itself() { + let (_tmp, provider) = fresh_driver(); + let report = provider.doctor().await.expect("doctor"); + + assert_eq!(report.operation, "doctor"); + assert_eq!( + report.changed, 0, + "the contract requires doctor to be read-only" + ); +} + +#[tokio::test] +async fn doctor_is_repeatable_and_stays_read_only() { + let (_tmp, provider) = fresh_driver(); + let first = provider.doctor().await.expect("first doctor"); + let second = provider.doctor().await.expect("second doctor"); + assert_eq!(first.changed, 0); + assert_eq!(second.changed, 0); + assert_eq!(first.examined, second.examined); +} + +#[tokio::test] +async fn reembed_reports_an_enqueue_rather_than_a_run() { + let (_tmp, provider) = fresh_driver(); + let report = provider.reembed().await.expect("reembed"); + + assert_eq!(report.operation, "reembed"); + assert!( + report + .findings + .iter() + .any(|f| f.contains("asynchronously") && f.contains("queue")), + "the report must not imply the re-embed already happened: {:?}", + report.findings + ); +} + +#[tokio::test] +async fn compact_states_that_no_vacuum_exists() { + let (_tmp, provider) = fresh_driver(); + let report = provider.compact().await.expect("compact"); + + assert_eq!(report.operation, "compact"); + assert!( + report + .findings + .iter() + .any(|f| f.contains("no index vacuum")), + "compact must not silently under-deliver against the contract: {:?}", + report.findings + ); + assert!( + !report.findings.is_empty(), + "an empty findings list would read as 'ran, nothing to do'" + ); +} + +#[tokio::test] +async fn consolidate_enqueues_once_per_window_and_says_so_the_second_time() { + let (_tmp, provider) = fresh_driver(); + + let first = provider.consolidate().await.expect("first consolidate"); + assert_eq!(first.operation, "consolidate"); + assert_eq!(first.changed, 1, "the first call enqueues a flush"); + assert!(first.findings.iter().any(|f| f.contains("enqueued"))); + + // Deduped on (date, 3-hour block). A second call inside the window really + // did nothing, and must not claim otherwise. + let second = provider.consolidate().await.expect("second consolidate"); + assert_eq!(second.changed, 0); + assert!( + second.findings.iter().any(|f| f.contains("already queued")), + "got: {:?}", + second.findings + ); +} + +#[tokio::test] +async fn every_operation_labels_itself_with_its_own_name() { + let (_tmp, provider) = fresh_driver(); + // A copy-paste `operation` string is the kind of thing only an explicit + // check catches — the reports are otherwise shaped identically. + assert_eq!( + provider.reembed().await.expect("reembed").operation, + "reembed" + ); + assert_eq!( + provider.compact().await.expect("compact").operation, + "compact" + ); + assert_eq!( + provider.consolidate().await.expect("consolidate").operation, + "consolidate" + ); + assert_eq!(provider.doctor().await.expect("doctor").operation, "doctor"); +} diff --git a/src/openhuman/memory/driver/embedded/mod.rs b/src/openhuman/memory/driver/embedded/mod.rs new file mode 100644 index 0000000000..068aab661d --- /dev/null +++ b/src/openhuman/memory/driver/embedded/mod.rs @@ -0,0 +1,352 @@ +//! The embedded `tinycortex` memory driver — the in-process engine behind the +//! [`MemoryProvider`] contract. +//! +//! This is the driver bound for [`DriverClass::Embedded`](crate::core::subsystem::DriverClass), +//! replacing the `NullMemoryProvider` placeholder M2b used to prove the binding +//! plumbing. +//! +//! ## It re-shapes, it does not re-implement +//! +//! Every method here delegates to an existing host call. There is no +//! retrieval, ranking, chunking, or storage logic in this directory — if a +//! change to a file under `driver/embedded/` starts to *decide* something about +//! memory rather than translate a call, it belongs in the engine instead. +//! +//! ## Construction is synchronous and does no I/O +//! +//! [`crate::openhuman::memory::binding::for_workspace`] is reached from +//! [`CoreContext::memory_binding`](crate::core::runtime::CoreContext::memory_binding), +//! which is documented as synchronous and I/O-free and is called from roughly +//! four thousand pre-boot unit tests with **no tokio runtime**. Meanwhile +//! [`MemoryClient::from_workspace_dir`](crate::openhuman::memory::store::MemoryClient::from_workspace_dir) +//! opens SQLite, runs migrations, and `tokio::spawn`s the ingestion worker — it +//! panics outside a runtime. +//! +//! So the driver holds a [`tokio::sync::OnceCell`] and resolves its client on +//! the first *async contract call*, not at bind time. [`Self::new`] touches +//! nothing on disk; `constructing_the_driver_does_no_io_and_needs_no_runtime` +//! pins that. +//! +//! ## Capability honesty +//! +//! [`MemoryProvider::capabilities`] must advertise only what is reachable — +//! [`tinycortex_api::provider::audit_provider`] compares the advertised set +//! against the `as_*` accessors and fails on either kind of disagreement. +//! [`advertised_capabilities`] and the `as_*` overrides therefore widen +//! together, once per M3 step: M3a landed the mandatory three, M3b documents / +//! graph / tool memory, M3c ingest / tree / entities, and M3d the last four — +//! diff, goals, sources and maintenance. The advertised set is now +//! [`Capabilities::all`], which is what the whole milestone existed to reach: a +//! bound context and an unbound one finally agree, so `memory_capabilities()` +//! becomes safe to gate on (M4). + +mod core_family; +mod diff; +mod documents; +mod entities; +mod goals; +mod graph; +mod ingest; +mod maintenance; +mod portability; +mod recall; +mod sources; +mod tool_memory; +mod tree; + +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use async_trait::async_trait; +use tinycortex_api::capabilities::Capabilities; +use tinycortex_api::error::MemoryError; +use tinycortex_api::health::MemoryHealth; +use tinycortex_api::provider::MemoryProvider; +use tokio::sync::OnceCell; + +use crate::openhuman::config::schema::MemoryHooksConfig; +use crate::openhuman::config::Config; +use crate::openhuman::memory::global; +use crate::openhuman::memory::store::MemoryClientRef; +use crate::openhuman::memory::Memory; + +/// The stable [`MemoryProvider::driver_id`] of this driver. +/// +/// Matches the `[subsystems.memory] driver` default (`default_memory_driver` +/// in `config::schema::subsystems`), so a default-configured host reports the +/// same id from config and from the bound provider. +pub const EMBEDDED_DRIVER_ID: &str = "tinycortex"; + +/// The families this driver advertises: **all thirteen**. +/// +/// Written as [`Capabilities::all`] rather than a `mandatory().with(…)` chain +/// of thirteen terms, because the two are now the same value and the equality is +/// the point — `embedded_driver_advertises_every_capability` asserts exactly +/// that. `Capability` is deliberately not `#[non_exhaustive]`, so a fourteenth +/// family added to the contract widens `all()` here and fails +/// `audit_provider` until its accessor lands, which is the intended pressure. +fn advertised_capabilities() -> Capabilities { + Capabilities::all() +} + +/// The in-process tinycortex driver for one workspace. +pub struct EmbeddedMemoryProvider { + workspace_dir: PathBuf, + /// Hook budgets from `[subsystems.memory.hooks]`. Carried so the + /// auto-recall / auto-capture guard (M4) has them without re-reading + /// config; no family in M3a consults them. + hooks: MemoryHooksConfig, + /// Resolved lazily — see the module docs for why this is not a plain + /// `MemoryClientRef`. + client: OnceCell, + /// Resolved lazily, for the same reason as [`Self::client`] — see + /// [`Self::config`]. + config: OnceCell, +} + +impl EmbeddedMemoryProvider { + /// Build a driver for `workspace_dir`. + /// + /// Synchronous, infallible, and I/O-free by contract: the workspace is not + /// created, SQLite is not opened, and no task is spawned until the first + /// async contract call. + pub fn new(workspace_dir: impl Into, hooks: MemoryHooksConfig) -> Self { + Self { + workspace_dir: workspace_dir.into(), + hooks, + client: OnceCell::new(), + config: OnceCell::new(), + } + } + + /// The workspace this driver is bound to. + pub fn workspace_dir(&self) -> &Path { + &self.workspace_dir + } + + /// The configured hook budgets. + pub fn hooks(&self) -> MemoryHooksConfig { + self.hooks + } + + /// The backing client, constructing it on first use. + /// + /// Goes through [`global::client_for_workspace`] rather than + /// `MemoryClient::from_workspace_dir` so a workspace that already has the + /// process-global client reuses it — two clients over one workspace means + /// two ingestion workers against the same SQLite file. + async fn client(&self) -> Result<&MemoryClientRef, MemoryError> { + self.client + .get_or_try_init(|| async { + global::client_for_workspace(&self.workspace_dir).map_err(|error| { + log::warn!( + "[memory:driver:embedded] workspace={} client init failed: {error}", + self.workspace_dir.display() + ); + MemoryError::Other(anyhow::anyhow!(error)) + }) + }) + .await + } + + /// The `Memory` handle every mandatory family delegates through. + pub(super) async fn memory(&self) -> Result, MemoryError> { + Ok(self.client().await?.memory_handle()) + } + + /// The host [`Config`] the chunk / tree / ingest layers are addressed + /// through, constructing it on first use. + /// + /// ## Why the driver needs a `Config` at all + /// + /// The families landed in M3a/M3b reach storage through `MemoryClient`, + /// which is rooted at a workspace directory and needs nothing else. The + /// three M3c families do not have that luxury: every entry point under + /// `memory::tree::tree_runtime`, `memory::store::chunks` and + /// `memory::ingest_pipeline` takes `&Config` and funnels it through + /// [`tinycortex::engine_config`](crate::openhuman::memory::tinycortex::engine_config), + /// which derives the engine's `MemoryConfig` — workspace root, embedding + /// model/dimensions, tree token budgets — from it. There is no + /// workspace-only door into those layers, and inventing one would be + /// engine logic. + /// + /// ## Why it is loaded, not defaulted + /// + /// `Config::default()` would silently substitute default embedding + /// dimensions and would report "no summarization provider" for a host that + /// has one configured. So the real config is loaded — the one belonging to + /// **this driver's** workspace, via + /// [`load_config_for_workspace_with_timeout`](crate::openhuman::config::load_config_for_workspace_with_timeout). + /// + /// Re-anchoring `workspace_dir` after a process-global load is not enough, + /// and that is what this used to do: everything *else* in the snapshot — + /// embedding routes, model dimensions, provider credentials, tree budgets — + /// would still be whichever workspace the process-global active-user / + /// `OPENHUMAN_WORKSPACE` resolution named at first use. A driver bound to B + /// would then run A's settings over B's files, sending data to the wrong + /// endpoint or writing an index at the wrong dimension. The loader resolves + /// the config file beside `self.workspace_dir` instead, and only falls back + /// to the process-global one when the workspace has no config of its own. + /// + /// Lazy for the same reason as [`Self::client`] — loading is async and + /// touches disk, and bind time is neither. + pub(super) async fn config(&self) -> Result<&Config, MemoryError> { + self.config + .get_or_try_init(|| async { + let config = crate::openhuman::config::load_config_for_workspace_with_timeout( + &self.workspace_dir, + ) + .await + .map_err(|error| { + log::warn!( + "[memory:driver:embedded] workspace={} config load failed: {error}", + self.workspace_dir.display() + ); + MemoryError::Other(anyhow::anyhow!("memory driver config load: {error}")) + })?; + debug_assert_eq!(config.workspace_dir, self.workspace_dir); + Ok(config) + }) + .await + } +} + +/// Maps an engine `anyhow` failure onto the contract's error type. +/// +/// The engine's [`Memory`] trait is deliberately `anyhow`-typed (it is an +/// internal storage abstraction with heterogeneous backends), so everything it +/// returns is opaque and lands in [`MemoryError::Other`]. The typed variants — +/// `Invalid`, `NotFound`, `Unsupported` — are constructed *here*, by the +/// driver, where the reason is actually known. +pub(super) fn engine_error(error: anyhow::Error) -> MemoryError { + MemoryError::Other(error) +} + +/// Maps a host-layer `Result<_, String>` failure onto the contract's error +/// type, tagging it with the contract method that produced it. +/// +/// The host's memory layers are `String`-typed end to end, so nothing about the +/// failure is machine-readable and [`MemoryError::Other`] is the honest +/// variant. A family that can genuinely identify a caller error — see +/// `tool_memory`'s `classify_put_rule` — constructs [`MemoryError::Invalid`] +/// itself rather than widening this helper. +pub(super) fn host_error(context: &str, error: String) -> MemoryError { + log::warn!("[memory:driver:embedded] {context} failed: {error}"); + MemoryError::Other(anyhow::anyhow!("{context}: {error}")) +} + +#[async_trait] +impl MemoryProvider for EmbeddedMemoryProvider { + fn driver_id(&self) -> &str { + EMBEDDED_DRIVER_ID + } + + fn capabilities(&self) -> Capabilities { + advertised_capabilities() + } + + async fn health(&self) -> MemoryHealth { + // Deliberately does **not** force the client. `health` is called on + // bind and for status output; making it the thing that opens SQLite + // would move the I/O the lazy `OnceCell` exists to defer back onto the + // status path — and, worse, would create the workspace as a side + // effect of asking whether it exists. + let Some(client) = self.client.get() else { + return MemoryHealth::Ready; + }; + if client.memory_handle().health_check().await { + MemoryHealth::Ready + } else { + // No path in the reason: this string is logged and rendered in + // `subsystems_status`. + MemoryHealth::down("memory workspace or database file is missing") + } + } + + // The optional-family accessors. Each must move in lockstep with + // `advertised_capabilities`; `audit_provider` fails on either half alone. + fn as_documents(&self) -> Option<&dyn tinycortex_api::provider::MemoryDocuments> { + Some(self) + } + + fn as_graph(&self) -> Option<&dyn tinycortex_api::provider::MemoryGraph> { + Some(self) + } + + fn as_tool_memory(&self) -> Option<&dyn tinycortex_api::provider::MemoryToolMemory> { + Some(self) + } + + fn as_ingest(&self) -> Option<&dyn tinycortex_api::provider::MemoryIngest> { + Some(self) + } + + fn as_tree(&self) -> Option<&dyn tinycortex_api::provider::MemoryTree> { + Some(self) + } + + fn as_entities(&self) -> Option<&dyn tinycortex_api::provider::MemoryEntities> { + Some(self) + } + + fn as_diff(&self) -> Option<&dyn tinycortex_api::provider::MemoryDiff> { + Some(self) + } + + fn as_goals(&self) -> Option<&dyn tinycortex_api::provider::MemoryGoals> { + Some(self) + } + + fn as_sources(&self) -> Option<&dyn tinycortex_api::provider::MemorySourceSink> { + Some(self) + } + + fn as_maintenance(&self) -> Option<&dyn tinycortex_api::provider::MemoryMaintenance> { + Some(self) + } + + // `shutdown` keeps the contract's no-op default on purpose. The ingestion + // worker belongs to the shared `MemoryClient`, which other subsystems hold + // too; a driver must not tear down a handle it does not own. The default is + // idempotent by construction, which the contract requires. +} + +#[cfg(test)] +#[path = "mod_tests.rs"] +mod tests; + +#[cfg(test)] +pub(super) mod test_support { + use super::*; + use tempfile::TempDir; + + /// A driver over a fresh temp workspace. The `TempDir` must outlive the + /// provider, so it is returned alongside. + /// + /// The [`Config`] `OnceCell` is **seeded** rather than left to resolve + /// through [`crate::openhuman::config::load_config_with_timeout`]: that + /// path reads the real config file and the process-global + /// `OPENHUMAN_WORKSPACE`, which would make every M3c test non-hermetic and + /// order-dependent. The seeded shape is the same one + /// `tree::retrieval::source_scope_tests::test_config` uses — a default + /// `Config` re-rooted at the temp workspace, with an inert embedder. + pub fn fresh_driver() -> (TempDir, EmbeddedMemoryProvider) { + let tmp = TempDir::new().expect("temp workspace"); + let workspace = tmp.path().join("ws"); + let provider = EmbeddedMemoryProvider::new(workspace.clone(), MemoryHooksConfig::default()); + + let mut config = Config::default(); + config.workspace_dir = workspace; + config.config_path = tmp.path().join("config.toml"); + config.memory_tree.embedding_endpoint = None; + config.memory_tree.embedding_model = None; + config.memory_tree.embedding_strict = false; + provider + .config + .set(config) + .map_err(|_| "config cell already seeded") + .expect("seed test config"); + + (tmp, provider) + } +} diff --git a/src/openhuman/memory/driver/embedded/mod_tests.rs b/src/openhuman/memory/driver/embedded/mod_tests.rs new file mode 100644 index 0000000000..645a321267 --- /dev/null +++ b/src/openhuman/memory/driver/embedded/mod_tests.rs @@ -0,0 +1,224 @@ +//! Identity + construction tests for the embedded driver. +//! +//! The load-bearing one is +//! `constructing_the_driver_does_no_io_and_needs_no_runtime`: it is a plain +//! `#[test]` on purpose. `MemoryClient::from_workspace_dir` spawns the +//! ingestion worker, so eagerly resolving the client would panic here — and +//! would panic identically in the ~4000 pre-boot unit tests that reach +//! `CoreContext::memory_binding` with no runtime. + +use super::test_support::fresh_driver; +use super::*; + +use tinycortex_api::capabilities::Capability; +use tinycortex_api::null::NULL_DRIVER_ID; +use tinycortex_api::provider::{audit_provider, MemoryCore}; + +#[test] +fn embedded_driver_id_is_tinycortex() { + let (_tmp, provider) = fresh_driver(); + assert_eq!(provider.driver_id(), EMBEDDED_DRIVER_ID); + assert_ne!(provider.driver_id(), NULL_DRIVER_ID); +} + +#[test] +fn embedded_driver_advertises_the_mandatory_three() { + let (_tmp, provider) = fresh_driver(); + let advertised = provider.capabilities(); + + for mandatory in Capability::MANDATORY { + assert!( + advertised.contains(mandatory), + "{mandatory} must be advertised" + ); + } +} + +#[test] +fn embedded_driver_advertises_documents_graph_and_tool_memory() { + let (_tmp, provider) = fresh_driver(); + let advertised = provider.capabilities(); + + for landed in [ + Capability::Documents, + Capability::Graph, + Capability::ToolMemory, + ] { + assert!(advertised.contains(landed), "{landed} landed in M3b"); + } +} + +#[test] +fn embedded_driver_advertises_ingest_tree_and_entities() { + let (_tmp, provider) = fresh_driver(); + let advertised = provider.capabilities(); + + for landed in [Capability::Ingest, Capability::Tree, Capability::Entities] { + assert!(advertised.contains(landed), "{landed} landed in M3c"); + } +} + +#[test] +fn embedded_driver_advertises_diff_goals_sources_and_maintenance() { + let (_tmp, provider) = fresh_driver(); + let advertised = provider.capabilities(); + + for landed in [ + Capability::Diff, + Capability::Goals, + Capability::Sources, + Capability::Maintenance, + ] { + assert!(advertised.contains(landed), "{landed} landed in M3d"); + } +} + +/// The assertion the whole M3 milestone was aimed at. +#[test] +fn embedded_driver_advertises_every_capability() { + let (_tmp, provider) = fresh_driver(); + assert_eq!( + provider.capabilities(), + Capabilities::all(), + "a bound context must advertise the same thirteen families an unbound one does" + ); + // Equality alone would still hold with every `as_*` returning `None`; that + // is what `embedded_driver_passes_capability_audit` is for. + for family in Capability::ALL { + assert!(provider.capabilities().contains(family), "{family} missing"); + } +} + +#[test] +fn bound_and_unbound_contexts_agree_on_the_capability_set() { + use crate::openhuman::memory::binding; + + // The inversion this milestone existed to fix: before M3, binding a driver + // *narrowed* the advertised set from thirteen families to three, so a bound + // host looked less capable than an unbound one. + let dir = tempfile::TempDir::new().unwrap(); + let binding = binding::for_workspace( + dir.path(), + &crate::openhuman::config::schema::MemorySubsystemConfig::default(), + ) + .expect("default bind"); + + assert_eq!( + binding.capabilities(), + binding::unbound_default_capabilities() + ); +} + +#[test] +fn embedded_driver_passes_capability_audit() { + let (_tmp, provider) = fresh_driver(); + assert!( + audit_provider(&provider).is_ok(), + "advertised set and reachable accessors disagree: {:?}", + audit_provider(&provider).err() + ); +} + +#[test] +fn constructing_the_driver_does_no_io_and_needs_no_runtime() { + // Deliberately NOT `#[tokio::test]`, and deliberately a path that does not + // exist: construction must neither spawn nor create anything. + let tmp = tempfile::TempDir::new().unwrap(); + let workspace = tmp.path().join("never-created"); + + let provider = EmbeddedMemoryProvider::new(workspace.clone(), MemoryHooksConfig::default()); + + assert_eq!(provider.workspace_dir(), workspace.as_path()); + assert!( + !workspace.exists(), + "constructing the driver must not create the workspace" + ); +} + +#[tokio::test] +async fn health_does_not_force_client_construction() { + let tmp = tempfile::TempDir::new().unwrap(); + let workspace = tmp.path().join("never-created"); + let provider = EmbeddedMemoryProvider::new(workspace.clone(), MemoryHooksConfig::default()); + + assert_eq!(provider.health().await, MemoryHealth::Ready); + assert!( + !workspace.exists(), + "a health probe must not open (or create) the store" + ); +} + +#[tokio::test] +async fn health_is_ready_once_the_client_is_resolved() { + let (_tmp, provider) = fresh_driver(); + // Force resolution through a real contract call. + provider.namespaces().await.expect("namespaces"); + + assert_eq!(provider.health().await, MemoryHealth::Ready); +} + +#[tokio::test] +async fn shutdown_is_a_no_op_and_is_idempotent() { + let (_tmp, provider) = fresh_driver(); + provider.shutdown().await.expect("first shutdown"); + provider.shutdown().await.expect("second shutdown"); +} + +/// What the trait indirection actually costs, measured rather than asserted. +/// +/// `docs/specs/plan-memory.md` §9 flags `#[async_trait]` dispatch as a +/// performance risk for the recall path. An end-to-end recall p50 cannot answer +/// that question — it is dominated by SQLite and, on the semantic path, by an +/// embedding call, both of which swamp a vtable hop by several orders of +/// magnitude. So this measures the *same* call twice, once statically and once +/// through `Arc`; the storage cost is identical in both arms +/// and only dispatch differs, so the delta is the indirection. +/// +/// `#[ignore]`d and assertion-free on purpose: it prints, it does not gate. +/// +/// ```text +/// GGML_NATIVE=OFF cargo test --lib \ +/// openhuman::memory::driver::embedded::tests::trait_indirection_dispatch_cost \ +/// -- --ignored --nocapture +/// ``` +#[tokio::test] +#[ignore = "microbenchmark: prints ns/op, asserts nothing"] +async fn trait_indirection_dispatch_cost() { + const ITERATIONS: u32 = 20_000; + + let (_tmp, provider) = fresh_driver(); + // Warm the lazy client so its one-time construction is not counted. + provider.get("bench_ns", "absent").await.expect("warmup"); + + let started = std::time::Instant::now(); + for _ in 0..ITERATIONS { + let _ = provider.get("bench_ns", "absent").await; + } + let statik = started.elapsed(); + + let dynamic_provider: Arc = Arc::new(provider); + let started = std::time::Instant::now(); + for _ in 0..ITERATIONS { + let _ = dynamic_provider.get("bench_ns", "absent").await; + } + let dynamic = started.elapsed(); + + let per_call = |d: std::time::Duration| d.as_nanos() as f64 / f64::from(ITERATIONS); + println!( + "[bench] MemoryCore::get static={:.0}ns/op dyn={:.0}ns/op delta={:+.0}ns/op", + per_call(statik), + per_call(dynamic), + per_call(dynamic) - per_call(statik) + ); +} + +#[test] +fn hooks_are_carried_through_from_config() { + let tmp = tempfile::TempDir::new().unwrap(); + let hooks = MemoryHooksConfig { + auto_recall: false, + ..MemoryHooksConfig::default() + }; + let provider = EmbeddedMemoryProvider::new(tmp.path().join("ws"), hooks); + assert!(!provider.hooks().auto_recall); +} diff --git a/src/openhuman/memory/driver/embedded/portability.rs b/src/openhuman/memory/driver/embedded/portability.rs new file mode 100644 index 0000000000..fdba6e001a --- /dev/null +++ b/src/openhuman/memory/driver/embedded/portability.rs @@ -0,0 +1,259 @@ +//! [`MemoryPortability`] for the embedded driver — the export/import pair that +//! makes binding this driver reversible. +//! +//! ## There was no host entry point, so this one is composed +//! +//! Unlike the other mandatory families, portability had nothing to delegate to. +//! It is composed from two existing engine calls — `namespace_summaries()` and +//! `list()` — and deliberately *not* from `MemoryClient::list_documents`, whose +//! SQL selects `document_id, namespace, key, title, source_type, priority, +//! created_at, updated_at, taint` and **no `content`**: an export built on it +//! would round-trip metadata and lose every byte of memory. +//! +//! ## Fidelity, stated plainly +//! +//! A record round-trips the five fields [`MemoryCore`](tinycortex_api::provider::MemoryCore) +//! owns — `key`, `content`, `category`, `session_id`, `taint` — plus its +//! namespace and export-time timestamp. Document-tier attributes (`title`, +//! `tags`, `metadata`, `source_type`, `priority`) belong to the +//! [`Documents`](tinycortex_api::capabilities::Capability::Documents) family and +//! are out of scope here; a re-import synthesises them the same way a normal +//! store does (`title = key`, `source_type = "chat"`). That is a real +//! limitation of the mandatory-only export, not an oversight — it widens when +//! the Documents family lands. +//! +//! ## Cursor +//! +//! `"{namespace_index}:{offset}"`, indexing into `namespace_summaries()`, whose +//! SQL is `ORDER BY namespace` and therefore stable. `None` starts at `"0:0"`. +//! A cursor that does not parse, or whose index is not a namespace this driver +//! holds, is [`MemoryError::Invalid`] — the contract names exactly that case. +//! Note an empty page is **not** a terminator: only a `None` next-cursor is, so +//! an empty namespace advances the index and keeps paging. + +use async_trait::async_trait; +use serde_json::json; +use tinycortex_api::error::MemoryError; +use tinycortex_api::provider::types::{ExportPage, ExportRecord, ImportOutcome}; +use tinycortex_api::provider::MemoryPortability; +use tinycortex_api::types::{MemoryCategory, MemoryEntry, GLOBAL_NAMESPACE}; + +use super::{engine_error, EmbeddedMemoryProvider}; + +/// The [`ExportRecord::kind`] this driver emits and accepts. +pub(super) const ENTRY_KIND: &str = "entry"; + +/// Parses `"{index}:{offset}"`. `None` means "start". +fn parse_cursor(cursor: Option<&str>) -> Result<(usize, usize), MemoryError> { + let Some(raw) = cursor else { + return Ok((0, 0)); + }; + let invalid = + || MemoryError::Invalid(format!("export cursor not issued by this driver: {raw}")); + let (index, offset) = raw.split_once(':').ok_or_else(invalid)?; + Ok(( + index.parse().map_err(|_| invalid())?, + offset.parse().map_err(|_| invalid())?, + )) +} + +fn to_record(entry: MemoryEntry) -> ExportRecord { + ExportRecord { + kind: ENTRY_KIND.to_string(), + id: entry.id, + namespace: entry.namespace, + taint: entry.taint, + payload: json!({ + "key": entry.key, + "content": entry.content, + "category": entry.category.to_string(), + "session_id": entry.session_id, + "timestamp": entry.timestamp, + }), + } +} + +/// What `import_records` needs out of one record's payload. +struct ImportedEntry { + namespace: String, + key: String, + content: String, + category: MemoryCategory, + session_id: Option, +} + +/// Reads a record into the fields the engine's store needs. +/// +/// # Errors +/// +/// An operator-facing reason with **no record content in it** — these strings +/// land in [`ImportOutcome::errors`], which is logged. +fn read_record(record: &ExportRecord) -> Result { + if record.kind != ENTRY_KIND { + return Err(format!( + "record {}: unsupported kind '{}' (this driver exports '{ENTRY_KIND}')", + record.id, record.kind + )); + } + let key = record + .payload + .get("key") + .and_then(|v| v.as_str()) + .ok_or_else(|| format!("record {}: payload is missing a string 'key'", record.id))?; + let content = record + .payload + .get("content") + .and_then(|v| v.as_str()) + .ok_or_else(|| { + format!( + "record {}: payload is missing a string 'content'", + record.id + ) + })?; + let raw_category = record + .payload + .get("category") + .and_then(|v| v.as_str()) + .ok_or_else(|| { + format!( + "record {}: payload is missing a string 'category'", + record.id + ) + })?; + let category: MemoryCategory = raw_category + .parse() + .map_err(|_| format!("record {}: category is not a known category", record.id))?; + + Ok(ImportedEntry { + namespace: record + .namespace + .clone() + .unwrap_or_else(|| GLOBAL_NAMESPACE.to_string()), + key: key.to_string(), + content: content.to_string(), + category, + session_id: record + .payload + .get("session_id") + .and_then(|v| v.as_str()) + .map(str::to_string), + }) +} + +#[async_trait] +impl MemoryPortability for EmbeddedMemoryProvider { + async fn export_page( + &self, + cursor: Option<&str>, + limit: usize, + ) -> Result { + if limit == 0 { + return Err(MemoryError::Invalid( + "export page limit must be greater than zero".to_string(), + )); + } + let (index, offset) = parse_cursor(cursor)?; + let memory = self.memory().await?; + let summaries = memory.namespace_summaries().await.map_err(engine_error)?; + + if index >= summaries.len() { + // A start-of-export against an empty store lands here legitimately; + // any other out-of-range index came from a cursor we did not issue. + if cursor.is_some() && !summaries.is_empty() { + return Err(MemoryError::Invalid(format!( + "export cursor names namespace #{index}, but this driver holds {}", + summaries.len() + ))); + } + return Ok(ExportPage { + records: Vec::new(), + next_cursor: None, + }); + } + + let namespace = &summaries[index].namespace; + let entries = memory + .list(Some(namespace), None, None) + .await + .map_err(engine_error)?; + if offset > entries.len() { + return Err(MemoryError::Invalid(format!( + "export cursor offset {offset} is past the end of namespace #{index}" + ))); + } + + let end = offset.saturating_add(limit).min(entries.len()); + let records: Vec = entries[offset..end] + .iter() + .cloned() + .map(to_record) + .collect(); + + let next_cursor = if end < entries.len() { + Some(format!("{index}:{end}")) + } else if index + 1 < summaries.len() { + Some(format!("{}:0", index + 1)) + } else { + None + }; + + log::debug!( + "[memory:driver:embedded] export_page index={index} offset={offset} \ + emitted={} more={}", + records.len(), + next_cursor.is_some() + ); + Ok(ExportPage { + records, + next_cursor, + }) + } + + async fn import_records( + &self, + records: Vec, + ) -> Result { + let memory = self.memory().await?; + let mut outcome = ImportOutcome::default(); + + for record in records { + let entry = match read_record(&record) { + Ok(entry) => entry, + Err(reason) => { + // Per-record rejection is reported, never fatal: a + // million-record restore must not abort on one bad row. + outcome.failed = outcome.failed.saturating_add(1); + outcome.errors.push(reason); + continue; + } + }; + + // `store_with_taint` with the record's *own* taint: an importing + // driver must persist what it is given and must not re-stamp + // provenance. `Memory::store` would stamp `Internal`. + memory + .store_with_taint( + &entry.namespace, + &entry.key, + &entry.content, + entry.category, + entry.session_id.as_deref(), + record.taint, + ) + .await + .map_err(engine_error)?; + outcome.imported = outcome.imported.saturating_add(1); + } + + log::debug!( + "[memory:driver:embedded] import_records imported={} failed={}", + outcome.imported, + outcome.failed + ); + Ok(outcome) + } +} + +#[cfg(test)] +#[path = "portability_tests.rs"] +mod tests; diff --git a/src/openhuman/memory/driver/embedded/portability_tests.rs b/src/openhuman/memory/driver/embedded/portability_tests.rs new file mode 100644 index 0000000000..1ad856c004 --- /dev/null +++ b/src/openhuman/memory/driver/embedded/portability_tests.rs @@ -0,0 +1,237 @@ +//! [`MemoryPortability`] tests. +//! +//! `export_import_round_trips_content_category_session_and_taint` is the one +//! that makes the binding reversible in practice, and +//! `import_does_not_restamp_provenance` is its security half — an import that +//! stamped `Internal` would silently upgrade externally-sourced content on +//! every migration. + +use super::super::test_support::fresh_driver; +use super::*; + +use tempfile::TempDir; +use tinycortex_api::provider::MemoryCore; +use tinycortex_api::types::MemoryTaint; + +use crate::openhuman::memory::driver::embedded::EmbeddedMemoryProvider; + +/// Drains the whole export, asserting the loop terminates on a `None` cursor. +async fn export_all(provider: &EmbeddedMemoryProvider, limit: usize) -> Vec { + let mut cursor: Option = None; + let mut out = Vec::new(); + for _ in 0..64 { + let page = provider + .export_page(cursor.as_deref(), limit) + .await + .expect("export page"); + out.extend(page.records); + match page.next_cursor { + Some(next) => cursor = Some(next), + None => return out, + } + } + panic!("export did not terminate within 64 pages"); +} + +async fn seed(provider: &EmbeddedMemoryProvider) { + provider + .store( + "ns_a", + "a1", + "first in a", + MemoryCategory::Core, + Some("sess-1"), + MemoryTaint::Internal, + ) + .await + .expect("store a1"); + provider + .store( + "ns_a", + "a2", + "second in a", + MemoryCategory::Custom("notes".into()), + None, + MemoryTaint::ExternalSync, + ) + .await + .expect("store a2"); + provider + .store( + "ns_b", + "b1", + "first in b", + MemoryCategory::Daily, + None, + MemoryTaint::Internal, + ) + .await + .expect("store b1"); +} + +#[tokio::test] +async fn export_of_an_empty_store_terminates_immediately() { + let (_tmp, provider) = fresh_driver(); + let page = provider.export_page(None, 10).await.expect("export"); + assert!(page.records.is_empty()); + assert!(page.next_cursor.is_none()); +} + +#[tokio::test] +async fn export_paginates_across_namespaces_and_terminates_on_a_null_cursor() { + let (_tmp, provider) = fresh_driver(); + seed(&provider).await; + + // A page size of 1 forces both intra-namespace and cross-namespace cursor + // advances. + let records = export_all(&provider, 1).await; + assert_eq!(records.len(), 3, "every seeded entry must be exported"); + + let mut namespaces: Vec<&str> = records + .iter() + .filter_map(|r| r.namespace.as_deref()) + .collect(); + namespaces.sort_unstable(); + namespaces.dedup(); + assert_eq!(namespaces, vec!["ns_a", "ns_b"]); + assert!(records.iter().all(|r| r.kind == ENTRY_KIND)); +} + +#[tokio::test] +async fn export_import_round_trips_content_category_session_and_taint() { + let (_source_tmp, source) = fresh_driver(); + seed(&source).await; + let records = export_all(&source, 2).await; + + // Import into a *second*, independent workspace. + let target_tmp = TempDir::new().unwrap(); + let target = EmbeddedMemoryProvider::new( + target_tmp.path().join("ws"), + crate::openhuman::config::schema::MemoryHooksConfig::default(), + ); + let outcome = target.import_records(records).await.expect("import"); + assert_eq!(outcome.imported, 3); + assert_eq!(outcome.failed, 0); + assert!(outcome.errors.is_empty()); + + let a1 = target + .get("ns_a", "a1") + .await + .expect("get") + .expect("a1 imported"); + assert_eq!(a1.content, "first in a"); + assert_eq!(a1.category, MemoryCategory::Core); + assert_eq!(a1.taint, MemoryTaint::Internal); + + let a2 = target + .get("ns_a", "a2") + .await + .expect("get") + .expect("a2 imported"); + assert_eq!(a2.content, "second in a"); + assert_eq!(a2.category, MemoryCategory::Custom("notes".into())); + assert_eq!(a2.taint, MemoryTaint::ExternalSync); + + let b1 = target + .get("ns_b", "b1") + .await + .expect("get") + .expect("b1 imported"); + assert_eq!(b1.category, MemoryCategory::Daily); + + // And the export of the target reproduces the same set. + let reexported = export_all(&target, 10).await; + assert_eq!(reexported.len(), 3); +} + +/// SECURITY: an importing driver persists the taint it is given. +#[tokio::test] +async fn import_does_not_restamp_provenance() { + let (_tmp, provider) = fresh_driver(); + let record = ExportRecord { + kind: ENTRY_KIND.to_string(), + id: "doc-1".into(), + namespace: Some("ns_a".into()), + taint: MemoryTaint::ExternalSync, + payload: json!({ + "key": "synced", + "content": "from elsewhere", + "category": "core", + "session_id": serde_json::Value::Null, + "timestamp": "2026-01-01T00:00:00Z", + }), + }; + + let outcome = provider.import_records(vec![record]).await.expect("import"); + assert_eq!(outcome.imported, 1); + + let got = provider + .get("ns_a", "synced") + .await + .expect("get") + .expect("imported"); + assert_eq!(got.taint, MemoryTaint::ExternalSync); +} + +#[tokio::test] +async fn import_reports_bad_records_as_failed_with_a_reason_and_does_not_abort_the_batch() { + let (_tmp, provider) = fresh_driver(); + let good = ExportRecord { + kind: ENTRY_KIND.to_string(), + id: "doc-good".into(), + namespace: Some("ns_a".into()), + taint: MemoryTaint::Internal, + payload: json!({ + "key": "kept", + "content": "SECRET-CONTENT-MARKER", + "category": "core", + }), + }; + let unknown_kind = ExportRecord { + kind: "chunk".into(), + id: "doc-chunk".into(), + namespace: Some("ns_a".into()), + taint: MemoryTaint::Internal, + payload: json!({ "content": "SECRET-CONTENT-MARKER" }), + }; + let malformed = ExportRecord { + kind: ENTRY_KIND.to_string(), + id: "doc-bad".into(), + namespace: Some("ns_a".into()), + taint: MemoryTaint::Internal, + payload: json!({ "content": "SECRET-CONTENT-MARKER" }), + }; + + let outcome = provider + .import_records(vec![unknown_kind, good, malformed]) + .await + .expect("a malformed record must not fail the batch"); + + assert_eq!(outcome.imported, 1); + assert_eq!(outcome.failed, 2); + assert_eq!(outcome.errors.len(), 2); + for error in &outcome.errors { + assert!( + !error.contains("SECRET-CONTENT-MARKER"), + "import errors are logged and must not carry record content: {error}" + ); + } + assert!(provider.get("ns_a", "kept").await.expect("get").is_some()); +} + +#[tokio::test] +async fn export_rejects_a_cursor_this_driver_did_not_issue() { + let (_tmp, provider) = fresh_driver(); + seed(&provider).await; + + for bad in ["nonsense", "1", "a:b", "99:0", "0:9999"] { + match provider.export_page(Some(bad), 10).await { + Err(MemoryError::Invalid(_)) => {} + Err(other) => panic!("cursor '{bad}' produced the wrong variant: {other:?}"), + Ok(page) => panic!( + "cursor '{bad}' must be rejected, got {} record(s)", + page.records.len() + ), + } + } +} diff --git a/src/openhuman/memory/driver/embedded/recall.rs b/src/openhuman/memory/driver/embedded/recall.rs new file mode 100644 index 0000000000..7f89fe1a38 --- /dev/null +++ b/src/openhuman/memory/driver/embedded/recall.rs @@ -0,0 +1,86 @@ +//! [`MemoryRecall`] for the embedded driver. +//! +//! The delegation itself is trivial — [`OwnedRecallOpts`] borrows into the +//! engine's `RecallOpts` for free, and `UnifiedMemory::recall` already returns +//! ranked, `min_score`-filtered, category-filtered, cross-session-merged +//! results. Nothing is re-ranked here. +//! +//! ## `scope` is refused, not ignored +//! +//! The contract's `scope` is a **query predicate the driver must apply +//! internally**: applying it after the fact would let `limit` be consumed by +//! rows the caller may not see, and an empty scope denies all source-attributed +//! content rather than waving it through. +//! +//! The embedded recall path has no such predicate today. `Memory::recall` → +//! `query_namespace_ranked_excluding_session` consults nothing resembling +//! [`SourceScope`]; the host's ambient equivalent +//! ([`crate::openhuman::memory::source_scope`]) is read only by the +//! tree-retrieval and chunk-search layers, which land with the +//! [`Tree`](tinycortex_api::capabilities::Capability::Tree) family. +//! +//! So a `Some(scope)` here has exactly three possible treatments, and two are +//! wrong: +//! +//! - *silently ignore it* — a scoped query answered in full. That is a leak, +//! and it is invisible. +//! - *post-filter the rows* — the failure mode the contract's own docs name. +//! - *refuse* — what this driver does, until the predicate exists. +//! +//! [`MemoryError::Invalid`] is the right variant, not `Unsupported`: +//! `Unsupported` names a whole capability *family*, and recall is advertised. +//! Nothing calls the driver's `recall` yet (the RPC surface still goes straight +//! to the engine), so the refusal costs no live behaviour — it just cannot be +//! mistaken for working. + +use async_trait::async_trait; +use tinycortex_api::error::MemoryError; +use tinycortex_api::provider::types::SourceScope; +use tinycortex_api::provider::MemoryRecall; +use tinycortex_api::recall::OwnedRecallOpts; +use tinycortex_api::types::{MemoryEntry, RecallOpts}; + +use super::{engine_error, EmbeddedMemoryProvider}; + +/// Refusal message for a scoped recall. A constant so the test asserts the same +/// string the caller sees. +pub(super) const SCOPE_UNAPPLIED: &str = + "source scope is not applied by the embedded recall path yet: the scope predicate lives in \ + the tree-retrieval layer and lands with the tree capability family"; + +#[async_trait] +impl MemoryRecall for EmbeddedMemoryProvider { + async fn recall( + &self, + query: &str, + limit: usize, + opts: &OwnedRecallOpts, + scope: Option<&SourceScope>, + ) -> Result, MemoryError> { + if scope.is_some() { + log::warn!("[memory:driver:embedded] recall refused: {SCOPE_UNAPPLIED}"); + return Err(MemoryError::Invalid(SCOPE_UNAPPLIED.to_string())); + } + + log::debug!( + "[memory:driver:embedded] recall query_len={} limit={limit} namespace={} \ + cross_session={}", + query.len(), + opts.namespace.as_deref().unwrap_or("-"), + opts.cross_session + ); + + // Zero-copy for the string filters; exhaustively destructured inside + // the contract crate so a new filter cannot be dropped silently. + let borrowed = RecallOpts::from(opts); + self.memory() + .await? + .recall(query, limit, borrowed) + .await + .map_err(engine_error) + } +} + +#[cfg(test)] +#[path = "recall_tests.rs"] +mod tests; diff --git a/src/openhuman/memory/driver/embedded/recall_tests.rs b/src/openhuman/memory/driver/embedded/recall_tests.rs new file mode 100644 index 0000000000..f54cab2fb0 --- /dev/null +++ b/src/openhuman/memory/driver/embedded/recall_tests.rs @@ -0,0 +1,147 @@ +//! [`MemoryRecall`] tests. +//! +//! `recall_with_a_source_scope_is_refused_until_the_predicate_exists` is the +//! one that matters: it pins the deliberate refusal so nobody "fixes" it by +//! quietly dropping the argument, which would answer a scoped query in full. + +use super::super::test_support::fresh_driver; +use super::*; + +use tinycortex_api::provider::types::SourceScope; +use tinycortex_api::provider::MemoryCore; +use tinycortex_api::types::{MemoryCategory, MemoryTaint}; + +async fn seed(provider: &EmbeddedMemoryProvider) { + provider + .store( + "ns_a", + "rust", + "the rust programming language is memory safe", + MemoryCategory::Core, + None, + MemoryTaint::Internal, + ) + .await + .expect("store rust"); + provider + .store( + "ns_a", + "sailing", + "sailing boats need wind", + MemoryCategory::Core, + None, + MemoryTaint::Internal, + ) + .await + .expect("store sailing"); +} + +fn opts_for(namespace: &str) -> OwnedRecallOpts { + OwnedRecallOpts { + namespace: Some(namespace.to_string()), + ..OwnedRecallOpts::default() + } +} + +#[tokio::test] +async fn recall_returns_ranked_results_for_a_query() { + let (_tmp, provider) = fresh_driver(); + seed(&provider).await; + + let hits = provider + .recall("rust programming language", 5, &opts_for("ns_a"), None) + .await + .expect("recall"); + + assert!(!hits.is_empty(), "expected at least one hit"); + let scores: Vec = hits.iter().map(|h| h.score.unwrap_or(0.0)).collect(); + assert!( + scores.windows(2).all(|w| w[0] >= w[1]), + "results must be most-relevant first: {scores:?}" + ); + assert_eq!(hits[0].key, "rust"); +} + +#[tokio::test] +async fn recall_of_an_empty_store_is_empty_not_an_error() { + let (_tmp, provider) = fresh_driver(); + let hits = provider + .recall("anything", 5, &opts_for("ns_a"), None) + .await + .expect("recall must not error on an empty store"); + assert!(hits.is_empty()); +} + +#[tokio::test] +async fn recall_honours_the_min_score_filter_from_owned_opts() { + let (_tmp, provider) = fresh_driver(); + seed(&provider).await; + + let opts = OwnedRecallOpts { + min_score: Some(1.1), + ..opts_for("ns_a") + }; + let hits = provider + .recall("rust programming language", 5, &opts, None) + .await + .expect("recall"); + assert!( + hits.is_empty(), + "an unreachable min_score must drop every hit, got {}", + hits.len() + ); +} + +#[tokio::test] +async fn recall_surfaces_external_sync_taint() { + let (_tmp, provider) = fresh_driver(); + provider + .store( + "ns_a", + "synced", + "kubernetes cluster autoscaling notes", + MemoryCategory::Core, + None, + MemoryTaint::ExternalSync, + ) + .await + .expect("store"); + + let hits = provider + .recall("kubernetes cluster autoscaling", 5, &opts_for("ns_a"), None) + .await + .expect("recall"); + let hit = hits.first().expect("expected a hit"); + assert_eq!(hit.taint, MemoryTaint::ExternalSync); +} + +#[tokio::test] +async fn recall_with_a_source_scope_is_refused_until_the_predicate_exists() { + let (_tmp, provider) = fresh_driver(); + seed(&provider).await; + + // The unscoped call still works … + provider + .recall("rust", 5, &opts_for("ns_a"), None) + .await + .expect("unscoped recall"); + + // … while a scoped one is refused loudly rather than answered in full. + let scope = SourceScope::new(["src-abc"]); + let error = provider + .recall("rust", 5, &opts_for("ns_a"), Some(&scope)) + .await + .expect_err("a scope this driver cannot apply must be refused"); + match error { + MemoryError::Invalid(reason) => assert_eq!(reason, SCOPE_UNAPPLIED), + other => panic!("expected MemoryError::Invalid, got {other:?}"), + } + + // An *empty* scope denies all source-attributed content, so it must be + // refused too rather than read as "unrestricted". + let empty = SourceScope::default(); + assert!(provider + .recall("rust", 5, &opts_for("ns_a"), Some(&empty)) + .await + .is_err()); +} diff --git a/src/openhuman/memory/driver/embedded/sources.rs b/src/openhuman/memory/driver/embedded/sources.rs new file mode 100644 index 0000000000..1d364625ed --- /dev/null +++ b/src/openhuman/memory/driver/embedded/sources.rs @@ -0,0 +1,214 @@ +//! [`MemorySourceSink`] for the embedded driver — the write seam the host's +//! sync machinery pushes already-fetched items through. +//! +//! ## The host keeps the loop; this file is one step of it +//! +//! `memory::sources::sync::sync_source` stays exactly where it is. It owns the +//! per-source mutex, the `emit_sync_stage` progress events, Composio billing, +//! OAuth, rate limits, and dispatch by `SourceKind` — none of which belongs +//! behind a trait a third-party driver implements. This family is only the +//! "persist these items" step at the end of that loop. +//! +//! ## TAINT — why this family writes documents and `MemoryIngest` refuses +//! +//! `taint` is host-stamped provenance a driver "must persist … and never assign +//! itself". The two write tiers differ on whether they can: +//! +//! - The **chunk** tier (`ingest_pipeline::ingest_document_with_scope`, what +//! `run_source_pipeline` writes through) has **no taint column** anywhere +//! along its path — not on `DocumentInput`, not on `CanonicalisedSource`, not +//! on `Chunk::metadata`. `MemoryIngest` (M3c) therefore *refuses* a non-default +//! taint rather than dropping it. +//! - The **namespace-document** tier (`MemoryClient::put_doc` → +//! `NamespaceDocumentInput.taint`) carries it as a real column. +//! +//! Refusal is the right answer for `MemoryIngest`, whose callers pass the +//! default. It is the *wrong* answer here: the contract says sync paths pass +//! [`MemoryTaint::ExternalSync`], so a sink that refuses non-default taint would +//! refuse its only intended caller. So this family writes through the tier that +//! can honour the argument. The alternative — call the chunk path and let the +//! `taint` parameter fall on the floor — is the exact failure the rule exists to +//! prevent: externally-synced content landing indistinguishable from +//! user-authored content across a prompt-injection trust boundary. +//! +//! **The behaviour difference this buys is real and is not hidden.** Items +//! accepted here land as namespace documents, so they are readable through +//! `MemoryDocuments` / `MemoryCore` and are queryable, but they do **not** flow +//! through canonicalisation, chunking and the summary-tree ingest the way +//! `run_source_pipeline` output does — so they do not appear in a source's +//! summary tree. Closing that gap needs a taint column on the chunk tier, not a +//! different call here. +//! +//! ## `skipped` is always 0, deliberately +//! +//! `put_doc` upserts and returns a document id whether the key was new or +//! already present; there is no already-present signal to read. Reporting a +//! guess would corrupt the one number the contract gives a caller for detecting +//! a silently-dropping driver, so the honest value is 0 and the count lands in +//! `written`. +//! +//! ## Naming hazard +//! +//! `tinycortex::memory::sources::SourceItem` also exists and is a **different +//! type** (an engine-side sync item). Everything here is +//! [`tinycortex_api::provider::types::SourceItem`]; nothing imports the other +//! one. + +use async_trait::async_trait; +use serde_json::json; +use tinycortex_api::error::MemoryError; +use tinycortex_api::provider::types::{IngestOutcome, SourceItem}; +use tinycortex_api::provider::MemorySourceSink; +use tinycortex_api::types::{MemoryTaint, NamespaceDocumentInput}; + +use crate::openhuman::memory::store::chunks::store as chunks; +use crate::openhuman::memory::store::chunks::types::SourceKind; + +use super::{host_error, EmbeddedMemoryProvider}; + +/// The namespace one logical source's accepted items live in. +/// +/// Keyed on `source_id` alone — **not** on `(source_kind, source_id)` — so +/// [`MemorySourceSink::forget_source`], which is given only the id, can address +/// exactly what [`MemorySourceSink::accept_source_items`] wrote. The kind is +/// carried on each document instead (`source_type` + metadata), where losing it +/// costs nothing. +fn namespace_for(source_id: &str) -> String { + format!("source:{source_id}") +} + +#[async_trait] +impl MemorySourceSink for EmbeddedMemoryProvider { + async fn accept_source_items( + &self, + source_id: &str, + source_kind: &str, + items: Vec, + taint: MemoryTaint, + ) -> Result { + log::debug!( + "[memory:driver:embedded] accept_source_items source_id={source_id} \ + source_kind={source_kind} items={} taint={}", + items.len(), + taint.as_db_str() + ); + + let namespace = namespace_for(source_id); + let client = self.client().await?; + let mut outcome = IngestOutcome::default(); + + for item in items { + if item.item_id.trim().is_empty() { + // The item id is the upsert key. An empty one would collide + // every such item onto a single document, silently losing all + // but the last — refuse instead. + return Err(MemoryError::Invalid( + "source item_id must not be empty: it is the dedupe key".to_string(), + )); + } + + let title = if item.title.trim().is_empty() { + item.item_id.clone() + } else { + item.title.clone() + }; + + let input = NamespaceDocumentInput { + namespace: namespace.clone(), + key: item.item_id, + title, + content: item.content, + source_type: source_kind.to_string(), + priority: "medium".to_string(), + tags: item.tags, + metadata: json!({ + "sourceId": source_id, + "sourceKind": source_kind, + // This is collection identity, deliberately separate + // from `item_id`, which is only the per-item dedupe key. + "path_scope": namespace, + "url": item.url, + "mime": item.mime, + "updatedAtMs": item.updated_at_ms, + }), + category: "core".to_string(), + session_id: None, + document_id: None, + // The whole point of this family. Never substituted, never + // defaulted. + taint, + }; + + let id = client + .put_doc(input) + .await + .map_err(|error| host_error("accept_source_items", error))?; + outcome.written = outcome.written.saturating_add(1); + outcome.ids.push(id); + } + + Ok(outcome) + } + + async fn forget_source(&self, source_id: &str) -> Result { + log::debug!("[memory:driver:embedded] forget_source source_id={source_id}"); + let namespace = namespace_for(source_id); + let client = self.client().await?; + + // Count before clearing: `clear_namespace` returns `()`, and the + // contract wants the number of units removed. + let listed = client + .list_documents(Some(&namespace)) + .await + .map_err(|error| host_error("forget_source", error))?; + // `list_documents` answers `{"documents": [...]}`, not a bare array. + let documents = listed + .get("documents") + .and_then(serde_json::Value::as_array) + .map(Vec::len) + .unwrap_or(0) as u64; + + if documents > 0 { + client + .clear_namespace(&namespace) + .await + .map_err(|error| host_error("forget_source", error))?; + } + + // Also drop chunk-tier content written for the same logical source + // through `MemoryIngest` — the disconnect path must not leave half the + // driver's copy of a source behind. + // + // **Exact match, never a prefix.** `sources::status::source_id_prefix` + // expands a Composio source to `{toolkit}:%`, which would take out every + // source sharing that toolkit. That over-deletion is tolerable in the + // host's own connection-teardown path, which knows it is tearing down + // the whole connection; behind a contract method that promises to drop + // *one* logical source it would be a surprise. Teardown of a Composio + // connection stays host-side in `integrations::composio::ops:: + // memory_cleanup`, which has the toolkit and connection id this family + // deliberately never sees. + let config = self.config().await?.clone(); + let id = source_id.to_string(); + let chunks_removed = tokio::task::spawn_blocking(move || -> anyhow::Result { + let removed = chunks::delete_chunks_by_source(&config, SourceKind::Document, &id)?; + // Finish off a tree left orphaned by an earlier partial delete; + // a no-op when the chunk delete above already cascaded it. + chunks::delete_orphaned_source_tree(&config, SourceKind::Document, &id)?; + Ok(removed) + }) + .await + .map_err(|error| host_error("forget_source", format!("join: {error}")))? + .map_err(|error| host_error("forget_source", format!("{error:#}")))?; + + log::debug!( + "[memory:driver:embedded] forget_source source_id={source_id} documents={documents} \ + chunks={chunks_removed}" + ); + Ok(documents + chunks_removed as u64) + } +} + +#[cfg(test)] +#[path = "sources_tests.rs"] +mod tests; diff --git a/src/openhuman/memory/driver/embedded/sources_tests.rs b/src/openhuman/memory/driver/embedded/sources_tests.rs new file mode 100644 index 0000000000..ab97cf2a8d --- /dev/null +++ b/src/openhuman/memory/driver/embedded/sources_tests.rs @@ -0,0 +1,245 @@ +//! [`MemorySourceSink`] tests. +//! +//! `accept_source_items_persists_the_caller_supplied_taint` is the security test +//! of this family, and the reason it writes through the document tier at all: a +//! sink that quietly downgraded `external_sync` to `internal` would erase a +//! prompt-injection trust boundary while every other assertion here still +//! passed. + +use super::super::test_support::fresh_driver; +use super::*; + +fn item(item_id: &str, title: &str, content: &str) -> SourceItem { + SourceItem { + item_id: item_id.to_string(), + title: title.to_string(), + content: content.to_string(), + mime: Some("text/plain".to_string()), + url: Some(format!("https://example.invalid/{item_id}")), + updated_at_ms: Some(1_700_000_000_000), + tags: vec!["synced".to_string()], + } +} + +#[tokio::test] +async fn accept_source_items_writes_one_document_per_item() { + let (_tmp, provider) = fresh_driver(); + + let outcome = provider + .accept_source_items( + "src_a", + "folder", + vec![ + item("i1", "First", "first body"), + item("i2", "Second", "second body"), + ], + MemoryTaint::ExternalSync, + ) + .await + .expect("accept_source_items"); + + assert_eq!(outcome.written, 2); + // See the module docs: `put_doc` gives no already-present signal, so a + // non-zero `skipped` here would be invented. + assert_eq!(outcome.skipped, 0); + assert_eq!(outcome.ids.len(), 2); +} + +#[tokio::test] +async fn accept_source_items_persists_the_caller_supplied_taint() { + use tinycortex_api::provider::MemoryDocuments; + + let (_tmp, provider) = fresh_driver(); + provider + .accept_source_items( + "src_a", + "folder", + vec![item("i1", "First", "first body")], + MemoryTaint::ExternalSync, + ) + .await + .expect("accept_source_items"); + + let stored = provider + .get_document("source:src_a", "i1") + .await + .expect("get_document") + .expect("document exists"); + assert_eq!( + stored.taint, + MemoryTaint::ExternalSync, + "the sink must persist the host-stamped provenance, never downgrade it" + ); + assert_eq!(stored.content, "first body"); +} + +#[tokio::test] +async fn accept_source_items_persists_a_source_level_path_scope() { + use tinycortex_api::provider::MemoryDocuments; + + let (_tmp, provider) = fresh_driver(); + provider + .accept_source_items( + "src_a", + "folder", + vec![ + item("first", "First", "one"), + item("second", "Second", "two"), + ], + MemoryTaint::ExternalSync, + ) + .await + .expect("accept_source_items"); + + for item_id in ["first", "second"] { + let stored = provider + .get_document("source:src_a", item_id) + .await + .expect("get_document") + .expect("document exists"); + assert_eq!( + stored + .metadata + .get("path_scope") + .and_then(serde_json::Value::as_str), + Some("source:src_a"), + "path scope must identify the collection rather than item `{item_id}`" + ); + } +} + +#[tokio::test] +async fn accept_source_items_upserts_on_the_item_id() { + use tinycortex_api::provider::MemoryDocuments; + + let (_tmp, provider) = fresh_driver(); + provider + .accept_source_items( + "src_a", + "folder", + vec![item("i1", "First", "v1")], + MemoryTaint::ExternalSync, + ) + .await + .expect("first accept"); + provider + .accept_source_items( + "src_a", + "folder", + vec![item("i1", "First", "v2")], + MemoryTaint::ExternalSync, + ) + .await + .expect("second accept"); + + let stored = provider + .get_document("source:src_a", "i1") + .await + .expect("get_document") + .expect("document exists"); + assert_eq!(stored.content, "v2", "item_id is the dedupe key"); +} + +#[tokio::test] +async fn accept_source_items_refuses_an_empty_item_id() { + let (_tmp, provider) = fresh_driver(); + let error = provider + .accept_source_items( + "src_a", + "folder", + vec![item("", "No id", "body")], + MemoryTaint::ExternalSync, + ) + .await + .expect_err("an empty dedupe key must be refused, not collapsed"); + assert!(matches!(error, MemoryError::Invalid(_)), "got: {error:?}"); +} + +#[tokio::test] +async fn accept_an_empty_batch_is_a_no_op() { + let (_tmp, provider) = fresh_driver(); + let outcome = provider + .accept_source_items("src_a", "folder", Vec::new(), MemoryTaint::ExternalSync) + .await + .expect("accept_source_items"); + assert_eq!(outcome, IngestOutcome::default()); +} + +#[tokio::test] +async fn forget_source_removes_what_the_sink_wrote_and_is_idempotent() { + use tinycortex_api::provider::MemoryDocuments; + + let (_tmp, provider) = fresh_driver(); + provider + .accept_source_items( + "src_a", + "folder", + vec![item("i1", "First", "a"), item("i2", "Second", "b")], + MemoryTaint::ExternalSync, + ) + .await + .expect("accept_source_items"); + + let removed = provider + .forget_source("src_a") + .await + .expect("forget_source"); + assert_eq!(removed, 2); + + assert!(provider + .get_document("source:src_a", "i1") + .await + .expect("get_document") + .is_none()); + + // Idempotent, per the contract. + assert_eq!( + provider + .forget_source("src_a") + .await + .expect("second forget"), + 0 + ); +} + +#[tokio::test] +async fn forget_source_on_an_unknown_source_is_zero_not_an_error() { + let (_tmp, provider) = fresh_driver(); + assert_eq!( + provider + .forget_source("never-synced") + .await + .expect("forget_source must be idempotent"), + 0 + ); +} + +#[tokio::test] +async fn forget_source_leaves_a_sibling_source_alone() { + use tinycortex_api::provider::MemoryDocuments; + + let (_tmp, provider) = fresh_driver(); + for source in ["src_a", "src_a_extra"] { + provider + .accept_source_items( + source, + "folder", + vec![item("i1", "First", "body")], + MemoryTaint::ExternalSync, + ) + .await + .expect("accept_source_items"); + } + + provider + .forget_source("src_a") + .await + .expect("forget_source"); + + // Exact match, never a prefix — `src_a_extra` shares a prefix with `src_a`. + assert!(provider + .get_document("source:src_a_extra", "i1") + .await + .expect("get_document") + .is_some()); +} diff --git a/src/openhuman/memory/driver/embedded/tool_memory.rs b/src/openhuman/memory/driver/embedded/tool_memory.rs new file mode 100644 index 0000000000..7a424be16a --- /dev/null +++ b/src/openhuman/memory/driver/embedded/tool_memory.rs @@ -0,0 +1,96 @@ +//! [`MemoryToolMemory`] for the embedded driver — per-tool learned rules. +//! +//! Backed by the engine's own [`ToolMemoryStore`], built over this driver's +//! `Arc` through the host wrapper +//! [`tool_memory_store`](crate::openhuman::memory::tool_memory::tool_memory_store). +//! The store is a single `Arc` behind a `#[derive(Clone)]` struct, so +//! constructing one per call costs an `Arc` clone and keeps the driver's lazy +//! client rule intact — no eager handle, no second `OnceCell`. +//! +//! ## Not through `memory::ops::tool_memory` +//! +//! Those are the RPC handlers, and their `open_store()` resolves the +//! **process-global** active memory client. This driver holds a +//! workspace-scoped client on purpose: routing through the global slot would +//! let a driver bound to workspace B write into workspace A, which is exactly +//! the property the workspace-keyed binding map buys. +//! +//! ## Taint +//! +//! `ToolMemoryStore::put_rule` writes through `Memory::store`, and that is +//! correct here. The taint trap this milestone warns about is the engine's +//! *default* `store` impl, which drops the taint argument; +//! `UnifiedMemory::store` is an explicit impl forwarding to +//! `store_with_taint(..., MemoryTaint::Internal)`. Tool rules are host-authored, +//! so `Internal` is the right provenance — and no method in this family takes a +//! taint argument to lose in the first place. + +use async_trait::async_trait; +use tinycortex_api::error::MemoryError; +use tinycortex_api::provider::MemoryToolMemory; +use tinycortex_api::tool_memory::ToolMemoryRule; + +use super::{host_error, EmbeddedMemoryProvider}; +use crate::openhuman::memory::tool_memory::{tool_memory_store, ToolMemoryStore}; + +/// The two rejections `ToolMemoryStore::put_rule` performs before touching +/// storage. Matched by value so a genuine backend failure is never mislabelled +/// as caller error. +const PUT_RULE_REJECTIONS: [&str; 2] = ["tool_name is required", "rule body is required"]; + +/// Classifies a `put_rule` failure: a validated rejection is +/// [`MemoryError::Invalid`], everything else is a backend failure. +fn classify_put_rule(error: String) -> MemoryError { + if PUT_RULE_REJECTIONS.contains(&error.as_str()) { + return MemoryError::Invalid(error); + } + host_error("put_tool_rule", error) +} + +impl EmbeddedMemoryProvider { + async fn tool_memory(&self) -> Result { + Ok(tool_memory_store(self.memory().await?)) + } +} + +#[async_trait] +impl MemoryToolMemory for EmbeddedMemoryProvider { + async fn tool_rules(&self, tool_name: &str) -> Result, MemoryError> { + log::debug!("[memory:driver:embedded] tool_rules tool={tool_name}"); + self.tool_memory() + .await? + .list_rules(tool_name) + .await + .map_err(|error| host_error("tool_rules", error)) + } + + async fn put_tool_rule(&self, rule: ToolMemoryRule) -> Result<(), MemoryError> { + log::debug!( + "[memory:driver:embedded] put_tool_rule tool={} priority={:?}", + rule.tool_name, + rule.priority + ); + // The stored copy (with `created_at` preserved and `updated_at` + // refreshed) is discarded: the contract returns unit, and re-reading it + // is `tool_rules`' job. + self.tool_memory() + .await? + .put_rule(rule) + .await + .map(|_| ()) + .map_err(classify_put_rule) + } + + async fn delete_tool_rule(&self, tool_name: &str, rule_id: &str) -> Result { + log::debug!("[memory:driver:embedded] delete_tool_rule tool={tool_name} rule={rule_id}"); + self.tool_memory() + .await? + .delete_rule(tool_name, rule_id) + .await + .map_err(|error| host_error("delete_tool_rule", error)) + } +} + +#[cfg(test)] +#[path = "tool_memory_tests.rs"] +mod tests; diff --git a/src/openhuman/memory/driver/embedded/tool_memory_tests.rs b/src/openhuman/memory/driver/embedded/tool_memory_tests.rs new file mode 100644 index 0000000000..c2f610a8d5 --- /dev/null +++ b/src/openhuman/memory/driver/embedded/tool_memory_tests.rs @@ -0,0 +1,197 @@ +//! [`MemoryToolMemory`] tests. +//! +//! Two carry weight beyond a round-trip: +//! +//! - `put_tool_rule_with_a_blank_tool_name_is_invalid_not_other` pins the error +//! classification. `ToolMemoryStore::put_rule` rejects a blank name before +//! touching storage; collapsing that into `Other` would tell a caller their +//! backend is broken when their input is. +//! - `put_tool_rule_through_the_contract_is_visible_to_an_independent_reader` +//! is the same-store proof: a second client built over the same workspace +//! sees the rule, so the contract write reached the workspace's real store +//! rather than something private to the driver's handle. + +use super::super::test_support::fresh_driver; +use super::*; + +use crate::openhuman::config::schema::MemoryHooksConfig; +use crate::openhuman::memory::driver::embedded::EmbeddedMemoryProvider; +use crate::openhuman::memory::tool_memory::{ToolMemoryPriority, ToolMemorySource}; + +fn rule(tool: &str, body: &str, priority: ToolMemoryPriority) -> ToolMemoryRule { + ToolMemoryRule::new(tool, body, priority, ToolMemorySource::Programmatic) +} + +#[tokio::test] +async fn put_tool_rule_then_tool_rules_returns_it() { + let (_tmp, provider) = fresh_driver(); + + let stored = rule("shell", "never rm -rf /", ToolMemoryPriority::Critical); + provider + .put_tool_rule(stored.clone()) + .await + .expect("put_tool_rule"); + + let rules = provider.tool_rules("shell").await.expect("tool_rules"); + assert_eq!(rules.len(), 1); + assert_eq!(rules[0].id, stored.id); + assert_eq!(rules[0].rule, "never rm -rf /"); + assert_eq!(rules[0].priority, ToolMemoryPriority::Critical); +} + +#[tokio::test] +async fn tool_rules_is_empty_for_a_tool_with_no_rules() { + let (_tmp, provider) = fresh_driver(); + assert!(provider + .tool_rules("never-used") + .await + .expect("tool_rules") + .is_empty()); +} + +#[tokio::test] +async fn tool_rules_orders_critical_before_high_before_normal() { + let (_tmp, provider) = fresh_driver(); + + for (body, priority) in [ + ("normal one", ToolMemoryPriority::Normal), + ("critical one", ToolMemoryPriority::Critical), + ("high one", ToolMemoryPriority::High), + ] { + provider + .put_tool_rule(rule("email", body, priority)) + .await + .expect("put_tool_rule"); + } + + let rules = provider.tool_rules("email").await.expect("tool_rules"); + let priorities: Vec = rules.iter().map(|r| r.priority).collect(); + assert_eq!( + priorities, + vec![ + ToolMemoryPriority::Critical, + ToolMemoryPriority::High, + ToolMemoryPriority::Normal + ], + "the contract says highest priority first" + ); +} + +#[tokio::test] +async fn put_tool_rule_upserts_on_the_same_id() { + let (_tmp, provider) = fresh_driver(); + + let mut existing = rule("shell", "first body", ToolMemoryPriority::Normal); + provider + .put_tool_rule(existing.clone()) + .await + .expect("first put"); + existing.rule = "second body".to_string(); + provider + .put_tool_rule(existing.clone()) + .await + .expect("second put"); + + let rules = provider.tool_rules("shell").await.expect("tool_rules"); + assert_eq!(rules.len(), 1, "same id must upsert, not duplicate"); + assert_eq!(rules[0].rule, "second body"); +} + +#[tokio::test] +async fn put_tool_rule_with_a_blank_tool_name_is_invalid_not_other() { + let (_tmp, provider) = fresh_driver(); + + let mut blank = rule("shell", "some body", ToolMemoryPriority::Normal); + blank.tool_name = " ".to_string(); + + let error = provider + .put_tool_rule(blank) + .await + .expect_err("a blank tool name must be rejected"); + assert!( + matches!(error, MemoryError::Invalid(_)), + "caller error must not be reported as a backend failure: {error:?}" + ); +} + +#[tokio::test] +async fn put_tool_rule_with_a_blank_body_is_invalid_not_other() { + let (_tmp, provider) = fresh_driver(); + + let mut blank = rule("shell", "placeholder", ToolMemoryPriority::Normal); + blank.rule = " ".to_string(); + + let error = provider + .put_tool_rule(blank) + .await + .expect_err("a blank rule body must be rejected"); + assert!(matches!(error, MemoryError::Invalid(_)), "{error:?}"); +} + +#[tokio::test] +async fn delete_tool_rule_reports_existence_then_is_idempotent() { + let (_tmp, provider) = fresh_driver(); + + let stored = rule("shell", "a rule", ToolMemoryPriority::Normal); + provider + .put_tool_rule(stored.clone()) + .await + .expect("put_tool_rule"); + + assert!( + provider + .delete_tool_rule("shell", &stored.id) + .await + .expect("first delete"), + "the first delete must report that the rule existed" + ); + assert!( + !provider + .delete_tool_rule("shell", &stored.id) + .await + .expect("second delete"), + "deleting twice is a successful no-op, not an error" + ); + assert!(provider + .tool_rules("shell") + .await + .expect("tool_rules") + .is_empty()); +} + +#[tokio::test] +async fn put_tool_rule_through_the_contract_is_visible_to_an_independent_reader() { + use crate::openhuman::memory::store::MemoryClient; + use crate::openhuman::memory::tool_memory::tool_memory_store; + + let tmp = tempfile::TempDir::new().expect("temp workspace"); + let workspace = tmp.path().join("ws"); + let provider = EmbeddedMemoryProvider::new(workspace.clone(), MemoryHooksConfig::default()); + + let stored = rule( + "driver_store_proof", + "reachable from a second handle", + ToolMemoryPriority::High, + ); + provider + .put_tool_rule(stored.clone()) + .await + .expect("put_tool_rule"); + + // A second, independently constructed client over the same workspace — + // exactly how `memory::ops::tool_memory::open_store` builds its store, but + // without the process-global slot. The RPC handler itself resolves that + // global, which any concurrently running test may rebind to its own + // workspace mid-body; dialling it here would make this proof flaky for a + // reason that has nothing to do with the driver. + let independent = MemoryClient::from_workspace_dir(workspace).expect("second client"); + let rules = tool_memory_store(independent.memory_handle()) + .list_rules("driver_store_proof") + .await + .expect("list_rules"); + + assert!( + rules.iter().any(|r| r.id == stored.id), + "contract write must land in the workspace store every other reader sees: {rules:?}" + ); +} diff --git a/src/openhuman/memory/driver/embedded/tree.rs b/src/openhuman/memory/driver/embedded/tree.rs new file mode 100644 index 0000000000..e6472cfef8 --- /dev/null +++ b/src/openhuman/memory/driver/embedded/tree.rs @@ -0,0 +1,259 @@ +//! [`MemoryTree`] for the embedded driver — the markdown time-summary tree. +//! +//! ## The family is `tree_runtime`, not `tree::retrieval` +//! +//! This is worth stating up front because the obvious reading of the method +//! names points at the wrong module. `tree::retrieval::{query_source, +//! drill_down}` are *retrieval* entry points: they return `QueryResponse<…>` / +//! `Vec` over the hybrid ranker, and the `TreeStatus` in +//! `memory::store::trees::types` is a **different type with the same name** (an +//! enum with an `Active` variant, describing a sealed source tree). +//! +//! The contract's [`IngestRequest`], [`QueryResult`], [`TreeNode`] and +//! [`TreeStatus`] are the *runtime* tree's types — literally so: +//! `memory::tree::tree_runtime` re-exports +//! `tinycortex::memory::tree::runtime::*`, and that module in turn is +//! `pub use tinycortex_api::tree as types`. Contract type and host type are +//! **the same type**, so three of the five methods below convert nothing. +//! +//! ## The source-scope hazard does not arise here +//! +//! The M3c brief warned that threading `scope` into `retrieval::query_source` +//! would apply the allowlist twice — once as the explicit parameter and once +//! through the `current_source_scope()` task-local that callee reads +//! internally. That warning is real, and this file avoids it by not going +//! there: [`Self::query_source`] reads the **chunk store** +//! (`store::chunks::store::list_chunks`), whose `ListChunksQuery.source_scope` +//! is already an explicit parameter applied **in SQL before `LIMIT`**. +//! +//! That SQL predicate is predicate 3 of the three pinned by +//! `tree::retrieval::source_scope_tests`, and it is the one +//! [`SourceScope::allows_source_id`] was written against — equality or +//! `mem_src:{allowed}:` prefix, untagged content fails open, an empty allow +//! list keeps only untagged rows. So no predicate changes, no task-local is +//! read, and all 25 characterization tests stay untouched. +//! +//! ## `namespace` has no home on the chunk tier +//! +//! `mem_tree_chunks` has no namespace column — chunks are keyed by +//! `(source_kind, source_id)`. [`Self::query_source`] therefore *validates* +//! `namespace` (so a traversal attempt is still refused) and otherwise ignores +//! it. Said out loud rather than dropped silently. +//! +//! ## Sealing needs a summarisation model +//! +//! [`Self::seal`] and [`Self::cascade`] drive the LLM fold, so they resolve a +//! provider through `tree_runtime::ops::create_provider` — the same resolver +//! the `tree_summarizer.*` RPC path and the memory doctor use. When the host +//! has neither local AI nor `memory_tree.cloud_summarization_opt_in`, that +//! resolver fails and the call surfaces as [`MemoryError::Invalid`] carrying +//! the existing operator-facing message. +//! +//! Both short-circuit when there is nothing to do — an empty buffer for +//! `seal`, an empty tree for `cascade` — and return the current status without +//! resolving a provider at all. The contract requires both to be idempotent +//! no-ops in exactly those cases, and a no-op must not need a model. + +use async_trait::async_trait; +use std::collections::HashSet; +use tinycortex_api::chunks::Chunk; +use tinycortex_api::error::MemoryError; +use tinycortex_api::provider::types::SourceScope; +use tinycortex_api::provider::MemoryTree; +use tinycortex_api::tree::{IngestRequest, QueryResult, TreeStatus}; + +use crate::openhuman::config::Config; +use crate::openhuman::memory::store::chunks::store::{list_chunks, ListChunksQuery}; +use crate::openhuman::memory::tree::tree_runtime::{engine, ops, store}; + +use super::{host_error, EmbeddedMemoryProvider}; + +/// Runs a blocking store call on the blocking pool with an owned `Config`. +/// +/// Every `tree_runtime::store` and `chunks::store` entry point is synchronous +/// and hits SQLite or the filesystem; calling one straight from an async +/// contract method would stall the reactor. +async fn blocking(config: &Config, context: &'static str, run: F) -> Result +where + T: Send + 'static, + F: FnOnce(&Config) -> anyhow::Result + Send + 'static, +{ + let config = config.clone(); + tokio::task::spawn_blocking(move || run(&config)) + .await + .map_err(|error| host_error(context, format!("join error: {error}")))? + .map_err(|error| host_error(context, format!("{error:#}"))) +} + +/// `validate_namespace` / `validate_node_id` failures are caller errors, so +/// they become [`MemoryError::Invalid`] rather than the opaque `Other` the +/// host's `String` channel would otherwise collapse to. +fn invalid(reason: String) -> MemoryError { + MemoryError::Invalid(reason) +} + +#[async_trait] +impl MemoryTree for EmbeddedMemoryProvider { + async fn append(&self, request: IngestRequest) -> Result<(), MemoryError> { + log::debug!( + "[memory:driver:embedded] tree_append namespace={} content_chars={} has_metadata={}", + request.namespace, + request.content.chars().count(), + request.metadata.is_some() + ); + store::validate_namespace(&request.namespace).map_err(invalid)?; + if request.content.trim().is_empty() { + return Err(invalid("content must not be empty".to_string())); + } + + let config = self.config().await?; + // Mirrors `ops::tree_summarizer_ingest` exactly: trimmed namespace, + // ingest-time fallback for the timestamp. The returned buffer path is + // an implementation detail and is dropped. + let namespace = request.namespace.trim().to_string(); + let timestamp = request.timestamp.unwrap_or_else(chrono::Utc::now); + let content = request.content; + let metadata = request.metadata; + blocking(config, "tree_append", move |config| { + store::buffer_write(config, &namespace, &content, ×tamp, metadata.as_ref()) + .map(|_path| ()) + }) + .await + } + + async fn query_source( + &self, + namespace: &str, + source_id: &str, + limit: usize, + scope: Option<&SourceScope>, + ) -> Result, MemoryError> { + log::debug!( + "[memory:driver:embedded] tree_query_source namespace={namespace} \ + source_id={source_id} limit={limit} scoped={}", + scope.is_some() + ); + // Validated but not used as a filter — see the module docs. + store::validate_namespace(namespace).map_err(invalid)?; + + let query = ListChunksQuery { + source_id: Some(source_id.to_string()), + // The allowlist travels into SQL, applied before `LIMIT`. This is + // the whole point of the contract taking `scope` as a parameter. + source_scope: scope + .map(|scope| scope.allow.iter().cloned().collect::>()), + limit: Some(limit), + exclude_dropped: true, + ..ListChunksQuery::default() + }; + + let config = self.config().await?; + // `ORDER BY timestamp_ms DESC` in `list_chunks` is the contract's + // "newest first"; no re-sorting here. + blocking(config, "tree_query_source", move |config| { + list_chunks(config, &query) + }) + .await + } + + async fn drill_down(&self, namespace: &str, node_id: &str) -> Result { + log::debug!( + "[memory:driver:embedded] tree_drill_down namespace={namespace} node_id={node_id}" + ); + store::validate_namespace(namespace).map_err(invalid)?; + store::validate_node_id(node_id).map_err(invalid)?; + + let config = self.config().await?; + let namespace = namespace.trim().to_string(); + let node_id = node_id.to_string(); + let found = { + let namespace = namespace.clone(); + let node_id = node_id.clone(); + blocking(config, "tree_drill_down", move |config| { + let Some(node) = store::read_node(config, &namespace, &node_id)? else { + return Ok(None); + }; + let children = store::read_children(config, &namespace, &node_id)?; + Ok(Some(QueryResult { node, children })) + }) + .await? + }; + + // The contract mandates `NotFound` here. The RPC path returns a + // `String` for the same case, which is why this is constructed in the + // driver rather than mapped from below. + found.ok_or_else(|| { + MemoryError::NotFound(format!("tree node '{node_id}' not found in '{namespace}'")) + }) + } + + async fn seal(&self, namespace: &str) -> Result { + log::debug!("[memory:driver:embedded] tree_seal namespace={namespace}"); + store::validate_namespace(namespace).map_err(invalid)?; + let config = self.config().await?; + let namespace = namespace.trim().to_string(); + + // Nothing buffered ⇒ nothing to seal. Short-circuited *before* the + // provider is resolved so a scheduler may call `seal` unconditionally + // on a host with no summarisation model without seeing an error. + let buffered = { + let namespace = namespace.clone(); + blocking(config, "tree_seal_buffer_read", move |config| { + store::buffer_read(config, &namespace) + }) + .await? + }; + if !buffered.is_empty() { + let provider = ops::create_provider(config) + .map_err(invalid) + .map(|(provider, _model)| provider)?; + // `Ok(None)` means the buffer emptied under us — still a success. + engine::run_summarization(config, provider.as_ref(), &namespace, chrono::Utc::now()) + .await + .map_err(|error| host_error("tree_seal", format!("{error:#}")))?; + } + + blocking(config, "tree_seal_status", move |config| { + store::get_tree_status(config, &namespace) + }) + .await + } + + async fn cascade(&self, namespace: &str) -> Result { + log::debug!("[memory:driver:embedded] tree_cascade namespace={namespace}"); + store::validate_namespace(namespace).map_err(invalid)?; + let config = self.config().await?; + let namespace = namespace.trim().to_string(); + + // An empty tree has no leaves to roll up. Same short-circuit rationale + // as `seal`. + let status = { + let namespace = namespace.clone(); + blocking(config, "tree_cascade_status", move |config| { + store::get_tree_status(config, &namespace) + }) + .await? + }; + if status.total_nodes == 0 { + return Ok(status); + } + + let provider = ops::create_provider(config) + .map_err(invalid) + .map(|(provider, _model)| provider)?; + // NOTE: `rebuild_tree` recomputes every parent level from the hour + // leaves rather than incrementally rolling up only what changed. Same + // direction and same resulting state as the contract's "roll sealed + // leaves up through the parent levels", and idempotent as required — + // but more expensive than the word "cascade" suggests. There is no + // incremental host entry point to delegate to, and writing one would + // be engine logic. + engine::rebuild_tree(config, provider.as_ref(), &namespace) + .await + .map_err(|error| host_error("tree_cascade", format!("{error:#}"))) + } +} + +#[cfg(test)] +#[path = "tree_tests.rs"] +mod tests; diff --git a/src/openhuman/memory/driver/embedded/tree_tests.rs b/src/openhuman/memory/driver/embedded/tree_tests.rs new file mode 100644 index 0000000000..a1495b09e1 --- /dev/null +++ b/src/openhuman/memory/driver/embedded/tree_tests.rs @@ -0,0 +1,378 @@ +//! [`MemoryTree`] tests for the embedded driver. +//! +//! The scope tests below deliberately mirror +//! `tree::retrieval::source_scope_tests`' predicate-3 cases, but reach them +//! *through the driver*. Those 25 characterization tests keep asserting the +//! host predicate directly and are untouched; these assert that routing a +//! contract `SourceScope` into `ListChunksQuery.source_scope` preserves it. + +use super::super::test_support::fresh_driver; + +use chrono::{TimeZone, Utc}; +use tinycortex_api::provider::types::SourceScope; +use tinycortex_api::provider::MemoryTree; +use tinycortex_api::tree::IngestRequest; + +use crate::openhuman::config::Config; +use crate::openhuman::memory::store::chunks::store::{ + upsert_chunks, upsert_staged_chunks_tx, with_connection, +}; +use crate::openhuman::memory::store::chunks::types::{ + chunk_id, Chunk, Metadata, SourceKind, SourceRef, +}; +use crate::openhuman::memory::store::content as content_store; + +const BASE_MS: i64 = 1_700_000_000_000; +const MEMORY_SOURCES: &str = "memory_sources"; + +fn request(namespace: &str, content: &str) -> IngestRequest { + IngestRequest { + namespace: namespace.to_string(), + content: content.to_string(), + timestamp: Some(Utc.timestamp_millis_opt(BASE_MS).unwrap()), + metadata: None, + } +} + +/// A chunk in `source`, tagged with `tags`, timestamped `ts_ms`. Same shape as +/// the `source_scope_tests` fixture so the two suites stay comparable. +fn src_chunk(source: &str, seq: u32, tags: &[&str], ts_ms: i64) -> Chunk { + let ts = Utc.timestamp_millis_opt(ts_ms).unwrap(); + Chunk { + id: chunk_id(SourceKind::Chat, source, seq, "driver-content"), + content: format!("content-{source}-{seq}"), + metadata: Metadata { + source_kind: SourceKind::Chat, + source_id: source.into(), + owner: "alice".into(), + timestamp: ts, + time_range: (ts, ts), + tags: tags.iter().map(|t| (*t).to_string()).collect(), + source_ref: Some(SourceRef::new(format!("slack://{source}/{seq}"))), + path_scope: None, + }, + token_count: 20, + seq_in_source: seq, + created_at: ts, + partial_message: false, + } +} + +fn seed_chunks(config: &Config, chunks: &[Chunk]) { + upsert_chunks(config, chunks).expect("upsert_chunks"); + let content_root = config.memory_tree_content_root(); + std::fs::create_dir_all(&content_root).expect("create content_root"); + let staged = content_store::stage_chunks(&content_root, chunks).expect("stage_chunks"); + with_connection(config, |conn| { + let tx = conn.unchecked_transaction()?; + upsert_staged_chunks_tx(&tx, &staged)?; + tx.commit()?; + Ok(()) + }) + .expect("persist staged chunk pointers"); +} + +// ── append ─────────────────────────────────────────────────────────────── + +#[tokio::test] +async fn tree_append_buffers_content_for_namespace() { + let (_tmp, provider) = fresh_driver(); + provider + .append(request("work", "phoenix launch is friday")) + .await + .expect("append"); + + let config = provider.config().await.expect("config"); + let buffered = crate::openhuman::memory::tree::tree_runtime::store::buffer_read(config, "work") + .expect("buffer_read"); + assert_eq!(buffered.len(), 1, "one buffered entry"); + assert!( + buffered[0].1.contains("phoenix launch is friday"), + "buffered body must carry the content, got {:?}", + buffered[0].1 + ); +} + +#[tokio::test] +async fn tree_append_rejects_empty_content_as_invalid() { + let (_tmp, provider) = fresh_driver(); + let error = provider + .append(request("work", " \n ")) + .await + .expect_err("whitespace-only content must be refused"); + assert!( + matches!(error, tinycortex_api::error::MemoryError::Invalid(_)), + "expected Invalid, got {error:?}" + ); +} + +#[tokio::test] +async fn tree_append_rejects_traversing_namespace_as_invalid() { + let (_tmp, provider) = fresh_driver(); + let error = provider + .append(request("../escape", "body")) + .await + .expect_err("traversal namespace must be refused"); + assert!( + matches!(error, tinycortex_api::error::MemoryError::Invalid(_)), + "expected Invalid, got {error:?}" + ); +} + +// ── drill_down ─────────────────────────────────────────────────────────── + +#[tokio::test] +async fn tree_drill_down_unknown_node_is_not_found() { + let (_tmp, provider) = fresh_driver(); + let error = provider + .drill_down("work", "2024/03/15/09") + .await + .expect_err("an absent node must not be Ok"); + assert!( + matches!(error, tinycortex_api::error::MemoryError::NotFound(_)), + "the contract mandates NotFound here, got {error:?}" + ); +} + +#[tokio::test] +async fn tree_drill_down_returns_node_with_direct_children() { + use crate::openhuman::memory::tree::tree_runtime::store::write_node; + use tinycortex::memory::tree::runtime::{NodeLevel, TreeNode}; + + let (_tmp, provider) = fresh_driver(); + let config = provider.config().await.expect("config").clone(); + let ts = Utc.timestamp_millis_opt(BASE_MS).unwrap(); + + let node = |node_id: &str, level: NodeLevel, parent: Option<&str>| TreeNode { + node_id: node_id.to_string(), + namespace: "work".to_string(), + level, + parent_id: parent.map(str::to_string), + summary: format!("summary for {node_id}"), + token_count: 10, + child_count: 0, + created_at: ts, + updated_at: ts, + metadata: None, + }; + + write_node(&config, &node("2024", NodeLevel::Year, Some("root"))).expect("write year"); + write_node(&config, &node("2024/03", NodeLevel::Month, Some("2024"))).expect("write month"); + + let result = provider + .drill_down("work", "2024") + .await + .expect("drill_down"); + assert_eq!(result.node.node_id, "2024"); + assert_eq!( + result + .children + .iter() + .map(|child| child.node_id.as_str()) + .collect::>(), + vec!["2024/03"], + ); +} + +#[tokio::test] +async fn tree_drill_down_rejects_traversing_node_id_as_invalid() { + let (_tmp, provider) = fresh_driver(); + let error = provider + .drill_down("work", "../../etc") + .await + .expect_err("traversal node id must be refused"); + assert!( + matches!(error, tinycortex_api::error::MemoryError::Invalid(_)), + "expected Invalid, got {error:?}" + ); +} + +// ── query_source + scope ───────────────────────────────────────────────── + +#[tokio::test] +async fn tree_query_source_returns_that_sources_chunks_newest_first() { + let (_tmp, provider) = fresh_driver(); + let config = provider.config().await.expect("config").clone(); + seed_chunks( + &config, + &[ + src_chunk("src-abc", 1, &[], BASE_MS), + src_chunk("src-abc", 2, &[], BASE_MS + 1_000), + src_chunk("src-xyz", 1, &[], BASE_MS + 2_000), + ], + ); + + let hits = provider + .query_source("work", "src-abc", 10, None) + .await + .expect("query_source"); + + assert_eq!(hits.len(), 2, "only src-abc's chunks"); + assert!( + hits[0].metadata.timestamp >= hits[1].metadata.timestamp, + "newest first" + ); +} + +#[tokio::test] +async fn tree_query_source_unknown_source_is_empty_not_an_error() { + let (_tmp, provider) = fresh_driver(); + let hits = provider + .query_source("work", "src-nope", 10, None) + .await + .expect("an unknown source must yield an empty vector, not an error"); + assert!(hits.is_empty()); +} + +#[tokio::test] +async fn tree_query_source_scope_admits_the_listed_source() { + let (_tmp, provider) = fresh_driver(); + let config = provider.config().await.expect("config").clone(); + seed_chunks( + &config, + &[src_chunk("src-abc", 1, &[MEMORY_SOURCES], BASE_MS)], + ); + + let scope = SourceScope::new(["src-abc"]); + let hits = provider + .query_source("work", "src-abc", 10, Some(&scope)) + .await + .expect("query_source"); + assert_eq!(hits.len(), 1); + + let other = SourceScope::new(["src-other"]); + let hits = provider + .query_source("work", "src-abc", 10, Some(&other)) + .await + .expect("query_source"); + assert!( + hits.is_empty(), + "a source-tagged chunk outside scope is denied" + ); +} + +#[tokio::test] +async fn tree_query_source_scope_admits_mem_src_prefix() { + let (_tmp, provider) = fresh_driver(); + let config = provider.config().await.expect("config").clone(); + seed_chunks( + &config, + &[src_chunk( + "mem_src:src-abc:item-1", + 1, + &[MEMORY_SOURCES], + BASE_MS, + )], + ); + + let scope = SourceScope::new(["src-abc"]); + let hits = provider + .query_source("work", "mem_src:src-abc:item-1", 10, Some(&scope)) + .await + .expect("query_source"); + assert_eq!( + hits.len(), + 1, + "the `mem_src:{{allowed}}:` prefix rule must survive the driver hop" + ); +} + +#[tokio::test] +async fn tree_query_source_empty_scope_keeps_only_untagged_chunks() { + let (_tmp, provider) = fresh_driver(); + let config = provider.config().await.expect("config").clone(); + seed_chunks( + &config, + &[ + src_chunk("src-abc", 1, &[MEMORY_SOURCES], BASE_MS), + src_chunk("src-plain", 1, &[], BASE_MS), + ], + ); + + let empty = SourceScope::default(); + assert!( + provider + .query_source("work", "src-abc", 10, Some(&empty)) + .await + .expect("query_source") + .is_empty(), + "an empty allow list denies all source-attributed content" + ); + assert_eq!( + provider + .query_source("work", "src-plain", 10, Some(&empty)) + .await + .expect("query_source") + .len(), + 1, + "untagged content fails open, exactly as the SQL predicate does" + ); +} + +#[tokio::test] +async fn tree_query_source_scope_is_applied_before_limit() { + let (_tmp, provider) = fresh_driver(); + let config = provider.config().await.expect("config").clone(); + // Two out-of-scope chunks are NEWER than the in-scope one. A post-filter + // would spend the limit on them and return nothing; a SQL predicate before + // LIMIT returns the in-scope row. + seed_chunks( + &config, + &[ + src_chunk("src-abc", 1, &[MEMORY_SOURCES], BASE_MS), + src_chunk("src-abc", 2, &[MEMORY_SOURCES], BASE_MS + 1_000), + src_chunk("src-abc", 3, &[MEMORY_SOURCES], BASE_MS + 2_000), + ], + ); + + let scope = SourceScope::new(["src-abc"]); + let hits = provider + .query_source("work", "src-abc", 1, Some(&scope)) + .await + .expect("query_source"); + assert_eq!(hits.len(), 1, "limit is honoured"); +} + +// ── seal / cascade ─────────────────────────────────────────────────────── + +#[tokio::test] +async fn tree_seal_on_empty_buffer_is_a_successful_noop() { + let (_tmp, provider) = fresh_driver(); + // The default test config resolves NO summarisation provider. Sealing + // nothing must still succeed, which is why the empty-buffer check runs + // before provider resolution. + let status = provider.seal("work").await.expect("seal an empty buffer"); + assert_eq!(status.namespace, "work"); + assert_eq!(status.total_nodes, 0); +} + +#[tokio::test] +async fn tree_seal_with_buffered_content_needs_a_summarization_provider() { + let (_tmp, provider) = fresh_driver(); + provider + .append(request("work", "something to summarise")) + .await + .expect("append"); + + let error = provider + .seal("work") + .await + .expect_err("no local AI and no cloud opt-in means no provider"); + match error { + tinycortex_api::error::MemoryError::Invalid(reason) => assert!( + reason.contains("summarization provider"), + "the operator-facing resolver message must survive, got {reason}" + ), + other => panic!("expected Invalid, got {other:?}"), + } +} + +#[tokio::test] +async fn tree_cascade_on_empty_tree_is_a_successful_noop() { + let (_tmp, provider) = fresh_driver(); + let status = provider + .cascade("work") + .await + .expect("cascade an empty tree"); + assert_eq!(status.total_nodes, 0); +} diff --git a/src/openhuman/memory/driver/mod.rs b/src/openhuman/memory/driver/mod.rs new file mode 100644 index 0000000000..d91ca01a80 --- /dev/null +++ b/src/openhuman/memory/driver/mod.rs @@ -0,0 +1,12 @@ +//! Memory-driver implementations of the [`tinycortex_api`] contract. +//! +//! One subdirectory per driver. Today there is exactly one — [`embedded`], +//! which wraps the in-process tinycortex engine — plus the reference +//! `NullMemoryProvider` that ships inside the contract crate itself. +//! +//! Drivers live *under* `memory/` rather than in a sibling top-level directory +//! so the "one directory equals one feature gate" family rule holds: a memory +//! driver is memory, and gating it separately from the domain it implements +//! would be meaningless. + +pub mod embedded; diff --git a/src/openhuman/memory/global.rs b/src/openhuman/memory/global.rs index 83efde88b3..e5c415cec1 100644 --- a/src/openhuman/memory/global.rs +++ b/src/openhuman/memory/global.rs @@ -16,7 +16,8 @@ //! client.put_doc(input).await?; //! ``` -use std::path::PathBuf; +use std::collections::HashMap; +use std::path::{Path, PathBuf}; use std::sync::{Arc, OnceLock, RwLock}; use crate::openhuman::memory::store::{MemoryClient, MemoryClientRef}; @@ -61,6 +62,28 @@ fn init_in_slot( } } + // Reuse the per-workspace cache before constructing anything. A desktop + // active-user switch A -> B -> A lands here with the global slot pointing + // at B, and building a *second* client for A would put two ingestion + // workers over A's SQLite file — duplicate graph extraction and duplicate + // embedding work — while any `MemoryBinding` cached for A still held the + // first one. `client_for_workspace` writes into the same map, so the two + // resolution paths converge on one client per workspace. + if let Some(cached) = cached_client(&workspace_dir)? { + log::debug!( + "[memory:global] reusing cached workspace client for {}", + workspace_dir.display() + ); + let mut guard = slot + .write() + .map_err(|e| format!("[memory:global] write lock poisoned: {e}"))?; + *guard = Some(GlobalMemoryClient { + workspace_dir, + client: Arc::clone(&cached), + }); + return Ok(cached); + } + log::info!( "[memory:global] initialising global MemoryClient workspace={}", workspace_dir.display() @@ -90,7 +113,9 @@ fn init_in_slot( .map_err(|e| format!("[memory:global] write lock poisoned: {e}"))?; if let Some(existing) = guard.as_ref() { if existing.workspace_dir == workspace_dir { - return Ok(Arc::clone(&existing.client)); + let client = Arc::clone(&existing.client); + cache_client(&workspace_dir, &client)?; + return Ok(client); } log::info!( @@ -100,6 +125,12 @@ fn init_in_slot( ); } + // Publish into the shared cache under the same client the global slot is + // about to hold, so a later `client_for_workspace(workspace)` — or a return + // to this workspace after a switch — reuses it rather than building a + // second engine over the same store. + let client = cache_client(&workspace_dir, &client)?; + *guard = Some(GlobalMemoryClient { workspace_dir, client: Arc::clone(&client), @@ -148,6 +179,109 @@ fn client_from(slot: &GlobalClientSlot) -> Result { }) } +/// The workspace the process-global client is currently bound to, or `None` +/// when [`init`] has not run yet. +/// +/// Exists so `memory::ops::guard::active_memory_guard` can resolve *the same* +/// workspace [`crate::openhuman::memory::ops::helpers::active_memory_client`] +/// would, in the pre-boot case where there is no ambient `CoreContext` to ask. +/// Reading the workspace rather than the client keeps the two resolutions +/// answering about the same store instead of drifting onto whatever +/// `Config::load_or_init` happens to say. +pub(crate) fn active_workspace_dir() -> Option { + global_slot() + .read() + .ok()? + .as_ref() + .map(|entry| entry.workspace_dir.clone()) +} + +/// Per-workspace client cache used by [`client_for_workspace`]. +/// +/// A *map*, not a slot, for the same reason +/// [`crate::openhuman::memory::binding`] caches bindings in a map: a subsystem +/// driver is resolved per workspace and must never be handed another +/// workspace's handle. +static WORKSPACE_CLIENTS: OnceLock>> = OnceLock::new(); + +/// The cached client for `workspace_dir`, if one has already been built by +/// either resolution path ([`init`] or [`client_for_workspace`]). +fn cached_client(workspace_dir: &Path) -> Result, String> { + Ok(WORKSPACE_CLIENTS + .get_or_init(Default::default) + .read() + .map_err(|e| format!("[memory:global] workspace cache read lock poisoned: {e}"))? + .get(workspace_dir) + .map(Arc::clone)) +} + +/// Publish `client` as *the* client for `workspace_dir`, returning whichever +/// client wins. +/// +/// A racing caller may have inserted first; theirs wins, so the "one ingestion +/// worker per workspace" property holds even when two paths construct +/// concurrently. Callers must use the returned handle, not the one they passed. +fn cache_client( + workspace_dir: &Path, + client: &MemoryClientRef, +) -> Result { + let mut guard = WORKSPACE_CLIENTS + .get_or_init(Default::default) + .write() + .map_err(|e| format!("[memory:global] workspace cache write lock poisoned: {e}"))?; + let entry = guard + .entry(workspace_dir.to_path_buf()) + .or_insert_with(|| Arc::clone(client)); + Ok(Arc::clone(entry)) +} + +/// The `MemoryClient` for `workspace_dir`, **reusing the process-global client +/// when it already owns that workspace**. +/// +/// Exists for the embedded memory driver +/// ([`crate::openhuman::memory::driver::embedded`]), which is constructed +/// synchronously at bind time and must resolve its client lazily on the first +/// contract call. +/// +/// The reuse check is load-bearing, not an optimisation: [`MemoryClient`] owns +/// a `UnifiedMemory` handle *and* spawns a background ingestion worker, so two +/// clients over one workspace means two workers doing duplicate graph +/// extraction and duplicate embedding work against the same SQLite file. +/// +/// # Errors +/// +/// Lock poisoning, or any failure constructing a fresh +/// [`MemoryClient::from_workspace_dir`] (directory creation, store open). +pub(crate) fn client_for_workspace(workspace_dir: &Path) -> Result { + if let Some(existing) = global_slot() + .read() + .map_err(|e| format!("[memory:global] read lock poisoned: {e}"))? + .as_ref() + { + if existing.workspace_dir == workspace_dir { + // Record it under the workspace too. Without this the global's + // client is invisible to the cache, so a switch away and back + // rebuilds a second client for this workspace while a binding + // cached here still holds the first. + return cache_client(workspace_dir, &existing.client); + } + } + + if let Some(existing) = cached_client(workspace_dir)? { + return Ok(existing); + } + + log::info!( + "[memory:global] building workspace-scoped MemoryClient workspace={}", + workspace_dir.display() + ); + let client: MemoryClientRef = Arc::new(MemoryClient::from_workspace_dir( + workspace_dir.to_path_buf(), + )?); + + cache_client(workspace_dir, &client) +} + /// Returns the global client if already initialised, without lazy init. pub fn client_if_ready() -> Option { global_slot() diff --git a/src/openhuman/memory/guard/audit.rs b/src/openhuman/memory/guard/audit.rs new file mode 100644 index 0000000000..92c105bc23 --- /dev/null +++ b/src/openhuman/memory/guard/audit.rs @@ -0,0 +1,115 @@ +//! Step 7: the guard's tracing span and its audit event. +//! +//! ## What may be logged, and what may never be +//! +//! Memory content is the most sensitive data in the product. Nothing in this +//! module — span field, log line, or bus event — carries a memory body, a +//! recall query, a namespace *key*, or a document title. What it carries is +//! **shapes**: the driver id, the contract method, the namespace, char counts, +//! and hit counts. A namespace is an operator-chosen bucket name and is already +//! logged verbatim by the embedded driver; a key is caller data and is not. +//! +//! That is also why this module publishes a purpose-built +//! [`DomainEvent::MemoryGuardDenied`] rather than reusing +//! [`DomainEvent::MemoryRecalled`], which carries the raw query string. Firing +//! that one from the guard would push user query text onto the bus on every +//! call. +//! +//! Where an identifier genuinely helps correlate a report — a key, a source id +//! — use [`redact`], which returns an 8-hex-char SHA-256 prefix: stable enough +//! to correlate two log lines, useless for recovering the value. Note that +//! [`redact`] is a **log** redactor only; the content-side scrubber for egress +//! is `store::safety::sanitize_text`, selected in +//! [`GuardPolicy::redact_outbound`](super::GuardPolicy::redact_outbound). +//! +//! ## Denials only on the bus +//! +//! Success is logged (at `debug`, into the file-only core log) but is **not** +//! published. One bus event per memory read would flood every subscriber on the +//! hot path for no operator benefit; a refusal is the rare, actionable event. + +use tinycortex_api::capabilities::Capability; + +use crate::core::event_bus::{publish_global, DomainEvent}; +use crate::openhuman::memory::util::redact::redact; + +use super::policy::GuardPolicy; + +/// Grep prefix for every guard log line, matching `[memory:binding]` and +/// `[memory:driver:embedded]`. +pub const LOG_PREFIX: &str = "[memory:guard]"; + +/// The tracing span every guarded call runs inside. +/// +/// Carries the three correlation fields the spec asks for — `driver_id`, the +/// capability family, and the namespace — plus the contract method, so two +/// calls into the same family are distinguishable in a trace. +pub fn guard_span( + policy: &GuardPolicy, + capability: Capability, + method: &str, + namespace: &str, +) -> tracing::Span { + tracing::debug_span!( + "memory_guard", + driver_id = policy.driver_id(), + capability = capability.as_str(), + method = method, + namespace = namespace, + ) +} + +/// Namespace placeholder for the contract methods that address no namespace +/// (maintenance, portability, goals). Better than an empty field, which reads +/// as "the namespace was lost". +pub const NO_NAMESPACE: &str = "-"; + +/// Log a call the guard let through. Shapes only. +pub fn trace_allowed(policy: &GuardPolicy, method: &str, namespace: &str, chars: usize) { + log::debug!( + "{LOG_PREFIX} allowed driver={} class={} method={method} namespace={namespace} \ + content_chars={chars}", + policy.driver_id(), + policy.class(), + ); +} + +/// Log the effect of a budget, when it actually bit. Silent when it did not, so +/// the log is a record of truncation rather than a per-call heartbeat. +pub fn trace_budget(policy: &GuardPolicy, method: &str, dropped: usize, trimmed_chars: usize) { + if dropped == 0 && trimmed_chars == 0 { + return; + } + log::debug!( + "{LOG_PREFIX} budget applied driver={} method={method} dropped={dropped} \ + trimmed_chars={trimmed_chars}", + policy.driver_id(), + ); +} + +/// Log and publish a refusal. +/// +/// Called from [`GuardPolicy::denied`](super::GuardPolicy::denied), so every +/// deny path audits by construction rather than by each call site remembering +/// to. `publish_global` is synchronous and a no-op before the bus is +/// initialised, so this is safe pre-boot with no `#[cfg(test)]` guard — the +/// same property `binding::build` relies on. +pub fn publish_guard_denied(policy: &GuardPolicy, method: &str, reason: &str) { + log::warn!( + "{LOG_PREFIX} DENIED driver={} class={} method={method}: {reason}", + policy.driver_id(), + policy.class(), + ); + publish_global(DomainEvent::MemoryGuardDenied { + driver_id: policy.driver_id().to_string(), + method: method.to_string(), + reason: reason.to_string(), + }); +} + +/// A caller-supplied identifier in a form that is safe to log: an 8-hex-char +/// digest, never the value. Use for keys, node ids, and source ids when a log +/// line genuinely needs to correlate two calls. +pub fn correlate(value: &str) -> String { + redact(value) +} diff --git a/src/openhuman/memory/guard/budget.rs b/src/openhuman/memory/guard/budget.rs new file mode 100644 index 0000000000..7a84e6581b --- /dev/null +++ b/src/openhuman/memory/guard/budget.rs @@ -0,0 +1,89 @@ +//! Char-budget truncation — step 6 of the guard's enforcement chain. +//! +//! Pure functions over already-materialised values, with no policy, no config +//! and no I/O, so the budget arithmetic is testable without constructing a +//! provider. +//! +//! ## Chars, not bytes +//! +//! Every count here is [`str::chars`], never [`str::len`]. `len()` is a byte +//! count, so a budget expressed in "chars" would truncate a Japanese or emoji +//! transcript to a third of its stated size — and, worse, `String::truncate` +//! on a byte index panics when that index is not a char boundary. Slicing at a +//! char boundary is the only form that is both correct and total. + +use tinycortex_api::types::MemoryEntry; + +/// Truncate `content` to at most `max_chars` characters. +/// +/// Returns the input unchanged (and unallocated) when it already fits, so the +/// common case costs nothing. +pub fn truncate_content(content: &str, max_chars: usize) -> std::borrow::Cow<'_, str> { + let mut chars = content.char_indices(); + match chars.nth(max_chars) { + // Fewer than `max_chars + 1` chars ⇒ it fits. + None => std::borrow::Cow::Borrowed(content), + Some((byte_idx, _)) => std::borrow::Cow::Owned(content[..byte_idx].to_string()), + } +} + +/// Outcome of applying a recall budget to a result set. +#[derive(Debug, Clone)] +pub struct BudgetOutcome { + /// Entries that survived, in input order. At most one of them has had its + /// `content` shortened — the entry that straddles the budget boundary. + pub entries: Vec, + /// How many entries were dropped whole because the budget was already + /// spent when they were reached. + pub dropped: usize, + /// How many characters of content were removed in total, across the + /// truncated entry and the dropped ones. + pub trimmed_chars: usize, +} + +/// Apply a **cumulative** char budget across a ranked recall result. +/// +/// The budget is spent in rank order, which is what makes truncation +/// least-destructive: recall returns most-relevant-first, so the entries that +/// lose content are the ones the caller was least likely to use. An entry that +/// straddles the boundary is kept with its content shortened rather than +/// dropped, because dropping it would silently change the hit count a caller +/// may be reporting. +/// +/// A zero-length budget is not special-cased here — the caller decides whether +/// `0` means "disabled" (it does; see `GuardPolicy::recall_budget`) before +/// calling. +pub fn truncate_entries(entries: Vec, max_chars: usize) -> BudgetOutcome { + let mut remaining = max_chars; + let mut kept: Vec = Vec::with_capacity(entries.len()); + let mut dropped = 0usize; + let mut trimmed_chars = 0usize; + + for mut entry in entries { + if remaining == 0 { + trimmed_chars += entry.content.chars().count(); + dropped += 1; + continue; + } + let len = entry.content.chars().count(); + if len <= remaining { + remaining -= len; + kept.push(entry); + } else { + trimmed_chars += len - remaining; + entry.content = truncate_content(&entry.content, remaining).into_owned(); + remaining = 0; + kept.push(entry); + } + } + + BudgetOutcome { + entries: kept, + dropped, + trimmed_chars, + } +} + +#[cfg(test)] +#[path = "budget_tests.rs"] +mod tests; diff --git a/src/openhuman/memory/guard/budget_tests.rs b/src/openhuman/memory/guard/budget_tests.rs new file mode 100644 index 0000000000..3562523aac --- /dev/null +++ b/src/openhuman/memory/guard/budget_tests.rs @@ -0,0 +1,77 @@ +//! Step 6 — the pure char-budget arithmetic. + +use super::*; +use tinycortex_api::types::{MemoryCategory, MemoryEntry, MemoryTaint}; + +fn entry(content: &str) -> MemoryEntry { + MemoryEntry { + id: "id".into(), + key: "key".into(), + content: content.into(), + namespace: Some("ns".into()), + category: MemoryCategory::Core, + timestamp: "2026-01-01T00:00:00Z".into(), + session_id: None, + score: None, + taint: MemoryTaint::Internal, + } +} + +#[test] +fn truncate_content_leaves_a_fitting_string_untouched() { + let out = truncate_content("hello", 5); + assert_eq!(out, "hello"); + assert!(matches!(out, std::borrow::Cow::Borrowed(_))); +} + +#[test] +fn truncate_content_cuts_to_the_budget() { + assert_eq!(truncate_content("hello world", 5), "hello"); +} + +#[test] +fn guard_budget_counts_chars_not_bytes() { + // Six chars, eighteen bytes. A byte-counting implementation would cut this + // to two chars — or panic on a non-char-boundary index. + let multibyte = "日本語です、はい"; + assert_eq!(truncate_content(multibyte, 3), "日本語"); + assert_eq!(truncate_content(multibyte, 100), multibyte); +} + +#[test] +fn guard_truncates_recall_results_to_the_budget() { + let out = truncate_entries(vec![entry("aaaa"), entry("bbbb"), entry("cccc")], 6); + assert_eq!( + out.entries.len(), + 2, + "the straddling entry is kept, trimmed" + ); + assert_eq!(out.entries[0].content, "aaaa"); + assert_eq!(out.entries[1].content, "bb"); + assert_eq!(out.dropped, 1); + assert_eq!(out.trimmed_chars, 2 + 4); +} + +#[test] +fn a_budget_that_fits_changes_nothing() { + let out = truncate_entries(vec![entry("aaaa"), entry("bbbb")], 100); + assert_eq!(out.entries.len(), 2); + assert_eq!(out.dropped, 0); + assert_eq!(out.trimmed_chars, 0); +} + +#[test] +fn a_zero_budget_drops_everything_when_the_caller_asks_for_one() { + // `GuardPolicy::recall_budget` never passes 0 (it reads 0 as "disabled"), + // but the pure function must still be total rather than panicking. + let out = truncate_entries(vec![entry("aaaa")], 0); + assert!(out.entries.is_empty()); + assert_eq!(out.dropped, 1); +} + +#[test] +fn budget_spends_in_rank_order_so_the_top_hit_survives_whole() { + let out = truncate_entries(vec![entry("top hit"), entry("second")], 7); + assert_eq!(out.entries[0].content, "top hit"); + assert_eq!(out.dropped, 1); +} diff --git a/src/openhuman/memory/guard/families.rs b/src/openhuman/memory/guard/families.rs new file mode 100644 index 0000000000..e4717f9afa --- /dev/null +++ b/src/openhuman/memory/guard/families.rs @@ -0,0 +1,676 @@ +//! The ten optional-family decorators — the load-bearing half of the guard. +//! +//! ## Why these exist at all +//! +//! [`MemoryProvider::as_tree`] and its nine siblings return a **borrow** of a +//! family trait object. If [`MemoryGuard`]'s override simply forwarded +//! `self.inner.as_tree()`, every caller that reached memory through a family +//! accessor would hold a raw, unguarded driver handle — and the guard's whole +//! reason to exist ("the only handle product code receives") would be +//! bypassable by one method call. Nine of the thirteen families are *only* +//! reachable that way. +//! +//! So each family gets its own decorator, and the accessor hands back a borrow +//! of that. Because the accessor returns a reference, the decorators cannot be +//! constructed on demand inside it — a reference to a temporary does not +//! outlive the call — so they are **fields on the guard, built once at +//! construction**. That is also what makes their presence mirror the inner +//! driver's exactly: a field exists iff `inner.provides(...)` said so, which is +//! what keeps `audit_provider` happy. +//! +//! ## Why each decorator holds the provider, not the family +//! +//! A `GuardedTree { inner: &dyn MemoryTree }` borrowed out of an +//! `Arc` the same struct owns is self-referential, and Rust +//! has no way to express that without unsafe pinning. Holding +//! `Arc` and re-deriving the family per call sidesteps it +//! entirely, at the cost of one `Option` unwrap that is structurally +//! unreachable — see [`family`](GuardedTree::family). +//! +//! [`MemoryGuard`]: super::MemoryGuard + +use std::sync::Arc; + +use async_trait::async_trait; +use tinycortex_api::capabilities::Capability; +use tinycortex_api::chunks::Chunk; +use tinycortex_api::error::MemoryError; +use tinycortex_api::goals::GoalsDoc; +use tinycortex_api::provider::types::{ + DiffReport, EntityHit, IngestItem, IngestOutcome, MaintenanceReport, SnapshotRef, SourceItem, + SourceScope, +}; +use tinycortex_api::provider::{ + MemoryDiff, MemoryDocuments, MemoryEntities, MemoryGoals, MemoryGraph, MemoryIngest, + MemoryMaintenance, MemoryProvider, MemorySourceSink, MemoryToolMemory, MemoryTree, +}; +use tinycortex_api::tool_memory::ToolMemoryRule; +use tinycortex_api::tree::{IngestRequest, QueryResult, TreeStatus}; +use tinycortex_api::types::{ + GraphRelationRecord, MemoryKvRecord, MemoryTaint, NamespaceDocumentInput, + NamespaceRetrievalContext, StoredMemoryDocument, +}; + +use super::audit::{trace_allowed, NO_NAMESPACE}; +use super::policy::GuardPolicy; + +/// Declares one decorator: the two shared fields, a constructor, and the +/// `family()` re-derivation. +macro_rules! decorator { + ($(#[$meta:meta])* $name:ident, $fam:ty, $accessor:ident, $cap:ident) => { + $(#[$meta])* + pub struct $name { + inner: Arc, + policy: Arc, + } + + impl $name { + pub(super) fn new(inner: Arc, policy: Arc) -> Self { + Self { inner, policy } + } + + /// The underlying family handle. + /// + /// The `Err` arm is **structurally unreachable**: `MemoryGuard::new` + /// only builds this decorator when the inner provider answered + /// `provides(Capability::$cap)`, and the contract documents the + /// capability set as fixed at bind time. It is written as a real + /// error rather than `.expect(...)` because a panic inside a memory + /// call is a strictly worse failure than an `Unsupported` a caller + /// can already handle. + fn family(&self) -> Result<&$fam, MemoryError> { + self.inner + .$accessor() + .ok_or_else(|| MemoryError::unsupported(Capability::$cap)) + } + } + }; +} + +decorator!( + /// Guarded [`MemoryIngest`]. + GuardedIngest, + dyn MemoryIngest, + as_ingest, + Ingest +); +decorator!( + /// Guarded [`MemoryDocuments`]. + GuardedDocuments, + dyn MemoryDocuments, + as_documents, + Documents +); +decorator!( + /// Guarded [`MemoryTree`] — the one family that carries step 2. + GuardedTree, + dyn MemoryTree, + as_tree, + Tree +); +decorator!( + /// Guarded [`MemoryEntities`]. + GuardedEntities, + dyn MemoryEntities, + as_entities, + Entities +); +decorator!( + /// Guarded [`MemoryGraph`]. + GuardedGraph, + dyn MemoryGraph, + as_graph, + Graph +); +decorator!( + /// Guarded [`MemoryDiff`]. + GuardedDiff, + dyn MemoryDiff, + as_diff, + Diff +); +decorator!( + /// Guarded [`MemoryGoals`]. + GuardedGoals, + dyn MemoryGoals, + as_goals, + Goals +); +decorator!( + /// Guarded [`MemoryToolMemory`]. + GuardedToolMemory, + dyn MemoryToolMemory, + as_tool_memory, + ToolMemory +); +decorator!( + /// Guarded [`MemorySourceSink`]. + GuardedSources, + dyn MemorySourceSink, + as_sources, + Sources +); +decorator!( + /// Guarded [`MemoryMaintenance`]. + GuardedMaintenance, + dyn MemoryMaintenance, + as_maintenance, + Maintenance +); + +// ── Ingest ─────────────────────────────────────────────────────────────────── + +impl GuardedIngest { + /// Steps 3 + 4 over one ingest item: stamp provenance, redact on egress. + fn admit(&self, mut item: IngestItem) -> IngestItem { + item.taint = self.policy.stamp_taint(item.taint); + item.content = self.policy.redact_outbound(&item.content).into_owned(); + item + } +} + +#[async_trait] +impl MemoryIngest for GuardedIngest { + async fn ingest_document(&self, item: IngestItem) -> Result { + let namespace = item.namespace.clone().unwrap_or_else(|| "-".to_string()); + self.policy.admit_write( + Capability::Ingest, + "ingest.ingest_document", + &namespace, + true, + )?; + let item = self.admit(item); + trace_allowed( + &self.policy, + "ingest.ingest_document", + &namespace, + item.content.chars().count(), + ); + self.family()?.ingest_document(item).await + } + + async fn ingest_chat(&self, messages: Vec) -> Result { + self.policy + .admit_write(Capability::Ingest, "ingest.ingest_chat", NO_NAMESPACE, true)?; + let messages: Vec = messages.into_iter().map(|m| self.admit(m)).collect(); + trace_allowed( + &self.policy, + "ingest.ingest_chat", + NO_NAMESPACE, + messages.iter().map(|m| m.content.chars().count()).sum(), + ); + self.family()?.ingest_chat(messages).await + } +} + +// ── Documents ──────────────────────────────────────────────────────────────── + +#[async_trait] +impl MemoryDocuments for GuardedDocuments { + async fn put_document(&self, mut input: NamespaceDocumentInput) -> Result { + self.policy.admit_write( + Capability::Documents, + "documents.put_document", + &input.namespace, + true, + )?; + input.taint = self.policy.stamp_taint(input.taint); + input.title = self.policy.redact_outbound(&input.title).into_owned(); + input.content = self.policy.redact_outbound(&input.content).into_owned(); + input.metadata = self.policy.redact_outbound_json(input.metadata); + trace_allowed( + &self.policy, + "documents.put_document", + &input.namespace, + input.content.chars().count(), + ); + self.family()?.put_document(input).await + } + + async fn get_document( + &self, + namespace: &str, + key: &str, + ) -> Result, MemoryError> { + self.policy.admit_read( + Capability::Documents, + "documents.get_document", + namespace, + false, + )?; + self.family()?.get_document(namespace, key).await + } + + async fn query_documents( + &self, + namespace: &str, + query: &str, + limit: usize, + ) -> Result { + // The query text itself crosses the boundary on an external driver. + self.policy.admit_read( + Capability::Documents, + "documents.query_documents", + namespace, + true, + )?; + let query = self.policy.redact_outbound(query).into_owned(); + self.family()? + .query_documents(namespace, &query, limit) + .await + } +} + +// ── Tree ───────────────────────────────────────────────────────────────────── + +#[async_trait] +impl MemoryTree for GuardedTree { + async fn append(&self, mut request: IngestRequest) -> Result<(), MemoryError> { + self.policy + .admit_write(Capability::Tree, "tree.append", &request.namespace, true)?; + request.content = self.policy.redact_outbound(&request.content).into_owned(); + trace_allowed( + &self.policy, + "tree.append", + &request.namespace, + request.content.chars().count(), + ); + self.family()?.append(request).await + } + + /// **Step 2 lives here.** This is the only contract method in the tree + /// today that both takes a [`SourceScope`] and applies it as a real query + /// predicate: the embedded driver pushes `scope.allow` into + /// `ListChunksQuery.source_scope`, which reaches SQL *before* `LIMIT`. + /// + /// The ambient allowlist + /// ([`source_scope::current_source_scope`](crate::openhuman::memory::source_scope::current_source_scope)) + /// is therefore read at this boundary and passed down, rather than being + /// applied to the returned rows. An explicit `scope` argument may only + /// *narrow* it: the two are intersected by + /// [`GuardPolicy::narrow_scope`](crate::openhuman::memory::guard::GuardPolicy::narrow_scope), + /// so a caller that computed a tighter scope than the task-local still wins, + /// while one that names a collection outside the ambient allowlist cannot + /// widen the turn back out. + /// + /// There is **no double application**: the embedded `query_source` does not + /// itself read the task-local (only the deeper `tree::retrieval` and + /// `list_chunks` paths do, and the guard does not sit in front of those), + /// so this fills a predicate that would otherwise be `None`. + async fn query_source( + &self, + namespace: &str, + source_id: &str, + limit: usize, + scope: Option<&SourceScope>, + ) -> Result, MemoryError> { + self.policy + .admit_read(Capability::Tree, "tree.query_source", namespace, false)?; + let ambient = self.policy.ambient_scope(); + let effective = self.policy.narrow_scope(scope); + log::debug!( + "[memory:guard] tree.query_source namespace={namespace} limit={limit} \ + scoped={} scope_from={}", + effective.is_some(), + match (scope.is_some(), ambient.is_some()) { + (true, true) => "argument∩ambient", + (true, false) => "argument", + (false, true) => "ambient", + (false, false) => "none", + } + ); + self.family()? + .query_source(namespace, source_id, limit, effective.as_ref()) + .await + } + + async fn drill_down(&self, namespace: &str, node_id: &str) -> Result { + self.policy + .admit_read(Capability::Tree, "tree.drill_down", namespace, false)?; + self.family()?.drill_down(namespace, node_id).await + } + + async fn seal(&self, namespace: &str) -> Result { + self.policy + .admit_write(Capability::Tree, "tree.seal", namespace, false)?; + self.family()?.seal(namespace).await + } + + async fn cascade(&self, namespace: &str) -> Result { + self.policy + .admit_write(Capability::Tree, "tree.cascade", namespace, false)?; + self.family()?.cascade(namespace).await + } +} + +// ── Entities ───────────────────────────────────────────────────────────────── + +#[async_trait] +impl MemoryEntities for GuardedEntities { + async fn entities( + &self, + namespace: &str, + query: Option<&str>, + limit: usize, + ) -> Result, MemoryError> { + self.policy.admit_read( + Capability::Entities, + "entities.entities", + namespace, + query.is_some(), + )?; + let redacted = query.map(|q| self.policy.redact_outbound(q).into_owned()); + self.family()? + .entities(namespace, redacted.as_deref(), limit) + .await + } + + async fn entity_edges( + &self, + namespace: &str, + entity_id: &str, + limit: usize, + ) -> Result, MemoryError> { + self.policy.admit_read( + Capability::Entities, + "entities.entity_edges", + namespace, + false, + )?; + self.family()? + .entity_edges(namespace, entity_id, limit) + .await + } + + async fn touch_entities( + &self, + namespace: &str, + entity_ids: &[String], + ) -> Result<(), MemoryError> { + self.policy.admit_write( + Capability::Entities, + "entities.touch_entities", + namespace, + false, + )?; + self.family()?.touch_entities(namespace, entity_ids).await + } +} + +// ── Graph ──────────────────────────────────────────────────────────────────── + +/// Namespace label for the graph family's `Option<&str>` namespace — `None` +/// addresses the global, namespace-less slice. +fn graph_ns(namespace: Option<&str>) -> &str { + namespace.unwrap_or(NO_NAMESPACE) +} + +#[async_trait] +impl MemoryGraph for GuardedGraph { + async fn kv_get( + &self, + namespace: Option<&str>, + key: &str, + ) -> Result, MemoryError> { + self.policy.admit_read( + Capability::Graph, + "graph.kv_get", + graph_ns(namespace), + false, + )?; + self.family()?.kv_get(namespace, key).await + } + + async fn kv_put( + &self, + namespace: Option<&str>, + key: &str, + value: serde_json::Value, + ) -> Result<(), MemoryError> { + self.policy + .admit_write(Capability::Graph, "graph.kv_put", graph_ns(namespace), true)?; + let value = self.policy.redact_outbound_json(value); + self.family()?.kv_put(namespace, key, value).await + } + + async fn kv_list( + &self, + namespace: Option<&str>, + prefix: Option<&str>, + limit: usize, + ) -> Result, MemoryError> { + self.policy.admit_read( + Capability::Graph, + "graph.kv_list", + graph_ns(namespace), + false, + )?; + self.family()?.kv_list(namespace, prefix, limit).await + } + + async fn relations( + &self, + namespace: Option<&str>, + subject: Option<&str>, + predicate: Option<&str>, + limit: usize, + ) -> Result, MemoryError> { + self.policy.admit_read( + Capability::Graph, + "graph.relations", + graph_ns(namespace), + false, + )?; + self.family()? + .relations(namespace, subject, predicate, limit) + .await + } + + async fn put_relation(&self, relation: GraphRelationRecord) -> Result<(), MemoryError> { + self.policy.admit_write( + Capability::Graph, + "graph.put_relation", + graph_ns(relation.namespace.as_deref()), + true, + )?; + self.family()?.put_relation(relation).await + } +} + +// ── Diff ───────────────────────────────────────────────────────────────────── + +#[async_trait] +impl MemoryDiff for GuardedDiff { + async fn capture_snapshot(&self, source_id: &str) -> Result { + self.policy.admit_write( + Capability::Diff, + "diff.capture_snapshot", + NO_NAMESPACE, + false, + )?; + self.family()?.capture_snapshot(source_id).await + } + + async fn snapshots( + &self, + source_id: &str, + limit: usize, + ) -> Result, MemoryError> { + self.policy + .admit_read(Capability::Diff, "diff.snapshots", NO_NAMESPACE, false)?; + self.family()?.snapshots(source_id, limit).await + } + + async fn diff( + &self, + source_id: &str, + from: Option<&str>, + to: &str, + ) -> Result { + self.policy + .admit_read(Capability::Diff, "diff.diff", NO_NAMESPACE, false)?; + self.family()?.diff(source_id, from, to).await + } +} + +// ── Goals ──────────────────────────────────────────────────────────────────── + +#[async_trait] +impl MemoryGoals for GuardedGoals { + async fn goals(&self) -> Result { + self.policy + .admit_read(Capability::Goals, "goals.goals", NO_NAMESPACE, false)?; + self.family()?.goals().await + } + + async fn set_goals(&self, goals: GoalsDoc) -> Result<(), MemoryError> { + self.policy + .admit_write(Capability::Goals, "goals.set_goals", NO_NAMESPACE, true)?; + // The goals document's own validating mutation surface (the PII and + // secret predicates) is host policy that already runs in + // `memory::goals` before a document reaches the contract, so the guard + // does not re-scrub item text here. If an external driver ever binds, + // M6 must decide whether that upstream scrub is sufficient for egress + // or whether item bodies need the same `redact_outbound` treatment the + // document and ingest paths get. + self.family()?.set_goals(goals).await + } +} + +// ── Tool memory ────────────────────────────────────────────────────────────── + +#[async_trait] +impl MemoryToolMemory for GuardedToolMemory { + async fn tool_rules(&self, tool_name: &str) -> Result, MemoryError> { + self.policy.admit_read( + Capability::ToolMemory, + "tool_memory.tool_rules", + NO_NAMESPACE, + false, + )?; + self.family()?.tool_rules(tool_name).await + } + + async fn put_tool_rule(&self, rule: ToolMemoryRule) -> Result<(), MemoryError> { + self.policy.admit_write( + Capability::ToolMemory, + "tool_memory.put_tool_rule", + NO_NAMESPACE, + true, + )?; + self.family()?.put_tool_rule(rule).await + } + + async fn delete_tool_rule(&self, tool_name: &str, rule_id: &str) -> Result { + self.policy.admit_write( + Capability::ToolMemory, + "tool_memory.delete_tool_rule", + NO_NAMESPACE, + false, + )?; + self.family()?.delete_tool_rule(tool_name, rule_id).await + } +} + +// ── Sources ────────────────────────────────────────────────────────────────── + +#[async_trait] +impl MemorySourceSink for GuardedSources { + async fn accept_source_items( + &self, + source_id: &str, + source_kind: &str, + items: Vec, + taint: MemoryTaint, + ) -> Result { + self.policy.admit_write( + Capability::Sources, + "sources.accept_source_items", + NO_NAMESPACE, + true, + )?; + // Step 3: the batch taint is the guard's to decide. `stamp_taint` never + // downgrades, so a sync path that already asked for `ExternalSync` keeps + // it whether or not a source scope is active. + let taint = self.policy.stamp_taint(taint); + let items: Vec = items + .into_iter() + .map(|mut item| { + item.title = self.policy.redact_outbound(&item.title).into_owned(); + item.content = self.policy.redact_outbound(&item.content).into_owned(); + item + }) + .collect(); + trace_allowed( + &self.policy, + "sources.accept_source_items", + NO_NAMESPACE, + items.iter().map(|i| i.content.chars().count()).sum(), + ); + self.family()? + .accept_source_items(source_id, source_kind, items, taint) + .await + } + + async fn forget_source(&self, source_id: &str) -> Result { + self.policy.admit_write( + Capability::Sources, + "sources.forget_source", + NO_NAMESPACE, + false, + )?; + self.family()?.forget_source(source_id).await + } +} + +// ── Maintenance ────────────────────────────────────────────────────────────── + +#[async_trait] +impl MemoryMaintenance for GuardedMaintenance { + async fn reembed(&self) -> Result { + self.policy.admit_write( + Capability::Maintenance, + "maintenance.reembed", + NO_NAMESPACE, + false, + )?; + self.family()?.reembed().await + } + + async fn compact(&self) -> Result { + self.policy.admit_write( + Capability::Maintenance, + "maintenance.compact", + NO_NAMESPACE, + false, + )?; + self.family()?.compact().await + } + + async fn consolidate(&self) -> Result { + self.policy.admit_write( + Capability::Maintenance, + "maintenance.consolidate", + NO_NAMESPACE, + false, + )?; + self.family()?.consolidate().await + } + + /// Read-only by contract, so this takes the **read** tier check: a + /// `readonly` operator must still be able to run `doctor`, which is exactly + /// the tier where diagnosing without mutating matters most. + async fn doctor(&self) -> Result { + self.policy.admit_read( + Capability::Maintenance, + "maintenance.doctor", + NO_NAMESPACE, + false, + )?; + self.family()?.doctor().await + } +} + +#[cfg(test)] +#[path = "families_tests.rs"] +mod tests; diff --git a/src/openhuman/memory/guard/families_tests.rs b/src/openhuman/memory/guard/families_tests.rs new file mode 100644 index 0000000000..25addc9a6c --- /dev/null +++ b/src/openhuman/memory/guard/families_tests.rs @@ -0,0 +1,238 @@ +//! The wrapped-accessor property — the reason this milestone exists — plus +//! step 2, which lives on `GuardedTree::query_source`. + +use tinycortex_api::provider::types::SourceScope; +use tinycortex_api::provider::{MemoryProvider, MemoryTree}; +use tinycortex_api::tree::IngestRequest; +use tinycortex_api::types::MemoryTaint; + +use crate::openhuman::memory::guard::test_support::{ + document, embedded_policy, external_policy, guarded, +}; +use crate::openhuman::memory::source_scope::with_source_scope; +use crate::openhuman::security::live_policy; +use crate::openhuman::security::policy::{AutonomyLevel, SecurityPolicy}; + +fn ingest_request(content: &str) -> IngestRequest { + IngestRequest { + namespace: "ns".into(), + content: content.into(), + timestamp: None, + metadata: None, + } +} + +// ── The wrapped-accessor property ─────────────────────────────────────────── + +#[tokio::test] +async fn guard_as_tree_is_not_the_raw_driver_handle() { + let (driver, guard) = guarded(embedded_policy()); + let via_guard = guard.as_tree().expect("tree family") as *const dyn MemoryTree; + let raw = driver.as_tree().expect("tree family") as *const dyn MemoryTree; + assert!( + !std::ptr::eq(via_guard, raw), + "the accessor handed out the driver's own handle — the guard is bypassable" + ); +} + +/// The assertion that actually matters. Pointer inequality only proves *some* +/// wrapper exists; this proves the wrapper still enforces. +#[tokio::test] +async fn guard_as_tree_still_applies_policy_reached_through_the_accessor() { + let dir = std::env::temp_dir(); + let _tier = live_policy::install_scoped( + std::sync::Arc::new(SecurityPolicy { + autonomy: AutonomyLevel::ReadOnly, + ..SecurityPolicy::default() + }), + dir.clone(), + dir, + ); + + let (driver, guard) = guarded(embedded_policy()); + let err = guard + .as_tree() + .expect("tree family") + .append(ingest_request("hello")) + .await + .expect_err("a readonly tier must refuse a tree write"); + assert!(err.to_string().contains("memory guard: "), "{err}"); + assert_eq!( + driver.call_count(), + 0, + "the driver must not be reached at all" + ); +} + +#[tokio::test] +async fn every_optional_family_accessor_enforces_the_tier() { + let dir = std::env::temp_dir(); + let _tier = live_policy::install_scoped( + std::sync::Arc::new(SecurityPolicy { + autonomy: AutonomyLevel::ReadOnly, + ..SecurityPolicy::default() + }), + dir.clone(), + dir, + ); + let (driver, guard) = guarded(embedded_policy()); + + // One representative *write* per optional family. Each must be refused + // before the driver sees it — a family whose decorator forwarded raw would + // record a call here. + guard + .as_ingest() + .unwrap() + .ingest_chat(vec![]) + .await + .expect_err("ingest"); + guard + .as_documents() + .unwrap() + .put_document(document("x", MemoryTaint::Internal)) + .await + .expect_err("documents"); + guard.as_tree().unwrap().seal("ns").await.expect_err("tree"); + guard + .as_entities() + .unwrap() + .touch_entities("ns", &[]) + .await + .expect_err("entities"); + guard + .as_graph() + .unwrap() + .kv_put(None, "k", serde_json::Value::Null) + .await + .expect_err("graph"); + guard + .as_diff() + .unwrap() + .capture_snapshot("src") + .await + .expect_err("diff"); + guard + .as_goals() + .unwrap() + .set_goals(Default::default()) + .await + .expect_err("goals"); + guard + .as_tool_memory() + .unwrap() + .delete_tool_rule("t", "r") + .await + .expect_err("tool_memory"); + guard + .as_sources() + .unwrap() + .forget_source("src") + .await + .expect_err("sources"); + guard + .as_maintenance() + .unwrap() + .compact() + .await + .expect_err("maintenance"); + + assert_eq!( + driver.call_count(), + 0, + "at least one family decorator forwarded an unguarded handle: {:?}", + driver.calls() + ); +} + +// ── Step 2 ────────────────────────────────────────────────────────────────── + +#[tokio::test] +async fn guard_fills_query_source_scope_from_the_task_local() { + let (driver, guard) = guarded(embedded_policy()); + with_source_scope(Some(vec!["slack:#eng".into()]), async { + guard + .as_tree() + .unwrap() + .query_source("ns", "src", 10, None) + .await + .expect("query_source"); + }) + .await; + let call = driver.only_call(); + assert_eq!(call.scoped, Some(true)); + assert_eq!(call.content.as_deref(), Some("slack:#eng")); +} + +#[tokio::test] +async fn guard_explicit_scope_argument_wins_over_the_ambient_one() { + let (driver, guard) = guarded(embedded_policy()); + let explicit = SourceScope::new(["gmail:me"]); + with_source_scope(Some(vec!["slack:#eng".into()]), async { + guard + .as_tree() + .unwrap() + .query_source("ns", "src", 10, Some(&explicit)) + .await + .expect("query_source"); + }) + .await; + assert_eq!( + driver.only_call().content.as_deref(), + Some("gmail:me"), + "a caller that computed a narrower scope must not be widened back" + ); +} + +#[tokio::test] +async fn guard_leaves_query_source_unscoped_outside_a_source_scope() { + let (driver, guard) = guarded(embedded_policy()); + guard + .as_tree() + .unwrap() + .query_source("ns", "src", 10, None) + .await + .expect("query_source"); + assert_eq!(driver.only_call().scoped, Some(false)); +} + +// ── Steps 3 + 4 through a family accessor ─────────────────────────────────── + +#[tokio::test] +async fn family_writes_are_taint_stamped_too() { + let (driver, guard) = guarded(embedded_policy()); + with_source_scope(Some(vec!["slack:#eng".into()]), async { + guard + .as_documents() + .unwrap() + .put_document(document("body", MemoryTaint::Internal)) + .await + .expect("put_document"); + }) + .await; + assert_eq!(driver.only_call().taint, Some(MemoryTaint::ExternalSync)); +} + +#[tokio::test] +async fn family_writes_are_not_redacted_for_an_embedded_driver() { + let secrety = "Authorization: Bearer abcdefghijklmnop"; + let (driver, guard) = guarded(embedded_policy()); + guard + .as_documents() + .unwrap() + .put_document(document(secrety, MemoryTaint::Internal)) + .await + .expect("put_document"); + assert_eq!(driver.only_call().content.as_deref(), Some(secrety)); +} + +#[tokio::test] +async fn family_calls_are_refused_for_an_untrusted_external_driver() { + let (driver, guard) = guarded(external_policy("untrusted")); + guard + .as_tree() + .unwrap() + .query_source("ns", "src", 10, None) + .await + .expect_err("fail-closed"); + assert_eq!(driver.call_count(), 0); +} diff --git a/src/openhuman/memory/guard/mandatory.rs b/src/openhuman/memory/guard/mandatory.rs new file mode 100644 index 0000000000..195791a629 --- /dev/null +++ b/src/openhuman/memory/guard/mandatory.rs @@ -0,0 +1,188 @@ +//! The three mandatory families on [`MemoryGuard`] — where steps 3, 4 and 6 +//! land for the always-present surface. + +use async_trait::async_trait; +use tinycortex_api::capabilities::Capability; +use tinycortex_api::error::MemoryError; +use tinycortex_api::provider::types::{ExportPage, ExportRecord, ImportOutcome, SourceScope}; +use tinycortex_api::provider::{MemoryCore, MemoryPortability, MemoryRecall}; +use tinycortex_api::recall::OwnedRecallOpts; +use tinycortex_api::types::{MemoryCategory, MemoryEntry, MemoryTaint, NamespaceSummary}; + +use super::audit::{trace_allowed, trace_budget, NO_NAMESPACE}; +use super::budget::{truncate_content, truncate_entries}; +use super::provider::MemoryGuard; + +#[async_trait] +impl MemoryCore for MemoryGuard { + /// Store, with steps 1, 3, 4, 5 and the capture half of 6 applied in that + /// order: refuse first, then stamp provenance, then redact, then trim. + /// + /// Trimming last is deliberate — redaction can lengthen content (a matched + /// secret becomes `[REDACTED_SECRET]`), so a budget applied before it could + /// be exceeded by the time the write leaves. + async fn store( + &self, + namespace: &str, + key: &str, + content: &str, + category: MemoryCategory, + session_id: Option<&str>, + taint: MemoryTaint, + ) -> Result<(), MemoryError> { + let policy = self.policy(); + policy.admit_write(Capability::Core, "core.store", namespace, true)?; + + let taint = policy.stamp_taint(taint); + let content = policy.redact_outbound(content); + let content = match policy.capture_budget() { + Some(max) => truncate_content(&content, max).into_owned(), + None => content.into_owned(), + }; + trace_allowed(policy, "core.store", namespace, content.chars().count()); + + self.inner() + .store(namespace, key, &content, category, session_id, taint) + .await + } + + async fn get(&self, namespace: &str, key: &str) -> Result, MemoryError> { + let policy = self.policy(); + policy.admit_read(Capability::Core, "core.get", namespace, false)?; + self.inner().get(namespace, key).await + } + + async fn forget(&self, namespace: &str, key: &str) -> Result { + let policy = self.policy(); + policy.admit_write(Capability::Core, "core.forget", namespace, false)?; + self.inner().forget(namespace, key).await + } + + async fn list( + &self, + namespace: Option<&str>, + category: Option<&MemoryCategory>, + session_id: Option<&str>, + ) -> Result, MemoryError> { + let policy = self.policy(); + policy.admit_read( + Capability::Core, + "core.list", + namespace.unwrap_or(NO_NAMESPACE), + false, + )?; + // No recall budget here on purpose: `list` is an enumeration surface + // (the UI's memory browser, export tooling) rather than the context + // block a turn injects, and silently truncating it would make the + // browser disagree with the store. `recall_max_chars` is a *recall* + // budget and is applied where recall happens. + self.inner().list(namespace, category, session_id).await + } + + async fn namespaces(&self) -> Result, MemoryError> { + let policy = self.policy(); + policy.admit_read(Capability::Core, "core.namespaces", NO_NAMESPACE, false)?; + self.inner().namespaces().await + } +} + +#[async_trait] +impl MemoryRecall for MemoryGuard { + /// Recall, with the recall char budget (step 6) applied to the driver's + /// result. + /// + /// ## `scope` is forwarded, never filled from the task-local + /// + /// This is the one place the "read the ambient scope at the guard boundary" + /// rule does **not** apply, and it is deliberate. The embedded driver + /// *refuses* a `Some(scope)` on recall — see `SCOPE_UNAPPLIED` in + /// `memory/driver/embedded/recall.rs`, which argues at length that + /// ignoring the scope is a silent leak and post-filtering is the named + /// anti-pattern, so refusing is the only honest answer until the recall + /// predicate exists. Filling `scope` here from + /// `current_source_scope()` would therefore turn **every** recall issued + /// inside a `with_source_scope` into a hard error against the only real + /// driver. + /// + /// So the guard passes the caller's `scope` through untouched. The ambient + /// allowlist is applied where a driver actually implements it as a query + /// predicate — `MemoryTree::query_source` — and recall joins that list when + /// the embedded recall path grows the predicate. + async fn recall( + &self, + query: &str, + limit: usize, + opts: &OwnedRecallOpts, + scope: Option<&SourceScope>, + ) -> Result, MemoryError> { + let policy = self.policy(); + // The query text itself crosses the boundary on an external driver. + policy.admit_read( + Capability::Recall, + "recall.recall", + opts.namespace.as_deref().unwrap_or(NO_NAMESPACE), + true, + )?; + let query = policy.redact_outbound(query); + + let hits = self.inner().recall(&query, limit, opts, scope).await?; + + match policy.recall_budget() { + None => Ok(hits), + Some(max) => { + let outcome = truncate_entries(hits, max); + trace_budget( + policy, + "recall.recall", + outcome.dropped, + outcome.trimmed_chars, + ); + Ok(outcome.entries) + } + } + } +} + +#[async_trait] +impl MemoryPortability for MemoryGuard { + /// Export is **not** budget-trimmed. A truncated export is a corrupt + /// backup, and portability exists so a binding is reversible — trimming it + /// would silently make it a one-way door, which is the exact failure the + /// contract made this family mandatory to avoid. + async fn export_page( + &self, + cursor: Option<&str>, + limit: usize, + ) -> Result { + let policy = self.policy(); + policy.admit_read( + Capability::Portability, + "portability.export_page", + NO_NAMESPACE, + false, + )?; + self.inner().export_page(cursor, limit).await + } + + /// Import **preserves** each record's taint rather than re-stamping it. + /// + /// The contract is explicit: "records carry their own taint; an importing + /// driver must persist what it is given and must not re-stamp provenance." + /// The guard holds to the same rule. Re-stamping here would rewrite an + /// entire restored store's provenance to whatever the ambient scope of the + /// restoring turn happened to be — laundering a million records in one + /// call, which is the opposite of what step 3 is for. + async fn import_records( + &self, + records: Vec, + ) -> Result { + let policy = self.policy(); + policy.admit_write( + Capability::Portability, + "portability.import_records", + NO_NAMESPACE, + true, + )?; + self.inner().import_records(records).await + } +} diff --git a/src/openhuman/memory/guard/mod.rs b/src/openhuman/memory/guard/mod.rs new file mode 100644 index 0000000000..40ff0aea8d --- /dev/null +++ b/src/openhuman/memory/guard/mod.rs @@ -0,0 +1,84 @@ +//! [`MemoryGuard`] — the kernel-owned policy decorator over the bound memory +//! driver (`docs/specs/plan-memory.md` §3.4, `docs/specs/kernel.md` §3.4). +//! +//! ## The shape, and why it is this shape +//! +//! The guard implements [`MemoryProvider`](tinycortex_api::provider::MemoryProvider) +//! over an `Arc`. That makes it *transparent* — a caller +//! writes the same code against the guard as against the driver — and it makes +//! the guard *unskippable by construction* for anyone holding it, because there +//! is no second, unguarded shape to reach for. +//! +//! The load-bearing detail is the ten `as_*` accessors. Nine of the thirteen +//! capability families are reachable **only** through them, so an override that +//! forwarded `self.inner.as_tree()` would hand out a raw driver handle and +//! defeat the entire design with one method call. Each family therefore gets +//! its own decorator, owned as a field on the guard (an accessor returns a +//! borrow, so it cannot build one on demand) and present exactly when the inner +//! driver provides that family. See [`families`]. +//! +//! ## The seven enforcement steps +//! +//! | # | Step | Where | +//! | - | ---- | ----- | +//! | 1 | `SecurityPolicy` tier | [`GuardPolicy::enforce_read`] / [`GuardPolicy::enforce_write`] | +//! | 1b | path rules | **no-op** — no contract method carries a path; see [`policy`] | +//! | 2 | source scope as a query predicate | [`GuardPolicy::ambient_scope`], applied in `GuardedTree::query_source` | +//! | 3 | taint stamping | [`GuardPolicy::stamp_taint`] | +//! | 4 | redaction | [`GuardPolicy::redact_outbound`] — a no-op for embedded drivers | +//! | 5 | egress + trust | [`GuardPolicy::check_egress`] | +//! | 6 | char budgets | [`budget`], driven by `MemoryHooksConfig` | +//! | 7 | audit + tracing | [`audit`] | +//! +//! Three of those departed from the milestone brief because the brief's version +//! would have been wrong against this tree; each departure is argued at its own +//! call site: +//! +//! - **Step 2 is not applied to `recall`.** The embedded driver *refuses* a +//! scoped recall (`SCOPE_UNAPPLIED` in `driver/embedded/recall.rs`), so +//! filling the parameter from the task-local would turn every recall inside a +//! `with_source_scope` into a hard error. The scope is filled on +//! `MemoryTree::query_source`, which is the one method that pushes it into +//! SQL before `LIMIT`. +//! - **Step 3 raises, it never overrides.** A plain override would rewrite a +//! caller's `ExternalSync` down to `Internal` outside a scope, which is the +//! laundering step the contract says the guard exists to prevent. +//! - **Step 1's path half is a no-op**, because nothing in the contract carries +//! a filesystem path to validate. +//! +//! ## What this milestone does NOT do +//! +//! M4a is **purely additive**. [`CoreContext::memory`] is new and nothing has +//! been migrated onto it; `CoreContext::memory_binding()` and +//! `MemoryBinding::unguarded_provider()` still exist and still hand out the bare driver. +//! The one production caller of `provider()` — the health probe in +//! `memory::ops::provider` — should keep bypassing the guard: a liveness probe +//! is not product code, and running it through the tier check would make an +//! autonomy setting able to break status output. +//! +//! ## Honesty clause: "the guard is the only path" is NOT yet true +//! +//! [`MemoryClient::profile_conn`](crate::openhuman::memory::store::MemoryClient::profile_conn) +//! hands out a raw `Arc>`. No decorator can wrap a +//! SQLite connection, so those callers reach the profile/facet tables beneath +//! every one of the seven steps above. It is explicitly out of scope for M4a +//! and must be closed before the invariant may be claimed. Current production +//! callers: +//! +//! - `memory/sync/composio/providers/profile.rs` +//! - `agent/learning/{tools,startup,schemas}.rs` +//! +//! `MemoryClient::memory_handle()` is already `pub(crate)`; do not widen it. + +pub mod audit; +pub mod budget; +pub mod families; +mod mandatory; +pub mod policy; +pub mod provider; + +#[cfg(test)] +mod test_support; + +pub use policy::GuardPolicy; +pub use provider::MemoryGuard; diff --git a/src/openhuman/memory/guard/policy.rs b/src/openhuman/memory/guard/policy.rs new file mode 100644 index 0000000000..9bd5268c00 --- /dev/null +++ b/src/openhuman/memory/guard/policy.rs @@ -0,0 +1,453 @@ +//! [`GuardPolicy`] — the resolved policy bundle [`MemoryGuard`] and all ten +//! family decorators share. +//! +//! [`MemoryGuard`]: super::MemoryGuard +//! +//! ## What is cached here, and what deliberately is not +//! +//! The *binding* facts — driver id, [`DriverClass`], the hook budgets, and the +//! configured `trust_state` — are resolved once, at bind time, and stored. They +//! cannot change without a rebind, and the binding is what constructs this +//! type. +//! +//! The [`SecurityPolicy`] is **not** cached. It is read from +//! [`live_policy::current`] on every call. That is a deliberate departure from +//! the obvious design: `MemoryBinding` is cached in a process-global, +//! workspace-keyed map for the life of the process, and `live_policy` is +//! installed *after* the first bind and hot-swapped on every autonomy change +//! (`reload_from`, `update_action_dir`). Caching an `Option>` +//! at construction would freeze whatever was current at first bind — in +//! practice the pre-boot `None` — and the tier check would then never fire +//! again for the rest of the process. A `RwLock` read per call is cheap and is +//! the only thing that stays correct across a hot swap. +//! +//! ## `None` policy means "no tier enforcement", never "deny" +//! +//! [`live_policy::current`] returns `None` before a session runtime has +//! installed one, which is the state roughly four thousand pre-boot unit tests +//! run in. Denying there would fail all of them at once, for the same reason +//! [`unbound_default_capabilities`] returns the full set and `core::all`'s +//! `group_allowed` returns `true` with no ambient context: denying is only ever +//! correct *after* something has actually answered. +//! +//! [`unbound_default_capabilities`]: crate::openhuman::memory::binding::unbound_default_capabilities + +use std::borrow::Cow; +use std::sync::Arc; + +use tinycortex_api::capabilities::Capability; +use tinycortex_api::error::MemoryError; +use tinycortex_api::provider::types::SourceScope; +use tinycortex_api::types::MemoryTaint; + +use crate::core::subsystem::DriverClass; +use crate::openhuman::config::schema::MemoryHooksConfig; +use crate::openhuman::memory::source_scope::current_source_scope; +use crate::openhuman::security::egress::emit_external_transfer; +use crate::openhuman::security::egress::types::{DataKind, EgressDescriptor, EgressReason}; +use crate::openhuman::security::live_policy; +use crate::openhuman::security::policy::ToolOperation; + +/// Prefix on every guard-authored error message, so a refusal that surfaces to +/// a caller is attributable to the guard rather than to the driver underneath. +pub const GUARD_DENIED_PREFIX: &str = "memory guard: "; + +/// The trust state an external driver must carry before the guard will let a +/// call reach it. Matches the value `binding::admit` requires at bind time. +pub const TRUSTED: &str = "trusted"; + +/// The policy bundle every guarded call consults. +/// +/// Cheap to clone-by-`Arc`: the guard holds one, and each of the ten family +/// decorators holds an `Arc` of the same value. +pub struct GuardPolicy { + driver_id: String, + class: DriverClass, + hooks: MemoryHooksConfig, + trust_state: String, +} + +impl GuardPolicy { + /// Build the policy for a bound driver. + pub fn new( + driver_id: impl Into, + class: DriverClass, + hooks: MemoryHooksConfig, + trust_state: impl Into, + ) -> Self { + Self { + driver_id: driver_id.into(), + class, + hooks, + trust_state: trust_state.into(), + } + } + + /// The bound driver's stable id — appears in every span and audit event. + pub fn driver_id(&self) -> &str { + &self.driver_id + } + + /// How the driver was reached. A host fact, never self-reported. + pub fn class(&self) -> DriverClass { + self.class + } + + /// The configured hook budgets. + pub fn hooks(&self) -> MemoryHooksConfig { + self.hooks + } + + /// The configured trust state for this driver binding. + pub fn trust_state(&self) -> &str { + &self.trust_state + } + + // ── The admission gate: steps 1, 5 and 7 in one call ───────────────────── + + /// Enter the guard's tracing span, run the tier (step 1) and egress + /// (step 5) checks inside it, and leave — all before the caller awaits + /// anything. + /// + /// The span is entered and exited **within this synchronous call** on + /// purpose. [`tracing::span::EnteredSpan`] is `!Send`, and every method on + /// the driver contract is an `#[async_trait]` method whose future must be + /// `Send`; holding an entered span across the `.await` of the forwarded + /// driver call makes the whole future `!Send` and fails to compile. Keeping + /// the span around the *decision* is also where it earns its keep — the + /// driver below emits its own instrumented line for the work itself. + /// + /// # Errors + /// + /// The first refusal, tier or egress. + pub fn admit_read( + &self, + capability: Capability, + method: &str, + namespace: &str, + carries_content: bool, + ) -> Result<(), MemoryError> { + let span = super::audit::guard_span(self, capability, method, namespace); + let _enter = span.enter(); + self.enforce_read(method)?; + self.check_egress(method, carries_content) + } + + /// [`Self::admit_read`] for a write, taking the `Act` tier check. + /// + /// # Errors + /// + /// The first refusal, tier or egress. + pub fn admit_write( + &self, + capability: Capability, + method: &str, + namespace: &str, + carries_content: bool, + ) -> Result<(), MemoryError> { + let span = super::audit::guard_span(self, capability, method, namespace); + let _enter = span.enter(); + self.enforce_write(method)?; + self.check_egress(method, carries_content) + } + + // ── Step 1: SecurityPolicy tier ───────────────────────────────────────── + + /// Tier check for a **read** operation. + /// + /// [`SecurityPolicy::enforce_tool_operation`] answers `Ok` unconditionally + /// for [`ToolOperation::Read`] — reads are never gated by autonomy tier or + /// by the action budget. The call is made anyway rather than skipped, so + /// that a future tier which *does* gate reads (privacy mode, a + /// read-quarantined tier) starts applying here without a new call site. + /// + /// # Errors + /// + /// Whatever the live policy refuses, prefixed with [`GUARD_DENIED_PREFIX`]. + pub fn enforce_read(&self, operation: &str) -> Result<(), MemoryError> { + self.enforce(ToolOperation::Read, operation) + } + + /// Tier check for a **write** operation: the `readonly`-tier refusal, and + /// deliberately **not** the hourly action budget. + /// + /// M4a routed this through [`ToolOperation::Act`], which is tier *plus* + /// `SecurityPolicy::record_action`. That was wrong in two ways, and both + /// bite the moment a call site migrates onto the guard (M4b): + /// + /// - **Double-count.** `memory/tools/store.rs` and `memory/tools/forget.rs` + /// already spend one budget unit each on `ToolOperation::Act`. + /// Re-pointing them at the guard would spend a second for the same call. + /// - **Wrong granularity.** The budget is denominated in agent *tool + /// calls*. The guard sits under `MemoryCore::store`, which one bulk + /// ingest or one sync pass calls hundreds of times — enough to exhaust a + /// budget with no agent having acted at all. + /// + /// So the guard takes the tier half only, via + /// [`SecurityPolicy::enforce_write_tier`]. That is the same shape the ~15 + /// acting tools which gate on bare `can_act()` already use + /// (`tools/impl/filesystem/file_write.rs`, + /// `tools/impl/system/python_exec.rs`, `cron/scheduler.rs`, …). Budget + /// accounting stays where it is denominated: at the tool boundary. + /// + /// # Errors + /// + /// Whatever the live policy refuses, prefixed with [`GUARD_DENIED_PREFIX`]. + pub fn enforce_write(&self, operation: &str) -> Result<(), MemoryError> { + // Read live, never cached — see the module docs. + let Some(policy) = live_policy::current() else { + return Ok(()); + }; + policy + .enforce_write_tier(operation) + .map_err(|reason| self.denied(operation, reason)) + } + + fn enforce(&self, op: ToolOperation, operation: &str) -> Result<(), MemoryError> { + // Read live, never cached — see the module docs. + let Some(policy) = live_policy::current() else { + return Ok(()); + }; + policy + .enforce_tool_operation(op, operation) + .map_err(|reason| self.denied(operation, reason)) + } + + /// Build the guard's canonical refusal error, publishing the audit event as + /// a side effect. Every deny path goes through here so a refusal can never + /// be raised without the operator seeing it. + pub fn denied(&self, method: &str, reason: impl Into) -> MemoryError { + let reason = reason.into(); + super::audit::publish_guard_denied(self, method, &reason); + MemoryError::Invalid(format!("{GUARD_DENIED_PREFIX}{reason}")) + } + + // ── Step 1b: path rules — deliberately a no-op ─────────────────────────── + // + // `validate_memory_relative_path` / `resolve_writable_memory_path` in + // `memory/ops/helpers.rs` validate `/memory/` *file* + // paths against the ambient workspace. Not one method on the driver + // contract carries a filesystem path: `MemoryCore` takes namespace/key, + // `MemoryTree` takes namespace/node_id, `MemoryDocuments` takes a + // `NamespaceDocumentInput`. There is nothing here to validate, and running + // a namespace string through a path validator would invent semantics + // neither side agreed to. The path half of step 1 belongs to whatever + // future family actually accepts a path. + + // ── Step 2: source scope as a query predicate ──────────────────────────── + + /// The ambient per-turn source allowlist, in contract form. + /// + /// `None` is unrestricted. `Some` — including `Some` over an empty set — + /// restricts: [`SourceScope`]'s own docs make an empty allow list deny all + /// source-attributed content, which matches + /// [`crate::openhuman::memory::source_scope`]'s empty-allowlist semantics + /// exactly. + /// + /// This is read at the guard boundary and passed **down** into the driver + /// so the allowlist becomes part of the query, never a post-filter over + /// rows a `limit` already truncated. + pub fn ambient_scope(&self) -> Option { + current_source_scope().map(SourceScope::new) + } + + /// The scope a query actually runs under, given what the caller asked for. + /// + /// The ambient allowlist is an **upper bound**, never a default that an + /// argument replaces. An earlier version returned `requested.or(ambient)`, + /// which let a source-restricted turn widen itself back out: passing an + /// explicit scope naming a collection the ambient allowlist did not contain + /// made that explicit scope the sole query predicate, and the restriction + /// the turn was running under vanished. + /// + /// So the two are intersected. Membership is decided by the ambient scope's + /// own [`SourceScope::allows_source_id`] rule (equality or the engine's + /// `mem_src:{allowed}:` prefix), so the guard and the driver's SQL agree on + /// what "in scope" means rather than the guard inventing a second rule. + /// + /// An empty intersection is returned as an empty `Some`, not `None`: an + /// empty allow list denies all source-attributed content, which is the + /// fail-closed reading [`SourceScope`] documents. Returning `None` there + /// would turn "you asked for nothing you are allowed to see" into + /// "unrestricted", the exact leak this method exists to close. + pub fn narrow_scope(&self, requested: Option<&SourceScope>) -> Option { + match (requested, self.ambient_scope()) { + (None, ambient) => ambient, + (Some(requested), None) => Some(requested.clone()), + (Some(requested), Some(ambient)) => { + let allow: Vec = requested + .allow + .iter() + .filter(|id| ambient.allows_source_id(id)) + .cloned() + .collect(); + if allow.len() != requested.allow.len() { + log::debug!( + "[memory:guard] explicit source scope narrowed by the ambient \ + allowlist requested={} admitted={}", + requested.allow.len(), + allow.len() + ); + } + Some(SourceScope { allow }) + } + } + } + + // ── Step 3: taint stamping ─────────────────────────────────────────────── + + /// The provenance the guard stamps on a write. + /// + /// The contract is explicit that the driver never assigns provenance, and + /// that the single failure mode the guard exists to prevent is *laundering* + /// externally-sourced content into internal-trust content. + /// + /// So this is a monotone raise, not a plain override: the result is + /// [`MemoryTaint::ExternalSync`] when the caller asked for it **or** when + /// the turn is running under a source scope (a source-restricted turn is + /// by definition handling source-attributed content), and + /// [`MemoryTaint::Internal`] only when neither holds. A pure override would + /// happily rewrite a caller's `ExternalSync` down to `Internal` outside a + /// scope, which is precisely the laundering step. + pub fn stamp_taint(&self, requested: MemoryTaint) -> MemoryTaint { + if requested == MemoryTaint::ExternalSync || current_source_scope().is_some() { + MemoryTaint::ExternalSync + } else { + MemoryTaint::Internal + } + } + + // ── Step 4: redaction ──────────────────────────────────────────────────── + + /// Content on its way to the driver, redacted when the driver is external. + /// + /// **No-op for [`DriverClass::Embedded`] and [`DriverClass::Null`]**: + /// nothing leaves the device, and scrubbing in-process memory writes would + /// silently destroy the user's own data. The borrowed arm is what makes + /// that a byte-identical pass-through rather than a re-allocation that + /// merely happens to compare equal. + /// + /// For [`DriverClass::External`] the content goes through the same + /// conservative secret/PII scrubber every other host write path uses + /// (`memory::store::safety::sanitize_text`, re-exported from the tinycortex + /// crate). + /// + /// **Do not substitute `memory::util::redact::redact` here.** That function + /// is a *log* redactor: it returns an 8-hex-character SHA-256 prefix, so + /// using it on egress content would not redact the write, it would delete + /// it. `redact` belongs in log lines only, and that is where + /// [`super::audit`] uses it. + /// + /// The external arm is unreachable today — `binding::admit` refuses every + /// external driver, so none can be bound — and is exercised for real in M6 + /// when the http adapter lands. It ships now so the branch exists at the + /// same time as the class check that selects it. + pub fn redact_outbound<'a>(&self, content: &'a str) -> Cow<'a, str> { + match self.class { + DriverClass::Embedded | DriverClass::Null => Cow::Borrowed(content), + DriverClass::External => { + Cow::Owned(crate::openhuman::memory::store::safety::sanitize_text(content).value) + } + } + } + + /// [`Self::redact_outbound`] for structured payloads (KV values, document + /// metadata), via the crate's `sanitize_json`. Same class rule: an + /// unmodified pass-through for embedded and null drivers. + pub fn redact_outbound_json(&self, value: serde_json::Value) -> serde_json::Value { + match self.class { + DriverClass::Embedded | DriverClass::Null => value, + DriverClass::External => { + crate::openhuman::memory::store::safety::sanitize_json(&value).value + } + } + } + + // ── Step 5: egress budget + trust state ────────────────────────────────── + + /// Per-call egress gate for an external driver. + /// + /// Bind-time refusal already covers the trust rule (`binding::admit` sends + /// every `trust_state != "trusted"` external driver to the fallback), so + /// this is the *per-call* seam and nothing more: it re-checks trust so a + /// guard constructed some other way cannot skip it, and it discloses the + /// transfer through the existing privacy-egress machinery. + /// + /// Deliberately minimal. There is no external driver in the tree yet, so + /// building byte counters and a rolling budget here would be accounting for + /// traffic that cannot exist; the real budget lands in M6 alongside the + /// transport that can spend it. + /// + /// `carries_content` selects the disclosed [`DataKind`]: + /// [`DataKind::FileContent`] for calls that hand raw memory bodies across + /// the boundary, [`DataKind::Metadata`] for the rest. Neither is a perfect + /// fit — there is no `MemoryContent` kind today — and adding one is an M6 + /// decision to take with the adapter, not a guess to bake in now. + /// + /// # Errors + /// + /// [`MemoryError::Invalid`] when the driver is external and its + /// `trust_state` has not been explicitly raised. + pub fn check_egress(&self, method: &str, carries_content: bool) -> Result<(), MemoryError> { + if self.class != DriverClass::External { + return Ok(()); + } + if self.trust_state != TRUSTED { + return Err(self.denied( + method, + format!( + "external driver '{}' is untrusted (trust_state = \"{}\"): \ + set trust_state = \"{TRUSTED}\" under [subsystems.memory.drivers] \ + before memory may cross the process boundary", + self.driver_id, self.trust_state + ), + )); + } + emit_external_transfer(EgressDescriptor::new( + self.driver_id.clone(), + method.to_string(), + true, + EgressReason::Integration, + if carries_content { + vec![DataKind::FileContent] + } else { + vec![DataKind::Metadata] + }, + )); + Ok(()) + } + + // ── Step 6: char budgets ───────────────────────────────────────────────── + + /// The recall char budget, or `None` when it is disabled. + /// + /// `0` reads as "no budget" rather than "return nothing": a zero-length + /// budget that silently emptied every recall would be indistinguishable + /// from a broken driver, and the config's own default is 1000. + pub fn recall_budget(&self) -> Option { + (self.hooks.recall_max_chars > 0).then_some(self.hooks.recall_max_chars) + } + + /// The capture char budget, or `None` when it is disabled. Same zero rule + /// as [`Self::recall_budget`]. + pub fn capture_budget(&self) -> Option { + (self.hooks.capture_max_chars > 0).then_some(self.hooks.capture_max_chars) + } + + // `max_context_tokens` is deliberately NOT enforced here. It is a + // context-*assembly* budget — how much recalled text an agent turn may + // inject into a prompt — and no method on the driver contract assembles a + // prompt. Enforcing it in the guard would mean inventing a tokenizer and + // applying it to a `Vec` that has not been rendered yet. It + // belongs to the auto-recall hook (M5), which is the thing that actually + // builds the context block. +} + +/// Wrap `policy` for sharing with the family decorators. +pub fn shared(policy: GuardPolicy) -> Arc { + Arc::new(policy) +} + +#[cfg(test)] +#[path = "policy_tests.rs"] +mod tests; diff --git a/src/openhuman/memory/guard/policy_tests.rs b/src/openhuman/memory/guard/policy_tests.rs new file mode 100644 index 0000000000..195a187098 --- /dev/null +++ b/src/openhuman/memory/guard/policy_tests.rs @@ -0,0 +1,333 @@ +//! Steps 1, 2, 3, 4, 5 and 6 at the policy level, without a provider. + +use super::*; +use std::sync::Arc; + +use crate::openhuman::memory::source_scope::with_source_scope; +use crate::openhuman::security::live_policy; +use crate::openhuman::security::policy::{AutonomyLevel, SecurityPolicy}; + +use crate::openhuman::memory::guard::test_support::{embedded_policy, external_policy}; + +/// Install `autonomy` as the live policy for this test thread only. +fn scoped_tier(autonomy: AutonomyLevel) -> live_policy::TestPolicyGuard { + let dir = std::env::temp_dir(); + live_policy::install_scoped( + Arc::new(SecurityPolicy { + autonomy, + ..SecurityPolicy::default() + }), + dir.clone(), + dir, + ) +} + +// ── Step 1 ─────────────────────────────────────────────────────────────────── + +#[test] +fn guard_with_scoped_full_security_policy_allows() { + // Other tests may install the process-global policy before this test runs. + // A thread-local full-tier override keeps the allow-path assertion isolated. + let _tier = scoped_tier(AutonomyLevel::Full); + let policy = embedded_policy(); + assert!(policy.enforce_read("core.get").is_ok()); + assert!(policy.enforce_write("core.store").is_ok()); +} + +#[test] +fn guard_denies_write_under_readonly_tier() { + let _tier = scoped_tier(AutonomyLevel::ReadOnly); + let err = embedded_policy() + .enforce_write("core.store") + .expect_err("readonly must refuse a write"); + let message = err.to_string(); + assert!( + message.contains(GUARD_DENIED_PREFIX), + "refusal must be attributable to the guard: {message}" + ); + assert!(matches!(err, MemoryError::Invalid(_))); +} + +#[test] +fn guard_allows_read_under_readonly_tier() { + let _tier = scoped_tier(AutonomyLevel::ReadOnly); + assert!(embedded_policy().enforce_read("core.get").is_ok()); +} + +#[test] +fn guard_allows_write_under_full_tier() { + let _tier = scoped_tier(AutonomyLevel::Full); + assert!(embedded_policy().enforce_write("core.store").is_ok()); +} + +/// As `scoped_tier`, but with the hourly action budget already exhausted. +fn scoped_tier_with_no_budget(autonomy: AutonomyLevel) -> live_policy::TestPolicyGuard { + let dir = std::env::temp_dir(); + live_policy::install_scoped( + Arc::new(SecurityPolicy { + autonomy, + max_actions_per_hour: 0, + ..SecurityPolicy::default() + }), + dir.clone(), + dir, + ) +} + +/// The guard's write check is the **tier**, not the agent-tool action budget. +/// +/// Fails before the fix: `enforce_write` mapped to `ToolOperation::Act`, whose +/// `record_action` refuses at `max_actions_per_hour = 0` with "Rate limit +/// exceeded". A budget denominated in tool calls must not be charged by a +/// decorator that sits under `MemoryCore::store` — one bulk ingest is hundreds +/// of calls. See `GuardPolicy::enforce_write`. +#[test] +fn guard_write_does_not_consume_the_agent_action_budget() { + let _tier = scoped_tier_with_no_budget(AutonomyLevel::Full); + assert!( + embedded_policy().enforce_write("core.store").is_ok(), + "an exhausted agent action budget must not refuse a kernel memory write" + ); +} + +/// …and the tier refusal the guard *does* own is unaffected by that change. +#[test] +fn guard_still_denies_write_under_readonly_tier_with_budget_exhausted() { + let _tier = scoped_tier_with_no_budget(AutonomyLevel::ReadOnly); + assert!(embedded_policy().enforce_write("core.store").is_err()); +} + +// ── Step 2 ─────────────────────────────────────────────────────────────────── + +#[tokio::test] +async fn ambient_scope_is_none_outside_a_source_scope() { + assert!(embedded_policy().ambient_scope().is_none()); +} + +#[tokio::test] +async fn ambient_scope_carries_the_task_local_allowlist() { + with_source_scope(Some(vec!["slack:#eng".into()]), async { + let scope = embedded_policy().ambient_scope().expect("scoped"); + assert!(scope.allows_source_id("slack:#eng")); + assert!(scope.allows_source_id("mem_src:slack:#eng:item-1")); + assert!(!scope.allows_source_id("gmail:me")); + }) + .await; +} + +#[tokio::test] +async fn an_empty_ambient_allowlist_stays_restrictive() { + // `Some(empty)` must not collapse to `None`: an empty allowlist denies all + // source-attributed content, per both `source_scope` and `SourceScope`. + with_source_scope(Some(vec![]), async { + let scope = embedded_policy().ambient_scope().expect("still restricted"); + assert!(scope.is_empty()); + assert!(!scope.allows_source_id("slack:#eng")); + }) + .await; +} + +#[tokio::test] +async fn narrow_scope_returns_the_ambient_scope_when_the_caller_asks_for_none() { + with_source_scope(Some(vec!["slack:#eng".into()]), async { + let scope = embedded_policy().narrow_scope(None).expect("scoped"); + assert_eq!(scope.allow, vec!["slack:#eng".to_string()]); + }) + .await; +} + +#[tokio::test] +async fn narrow_scope_passes_an_explicit_scope_through_when_unrestricted() { + let requested = SourceScope::new(["gmail:me"]); + let scope = embedded_policy() + .narrow_scope(Some(&requested)) + .expect("explicit scope survives"); + assert_eq!(scope.allow, vec!["gmail:me".to_string()]); +} + +#[tokio::test] +async fn narrow_scope_keeps_an_explicit_scope_that_is_a_subset_of_the_ambient_one() { + // The narrowing direction is the legitimate one: a caller that computed a + // tighter scope than the task-local still wins. + with_source_scope( + Some(vec!["slack:#eng".into(), "slack:#ops".into()]), + async { + let requested = SourceScope::new(["slack:#eng"]); + let scope = embedded_policy() + .narrow_scope(Some(&requested)) + .expect("scoped"); + assert_eq!(scope.allow, vec!["slack:#eng".to_string()]); + }, + ) + .await; +} + +#[tokio::test] +async fn narrow_scope_drops_explicit_sources_outside_the_ambient_allowlist() { + // The leak this exists to close: an explicit scope must not widen a + // source-restricted turn back out. + with_source_scope(Some(vec!["slack:#eng".into()]), async { + let requested = SourceScope::new(["slack:#eng", "gmail:me"]); + let scope = embedded_policy() + .narrow_scope(Some(&requested)) + .expect("scoped"); + assert_eq!(scope.allow, vec!["slack:#eng".to_string()]); + assert!(!scope.allows_source_id("gmail:me")); + }) + .await; +} + +#[tokio::test] +async fn an_entirely_out_of_scope_request_denies_rather_than_unrestricts() { + // Empty `Some`, never `None`: an empty allowlist denies all + // source-attributed content, which is the fail-closed reading. + with_source_scope(Some(vec!["slack:#eng".into()]), async { + let requested = SourceScope::new(["gmail:me"]); + let scope = embedded_policy() + .narrow_scope(Some(&requested)) + .expect("must stay restricted, not become unrestricted"); + assert!(scope.is_empty()); + }) + .await; +} + +// ── Step 3 ─────────────────────────────────────────────────────────────────── + +#[tokio::test] +async fn guard_stamps_internal_taint_outside_a_source_scope() { + assert_eq!( + embedded_policy().stamp_taint(MemoryTaint::Internal), + MemoryTaint::Internal + ); +} + +#[tokio::test] +async fn guard_stamps_external_sync_taint_inside_a_source_scope() { + with_source_scope(Some(vec!["slack:#eng".into()]), async { + assert_eq!( + embedded_policy().stamp_taint(MemoryTaint::Internal), + MemoryTaint::ExternalSync, + "a source-restricted turn is handling source-attributed content" + ); + }) + .await; +} + +#[tokio::test] +async fn guard_never_downgrades_a_caller_supplied_taint() { + // The laundering step the contract says the guard exists to prevent: a + // plain override would rewrite this to `Internal`. + assert_eq!( + embedded_policy().stamp_taint(MemoryTaint::ExternalSync), + MemoryTaint::ExternalSync + ); +} + +// ── Step 4 ─────────────────────────────────────────────────────────────────── + +/// Content that the secret scrubber definitely rewrites, so "no-op" is a real +/// claim rather than an accident of the fixture. +const SECRETY: &str = "Authorization: Bearer abcdefghijklmnop"; + +#[test] +fn guard_does_not_redact_for_an_embedded_driver() { + let out = embedded_policy().redact_outbound(SECRETY); + assert_eq!(out, SECRETY, "embedded traffic must be byte-identical"); + assert!( + matches!(out, std::borrow::Cow::Borrowed(_)), + "and must not even be re-allocated" + ); +} + +#[test] +fn guard_does_not_redact_for_a_null_driver() { + let policy = GuardPolicy::new( + "null", + DriverClass::Null, + MemoryHooksConfig::default(), + TRUSTED, + ); + assert_eq!(policy.redact_outbound(SECRETY), SECRETY); +} + +#[test] +fn guard_redacts_content_for_an_external_driver() { + let out = external_policy(TRUSTED).redact_outbound(SECRETY); + assert_ne!(out, SECRETY); + assert!( + out.contains("[REDACTED]"), + "expected the crate scrubber's placeholder, got: {out}" + ); +} + +#[test] +fn guard_redacts_json_only_for_an_external_driver() { + let value = serde_json::json!({ "token": SECRETY }); + assert_eq!( + embedded_policy().redact_outbound_json(value.clone()), + value, + "embedded JSON is untouched" + ); + assert_ne!( + external_policy(TRUSTED).redact_outbound_json(value.clone()), + value + ); +} + +// ── Step 5 ─────────────────────────────────────────────────────────────────── + +#[test] +fn guard_never_gates_egress_for_an_embedded_driver() { + assert!(embedded_policy().check_egress("core.store", true).is_ok()); +} + +#[test] +fn guard_refuses_an_external_driver_with_untrusted_state() { + let err = external_policy("untrusted") + .check_egress("core.store", true) + .expect_err("fail-closed"); + let message = err.to_string(); + assert!(message.contains(GUARD_DENIED_PREFIX), "{message}"); + assert!(message.contains("untrusted"), "{message}"); +} + +#[test] +fn guard_admits_an_external_driver_whose_trust_was_raised() { + assert!(external_policy(TRUSTED) + .check_egress("core.store", true) + .is_ok()); +} + +// ── Step 6 ─────────────────────────────────────────────────────────────────── + +#[test] +fn budgets_come_from_the_hooks_config() { + let policy = GuardPolicy::new( + "tinycortex", + DriverClass::Embedded, + MemoryHooksConfig { + recall_max_chars: 42, + capture_max_chars: 7, + ..MemoryHooksConfig::default() + }, + TRUSTED, + ); + assert_eq!(policy.recall_budget(), Some(42)); + assert_eq!(policy.capture_budget(), Some(7)); +} + +#[test] +fn a_zero_budget_reads_as_disabled_not_as_deny_everything() { + let policy = GuardPolicy::new( + "tinycortex", + DriverClass::Embedded, + MemoryHooksConfig { + recall_max_chars: 0, + capture_max_chars: 0, + ..MemoryHooksConfig::default() + }, + TRUSTED, + ); + assert_eq!(policy.recall_budget(), None); + assert_eq!(policy.capture_budget(), None); +} diff --git a/src/openhuman/memory/guard/provider.rs b/src/openhuman/memory/guard/provider.rs new file mode 100644 index 0000000000..29353bcf61 --- /dev/null +++ b/src/openhuman/memory/guard/provider.rs @@ -0,0 +1,162 @@ +//! [`MemoryGuard`] — the kernel-owned policy decorator over a bound driver. + +use std::sync::Arc; + +use async_trait::async_trait; +use tinycortex_api::capabilities::{Capabilities, Capability}; +use tinycortex_api::error::MemoryError; +use tinycortex_api::health::MemoryHealth; +use tinycortex_api::provider::{ + MemoryDiff, MemoryDocuments, MemoryEntities, MemoryGoals, MemoryGraph, MemoryIngest, + MemoryMaintenance, MemoryProvider, MemorySourceSink, MemoryToolMemory, MemoryTree, +}; + +use super::families::{ + GuardedDiff, GuardedDocuments, GuardedEntities, GuardedGoals, GuardedGraph, GuardedIngest, + GuardedMaintenance, GuardedSources, GuardedToolMemory, GuardedTree, +}; +use super::policy::GuardPolicy; + +/// The policy decorator every product caller receives instead of the raw +/// driver (`docs/specs/plan-memory.md` §3.4, `docs/specs/kernel.md` §3.4). +/// +/// It implements [`MemoryProvider`], so it is transparent to callers and cannot +/// be "skipped" by a caller that simply keeps using the contract — there is no +/// second, unguarded shape to hold. Its ten `as_*` overrides hand back +/// **guarded** family handles rather than the inner driver's, which is what +/// closes the accessor bypass; see [`super::families`] for why that forces the +/// decorators to be owned fields. +pub struct MemoryGuard { + inner: Arc, + policy: Arc, + + // The ten optional families. Each is `Some` **iff** the inner driver + // provides it, so `provides()` — which the contract's `audit_provider` + // compares against `capabilities()` — answers identically for the guard and + // for the driver underneath it. + ingest: Option, + documents: Option, + tree: Option, + entities: Option, + graph: Option, + diff: Option, + goals: Option, + tool_memory: Option, + sources: Option, + maintenance: Option, +} + +impl MemoryGuard { + /// Wrap `inner` in `policy`. + /// + /// Builds all ten decorators up front. That is not an optimisation: the + /// `as_*` accessors return borrows, so a decorator constructed inside an + /// accessor could not outlive the call. + pub fn new(inner: Arc, policy: Arc) -> Self { + macro_rules! family { + ($cap:ident, $ty:ident) => { + inner + .provides(Capability::$cap) + .then(|| $ty::new(Arc::clone(&inner), Arc::clone(&policy))) + }; + } + Self { + ingest: family!(Ingest, GuardedIngest), + documents: family!(Documents, GuardedDocuments), + tree: family!(Tree, GuardedTree), + entities: family!(Entities, GuardedEntities), + graph: family!(Graph, GuardedGraph), + diff: family!(Diff, GuardedDiff), + goals: family!(Goals, GuardedGoals), + tool_memory: family!(ToolMemory, GuardedToolMemory), + sources: family!(Sources, GuardedSources), + maintenance: family!(Maintenance, GuardedMaintenance), + inner, + policy, + } + } + + /// The policy this guard enforces. + pub fn policy(&self) -> &Arc { + &self.policy + } + + /// The wrapped driver. + /// + /// `pub(crate)` on purpose: handing this out is exactly the bypass the + /// guard exists to prevent, and the only legitimate use is inside the + /// memory subsystem itself (identity, health, tests). Do not widen it. + pub(crate) fn inner(&self) -> &Arc { + &self.inner + } +} + +#[async_trait] +impl MemoryProvider for MemoryGuard { + /// The **wrapped driver's** id, not a synthetic `"guard"`. The guard is a + /// policy layer, not a driver: status output, spans, and audit events all + /// name the thing that actually stores the bytes. + fn driver_id(&self) -> &str { + self.inner.driver_id() + } + + fn capabilities(&self) -> Capabilities { + self.inner.capabilities() + } + + async fn health(&self) -> MemoryHealth { + self.inner.health().await + } + + async fn shutdown(&self) -> Result<(), MemoryError> { + self.inner.shutdown().await + } + + fn as_ingest(&self) -> Option<&dyn MemoryIngest> { + self.ingest.as_ref().map(|g| g as &dyn MemoryIngest) + } + + fn as_documents(&self) -> Option<&dyn MemoryDocuments> { + self.documents.as_ref().map(|g| g as &dyn MemoryDocuments) + } + + fn as_tree(&self) -> Option<&dyn MemoryTree> { + self.tree.as_ref().map(|g| g as &dyn MemoryTree) + } + + fn as_entities(&self) -> Option<&dyn MemoryEntities> { + self.entities.as_ref().map(|g| g as &dyn MemoryEntities) + } + + fn as_graph(&self) -> Option<&dyn MemoryGraph> { + self.graph.as_ref().map(|g| g as &dyn MemoryGraph) + } + + fn as_diff(&self) -> Option<&dyn MemoryDiff> { + self.diff.as_ref().map(|g| g as &dyn MemoryDiff) + } + + fn as_goals(&self) -> Option<&dyn MemoryGoals> { + self.goals.as_ref().map(|g| g as &dyn MemoryGoals) + } + + fn as_tool_memory(&self) -> Option<&dyn MemoryToolMemory> { + self.tool_memory + .as_ref() + .map(|g| g as &dyn MemoryToolMemory) + } + + fn as_sources(&self) -> Option<&dyn MemorySourceSink> { + self.sources.as_ref().map(|g| g as &dyn MemorySourceSink) + } + + fn as_maintenance(&self) -> Option<&dyn MemoryMaintenance> { + self.maintenance + .as_ref() + .map(|g| g as &dyn MemoryMaintenance) + } +} + +#[cfg(test)] +#[path = "provider_tests.rs"] +mod tests; diff --git a/src/openhuman/memory/guard/provider_tests.rs b/src/openhuman/memory/guard/provider_tests.rs new file mode 100644 index 0000000000..f716454630 --- /dev/null +++ b/src/openhuman/memory/guard/provider_tests.rs @@ -0,0 +1,264 @@ +//! The guard as a [`MemoryProvider`]: capability mirroring, the mandatory +//! three, and step 7's audit event. + +use super::*; +use std::sync::Arc; + +use tinycortex_api::capabilities::{Capabilities, Capability}; +use tinycortex_api::null::NullMemoryProvider; +use tinycortex_api::provider::types::SourceScope; +use tinycortex_api::provider::{ + audit_provider, MemoryCore, MemoryPortability, MemoryProvider, MemoryRecall, +}; +use tinycortex_api::recall::OwnedRecallOpts; +use tinycortex_api::types::{MemoryCategory, MemoryTaint}; + +use crate::core::event_bus::{init_global, DomainEvent, DEFAULT_CAPACITY}; +use crate::core::subsystem::DriverClass; +use crate::openhuman::config::schema::MemoryHooksConfig; +use crate::openhuman::memory::guard::policy::TRUSTED; +use crate::openhuman::memory::guard::test_support::{ + embedded_policy, entry, export_record, external_policy, guarded, guarded_with, + RecordingProvider, +}; +use crate::openhuman::memory::guard::GuardPolicy; +use crate::openhuman::memory::source_scope::with_source_scope; + +fn budgeted(recall_max_chars: usize, capture_max_chars: usize) -> GuardPolicy { + GuardPolicy::new( + "recording", + DriverClass::Embedded, + MemoryHooksConfig { + recall_max_chars, + capture_max_chars, + ..MemoryHooksConfig::default() + }, + TRUSTED, + ) +} + +// ── Identity + capability mirroring ───────────────────────────────────────── + +#[tokio::test] +async fn guard_reports_the_wrapped_drivers_identity() { + let (_driver, guard) = guarded(embedded_policy()); + assert_eq!( + guard.driver_id(), + "recording", + "the guard is a policy layer, not a driver" + ); + assert_eq!(guard.capabilities(), Capabilities::all()); +} + +#[tokio::test] +async fn guard_passes_audit_provider_against_its_own_capabilities() { + let (_driver, guard) = guarded(embedded_policy()); + audit_provider(&guard).expect("advertised set and reachable accessors must agree"); +} + +#[tokio::test] +async fn guard_accessor_presence_mirrors_inner_provides_for_all_ten_families() { + let (_driver, guard) = guarded(embedded_policy()); + for capability in Capability::ALL { + assert!( + guard.provides(capability), + "{capability} must be reachable through the guard" + ); + } + + // The other direction: a driver with only the mandatory three must not + // acquire families merely by being guarded. + let inner = Arc::new(NullMemoryProvider::new()); + let null = MemoryGuard::new( + Arc::clone(&inner) as Arc, + Arc::new(embedded_policy()), + ); + for capability in Capability::ALL { + assert_eq!( + null.provides(capability), + inner.provides(capability), + "{capability} presence must mirror the inner driver exactly" + ); + } + audit_provider(&null).expect("mandatory-only driver stays consistent when guarded"); +} + +// ── The mandatory three ───────────────────────────────────────────────────── + +#[tokio::test] +async fn guard_stamps_taint_on_store_rather_than_trusting_the_caller() { + let (driver, guard) = guarded(embedded_policy()); + with_source_scope(Some(vec!["slack:#eng".into()]), async { + guard + .store( + "ns", + "k", + "hello", + MemoryCategory::Core, + None, + MemoryTaint::Internal, + ) + .await + .expect("store"); + }) + .await; + assert_eq!(driver.only_call().taint, Some(MemoryTaint::ExternalSync)); +} + +#[tokio::test] +async fn guard_truncates_stored_content_to_capture_max_chars() { + let (driver, guard) = guarded(budgeted(1000, 5)); + guard + .store( + "ns", + "k", + "hello world", + MemoryCategory::Core, + None, + MemoryTaint::Internal, + ) + .await + .expect("store"); + assert_eq!(driver.only_call().content.as_deref(), Some("hello")); +} + +#[tokio::test] +async fn guard_truncates_recall_results_to_recall_max_chars() { + let (_driver, guard) = guarded_with( + RecordingProvider::new().with_recall_result(vec![ + entry("aaaa"), + entry("bbbb"), + entry("cccc"), + ]), + budgeted(6, 500), + ); + let hits = guard + .recall("q", 10, &OwnedRecallOpts::default(), None) + .await + .expect("recall"); + assert_eq!(hits.len(), 2); + assert_eq!(hits[0].content, "aaaa"); + assert_eq!(hits[1].content, "bb"); +} + +/// Pins the departure argued in `mandatory.rs`: the embedded driver *refuses* a +/// `Some(scope)` on recall, so the guard must NOT fill it from the task-local. +#[tokio::test] +async fn guard_never_fills_scope_on_recall() { + let (driver, guard) = guarded(embedded_policy()); + with_source_scope(Some(vec!["slack:#eng".into()]), async { + guard + .recall("q", 10, &OwnedRecallOpts::default(), None) + .await + .expect("recall must not become an error merely by being scoped"); + }) + .await; + assert_eq!(driver.only_call().scoped, Some(false)); +} + +#[tokio::test] +async fn guard_forwards_an_explicit_recall_scope_untouched() { + let (driver, guard) = guarded(embedded_policy()); + let scope = SourceScope::new(["slack:#eng"]); + let _ = guard + .recall("q", 10, &OwnedRecallOpts::default(), Some(&scope)) + .await; + assert_eq!(driver.only_call().scoped, Some(true)); +} + +#[tokio::test] +async fn guard_preserves_import_taint_rather_than_restamping_it() { + let (driver, guard) = guarded(embedded_policy()); + // Inside a source scope, so a naive "stamp everything" would show up. + with_source_scope(Some(vec!["slack:#eng".into()]), async { + guard + .import_records(vec![export_record(MemoryTaint::Internal)]) + .await + .expect("import"); + }) + .await; + assert_eq!( + driver.only_call().taint, + Some(MemoryTaint::Internal), + "a restore must not have its provenance rewritten wholesale" + ); +} + +#[tokio::test] +async fn guard_does_not_budget_trim_an_export() { + let (driver, guard) = guarded(budgeted(1, 1)); + guard.export_page(None, 10).await.expect("export"); + assert_eq!(driver.only_call().method, "portability.export_page"); +} + +// ── Step 7 ────────────────────────────────────────────────────────────────── + +#[tokio::test] +async fn guard_publishes_memory_guard_denied_on_refusal() { + let mut rx = init_global(DEFAULT_CAPACITY).raw_receiver(); + let (driver, guard) = guarded(external_policy("untrusted")); + let err = guard + .store( + "ns", + "k", + "hello", + MemoryCategory::Core, + None, + MemoryTaint::Internal, + ) + .await + .expect_err("untrusted external driver must be refused"); + assert!(err.to_string().contains("memory guard: ")); + assert_eq!(driver.call_count(), 0, "the driver must never be reached"); + + let mut seen = None; + while let Ok(event) = rx.try_recv() { + if let DomainEvent::MemoryGuardDenied { + driver_id, + method, + reason, + } = event + { + seen = Some((driver_id, method, reason)); + break; + } + } + let (driver_id, method, reason) = seen.expect("a MemoryGuardDenied event"); + assert_eq!(driver_id, "supermemory"); + assert_eq!(method, "core.store"); + assert!(!reason.contains("hello"), "must never carry content"); +} + +#[tokio::test] +async fn guard_publishes_nothing_on_the_success_path() { + let mut rx = init_global(DEFAULT_CAPACITY).raw_receiver(); + let (_driver, guard) = guarded(embedded_policy()); + guard + .store( + "ns", + "k", + "hello", + MemoryCategory::Core, + None, + MemoryTaint::Internal, + ) + .await + .expect("store"); + guard + .recall("q", 3, &OwnedRecallOpts::default(), None) + .await + .expect("recall"); + + // Sibling tests share the process-global bus and run in parallel, so + // filter to *this* guard's driver id rather than asserting the channel is + // empty — `guard_publishes_memory_guard_denied_on_refusal` legitimately + // publishes one (for `supermemory`) at the same time. + while let Ok(event) = rx.try_recv() { + if let DomainEvent::MemoryGuardDenied { driver_id, .. } = &event { + assert_ne!( + driver_id, "recording", + "a guarded read/write must not publish on success" + ); + } + } +} diff --git a/src/openhuman/memory/guard/test_support.rs b/src/openhuman/memory/guard/test_support.rs new file mode 100644 index 0000000000..44c2eaa479 --- /dev/null +++ b/src/openhuman/memory/guard/test_support.rs @@ -0,0 +1,654 @@ +//! A recording fake [`MemoryProvider`] for the guard's tests. +//! +//! `NullMemoryProvider` cannot serve here: it advertises only the mandatory +//! three and returns `None` from every `as_*` accessor, so a guard built over +//! it would have no family decorators at all — which is precisely what the +//! interesting tests are about. This fake implements **all thirteen** families +//! and records what actually reached it, so a test can assert both "the driver +//! saw the value the guard rewrote" and "the driver saw nothing at all". + +#![cfg(test)] + +use std::sync::{Arc, Mutex}; + +use async_trait::async_trait; +use tinycortex_api::capabilities::Capabilities; +use tinycortex_api::chunks::Chunk; +use tinycortex_api::error::MemoryError; +use tinycortex_api::goals::GoalsDoc; +use tinycortex_api::health::MemoryHealth; +use tinycortex_api::provider::types::{ + DiffReport, EntityHit, ExportPage, ExportRecord, ImportOutcome, IngestItem, IngestOutcome, + MaintenanceReport, SnapshotRef, SourceItem, SourceScope, +}; +use tinycortex_api::provider::{ + MemoryCore, MemoryDiff, MemoryDocuments, MemoryEntities, MemoryGoals, MemoryGraph, + MemoryIngest, MemoryMaintenance, MemoryPortability, MemoryProvider, MemoryRecall, + MemorySourceSink, MemoryToolMemory, MemoryTree, +}; +use tinycortex_api::recall::OwnedRecallOpts; +use tinycortex_api::tool_memory::ToolMemoryRule; +use tinycortex_api::tree::{IngestRequest, QueryResult, TreeStatus}; +use tinycortex_api::types::{ + GraphRelationRecord, MemoryCategory, MemoryEntry, MemoryKvRecord, MemoryTaint, + NamespaceDocumentInput, NamespaceRetrievalContext, NamespaceSummary, StoredMemoryDocument, +}; + +/// One call that reached the driver. +#[derive(Debug, Clone, PartialEq)] +pub struct Call { + pub method: String, + /// Content the driver was handed, when the method carries any. + pub content: Option, + /// Provenance the driver was handed, when the method carries any. + pub taint: Option, + /// Whether the method received a `Some(scope)`. + pub scoped: Option, +} + +impl Call { + fn plain(method: &str) -> Self { + Self { + method: method.into(), + content: None, + taint: None, + scoped: None, + } + } +} + +/// A provider that records and answers with empties. +pub struct RecordingProvider { + calls: Mutex>, + /// What `recall` returns, so budget tests can drive a known result set. + recall_result: Mutex>, +} + +impl Default for RecordingProvider { + fn default() -> Self { + Self::new() + } +} + +impl RecordingProvider { + pub fn new() -> Self { + Self { + calls: Mutex::new(Vec::new()), + recall_result: Mutex::new(Vec::new()), + } + } + + pub fn with_recall_result(self, entries: Vec) -> Self { + *self.recall_result.lock().unwrap() = entries; + self + } + + fn record(&self, call: Call) { + self.calls.lock().unwrap().push(call); + } + + pub fn calls(&self) -> Vec { + self.calls.lock().unwrap().clone() + } + + pub fn call_count(&self) -> usize { + self.calls.lock().unwrap().len() + } + + /// The single recorded call, panicking when there is not exactly one. + pub fn only_call(&self) -> Call { + let calls = self.calls(); + assert_eq!( + calls.len(), + 1, + "expected exactly one driver call: {calls:?}" + ); + calls.into_iter().next().unwrap() + } +} + +/// A [`GuardPolicy`](super::GuardPolicy) over an embedded driver with default +/// budgets — the shipped configuration. +pub fn embedded_policy() -> super::GuardPolicy { + super::GuardPolicy::new( + "recording", + crate::core::subsystem::DriverClass::Embedded, + crate::openhuman::config::schema::MemoryHooksConfig::default(), + super::policy::TRUSTED, + ) +} + +/// A policy over an *external* driver. No such driver can bind today +/// (`binding::admit` refuses them), so this is the only way to reach the class +/// branches that land for real in M6. +pub fn external_policy(trust_state: &str) -> super::GuardPolicy { + super::GuardPolicy::new( + "supermemory", + crate::core::subsystem::DriverClass::External, + crate::openhuman::config::schema::MemoryHooksConfig::default(), + trust_state, + ) +} + +/// An [`ExportRecord`] fixture. +pub fn export_record(taint: MemoryTaint) -> ExportRecord { + ExportRecord { + kind: "entry".into(), + id: "r1".into(), + namespace: Some("ns".into()), + taint, + payload: serde_json::Value::Null, + } +} + +/// A guard over a fresh recording provider, plus a handle on that provider. +pub fn guarded(policy: super::GuardPolicy) -> (Arc, super::MemoryGuard) { + guarded_with(RecordingProvider::new(), policy) +} + +/// As [`guarded`], over a caller-configured provider. +pub fn guarded_with( + provider: RecordingProvider, + policy: super::GuardPolicy, +) -> (Arc, super::MemoryGuard) { + let provider = Arc::new(provider); + let guard = super::MemoryGuard::new( + Arc::clone(&provider) as Arc, + Arc::new(policy), + ); + (provider, guard) +} + +/// A [`MemoryEntry`] fixture. +pub fn entry(content: &str) -> MemoryEntry { + MemoryEntry { + id: "id".into(), + key: "key".into(), + content: content.into(), + namespace: Some("ns".into()), + category: MemoryCategory::Core, + timestamp: "2026-01-01T00:00:00Z".into(), + session_id: None, + score: None, + taint: MemoryTaint::Internal, + } +} + +/// A [`TreeStatus`] fixture. +fn tree_status(namespace: &str) -> TreeStatus { + TreeStatus { + namespace: namespace.to_string(), + total_nodes: 0, + depth: 0, + oldest_entry: None, + newest_entry: None, + last_run_at: None, + } +} + +/// A [`NamespaceDocumentInput`] fixture. +pub fn document(content: &str, taint: MemoryTaint) -> NamespaceDocumentInput { + NamespaceDocumentInput { + namespace: "ns".into(), + key: "k".into(), + title: "t".into(), + content: content.into(), + source_type: "chat".into(), + priority: "normal".into(), + tags: vec![], + metadata: serde_json::Value::Null, + category: "core".into(), + session_id: None, + document_id: None, + taint, + } +} + +#[async_trait] +impl MemoryCore for RecordingProvider { + async fn store( + &self, + _namespace: &str, + _key: &str, + content: &str, + _category: MemoryCategory, + _session_id: Option<&str>, + taint: MemoryTaint, + ) -> Result<(), MemoryError> { + self.record(Call { + method: "core.store".into(), + content: Some(content.to_string()), + taint: Some(taint), + scoped: None, + }); + Ok(()) + } + + async fn get(&self, _namespace: &str, _key: &str) -> Result, MemoryError> { + self.record(Call::plain("core.get")); + Ok(None) + } + + async fn forget(&self, _namespace: &str, _key: &str) -> Result { + self.record(Call::plain("core.forget")); + Ok(false) + } + + async fn list( + &self, + _namespace: Option<&str>, + _category: Option<&MemoryCategory>, + _session_id: Option<&str>, + ) -> Result, MemoryError> { + self.record(Call::plain("core.list")); + Ok(vec![]) + } + + async fn namespaces(&self) -> Result, MemoryError> { + self.record(Call::plain("core.namespaces")); + Ok(vec![]) + } +} + +#[async_trait] +impl MemoryRecall for RecordingProvider { + async fn recall( + &self, + query: &str, + _limit: usize, + _opts: &OwnedRecallOpts, + scope: Option<&SourceScope>, + ) -> Result, MemoryError> { + self.record(Call { + method: "recall.recall".into(), + content: Some(query.to_string()), + taint: None, + scoped: Some(scope.is_some()), + }); + Ok(self.recall_result.lock().unwrap().clone()) + } +} + +#[async_trait] +impl MemoryPortability for RecordingProvider { + async fn export_page( + &self, + _cursor: Option<&str>, + _limit: usize, + ) -> Result { + self.record(Call::plain("portability.export_page")); + Ok(ExportPage::default()) + } + + async fn import_records( + &self, + records: Vec, + ) -> Result { + self.record(Call { + method: "portability.import_records".into(), + content: None, + taint: records.first().map(|r| r.taint), + scoped: None, + }); + Ok(ImportOutcome::default()) + } +} + +#[async_trait] +impl MemoryIngest for RecordingProvider { + async fn ingest_document(&self, item: IngestItem) -> Result { + self.record(Call { + method: "ingest.ingest_document".into(), + content: Some(item.content), + taint: Some(item.taint), + scoped: None, + }); + Ok(IngestOutcome::default()) + } + + async fn ingest_chat(&self, messages: Vec) -> Result { + self.record(Call { + method: "ingest.ingest_chat".into(), + content: messages.first().map(|m| m.content.clone()), + taint: messages.first().map(|m| m.taint), + scoped: None, + }); + Ok(IngestOutcome::default()) + } +} + +#[async_trait] +impl MemoryDocuments for RecordingProvider { + async fn put_document(&self, input: NamespaceDocumentInput) -> Result { + self.record(Call { + method: "documents.put_document".into(), + content: Some(input.content), + taint: Some(input.taint), + scoped: None, + }); + Ok("doc".into()) + } + + async fn get_document( + &self, + _namespace: &str, + _key: &str, + ) -> Result, MemoryError> { + self.record(Call::plain("documents.get_document")); + Ok(None) + } + + async fn query_documents( + &self, + namespace: &str, + query: &str, + _limit: usize, + ) -> Result { + self.record(Call { + method: "documents.query_documents".into(), + content: Some(query.to_string()), + taint: None, + scoped: None, + }); + Ok(NamespaceRetrievalContext { + namespace: namespace.to_string(), + query: Some(query.to_string()), + context_text: String::new(), + hits: vec![], + }) + } +} + +#[async_trait] +impl MemoryTree for RecordingProvider { + async fn append(&self, request: IngestRequest) -> Result<(), MemoryError> { + self.record(Call { + method: "tree.append".into(), + content: Some(request.content), + taint: None, + scoped: None, + }); + Ok(()) + } + + async fn query_source( + &self, + _namespace: &str, + _source_id: &str, + _limit: usize, + scope: Option<&SourceScope>, + ) -> Result, MemoryError> { + self.record(Call { + method: "tree.query_source".into(), + // The scope's allow list, rendered so a test can assert which one + // arrived. Sorted because it comes from a `HashSet`. + content: scope.map(|s| { + let mut allow = s.allow.clone(); + allow.sort(); + allow.join(",") + }), + taint: None, + scoped: Some(scope.is_some()), + }); + Ok(vec![]) + } + + async fn drill_down( + &self, + _namespace: &str, + _node_id: &str, + ) -> Result { + self.record(Call::plain("tree.drill_down")); + Err(MemoryError::NotFound("node".into())) + } + + async fn seal(&self, namespace: &str) -> Result { + self.record(Call::plain("tree.seal")); + Ok(tree_status(namespace)) + } + + async fn cascade(&self, namespace: &str) -> Result { + self.record(Call::plain("tree.cascade")); + Ok(tree_status(namespace)) + } +} + +#[async_trait] +impl MemoryEntities for RecordingProvider { + async fn entities( + &self, + _namespace: &str, + _query: Option<&str>, + _limit: usize, + ) -> Result, MemoryError> { + self.record(Call::plain("entities.entities")); + Ok(vec![]) + } + + async fn entity_edges( + &self, + _namespace: &str, + _entity_id: &str, + _limit: usize, + ) -> Result, MemoryError> { + self.record(Call::plain("entities.entity_edges")); + Ok(vec![]) + } + + async fn touch_entities( + &self, + _namespace: &str, + _entity_ids: &[String], + ) -> Result<(), MemoryError> { + self.record(Call::plain("entities.touch_entities")); + Ok(()) + } +} + +#[async_trait] +impl MemoryGraph for RecordingProvider { + async fn kv_get( + &self, + _namespace: Option<&str>, + _key: &str, + ) -> Result, MemoryError> { + self.record(Call::plain("graph.kv_get")); + Ok(None) + } + + async fn kv_put( + &self, + _namespace: Option<&str>, + _key: &str, + value: serde_json::Value, + ) -> Result<(), MemoryError> { + self.record(Call { + method: "graph.kv_put".into(), + content: Some(value.to_string()), + taint: None, + scoped: None, + }); + Ok(()) + } + + async fn kv_list( + &self, + _namespace: Option<&str>, + _prefix: Option<&str>, + _limit: usize, + ) -> Result, MemoryError> { + self.record(Call::plain("graph.kv_list")); + Ok(vec![]) + } + + async fn relations( + &self, + _namespace: Option<&str>, + _subject: Option<&str>, + _predicate: Option<&str>, + _limit: usize, + ) -> Result, MemoryError> { + self.record(Call::plain("graph.relations")); + Ok(vec![]) + } + + async fn put_relation(&self, _relation: GraphRelationRecord) -> Result<(), MemoryError> { + self.record(Call::plain("graph.put_relation")); + Ok(()) + } +} + +#[async_trait] +impl MemoryDiff for RecordingProvider { + async fn capture_snapshot(&self, _source_id: &str) -> Result { + self.record(Call::plain("diff.capture_snapshot")); + Err(MemoryError::NotFound("source".into())) + } + + async fn snapshots( + &self, + _source_id: &str, + _limit: usize, + ) -> Result, MemoryError> { + self.record(Call::plain("diff.snapshots")); + Ok(vec![]) + } + + async fn diff( + &self, + _source_id: &str, + _from: Option<&str>, + _to: &str, + ) -> Result { + self.record(Call::plain("diff.diff")); + Err(MemoryError::NotFound("snapshot".into())) + } +} + +#[async_trait] +impl MemoryGoals for RecordingProvider { + async fn goals(&self) -> Result { + self.record(Call::plain("goals.goals")); + Ok(GoalsDoc::default()) + } + + async fn set_goals(&self, _goals: GoalsDoc) -> Result<(), MemoryError> { + self.record(Call::plain("goals.set_goals")); + Ok(()) + } +} + +#[async_trait] +impl MemoryToolMemory for RecordingProvider { + async fn tool_rules(&self, _tool_name: &str) -> Result, MemoryError> { + self.record(Call::plain("tool_memory.tool_rules")); + Ok(vec![]) + } + + async fn put_tool_rule(&self, _rule: ToolMemoryRule) -> Result<(), MemoryError> { + self.record(Call::plain("tool_memory.put_tool_rule")); + Ok(()) + } + + async fn delete_tool_rule( + &self, + _tool_name: &str, + _rule_id: &str, + ) -> Result { + self.record(Call::plain("tool_memory.delete_tool_rule")); + Ok(false) + } +} + +#[async_trait] +impl MemorySourceSink for RecordingProvider { + async fn accept_source_items( + &self, + _source_id: &str, + _source_kind: &str, + items: Vec, + taint: MemoryTaint, + ) -> Result { + self.record(Call { + method: "sources.accept_source_items".into(), + content: items.first().map(|i| i.content.clone()), + taint: Some(taint), + scoped: None, + }); + Ok(IngestOutcome::default()) + } + + async fn forget_source(&self, _source_id: &str) -> Result { + self.record(Call::plain("sources.forget_source")); + Ok(0) + } +} + +#[async_trait] +impl MemoryMaintenance for RecordingProvider { + async fn reembed(&self) -> Result { + self.record(Call::plain("maintenance.reembed")); + Ok(MaintenanceReport::default()) + } + + async fn compact(&self) -> Result { + self.record(Call::plain("maintenance.compact")); + Ok(MaintenanceReport::default()) + } + + async fn consolidate(&self) -> Result { + self.record(Call::plain("maintenance.consolidate")); + Ok(MaintenanceReport::default()) + } + + async fn doctor(&self) -> Result { + self.record(Call::plain("maintenance.doctor")); + Ok(MaintenanceReport::default()) + } +} + +#[async_trait] +impl MemoryProvider for RecordingProvider { + fn driver_id(&self) -> &str { + "recording" + } + + fn capabilities(&self) -> Capabilities { + Capabilities::all() + } + + async fn health(&self) -> MemoryHealth { + MemoryHealth::Ready + } + + fn as_ingest(&self) -> Option<&dyn MemoryIngest> { + Some(self) + } + fn as_documents(&self) -> Option<&dyn MemoryDocuments> { + Some(self) + } + fn as_tree(&self) -> Option<&dyn MemoryTree> { + Some(self) + } + fn as_entities(&self) -> Option<&dyn MemoryEntities> { + Some(self) + } + fn as_graph(&self) -> Option<&dyn MemoryGraph> { + Some(self) + } + fn as_diff(&self) -> Option<&dyn MemoryDiff> { + Some(self) + } + fn as_goals(&self) -> Option<&dyn MemoryGoals> { + Some(self) + } + fn as_tool_memory(&self) -> Option<&dyn MemoryToolMemory> { + Some(self) + } + fn as_sources(&self) -> Option<&dyn MemorySourceSink> { + Some(self) + } + fn as_maintenance(&self) -> Option<&dyn MemoryMaintenance> { + Some(self) + } +} diff --git a/src/openhuman/memory/mod.rs b/src/openhuman/memory/mod.rs index feda50253f..28ac4abebc 100644 --- a/src/openhuman/memory/mod.rs +++ b/src/openhuman/memory/mod.rs @@ -11,10 +11,13 @@ // Legacy memory modules pub mod agent; +pub mod binding; pub mod conversations; pub mod diff; +pub mod driver; pub mod global; pub mod goals; +pub mod guard; pub mod ingestion; pub mod ops; pub mod people; @@ -49,6 +52,8 @@ pub mod util; pub mod tree_policy; pub mod tree_source; +#[cfg(test)] +mod bypass_allowlist_tests; #[cfg(test)] mod sync_pipeline_e2e_tests; #[cfg(test)] @@ -63,7 +68,25 @@ pub use ops::*; pub use rpc_models::*; pub use schemas::{ all_controller_schemas as all_memory_controller_schemas, + all_core_recall_controller_schemas as all_memory_core_recall_controller_schemas, + all_core_recall_registered_controllers as all_memory_core_recall_registered_controllers, + all_documents_controller_schemas as all_memory_documents_controller_schemas, + all_documents_registered_controllers as all_memory_documents_registered_controllers, + all_files_controller_schemas as all_memory_files_controller_schemas, + all_files_registered_controllers as all_memory_files_registered_controllers, + all_ingest_controller_schemas as all_memory_ingest_controller_schemas, + all_ingest_registered_controllers as all_memory_ingest_registered_controllers, + all_kv_graph_controller_schemas as all_memory_kv_graph_controller_schemas, + all_kv_graph_registered_controllers as all_memory_kv_graph_registered_controllers, + all_learn_controller_schemas as all_memory_learn_controller_schemas, + all_learn_registered_controllers as all_memory_learn_registered_controllers, + all_provider_controller_schemas as all_memory_provider_controller_schemas, + all_provider_registered_controllers as all_memory_provider_registered_controllers, all_registered_controllers as all_memory_registered_controllers, + all_sync_controller_schemas as all_memory_sync_controller_schemas, + all_sync_registered_controllers as all_memory_sync_registered_controllers, + all_tool_memory_controller_schemas as all_memory_tool_memory_controller_schemas, + all_tool_memory_registered_controllers as all_memory_tool_memory_registered_controllers, }; pub use traits::{Memory, MemoryCategory, MemoryEntry, MemoryTaint, NamespaceSummary, RecallOpts}; diff --git a/src/openhuman/memory/ops/documents.rs b/src/openhuman/memory/ops/documents.rs index 6ec104f4ab..29782b64b6 100644 --- a/src/openhuman/memory/ops/documents.rs +++ b/src/openhuman/memory/ops/documents.rs @@ -5,6 +5,7 @@ use serde::{Deserialize, Serialize}; +use crate::core::subsystem::DriverClass; use crate::openhuman::memory::store::{NamespaceDocumentInput, NamespaceRetrievalContext}; use crate::openhuman::memory::{ ApiEnvelope, DeleteDocumentRequest, DeleteDocumentResponse, EmptyRequest, ListDocumentsRequest, @@ -14,8 +15,10 @@ use crate::openhuman::memory::{ RecallMemoriesRequest, RecallMemoriesResponse, }; use crate::rpc::RpcOutcome; +use tinycortex_api::provider::MemoryProvider; use super::envelope::{envelope, error_envelope, memory_counts}; +use super::guard::active_memory_guard; use super::helpers::{ active_memory_client, build_retrieval_context, current_workspace_dir, filter_hits_by_document_ids, format_llm_context_message, maybe_retrieval_context, @@ -171,10 +174,27 @@ pub async fn namespace_list() -> Result>, String> { } /// Upserts a document into a namespace. +/// +/// Routed through [`MemoryGuard`](crate::openhuman::memory::guard::MemoryGuard). +/// `MemoryDocuments::put_document` on the embedded driver is +/// `client.put_doc(input)` — deliberately the full pipeline, not the +/// `put_doc_light` shortcut — so the store, the input type and the background +/// graph-extraction enqueue are all unchanged. What the guard adds: the tier +/// check, redaction (a byte-identical pass-through for an embedded driver), and +/// taint stamping. +/// +/// The `taint: Internal` literal below stays: the contract says the *caller* +/// supplies provenance and the driver never assigns it. `GuardPolicy::stamp_taint` +/// is a monotone raise over that value — it can promote this write to +/// `ExternalSync` when the turn runs under a source scope, but it can never +/// launder an `ExternalSync` caller down to `Internal`. pub async fn doc_put(params: PutDocParams) -> Result, String> { - let client = active_memory_client().await?; - let document_id = client - .put_doc(NamespaceDocumentInput { + let guard = active_memory_guard().await?; + let documents = guard + .as_documents() + .ok_or_else(|| "memory driver does not support the documents family".to_string())?; + let document_id = documents + .put_document(NamespaceDocumentInput { namespace: params.namespace, key: params.key, title: params.title, @@ -191,7 +211,8 @@ pub async fn doc_put(params: PutDocParams) -> Result, S // `store_skill_sync` directly with their own taint label. taint: crate::openhuman::memory::MemoryTaint::Internal, }) - .await?; + .await + .map_err(|e| e.to_string())?; Ok(RpcOutcome::single_log( PutDocResult { document_id }, "memory document upserted", @@ -240,8 +261,37 @@ pub async fn doc_list( Ok(RpcOutcome::single_log(docs, "memory documents listed")) } +/// Refuse an embedded-store-only operation when the bound driver is not the +/// embedded engine. +/// +/// `delete_document` / `clear_namespace` / `doc_delete` operate on the local +/// embedded SQLite store through `active_memory_client` and have **no contract +/// twin** — `MemoryDocuments` has no `delete_document` or `clear_namespace` +/// method — so they cannot be routed through the contract. Under a null or +/// fallback binding the operator asked for memory to be disabled, yet these +/// handlers would otherwise still reach the store boot initialised and delete +/// persisted rows. This is the RPC half of the CLI's legacy-client gate +/// (`core::cli_capability::legacy_client_verdict`): an embedded-only operation +/// is only valid when the bound driver actually is the embedded engine. The +/// check runs through the guarded binding (`active_memory_guard`), so the +/// verdict always reflects the driver that bound, never a global slot. +async fn ensure_embedded_driver(operation: &str) -> Result<(), String> { + let guard = active_memory_guard().await?; + if guard.policy().class() == DriverClass::Embedded { + return Ok(()); + } + Err(format!( + "memory driver `{}` is not the embedded TinyCortex driver, so `{operation}` is \ + unavailable: it operates on the local embedded store directly, and this \ + configuration bound a different driver. Change `[subsystems.memory] driver` in \ + your config.", + guard.driver_id() + )) +} + /// Deletes a document from a namespace. pub async fn doc_delete(params: DeleteDocParams) -> Result, String> { + ensure_embedded_driver("doc_delete").await?; let client = active_memory_client().await?; let result = client .delete_document(¶ms.namespace, ¶ms.document_id) @@ -253,6 +303,7 @@ pub async fn doc_delete(params: DeleteDocParams) -> Result Result, String> { + ensure_embedded_driver("clear_namespace").await?; let client = active_memory_client().await?; log::debug!("[memory] clear_namespace RPC invoked"); client.clear_namespace(¶ms.namespace).await?; @@ -354,6 +405,7 @@ pub async fn memory_list_namespaces( pub async fn memory_delete_document( request: DeleteDocumentRequest, ) -> Result>, String> { + ensure_embedded_driver("delete_document").await?; let client = active_memory_client().await?; let raw = client .delete_document(&request.namespace, &request.document_id) @@ -755,4 +807,107 @@ mod tests { let after_data = listed_after.value.data.expect("after clear data"); assert_eq!(after_data.count, 0); } + + /// Same store property as `kv_set_through_the_guard_…`: the guarded + /// `doc_put` must be readable by the unguarded client, not merely by the + /// sibling handler. + /// + /// The taint half of this re-point is not asserted here because no read + /// path in `MemoryClient` projects the stored taint column back out. + /// `GuardPolicy::stamp_taint`'s monotone-raise behaviour is pinned in + /// `memory::guard::policy_tests` instead. + #[tokio::test] + async fn doc_put_through_the_guard_is_visible_to_the_unguarded_client() { + let _serial = crate::openhuman::memory::ops::GLOBAL_MEMORY_TEST_LOCK + .lock() + .await; + let _env = ensure_memory_client(); + let namespace = unique_namespace("memory-docs-guard"); + let key = format!( + "guarded{}", + &uuid::Uuid::new_v4().as_simple().to_string()[..12] + ); + + let put = doc_put(sample_put( + namespace.clone(), + key.clone(), + "Guarded write", + "This document was written through the memory guard.", + )) + .await + .expect("guarded doc_put"); + assert!(!put.value.document_id.is_empty()); + + let client = active_memory_client().await.expect("client"); + let raw = client + .list_documents(Some(namespace.as_str())) + .await + .expect("unguarded list_documents"); + let docs = raw + .get("documents") + .and_then(|v| v.as_array()) + .expect("documents array"); + assert!( + docs.iter().any(|doc| doc["key"] == key), + "the unguarded client must see the guarded write" + ); + } + + /// Pins the null-binding refusal for the destructive embedded-only ops + /// (the `all.rs` registration thread). Under `driver = "null"` the operator + /// asked for memory to be disabled, yet `clear_namespace` / + /// `delete_document` would still reach the embedded store boot initialised + /// and delete persisted rows. They must refuse with a config-fact message, + /// exactly as the CLI's legacy-client gate does. + #[tokio::test] + async fn destructive_ops_refuse_when_bound_driver_is_null() { + let _serial = crate::openhuman::memory::ops::GLOBAL_MEMORY_TEST_LOCK + .lock() + .await; + let _env = ensure_memory_client(); + let workspace = tempfile::tempdir().unwrap(); + let null_cfg = crate::openhuman::config::schema::MemorySubsystemConfig { + driver: "null".into(), + ..Default::default() + }; + let ctx = crate::core::runtime::context::CoreContext::for_test( + crate::core::runtime::DomainSet::full(), + Some(workspace.path().to_path_buf()), + Some(null_cfg), + ); + crate::core::runtime::context::CoreContext::scope(ctx, async { + let err = clear_namespace(ClearNamespaceParams { + namespace: unique_namespace("null-driver"), + }) + .await + .expect_err("clear_namespace must refuse under a null binding"); + assert!( + err.contains("not the embedded TinyCortex driver"), + "refusal must explain the binding: {err}" + ); + + let err = doc_delete(DeleteDocParams { + namespace: unique_namespace("null-driver"), + document_id: "any".into(), + }) + .await + .expect_err("doc_delete must refuse under a null binding"); + assert!( + err.contains("not the embedded TinyCortex driver"), + "refusal must explain the binding: {err}" + ); + + let err = memory_delete_document(DeleteDocumentRequest { + namespace: unique_namespace("null-driver"), + document_id: "any".into(), + }) + .await + .expect_err("memory_delete_document must refuse under a null binding"); + assert!( + err.contains("not the embedded TinyCortex driver"), + "refusal must explain the binding: {err}" + ); + }) + .await; + } } diff --git a/src/openhuman/memory/ops/guard.rs b/src/openhuman/memory/ops/guard.rs new file mode 100644 index 0000000000..31adee0164 --- /dev/null +++ b/src/openhuman/memory/ops/guard.rs @@ -0,0 +1,77 @@ +//! [`active_memory_guard`] — how a memory RPC handler reaches the **guarded** +//! driver. +//! +//! This is the read-write twin of +//! [`helpers::active_memory_client`](super::helpers::active_memory_client): the +//! same store, reached through +//! [`MemoryGuard`](crate::openhuman::memory::guard::MemoryGuard) so the seven +//! policy steps in `docs/specs/kernel.md` §3.4 actually run. A handler that has +//! a typed contract twin for what it does calls this; one that does not stays +//! on `active_memory_client` and is listed, with its reason, in +//! `docs/specs/memory-guard-allowlist.md`. +//! +//! ## Two resolution paths, and why the second one exists +//! +//! The primary path is the ambient [`CoreContext`], exactly as +//! [`ops::provider`](super::provider) already resolves the binding for the +//! health probe. Under RPC dispatch a context is always present and its +//! workspace is always bound, so that is the only path production takes. +//! +//! The fallback path exists for callers that run *before* a context is built — +//! in practice the roughly four thousand pre-boot unit tests, which bind the +//! process-global client through `ensure_shared_memory_client()` and never +//! build a `CoreContext`. `CoreContext::memory()` cannot serve them: +//! `memory_binding()` goes through `workspace_dir()`, which errors outright +//! when the context has no bound workspace. `active_memory_client()` already +//! has an answer for that state — it lazily initialises — so this mirrors it, +//! with one refinement: it prefers the workspace the global client is +//! **already** bound to over whatever `Config::load_or_init` reports. Resolving +//! through the config instead would hand a test a binding over a *different* +//! workspace than the client its fixtures wrote to, which is a silently wrong +//! store rather than a visible failure. +//! +//! The fallback binds with [`MemorySubsystemConfig::default`] (driver +//! `"tinycortex"`, default hook budgets). That is the right default precisely +//! because it is only reachable with no context: a context always carries the +//! operator's real `[subsystems.memory]` block and takes the first path. + +use std::sync::Arc; + +use crate::core::runtime::context::CoreContext; +use crate::openhuman::config::schema::MemorySubsystemConfig; +use crate::openhuman::memory::binding; +use crate::openhuman::memory::global; +use crate::openhuman::memory::guard::MemoryGuard; + +/// The guarded memory driver for this dispatch. +/// +/// # Errors +/// +/// When neither resolution path can name a workspace — no ambient context and +/// no global client, with `Config::load_or_init` also failing — or when the +/// binding cache lock is poisoned. +pub(crate) async fn active_memory_guard() -> Result, String> { + if let Some(ctx) = CoreContext::current() { + match ctx.memory() { + Ok(guard) => return Ok(guard), + Err(error) => log::debug!( + "[memory:guard] ambient context has no bound workspace ({error}); \ + resolving as active_memory_client does" + ), + } + } + + let workspace_dir = match global::active_workspace_dir() { + Some(dir) => dir, + None => super::helpers::current_workspace_dir().await?, + }; + log::debug!( + "[memory:guard] no context binding; guarding workspace={}", + workspace_dir.display() + ); + Ok(binding::for_workspace(&workspace_dir, &MemorySubsystemConfig::default())?.guard()) +} + +#[cfg(test)] +#[path = "guard_tests.rs"] +mod tests; diff --git a/src/openhuman/memory/ops/guard_tests.rs b/src/openhuman/memory/ops/guard_tests.rs new file mode 100644 index 0000000000..c655b980f8 --- /dev/null +++ b/src/openhuman/memory/ops/guard_tests.rs @@ -0,0 +1,70 @@ +//! Tests for [`super::active_memory_guard`]. +//! +//! The bypass allowlist ratchet lives in +//! [`crate::openhuman::memory::bypass_allowlist_tests`] — see the note at the +//! foot of this file. + +use super::*; + +/// The pre-boot fallback resolves *the same* workspace the global client is +/// bound to, not whatever `Config::load_or_init` reports. That is the property +/// the four re-pointed handlers rest on: their existing tests bind a temp +/// workspace through `ensure_shared_memory_client()` and never build a +/// `CoreContext`, so a config-derived fallback would silently guard a +/// different store. +#[tokio::test] +async fn falls_back_to_the_globally_bound_workspace_when_there_is_no_context() { + let _serial = crate::openhuman::memory::ops::GLOBAL_MEMORY_TEST_LOCK + .lock() + .await; + let workspace = crate::openhuman::memory::ops::ensure_shared_memory_client(); + assert_eq!( + global::active_workspace_dir().as_deref(), + Some(workspace.as_path()), + "the fixture must leave the global client bound to its own workspace" + ); + + let guard = active_memory_guard().await.expect("guard resolves"); + let bound = binding::for_workspace(&workspace, &MemorySubsystemConfig::default()) + .expect("binding resolves"); + + // Same cached binding ⇒ same driver ⇒ same store. + assert!( + Arc::ptr_eq(&guard, &bound.guard()), + "the fallback must reuse the binding cached for that workspace" + ); +} + +/// The guard advertises the driver underneath it, not a synthetic id — so a +/// handler routed through it still reports the embedded driver in status and +/// spans. +#[tokio::test] +async fn guards_the_embedded_driver_and_keeps_its_identity() { + use tinycortex_api::provider::MemoryProvider; + + let _serial = crate::openhuman::memory::ops::GLOBAL_MEMORY_TEST_LOCK + .lock() + .await; + crate::openhuman::memory::ops::ensure_shared_memory_client(); + + let guard = active_memory_guard().await.expect("guard resolves"); + assert_eq!( + guard.driver_id(), + crate::openhuman::memory::driver::embedded::EMBEDDED_DRIVER_ID + ); + assert!(guard.as_documents().is_some()); + assert!(guard.as_graph().is_some()); + assert!(guard.as_tool_memory().is_some()); +} + +// ── Where the allowlist drift guard lives ──────────────────────────────────── +// +// M4b shipped a provisional file-keyed drift guard here. M4c replaced it with a +// `(file, pattern)`-keyed lint in `memory::bypass_allowlist_tests`, which covers +// a strict superset of the call shapes (adding direct driver/engine construction +// and raw `MemoryBinding` reach-through) and reports which needle tripped rather +// than only which file. +// +// It was deleted rather than kept alongside: two allowlists over the same tree +// must both be struck when a bypass is cleaned up, and the one nobody remembers +// is exactly the dead-string rot the ratchet exists to prevent. One list. diff --git a/src/openhuman/memory/ops/kv_graph.rs b/src/openhuman/memory/ops/kv_graph.rs index 09f9bd4c51..8cb9bad3bc 100644 --- a/src/openhuman/memory/ops/kv_graph.rs +++ b/src/openhuman/memory/ops/kv_graph.rs @@ -2,8 +2,11 @@ use serde::Deserialize; +use tinycortex_api::provider::MemoryProvider; + use crate::rpc::RpcOutcome; +use super::guard::active_memory_guard; use super::helpers::active_memory_client; /// Parameters for the `kv_set` RPC method. @@ -64,11 +67,24 @@ pub struct GraphQueryParams { // --------------------------------------------------------------------------- /// Sets a key-value pair in the memory store. +/// +/// Routed through [`MemoryGuard`](crate::openhuman::memory::guard::MemoryGuard) +/// rather than the bare client. `MemoryGraph::kv_put` on the embedded driver is +/// `client.kv_set(namespace, key, &value)` — the same call on the same store, +/// so the only differences are the policy steps the guard adds (tier check, +/// audit span) and an error string that gains a `"kv_put: "` prefix. +/// +/// Its three siblings in this file deliberately stay on the bare client; see +/// `docs/specs/memory-guard-allowlist.md`. pub async fn kv_set(params: KvSetParams) -> Result, String> { - let client = active_memory_client().await?; - client - .kv_set(params.namespace.as_deref(), ¶ms.key, ¶ms.value) - .await?; + let guard = active_memory_guard().await?; + let graph = guard + .as_graph() + .ok_or_else(|| "memory driver does not support the graph family".to_string())?; + graph + .kv_put(params.namespace.as_deref(), ¶ms.key, params.value) + .await + .map_err(|e| e.to_string())?; Ok(RpcOutcome::single_log(true, "memory kv set")) } @@ -241,4 +257,38 @@ mod tests { assert_eq!(queried.value[0]["predicate"], "OWNS"); assert_eq!(queried.value[0]["object"], "ATLAS"); } + + /// The guarded `kv_set` must land in the **same** store the unguarded + /// readers use. This is the failure a re-point can hide: routing through a + /// binding over a different workspace still returns `Ok`, it just writes + /// somewhere nobody reads. Asserted against the raw client rather than the + /// sibling handler so a shared bug in `active_memory_client` cannot make + /// both halves agree while both are wrong. + #[tokio::test] + async fn kv_set_through_the_guard_is_visible_to_the_unguarded_client() { + let _serial = crate::openhuman::memory::ops::GLOBAL_MEMORY_TEST_LOCK + .lock() + .await; + ensure_memory_client(); + let namespace = unique_namespace("kv-guard"); + let key = format!( + "guarded{}", + &uuid::Uuid::new_v4().as_simple().to_string()[..12] + ); + + kv_set(KvSetParams { + namespace: Some(namespace.clone()), + key: key.clone(), + value: serde_json::json!({"via": "guard"}), + }) + .await + .expect("guarded kv set"); + + let client = active_memory_client().await.expect("client"); + let raw = client + .kv_get(Some(namespace.as_str()), &key) + .await + .expect("unguarded kv get"); + assert_eq!(raw, Some(serde_json::json!({"via": "guard"}))); + } } diff --git a/src/openhuman/memory/ops/mod.rs b/src/openhuman/memory/ops/mod.rs index d177e1262e..c84b746bde 100644 --- a/src/openhuman/memory/ops/mod.rs +++ b/src/openhuman/memory/ops/mod.rs @@ -11,20 +11,26 @@ //! envelope-style handler. //! - [`helpers`] — formatting, default constants, path validators, and the //! active memory-client lookup. +//! - [`guard`] — the guarded-driver lookup handlers use instead of +//! `helpers::active_memory_client` when their operation has a typed contract +//! twin (`docs/specs/memory-guard-allowlist.md`). //! - [`documents`] — document/namespace direct API and the envelope-style //! façade (`memory_init`, `memory_list_documents`, `memory_query_namespace`, //! recall_*). //! - [`kv_graph`] — key-value and knowledge-graph handlers. //! - [`sync`] — `memory_sync_*` and `memory_ingestion_status`. //! - [`learn`] — `memory_learn_all`. +//! - [`provider`] — `memory_provider_status` (the bound memory driver). //! - [`files`] — `ai_*_memory_file` handlers (use `tokio::fs`). pub mod documents; pub mod envelope; pub mod files; +pub mod guard; pub mod helpers; pub mod kv_graph; pub mod learn; +pub mod provider; pub mod sync; pub mod tool_memory; @@ -45,6 +51,7 @@ pub use kv_graph::{ GraphUpsertParams, KvGetDeleteParams, KvSetParams, }; pub use learn::{memory_learn_all, LearnAllParams, LearnAllResult, NamespaceLearnResult}; +pub use provider::{memory_provider_status, memory_subsystem_status}; pub use sync::{ memory_ingestion_status, memory_sync_all, memory_sync_channel, IngestionStatusResult, SyncAllResult, SyncChannelParams, SyncChannelResult, diff --git a/src/openhuman/memory/ops/provider.rs b/src/openhuman/memory/ops/provider.rs new file mode 100644 index 0000000000..597d3261a2 --- /dev/null +++ b/src/openhuman/memory/ops/provider.rs @@ -0,0 +1,231 @@ +//! `memory.provider_status` — what is bound in the memory subsystem slot +//! (`docs/specs/plan-memory.md` §5, `docs/specs/kernel.md` §6 item 6). +//! +//! This is the **memory adapter for status**: it resolves the context's +//! [`MemoryBinding`], converts it into the kernel's generic +//! [`BoundDriver`](crate::core::subsystem::BoundDriver) vocabulary, probes the +//! driver's live health, and projects the result onto the wire shape +//! [`SubsystemStatus`]. `subsystems.status` renders the same value for the +//! `memory` slot — deliberately, so the frontend can read driver capabilities +//! from the memory namespace without knowing the kernel namespace exists. +//! +//! Nothing here is a mutation and nothing here binds: a status call must never +//! be the thing that constructs a driver. It does resolve the binding, which +//! *is* lazily constructing on first use — that is +//! [`CoreContext::memory_binding`]'s existing cached behaviour and identical +//! to what any other memory RPC would trigger. + +use crate::core::runtime::context::CoreContext; +use crate::core::subsystem::{DriverHealth, SubsystemStatus}; +use crate::openhuman::memory::binding::{to_driver_health, MemoryBinding}; +use crate::rpc::RpcOutcome; + +/// The status of the memory slot for the current dispatch context. +/// +/// Infallible by design: an unresolvable binding (no workspace bound, e.g. a +/// pre-login core) is *reported*, not raised. A status surface that errors +/// exactly when something is wrong is the opposite of useful. +/// +/// A standalone CLI subcommand (`openhuman subsystems status`, +/// `openhuman memory status`) never builds a [`CoreContext`] — the generic +/// namespace dispatcher runs without one — so this resolves the configured +/// workspace's binding directly via [`standalone_status`] instead of reporting +/// an unresolved row. In an RPC host a context is always ambient, so that path +/// is unaffected. +pub async fn memory_subsystem_status() -> SubsystemStatus { + match CoreContext::current().map(|ctx| ctx.memory_binding()) { + Some(Ok(binding)) => status_from_binding(&binding).await, + Some(Err(err)) => unresolved_status(err), + None => standalone_status().await, + } +} + +/// Resolve status from the on-disk config when no [`CoreContext`] is ambient +/// (a bare CLI invocation). Reads the configured workspace's binding the same +/// way `cli_capability::bound_memory_driver_for` does, so `openhuman subsystems +/// status` shows the same resolved row as the table and as an RPC with a live +/// context. +/// +/// Never errors: on a config load or bind failure this reports an unresolved +/// row (with a reason) rather than refusing to render — mirroring the +/// capability gate's default-OPEN posture, where a status command that refuses +/// to run because it cannot read config is worse than one that shows the +/// unresolved row. +async fn standalone_status() -> SubsystemStatus { + let config = match crate::openhuman::config::Config::load_or_init().await { + Ok(config) => config, + Err(err) => { + log::debug!( + "[memory:provider] standalone status: config unresolved ({err}); reporting unresolved" + ); + return unresolved_status(format!("no core context; config load failed: {err}")); + } + }; + + match crate::openhuman::memory::binding::for_workspace( + &config.workspace_dir, + &config.subsystems.memory, + ) { + Ok(binding) => status_from_binding(&binding).await, + Err(err) => { + log::debug!( + "[memory:provider] standalone status: binding unresolved ({err}); reporting unresolved" + ); + unresolved_status(format!("no core context; binding failed: {err}")) + } + } +} + +/// Project one resolved binding. Separate from [`memory_subsystem_status`] so +/// the bound case is testable without standing up a [`CoreContext`]. +pub async fn status_from_binding(binding: &MemoryBinding) -> SubsystemStatus { + let bound = binding.to_bound_driver(); + let health = to_driver_health(binding.unguarded_provider().health().await); + let last_error = binding + .fallback() + .map(|fallback| format!("{}: {}", fallback.configured_driver, fallback.reason)); + + log::debug!( + "[memory:provider] status driver='{}' class={} health={} capabilities=[{}] fallback={}", + bound.id, + bound.class, + health.as_str(), + bound.capabilities.iter().collect::>().join(","), + bound.is_fallback() + ); + + SubsystemStatus::from_bound_with_health(&bound, health).with_last_error(last_error) +} + +/// The status reported when no driver could be resolved at all. Distinct from +/// a *fallback*: nothing was bound, so there is no driver id to name and no +/// capability set to advertise. +fn unresolved_status(reason: String) -> SubsystemStatus { + log::debug!("[memory:provider] status unresolved: {reason}"); + SubsystemStatus { + slot: crate::core::subsystem::SubsystemSlot::Memory + .as_str() + .to_string(), + driver: String::new(), + class: crate::core::subsystem::DriverClass::Null + .as_str() + .to_string(), + health: DriverHealth::down(reason.clone()).as_str().to_string(), + health_reason: Some(reason.clone()), + contract_version: crate::core::subsystem::format_contract_version( + tinycortex_api::CONTRACT_VERSION, + ), + capabilities: Vec::new(), + fell_back_from: None, + last_error: Some(reason), + } +} + +/// RPC handler body for `memory.provider_status`. +pub async fn memory_provider_status() -> RpcOutcome { + RpcOutcome::new(memory_subsystem_status().await, vec![]) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn status_without_a_context_reports_an_unresolved_slot() { + let status = unresolved_status("no workspace".to_string()); + assert_eq!(status.slot, "memory"); + assert_eq!(status.class, "null"); + assert_eq!(status.health, "down"); + assert!(status.capabilities.is_empty()); + assert_eq!(status.last_error.as_deref(), Some("no workspace")); + // Still reports the contract version this build speaks — that is a + // build fact, independent of whether anything bound. + assert_eq!( + status.contract_version, + crate::core::subsystem::format_contract_version(tinycortex_api::CONTRACT_VERSION) + ); + } + + #[tokio::test] + async fn bound_driver_status_reports_id_class_contract_and_capabilities() { + let workspace = tempfile::tempdir().expect("tempdir"); + let cfg = crate::openhuman::config::schema::MemorySubsystemConfig::default(); + let binding = crate::openhuman::memory::binding::for_workspace(workspace.path(), &cfg) + .expect("binding resolves"); + + let status = status_from_binding(&binding).await; + assert_eq!(status.slot, "memory"); + // The default `[subsystems.memory] driver` is the embedded tinycortex + // driver. + assert_eq!(status.driver, cfg.driver); + assert_eq!(status.class, "embedded"); + assert_eq!(status.health, "ready"); + assert_eq!( + status.contract_version, + crate::core::subsystem::format_contract_version(tinycortex_api::CONTRACT_VERSION) + ); + // All thirteen families, as of M3d. Spelled out rather than derived + // from `Capabilities::all()` on purpose: this is the wire surface the + // frontend reads, so the strings themselves are the assertion. + assert_eq!( + status.capabilities, + vec![ + "core", + "diff", + "documents", + "entities", + "goals", + "graph", + "ingest", + "maintenance", + "portability", + "recall", + "sources", + "tool_memory", + "tree" + ] + ); + assert_eq!(status.fell_back_from, None); + assert_eq!(status.last_error, None); + } + + #[tokio::test] + async fn a_refused_driver_reports_the_fallback_and_its_reason() { + let workspace = tempfile::tempdir().expect("tempdir"); + let mut cfg = crate::openhuman::config::schema::MemorySubsystemConfig { + driver: "supermemory".into(), + ..Default::default() + }; + cfg.drivers.insert( + "supermemory".into(), + crate::openhuman::config::schema::MemoryDriverConfig { + class: Some("external".into()), + transport: Some("http".into()), + endpoint: Some("https://api.supermemory.ai".into()), + credential_ref: Some("keychain:supermemory".into()), + trust_state: "untrusted".into(), + }, + ); + let binding = crate::openhuman::memory::binding::for_workspace(workspace.path(), &cfg) + .expect("binding falls back rather than failing"); + + let status = status_from_binding(&binding).await; + assert_eq!(status.driver, "null"); + assert_eq!(status.class, "null"); + assert_eq!(status.fell_back_from.as_deref(), Some("supermemory")); + let last_error = status.last_error.expect("a refused bind records why"); + assert!(last_error.contains("supermemory"), "{last_error}"); + assert!(last_error.contains("untrusted"), "{last_error}"); + // The refusal reason must never leak the credential reference or the + // endpoint — same rule the binding's own tests pin. + assert!(!last_error.contains("keychain:supermemory"), "{last_error}"); + assert!(!last_error.contains("api.supermemory.ai"), "{last_error}"); + } + + #[tokio::test] + async fn provider_status_wraps_the_snapshot_with_no_logs() { + let outcome = memory_provider_status().await; + assert!(outcome.logs.is_empty()); + assert_eq!(outcome.value.slot, "memory"); + } +} diff --git a/src/openhuman/memory/ops/tool_memory.rs b/src/openhuman/memory/ops/tool_memory.rs index 6691bf3433..a87845872e 100644 --- a/src/openhuman/memory/ops/tool_memory.rs +++ b/src/openhuman/memory/ops/tool_memory.rs @@ -1,14 +1,24 @@ //! RPC handlers for the tool-scoped memory layer (see //! [`crate::openhuman::memory::tool_memory`]). //! -//! All handlers go through [`active_memory_client`] so they hit the -//! same `UnifiedMemory` backend the rest of the memory RPCs use, and -//! the namespace they touch is exactly `tool-{tool_name}` — never -//! `global` or `tool_effectiveness`. +//! All handlers hit the same `UnifiedMemory` backend the rest of the memory +//! RPCs use, and the namespace they touch is exactly `tool-{tool_name}` — +//! never `global` or `tool_effectiveness`. +//! +//! Two of them — [`tool_rule_list`] and [`tool_rule_delete`] — reach it through +//! [`MemoryGuard`](crate::openhuman::memory::guard::MemoryGuard) because their +//! contract twins are literal delegations to the same store. The other four +//! stay on [`open_store`]: `tool_rule_put` returns the *stored* rule (with +//! `created_at` preserved and `updated_at` refreshed) while the contract's +//! `put_tool_rule` returns unit, and `get_rule` / `list_rules_json` / +//! `rules_for_prompt` have no contract equivalent at all. See +//! `docs/specs/memory-guard-allowlist.md`. use serde::Deserialize; use serde_json::Value; +use tinycortex_api::provider::MemoryProvider; + use crate::openhuman::memory::tool_memory::{ tool_memory_store, ToolMemoryPriority, ToolMemoryRule, ToolMemorySource, ToolMemoryStore, }; @@ -101,25 +111,53 @@ pub async fn tool_rule_get( Ok(RpcOutcome::single_log(rule, "tool memory rule fetched")) } +/// The reason a guarded handler in this file cannot proceed. +/// +/// A driver that does not advertise `Capability::ToolMemory` returns `None` +/// from `as_tool_memory()`; the embedded driver always advertises it, so this +/// is reachable only under a null / fallback binding. +const NO_TOOL_MEMORY: &str = "memory driver does not support the tool_memory family"; + /// List every tool-scoped rule for a tool. +/// +/// Routed through [`MemoryGuard`](crate::openhuman::memory::guard::MemoryGuard). +/// `MemoryToolMemory::tool_rules` on the embedded driver is +/// `tool_memory_store(self.memory()).list_rules(tool_name)` — the same store +/// over the same `Arc` [`open_store`] builds. The wire type matches +/// by identity, not conversion: `memory::tool_memory::ToolMemoryRule` **is** +/// `tinycortex_api::tool_memory::ToolMemoryRule`. pub async fn tool_rule_list( params: ToolRuleListParams, ) -> Result>, String> { log::debug!("[tool-memory] rpc tool_rule_list tool={}", params.tool_name); - let store = open_store().await?; - let rules = store.list_rules(¶ms.tool_name).await?; + let guard = super::guard::active_memory_guard().await?; + let rules = guard + .as_tool_memory() + .ok_or_else(|| NO_TOOL_MEMORY.to_string())? + .tool_rules(¶ms.tool_name) + .await + .map_err(|e| e.to_string())?; Ok(RpcOutcome::single_log(rules, "tool memory rules listed")) } /// Delete a tool-scoped rule by id. +/// +/// Routed through [`MemoryGuard`](crate::openhuman::memory::guard::MemoryGuard); +/// `MemoryToolMemory::delete_tool_rule` delegates to the same +/// `ToolMemoryStore::delete_rule` [`open_store`] would reach. pub async fn tool_rule_delete(params: ToolRuleRefParams) -> Result, String> { log::debug!( "[tool-memory] rpc tool_rule_delete tool={} id={}", params.tool_name, params.id ); - let store = open_store().await?; - let deleted = store.delete_rule(¶ms.tool_name, ¶ms.id).await?; + let guard = super::guard::active_memory_guard().await?; + let deleted = guard + .as_tool_memory() + .ok_or_else(|| NO_TOOL_MEMORY.to_string())? + .delete_tool_rule(¶ms.tool_name, ¶ms.id) + .await + .map_err(|e| e.to_string())?; Ok(RpcOutcome::single_log(deleted, "tool memory rule deleted")) } @@ -327,4 +365,59 @@ mod tests { }) .await; } + + /// The two guarded handlers and the four unguarded ones share one store. + /// `tool_rule_put` writes through `open_store()` (the bare client); + /// `tool_rule_list` and `tool_rule_delete` read and write through the + /// guard; `open_store()` is then asked directly whether the delete + /// actually happened. + #[tokio::test] + async fn guarded_list_and_delete_share_the_store_with_the_unguarded_put() { + let _serial = crate::openhuman::memory::ops::GLOBAL_MEMORY_TEST_LOCK + .lock() + .await; + ensure_memory_client(); + let tool_name = unique_tool_name(); + + let stored = tool_rule_put(ToolRulePutParams { + tool_name: tool_name.clone(), + rule: "Prefer the guarded path".into(), + priority: None, + source: None, + tags: vec![], + id: None, + }) + .await + .expect("unguarded put") + .value; + + let listed = tool_rule_list(ToolRuleListParams { + tool_name: tool_name.clone(), + }) + .await + .expect("guarded list") + .value; + assert_eq!(listed.len(), 1, "the guard must see the unguarded write"); + assert_eq!(listed[0].id, stored.id); + + let deleted = tool_rule_delete(ToolRuleRefParams { + tool_name: tool_name.clone(), + id: stored.id.clone(), + }) + .await + .expect("guarded delete") + .value; + assert!(deleted); + + let remaining = open_store() + .await + .expect("unguarded store") + .list_rules(&tool_name) + .await + .expect("unguarded list"); + assert!( + remaining.is_empty(), + "the unguarded store must observe the guarded delete" + ); + } } diff --git a/src/openhuman/memory/query/cover_window.rs b/src/openhuman/memory/query/cover_window.rs index 4b0dbbb600..688b7de41a 100644 --- a/src/openhuman/memory/query/cover_window.rs +++ b/src/openhuman/memory/query/cover_window.rs @@ -71,9 +71,9 @@ impl Tool for MemoryTreeCoverWindowTool { req.source_kind.is_some(), req.limit.is_some() ); - let cfg = config_rpc::load_config_with_timeout() - .await - .map_err(|e| anyhow::anyhow!("memory_tree_cover_window: load config failed: {e}"))?; + // Validate arguments before touching config/disk — `SourceKind::parse` + // is pure, so a bad `source_kind` must fail with the parse error + // regardless of workspace state. let source_kind = match req.source_kind.as_deref() { Some(s) => { log::trace!("[tool][memory_tree] cover_window parse_source_kind"); @@ -84,6 +84,9 @@ impl Tool for MemoryTreeCoverWindowTool { } None => None, }; + let cfg = config_rpc::load_config_with_timeout() + .await + .map_err(|e| anyhow::anyhow!("memory_tree_cover_window: load config failed: {e}"))?; log::trace!( "[tool][memory_tree] cover_window dispatch limit={}", req.limit.unwrap_or(0) @@ -139,6 +142,10 @@ mod tests { .execute(json!({ "since_ms": 0, "until_ms": 1, "source_kind": "not-real" })) .await .expect_err("invalid source kind should fail"); - assert!(err.to_string().contains("memory_tree_cover_window:")); + let msg = err.to_string(); + assert!( + msg.contains("memory_tree_cover_window:") && !msg.contains("load config failed"), + "expected a source-kind parse error, got: {msg}" + ); } } diff --git a/src/openhuman/memory/query/mod.rs b/src/openhuman/memory/query/mod.rs index 1efef4af06..bef076f441 100644 --- a/src/openhuman/memory/query/mod.rs +++ b/src/openhuman/memory/query/mod.rs @@ -14,6 +14,8 @@ mod fetch_leaves; mod ingest_document; mod query_source; mod search_entities; +#[cfg(test)] +mod test_workspace; // Re-export individual tool types for callers that need them directly // (e.g. tool registration in ops.rs). @@ -170,8 +172,10 @@ impl Tool for MemoryTreeTool { #[cfg(test)] mod memory_tree_dispatcher_tests { use super::*; + use crate::openhuman::memory::query::test_workspace::isolated_config; use crate::openhuman::tools::traits::Tool; use serde_json::json; + use tempfile::TempDir; #[test] fn memory_tree_tool_name_is_correct() { @@ -244,6 +248,12 @@ mod memory_tree_dispatcher_tests { #[tokio::test] async fn memory_tree_fetch_leaves_mode_dispatches_successfully() { + // `fetch_leaves` loads config from `OPENHUMAN_WORKSPACE`. Without an + // isolated workspace this races sibling tests whose `TempDir` is + // deleted mid-call ("Failed to create temporary config file ... No + // such file or directory"). + let tmp = TempDir::new().expect("tempdir"); + let (_workspace, _cfg) = isolated_config(&tmp).await; let result = MemoryTreeTool .execute(json!({ "mode": "fetch_leaves", diff --git a/src/openhuman/memory/query/query_source.rs b/src/openhuman/memory/query/query_source.rs index 80e4e87f8e..c272a1aea9 100644 --- a/src/openhuman/memory/query/query_source.rs +++ b/src/openhuman/memory/query/query_source.rs @@ -57,9 +57,9 @@ impl Tool for MemoryTreeQuerySourceTool { log::debug!("[tool][memory_tree] query_source invoked"); let req: QuerySourceRequest = serde_json::from_value(args) .map_err(|e| anyhow::anyhow!("invalid arguments for memory_tree_query_source: {e}"))?; - let cfg = config_rpc::load_config_with_timeout() - .await - .map_err(|e| anyhow::anyhow!("memory_tree_query_source: load config failed: {e}"))?; + // Validate arguments before touching config/disk — `SourceKind::parse` + // is pure, so a bad `source_kind` must fail with the parse error + // regardless of workspace state. let source_kind = match req.source_kind.as_deref() { Some(s) => Some( SourceKind::parse(s) @@ -67,6 +67,9 @@ impl Tool for MemoryTreeQuerySourceTool { ), None => None, }; + let cfg = config_rpc::load_config_with_timeout() + .await + .map_err(|e| anyhow::anyhow!("memory_tree_query_source: load config failed: {e}"))?; let resp = match req.source_id.as_deref() { Some(source_id) => { backend::query_source_scope( @@ -164,7 +167,11 @@ mod tests { })) .await .expect_err("invalid source kind should fail"); - assert!(err.to_string().contains("memory_tree_query_source:")); + let msg = err.to_string(); + assert!( + msg.contains("memory_tree_query_source:") && !msg.contains("load config failed"), + "expected a source-kind parse error, got: {msg}" + ); } #[tokio::test] diff --git a/src/openhuman/memory/query/search_entities.rs b/src/openhuman/memory/query/search_entities.rs index eed23fee83..6cea217e87 100644 --- a/src/openhuman/memory/query/search_entities.rs +++ b/src/openhuman/memory/query/search_entities.rs @@ -57,9 +57,9 @@ impl Tool for MemoryTreeSearchEntitiesTool { let req: SearchEntitiesRequest = serde_json::from_value(args).map_err(|e| { anyhow::anyhow!("invalid arguments for memory_tree_search_entities: {e}") })?; - let cfg = config_rpc::load_config_with_timeout() - .await - .map_err(|e| anyhow::anyhow!("memory_tree_search_entities: load config failed: {e}"))?; + // Validate arguments before touching config/disk — `EntityKind::parse` + // is pure, and a bad `kinds` value must fail with the kind error + // regardless of workspace state. let kinds = match req.kinds { None => None, Some(list) => { @@ -70,6 +70,9 @@ impl Tool for MemoryTreeSearchEntitiesTool { })?) } }; + let cfg = config_rpc::load_config_with_timeout() + .await + .map_err(|e| anyhow::anyhow!("memory_tree_search_entities: load config failed: {e}"))?; let limit = req.limit.unwrap_or(5).min(100); let matches = retrieval::search_entities(&cfg, &req.query, kinds, limit).await?; log::debug!( diff --git a/src/openhuman/memory/query/test_workspace.rs b/src/openhuman/memory/query/test_workspace.rs new file mode 100644 index 0000000000..d08901b191 --- /dev/null +++ b/src/openhuman/memory/query/test_workspace.rs @@ -0,0 +1,47 @@ +//! Shared test-only workspace isolation for `memory::query` tests. +//! +//! Any test in this module tree that reaches +//! `config_rpc::load_config_with_timeout()` MUST hold one of these guards. +//! Without it the test reads whatever `OPENHUMAN_WORKSPACE` a concurrently +//! running sibling has set, and fails when that sibling's `TempDir` is +//! dropped out from under it ("Failed to create temporary config file ... +//! No such file or directory"). + +use std::ffi::OsString; + +use tempfile::TempDir; + +use crate::openhuman::config::{Config, TEST_ENV_LOCK}; + +pub(crate) struct WorkspaceEnvGuard { + _lock: std::sync::MutexGuard<'static, ()>, + previous: Option, +} + +impl WorkspaceEnvGuard { + pub(crate) fn set(path: &std::path::Path) -> Self { + let lock = TEST_ENV_LOCK.lock().unwrap_or_else(|err| err.into_inner()); + let previous = std::env::var_os("OPENHUMAN_WORKSPACE"); + std::env::set_var("OPENHUMAN_WORKSPACE", path); + Self { + _lock: lock, + previous, + } + } +} + +impl Drop for WorkspaceEnvGuard { + fn drop(&mut self) { + if let Some(previous) = self.previous.as_ref() { + std::env::set_var("OPENHUMAN_WORKSPACE", previous); + } else { + std::env::remove_var("OPENHUMAN_WORKSPACE"); + } + } +} + +pub(crate) async fn isolated_config(tmp: &TempDir) -> (WorkspaceEnvGuard, Config) { + let guard = WorkspaceEnvGuard::set(tmp.path()); + let config = Config::load_or_init().await.expect("load config"); + (guard, config) +} diff --git a/src/openhuman/memory/queue/scheduler.rs b/src/openhuman/memory/queue/scheduler.rs index b8e0dddddc..dbaf1223de 100644 --- a/src/openhuman/memory/queue/scheduler.rs +++ b/src/openhuman/memory/queue/scheduler.rs @@ -55,7 +55,22 @@ fn retry_transient_failures(config: &Config) { } } -fn enqueue_flush_stale(config: &Config) { +/// Enqueue one stale-buffer flush job for the current 3-hour UTC window and +/// wake the workers, reporting whether a job was actually created. +/// +/// `Ok(false)` means the window already had one — the enqueue is deduped on +/// `(date, hour_block)`, so this is the normal repeat-call outcome and not a +/// failure. +/// +/// `pub(crate)` (it was private) so the embedded memory driver's +/// `MemoryMaintenance::consolidate` can drive the **same** path the periodic +/// scheduler drives. That matters: the flush job fans out into per-tree `Seal` +/// jobs whose label strategy is derived per tree by the queue worker +/// (`TreeFactory::from_tree(&tree).label_strategy(...)`). The alternative entry +/// point, `tree::tree::flush::flush_stale_buffers_default`, takes **one** +/// `LabelStrategy` for every tree, which no production caller uses and which +/// would apply one tree kind's labelling to all of them. +pub(crate) fn enqueue_flush_stale_job(config: &Config) -> Result { let memory = crate::openhuman::memory::tinycortex::memory_config_from( config, config.workspace_dir.clone(), @@ -63,11 +78,16 @@ fn enqueue_flush_stale(config: &Config) { match tinycortex::memory::queue::scheduler::enqueue_flush_stale(&memory) { Ok(Some(_)) => { super::worker::wake_workers(); + Ok(true) } - Ok(None) => {} - Err(err) => { - log::warn!("[memory::jobs] periodic flush_stale enqueue failed: {err:#}"); - } + Ok(None) => Ok(false), + Err(err) => Err(format!("{err:#}")), + } +} + +fn enqueue_flush_stale(config: &Config) { + if let Err(err) = enqueue_flush_stale_job(config) { + log::warn!("[memory::jobs] periodic flush_stale enqueue failed: {err}"); } } diff --git a/src/openhuman/memory/queue/store.rs b/src/openhuman/memory/queue/store.rs index 6b296c5dc2..1a0860dcc4 100644 --- a/src/openhuman/memory/queue/store.rs +++ b/src/openhuman/memory/queue/store.rs @@ -7,15 +7,12 @@ use crate::openhuman::config::Config; use crate::openhuman::memory::tree::health::PipelineFailure; use super::types::{Job, JobFailure, JobStatus, NewJob}; +use crate::openhuman::memory::tinycortex::engine_config; pub use tinycortex::memory::queue::DEFAULT_LOCK_DURATION_MS; -fn memory_config(config: &Config) -> tinycortex::memory::MemoryConfig { - crate::openhuman::memory::tinycortex::memory_config_from(config, config.workspace_dir.clone()) -} - pub fn enqueue(config: &Config, job: &NewJob) -> Result> { - tinycortex::memory::queue::enqueue(&memory_config(config), job) + tinycortex::memory::queue::enqueue(&engine_config(config), job) } pub fn enqueue_tx(tx: &Transaction<'_>, job: &NewJob) -> Result> { @@ -23,15 +20,15 @@ pub fn enqueue_tx(tx: &Transaction<'_>, job: &NewJob) -> Result> } pub fn claim_next(config: &Config, lock_duration_ms: i64) -> Result> { - tinycortex::memory::queue::claim_next(&memory_config(config), lock_duration_ms) + tinycortex::memory::queue::claim_next(&engine_config(config), lock_duration_ms) } pub fn mark_done(config: &Config, job: &Job) -> Result<()> { - tinycortex::memory::queue::mark_done(&memory_config(config), job) + tinycortex::memory::queue::mark_done(&engine_config(config), job) } pub fn mark_failed(config: &Config, job: &Job, error: &str) -> Result<()> { - tinycortex::memory::queue::mark_failed(&memory_config(config), job, error) + tinycortex::memory::queue::mark_failed(&engine_config(config), job, error) } pub fn mark_failed_typed( @@ -45,7 +42,7 @@ pub fn mark_failed_typed( class: failure.class.as_str(), }); tinycortex::memory::queue::mark_failed_typed( - &memory_config(config), + &engine_config(config), job, error, failure.as_ref(), @@ -53,41 +50,41 @@ pub fn mark_failed_typed( } pub fn mark_deferred(config: &Config, job: &Job, until_ms: i64, reason: &str) -> Result<()> { - tinycortex::memory::queue::mark_deferred(&memory_config(config), job, until_ms, reason) + tinycortex::memory::queue::mark_deferred(&engine_config(config), job, until_ms, reason) } pub fn recover_stale_locks(config: &Config) -> Result { - tinycortex::memory::queue::recover_stale_locks(&memory_config(config)) + tinycortex::memory::queue::recover_stale_locks(&engine_config(config)) } pub fn requeue_failed(config: &Config) -> Result { - tinycortex::memory::queue::requeue_failed(&memory_config(config)) + tinycortex::memory::queue::requeue_failed(&engine_config(config)) } pub fn requeue_transient_failed(config: &Config) -> Result { - tinycortex::memory::queue::requeue_transient_failed(&memory_config(config)) + tinycortex::memory::queue::requeue_transient_failed(&engine_config(config)) } pub fn release_running_locks(config: &Config) -> Result { - tinycortex::memory::queue::release_running_locks(&memory_config(config)) + tinycortex::memory::queue::release_running_locks(&engine_config(config)) } pub fn count_by_status(config: &Config, status: JobStatus) -> Result { - tinycortex::memory::queue::count_by_status(&memory_config(config), status) + tinycortex::memory::queue::count_by_status(&engine_config(config), status) } pub fn count_failed_unrecoverable(config: &Config) -> Result { - tinycortex::memory::queue::count_failed_unrecoverable(&memory_config(config)) + tinycortex::memory::queue::count_failed_unrecoverable(&engine_config(config)) } pub fn count_total(config: &Config) -> Result { - tinycortex::memory::queue::count_total(&memory_config(config)) + tinycortex::memory::queue::count_total(&engine_config(config)) } pub fn retry_all_failed(config: &Config) -> Result { - tinycortex::memory::queue::retry_all_failed(&memory_config(config)) + tinycortex::memory::queue::retry_all_failed(&engine_config(config)) } pub fn get_job(config: &Config, id: &str) -> Result> { - tinycortex::memory::queue::get_job(&memory_config(config), id) + tinycortex::memory::queue::get_job(&engine_config(config), id) } diff --git a/src/openhuman/memory/schema/registry.rs b/src/openhuman/memory/schema/registry.rs index fc918362b9..e96201ce8e 100644 --- a/src/openhuman/memory/schema/registry.rs +++ b/src/openhuman/memory/schema/registry.rs @@ -1,5 +1,17 @@ //! Registry: lists of all `memory_tree` controller schemas and registered //! controller pairs wired into `core::all`. +//! +//! **Deliberately NOT split per family (M5.1).** `memory/schemas/` was split +//! into seven per-capability-family pairs because a single aggregator there +//! fanned seven unrelated families (documents / files / kv_graph / sync / learn +//! / provider / tool_memory) into one `Vec` behind one push site. This registry +//! is the opposite shape: it is one family — the `memory_tree` chunk store — +//! under its own namespace, already registered from its own push site in +//! `src/core/all.rs`, and the tree domain's other halves (`retrieval`, +//! `tree_runtime`'s summarizer) are already separate registries with separate +//! push sites. A per-family filter can therefore already be applied here +//! without any split; carving these functions up further would invent +//! boundaries the domain does not have and add drift surface for no gain. use crate::core::all::RegisteredController; use crate::core::ControllerSchema; diff --git a/src/openhuman/memory/schemas/documents.rs b/src/openhuman/memory/schemas/documents.rs index ce121ce260..415b249ff1 100644 --- a/src/openhuman/memory/schemas/documents.rs +++ b/src/openhuman/memory/schemas/documents.rs @@ -16,7 +16,33 @@ use crate::openhuman::memory::{ use super::{parse_params, to_json}; -pub(super) const FUNCTIONS: &[&str] = &[ +// --------------------------------------------------------------------------- +// Capability partitions +// --------------------------------------------------------------------------- +// +// This file is ONE RPC family by directory layout but THREE capability families +// by contract (`tinycortex_api::capabilities::Capability`), so M5.2 partitions +// it rather than tagging the whole file with a single capability: +// +// * core/recall — `Capability::Core` + `Capability::Recall`, both MANDATORY. +// Every bindable driver advertises them (`Capabilities::validate`), so +// against a driver's advertised set this gate never fires. It is registered +// tagged `Capability::Core` regardless, because one host decision answers +// below the driver: `CoreContext::memory_capabilities` returns the empty +// set for a deliberate `[subsystems.memory] driver = "null"`, and that is +// how these methods disappear when an operator turns memory off. Tagging +// the whole file `Documents` would instead have made +// `memory.recall_memories` vanish under a driver that merely lacks the +// document tier — gating a mandatory family on an optional one. +// * documents — `Capability::Documents`, the namespace-document tier. +// * ingest — `Capability::Ingest`, where the DRIVER owns chunking/embedding. +// `doc_ingest` is the whole of that surface; it lives in this file only +// because it shares the `memory` namespace. +// +// `schema()` and every handler below are shared and unpartitioned. + +/// Mandatory core + recall surface. Never capability-gated — see above. +pub(super) const FUNCTIONS_CORE_RECALL: &[&str] = &[ "init", "list_documents", "list_namespaces", @@ -25,16 +51,18 @@ pub(super) const FUNCTIONS: &[&str] = &[ "recall_context", "recall_memories", "namespace_list", - "doc_put", - "doc_ingest", - "doc_list", - "doc_delete", "context_query", "context_recall", "clear_namespace", ]; -pub(super) fn controllers() -> Vec { +/// The namespace-document tier — `Capability::Documents`. +pub(super) const FUNCTIONS_DOCUMENTS: &[&str] = &["doc_put", "doc_list", "doc_delete"]; + +/// Driver-owned ingestion — `Capability::Ingest`. +pub(super) const FUNCTIONS_INGEST: &[&str] = &["doc_ingest"]; + +pub(super) fn controllers_core_recall() -> Vec { vec![ RegisteredController { schema: schema("init").unwrap(), @@ -68,22 +96,6 @@ pub(super) fn controllers() -> Vec { schema: schema("namespace_list").unwrap(), handler: handle_namespace_list, }, - RegisteredController { - schema: schema("doc_put").unwrap(), - handler: handle_doc_put, - }, - RegisteredController { - schema: schema("doc_ingest").unwrap(), - handler: handle_doc_ingest, - }, - RegisteredController { - schema: schema("doc_list").unwrap(), - handler: handle_doc_list, - }, - RegisteredController { - schema: schema("doc_delete").unwrap(), - handler: handle_doc_delete, - }, RegisteredController { schema: schema("context_query").unwrap(), handler: handle_context_query, @@ -99,6 +111,30 @@ pub(super) fn controllers() -> Vec { ] } +pub(super) fn controllers_documents() -> Vec { + vec![ + RegisteredController { + schema: schema("doc_put").unwrap(), + handler: handle_doc_put, + }, + RegisteredController { + schema: schema("doc_list").unwrap(), + handler: handle_doc_list, + }, + RegisteredController { + schema: schema("doc_delete").unwrap(), + handler: handle_doc_delete, + }, + ] +} + +pub(super) fn controllers_ingest() -> Vec { + vec![RegisteredController { + schema: schema("doc_ingest").unwrap(), + handler: handle_doc_ingest, + }] +} + pub(super) fn schema(function: &str) -> Option { Some(match function { "init" => ControllerSchema { @@ -542,10 +578,32 @@ mod tests { #[test] fn documents_schema_exposes_all_functions() { - assert_eq!(controllers().len(), FUNCTIONS.len()); - assert!(FUNCTIONS.contains(&"init")); - assert!(FUNCTIONS.contains(&"doc_ingest")); - assert!(FUNCTIONS.contains(&"clear_namespace")); + assert_eq!(controllers_core_recall().len(), FUNCTIONS_CORE_RECALL.len()); + assert_eq!(controllers_documents().len(), FUNCTIONS_DOCUMENTS.len()); + assert_eq!(controllers_ingest().len(), FUNCTIONS_INGEST.len()); + assert!(FUNCTIONS_CORE_RECALL.contains(&"init")); + assert!(FUNCTIONS_CORE_RECALL.contains(&"clear_namespace")); + assert!(FUNCTIONS_DOCUMENTS.contains(&"doc_put")); + assert!(FUNCTIONS_INGEST.contains(&"doc_ingest")); + } + + /// The three partitions must be disjoint and must cover the file — a + /// function that fell out of all three would silently lose its + /// registration, since `core::all` now pushes the parts, not the whole. + #[test] + fn capability_partitions_are_disjoint_and_total() { + let mut all: Vec<&str> = Vec::new(); + all.extend(FUNCTIONS_CORE_RECALL); + all.extend(FUNCTIONS_DOCUMENTS); + all.extend(FUNCTIONS_INGEST); + let mut sorted = all.clone(); + sorted.sort_unstable(); + sorted.dedup(); + assert_eq!(sorted.len(), all.len(), "a function appears in two parts"); + assert_eq!(all.len(), 15, "the documents file advertises 15 functions"); + for f in &all { + assert!(schema(f).is_some(), "{f} has no schema"); + } } #[test] diff --git a/src/openhuman/memory/schemas/mod.rs b/src/openhuman/memory/schemas/mod.rs index de11f2de54..b1a3cfcc52 100644 --- a/src/openhuman/memory/schemas/mod.rs +++ b/src/openhuman/memory/schemas/mod.rs @@ -7,11 +7,22 @@ //! Internally the schemas are organised into family submodules that mirror //! [`crate::openhuman::memory::ops`]: //! -//! - [`documents`] — doc/namespace/recall/clear schemas + handlers. +//! - [`documents`] — doc/namespace/recall/clear schemas + handlers. Partitioned +//! three ways by capability family (core+recall / documents / ingest); see +//! that module's header for why. //! - [`kv_graph`] — key-value and knowledge-graph schemas + handlers. //! - [`sync`] — `sync_channel`, `sync_all`, `ingestion_status`. //! - [`learn`] — `learn_all`. +//! - [`provider`] — `provider_status` (the bound memory driver). //! - [`files`] — file-based memory schemas + handlers. +//! - [`tool_memory`] — tool-scoped memory rules (#1400). +//! +//! Every family publishes its own `all__controller_schemas()` / +//! `all__registered_controllers()` pair; [`all_controller_schemas`] and +//! [`all_registered_controllers`] are thin fan-outs over the **nine** parts, in +//! a fixed order. The split exists so `core::all` can register (or decline to +//! register) one capability family at a time — the parts themselves skip +//! nothing. use serde::de::DeserializeOwned; use serde_json::{Map, Value}; @@ -24,34 +35,166 @@ mod documents; mod files; mod kv_graph; mod learn; +mod provider; mod sync; mod tool_memory; // --------------------------------------------------------------------------- -// Public entry points +// Per-family entry points +// --------------------------------------------------------------------------- +// +// Each capability family exposes its own `all__controller_schemas()` / +// `all__registered_controllers()` pair so a caller can register (or, in +// a later slice, decline to register) one family at a time. The aggregators +// below fan these out in a fixed order — core_recall, documents, ingest, files, +// kv_graph, sync, learn, provider, tool_memory. Do not reorder: `src/core/all.rs` +// pushes the nine parts in exactly this sequence and +// `registered_controller_order_is_pinned_to_the_capability_partition_snapshot` in +// `schemas_tests.rs` fails if it drifts. + +/// Controller schemas for the mandatory core + recall surface. Never +/// capability-gated — see [`documents`]'s header. +pub fn all_core_recall_controller_schemas() -> Vec { + documents::FUNCTIONS_CORE_RECALL + .iter() + .map(|f| schemas(f)) + .collect() +} + +/// Registered controllers for the mandatory core + recall surface. +pub fn all_core_recall_registered_controllers() -> Vec { + documents::controllers_core_recall() +} + +/// Controller schemas for the namespace-document tier +/// (`Capability::Documents`). +pub fn all_documents_controller_schemas() -> Vec { + documents::FUNCTIONS_DOCUMENTS + .iter() + .map(|f| schemas(f)) + .collect() +} + +/// Registered controllers for the namespace-document tier. +pub fn all_documents_registered_controllers() -> Vec { + documents::controllers_documents() +} + +/// Controller schemas for driver-owned ingestion (`Capability::Ingest`). +pub fn all_ingest_controller_schemas() -> Vec { + documents::FUNCTIONS_INGEST + .iter() + .map(|f| schemas(f)) + .collect() +} + +/// Registered controllers for driver-owned ingestion. +pub fn all_ingest_registered_controllers() -> Vec { + documents::controllers_ingest() +} + +/// Controller schemas for the file-backed memory family. +pub fn all_files_controller_schemas() -> Vec { + files::FUNCTIONS.iter().map(|f| schemas(f)).collect() +} + +/// Registered controllers for the file-backed memory family. +pub fn all_files_registered_controllers() -> Vec { + files::controllers() +} + +/// Controller schemas for the key-value + knowledge-graph family. +pub fn all_kv_graph_controller_schemas() -> Vec { + kv_graph::FUNCTIONS.iter().map(|f| schemas(f)).collect() +} + +/// Registered controllers for the key-value + knowledge-graph family. +pub fn all_kv_graph_registered_controllers() -> Vec { + kv_graph::controllers() +} + +/// Controller schemas for the channel/ingestion sync family. +pub fn all_sync_controller_schemas() -> Vec { + sync::FUNCTIONS.iter().map(|f| schemas(f)).collect() +} + +/// Registered controllers for the channel/ingestion sync family. +pub fn all_sync_registered_controllers() -> Vec { + sync::controllers() +} + +/// Controller schemas for the `learn_all` family. +pub fn all_learn_controller_schemas() -> Vec { + learn::FUNCTIONS.iter().map(|f| schemas(f)).collect() +} + +/// Registered controllers for the `learn_all` family. +pub fn all_learn_registered_controllers() -> Vec { + learn::controllers() +} + +/// Controller schemas for the bound-driver status family. +/// +/// This family is the one that *reports* the driver's advertised capability +/// set, so it must never itself be gated on a capability — doing so would be +/// self-referential and would blind the UI, which reads the capability set +/// from `_status` (kernel.md §3.3). +pub fn all_provider_controller_schemas() -> Vec { + provider::FUNCTIONS.iter().map(|f| schemas(f)).collect() +} + +/// Registered controllers for the bound-driver status family. +/// +/// Never gated — see [`all_provider_controller_schemas`]. +pub fn all_provider_registered_controllers() -> Vec { + provider::controllers() +} + +/// Controller schemas for the tool-scoped memory family (#1400). +pub fn all_tool_memory_controller_schemas() -> Vec { + tool_memory::FUNCTIONS.iter().map(|f| schemas(f)).collect() +} + +/// Registered controllers for the tool-scoped memory family (#1400). +pub fn all_tool_memory_registered_controllers() -> Vec { + tool_memory::controllers() +} + +// --------------------------------------------------------------------------- +// Aggregated entry points // --------------------------------------------------------------------------- /// Returns all controller schemas for the memory system. +/// +/// Thin fan-out over the nine per-family pairs above, in their pinned order. pub fn all_controller_schemas() -> Vec { let mut out = Vec::new(); - out.extend(documents::FUNCTIONS.iter().map(|f| schemas(f))); - out.extend(files::FUNCTIONS.iter().map(|f| schemas(f))); - out.extend(kv_graph::FUNCTIONS.iter().map(|f| schemas(f))); - out.extend(sync::FUNCTIONS.iter().map(|f| schemas(f))); - out.extend(learn::FUNCTIONS.iter().map(|f| schemas(f))); - out.extend(tool_memory::FUNCTIONS.iter().map(|f| schemas(f))); + out.extend(all_core_recall_controller_schemas()); + out.extend(all_documents_controller_schemas()); + out.extend(all_ingest_controller_schemas()); + out.extend(all_files_controller_schemas()); + out.extend(all_kv_graph_controller_schemas()); + out.extend(all_sync_controller_schemas()); + out.extend(all_learn_controller_schemas()); + out.extend(all_provider_controller_schemas()); + out.extend(all_tool_memory_controller_schemas()); out } /// Returns all registered controllers for the memory system, mapping schemas to handlers. +/// +/// Thin fan-out over the nine per-family pairs above, in their pinned order. pub fn all_registered_controllers() -> Vec { let mut out = Vec::new(); - out.extend(documents::controllers()); - out.extend(files::controllers()); - out.extend(kv_graph::controllers()); - out.extend(sync::controllers()); - out.extend(learn::controllers()); - out.extend(tool_memory::controllers()); + out.extend(all_core_recall_registered_controllers()); + out.extend(all_documents_registered_controllers()); + out.extend(all_ingest_registered_controllers()); + out.extend(all_files_registered_controllers()); + out.extend(all_kv_graph_registered_controllers()); + out.extend(all_sync_registered_controllers()); + out.extend(all_learn_registered_controllers()); + out.extend(all_provider_registered_controllers()); + out.extend(all_tool_memory_registered_controllers()); out } @@ -72,6 +215,9 @@ pub fn schemas(function: &str) -> ControllerSchema { if let Some(schema) = learn::schema(function) { return schema; } + if let Some(schema) = provider::schema(function) { + return schema; + } if let Some(schema) = tool_memory::schema(function) { return schema; } diff --git a/src/openhuman/memory/schemas/provider.rs b/src/openhuman/memory/schemas/provider.rs new file mode 100644 index 0000000000..b7ecb95e86 --- /dev/null +++ b/src/openhuman/memory/schemas/provider.rs @@ -0,0 +1,135 @@ +//! Schema and handler for the `memory.provider_status` RPC method +//! (`docs/specs/plan-memory.md` §5). + +use serde_json::{Map, Value}; + +use crate::core::all::{ControllerFuture, RegisteredController}; +use crate::core::{ControllerSchema, FieldSchema, TypeSchema}; +use crate::openhuman::memory::rpc; + +use super::to_json; + +pub(super) const FUNCTIONS: &[&str] = &["provider_status"]; + +pub(super) fn controllers() -> Vec { + vec![RegisteredController { + schema: schema("provider_status").unwrap(), + handler: handle_provider_status, + }] +} + +pub(super) fn schema(function: &str) -> Option { + Some(match function { + "provider_status" => ControllerSchema { + namespace: "memory", + function: "provider_status", + description: "Status of the bound memory driver: id, class, live health, contract version, advertised capability families, and any fallback.", + inputs: vec![], + outputs: vec![ + FieldSchema { + name: "slot", + ty: TypeSchema::String, + comment: "Subsystem slot name; always \"memory\" here.", + required: true, + }, + FieldSchema { + name: "driver", + ty: TypeSchema::String, + comment: "Bound driver id (tinycortex, supermemory, null). Empty when nothing could be bound.", + required: true, + }, + FieldSchema { + name: "class", + ty: TypeSchema::String, + comment: "How the host bound the driver: embedded | external | null.", + required: true, + }, + FieldSchema { + name: "health", + ty: TypeSchema::String, + comment: "Liveness as the driver reports it: ready | degraded | down.", + required: true, + }, + FieldSchema { + name: "health_reason", + ty: TypeSchema::Option(Box::new(TypeSchema::String)), + comment: "Operator-facing reason when degraded or down; null when ready.", + required: false, + }, + FieldSchema { + name: "contract_version", + ty: TypeSchema::String, + comment: "Memory contract version this build speaks, as \".\".", + required: true, + }, + FieldSchema { + name: "capabilities", + ty: TypeSchema::Array(Box::new(TypeSchema::String)), + comment: "Advertised capability families as opaque snake_case strings.", + required: true, + }, + FieldSchema { + name: "fell_back_from", + ty: TypeSchema::Option(Box::new(TypeSchema::String)), + comment: "The driver id that was asked for and refused, when this binding is a fallback.", + required: false, + }, + FieldSchema { + name: "last_error", + ty: TypeSchema::Option(Box::new(TypeSchema::String)), + comment: "Last bind or call failure, operator-facing; null when clean.", + required: false, + }, + ], + }, + _ => return None, + }) +} + +fn handle_provider_status(_params: Map) -> ControllerFuture { + Box::pin(async move { to_json(rpc::memory_provider_status().await) }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn provider_schema_only_exposes_provider_status() { + assert_eq!(FUNCTIONS, &["provider_status"]); + assert_eq!(controllers().len(), 1); + } + + #[test] + fn unknown_provider_schema_returns_none() { + assert!(schema("not_real").is_none()); + } + + #[test] + fn provider_status_schema_has_no_inputs_and_names_the_status_fields() { + let schema = schema("provider_status").unwrap(); + assert_eq!(schema.namespace, "memory"); + assert!(schema.inputs.is_empty()); + let names: Vec<&str> = schema.outputs.iter().map(|f| f.name).collect(); + for expected in [ + "slot", + "driver", + "class", + "health", + "contract_version", + "capabilities", + ] { + assert!(names.contains(&expected), "missing output {expected}"); + } + } + + #[tokio::test] + async fn handler_returns_driver_and_capability_fields() { + let value = handle_provider_status(Map::new()) + .await + .expect("handler succeeds"); + assert!(value["driver"].is_string()); + assert!(value["capabilities"].is_array()); + assert!(value["contract_version"].is_string()); + } +} diff --git a/src/openhuman/memory/schemas_tests.rs b/src/openhuman/memory/schemas_tests.rs index 9781c2eb2b..9ebe7f69c4 100644 --- a/src/openhuman/memory/schemas_tests.rs +++ b/src/openhuman/memory/schemas_tests.rs @@ -33,6 +33,8 @@ const ALL_FUNCTIONS: &[&str] = &[ "sync_all", "learn_all", "ingestion_status", + // The bound memory driver (kernel.md §6 item 6, plan-memory.md §5) + "provider_status", // Tool-scoped memory (#1400) "tool_rule_put", "tool_rule_get", @@ -42,6 +44,207 @@ const ALL_FUNCTIONS: &[&str] = &[ "tool_rules_json", ]; +/// The exact ordered `memory.*` registration sequence. Order matters: it is the +/// order `src/core/all.rs` pushes controllers in, which is the order `/schema` +/// and the CLI catalog advertise them in. +/// +/// Unlike [`ALL_FUNCTIONS`] — an unordered membership list — this is ordered +/// and must not be edited to accommodate a refactor. If a refactor makes this +/// fail, the refactor is a behaviour change. +/// +/// **Edited once, deliberately, by M5.2.** The `documents` family was split +/// into three capability partitions (core+recall / documents / ingest) so the +/// gated ones can be registered independently of the MANDATORY ones — tagging +/// the whole file `Capability::Documents` would have made `recall_memories` +/// vanish under a driver that merely lacks the document tier. The only +/// consequence visible here is that `doc_ingest` moved from between `doc_put` +/// and `doc_list` to after `doc_delete`, and the three `context_*`/`clear_*` +/// functions moved ahead of the `doc_*` block. Membership is unchanged, every +/// method name is unchanged, and registration order within a namespace carries +/// no wire semantics (dispatch is by method name; `/schema` is a list). This is +/// the M5.2 change, not licence to re-edit the list for the next refactor. +const REGISTRATION_ORDER: &[&str] = &[ + // documents — core + recall partition (mandatory, never capability-gated) + "init", + "list_documents", + "list_namespaces", + "delete_document", + "query_namespace", + "recall_context", + "recall_memories", + "namespace_list", + "context_query", + "context_recall", + "clear_namespace", + // documents — namespace-document tier (Capability::Documents) + "doc_put", + "doc_list", + "doc_delete", + // documents — driver-owned ingestion (Capability::Ingest) + "doc_ingest", + // files + "list_files", + "read_file", + "write_file", + // kv_graph + "kv_set", + "kv_get", + "kv_delete", + "kv_list_namespace", + "graph_upsert", + "graph_query", + // sync + "sync_channel", + "sync_all", + "ingestion_status", + // learn + "learn_all", + // provider + "provider_status", + // tool_memory + "tool_rule_put", + "tool_rule_get", + "tool_rule_list", + "tool_rule_delete", + "tool_rules_for_prompt", + "tool_rules_json", +]; + +fn functions_of(controllers: &[RegisteredController]) -> Vec<&'static str> { + controllers.iter().map(|c| c.schema.function).collect() +} + +#[test] +fn registered_controller_order_is_pinned_to_the_capability_partition_snapshot() { + assert_eq!( + ALL_FUNCTIONS.len(), + REGISTRATION_ORDER.len(), + "the membership list and the ordered list have drifted apart" + ); + assert_eq!( + functions_of(&all_registered_controllers()), + REGISTRATION_ORDER, + "memory controller registration order changed — this is a behaviour change, not a refactor" + ); +} + +#[test] +fn controller_schema_order_is_pinned_to_pre_split_snapshot() { + let names: Vec<_> = all_controller_schemas() + .into_iter() + .map(|s| s.function) + .collect(); + assert_eq!(names, REGISTRATION_ORDER); +} + +#[test] +fn aggregator_is_exactly_the_nine_parts_concatenated_in_order() { + let mut expected = Vec::new(); + expected.extend(functions_of(&all_core_recall_registered_controllers())); + expected.extend(functions_of(&all_documents_registered_controllers())); + expected.extend(functions_of(&all_ingest_registered_controllers())); + expected.extend(functions_of(&all_files_registered_controllers())); + expected.extend(functions_of(&all_kv_graph_registered_controllers())); + expected.extend(functions_of(&all_sync_registered_controllers())); + expected.extend(functions_of(&all_learn_registered_controllers())); + expected.extend(functions_of(&all_provider_registered_controllers())); + expected.extend(functions_of(&all_tool_memory_registered_controllers())); + + assert_eq!(functions_of(&all_registered_controllers()), expected); + assert_eq!(expected, REGISTRATION_ORDER); +} + +#[test] +fn schema_aggregator_is_exactly_the_nine_parts_concatenated_in_order() { + let mut expected: Vec<&'static str> = Vec::new(); + for family in [ + all_core_recall_controller_schemas(), + all_documents_controller_schemas(), + all_ingest_controller_schemas(), + all_files_controller_schemas(), + all_kv_graph_controller_schemas(), + all_sync_controller_schemas(), + all_learn_controller_schemas(), + all_provider_controller_schemas(), + all_tool_memory_controller_schemas(), + ] { + expected.extend(family.into_iter().map(|s| s.function)); + } + let actual: Vec<_> = all_controller_schemas() + .into_iter() + .map(|s| s.function) + .collect(); + assert_eq!(actual, expected); +} + +#[test] +fn each_family_pairs_its_schemas_with_its_controllers() { + let families: [(&str, Vec, Vec); 9] = [ + ( + "core_recall", + all_core_recall_controller_schemas(), + all_core_recall_registered_controllers(), + ), + ( + "documents", + all_documents_controller_schemas(), + all_documents_registered_controllers(), + ), + ( + "ingest", + all_ingest_controller_schemas(), + all_ingest_registered_controllers(), + ), + ( + "files", + all_files_controller_schemas(), + all_files_registered_controllers(), + ), + ( + "kv_graph", + all_kv_graph_controller_schemas(), + all_kv_graph_registered_controllers(), + ), + ( + "sync", + all_sync_controller_schemas(), + all_sync_registered_controllers(), + ), + ( + "learn", + all_learn_controller_schemas(), + all_learn_registered_controllers(), + ), + ( + "provider", + all_provider_controller_schemas(), + all_provider_registered_controllers(), + ), + ( + "tool_memory", + all_tool_memory_controller_schemas(), + all_tool_memory_registered_controllers(), + ), + ]; + + let mut total = 0; + for (name, schemas, controllers) in families { + assert!(!schemas.is_empty(), "family {name} advertises no schemas"); + let schema_fns: Vec<_> = schemas.iter().map(|s| s.function).collect(); + assert_eq!( + schema_fns, + functions_of(&controllers), + "family {name} schema order diverges from its handler order" + ); + total += controllers.len(); + } + assert_eq!( + total, + REGISTRATION_ORDER.len(), + "the nine parts must cover the whole memory surface — no function may be orphaned" + ); +} + #[test] fn all_controller_schemas_has_entry_per_supported_function() { let names: Vec<_> = all_controller_schemas() diff --git a/src/openhuman/memory/search/tools/vector_search.rs b/src/openhuman/memory/search/tools/vector_search.rs index 3acfd2bd1d..acb3fff56a 100644 --- a/src/openhuman/memory/search/tools/vector_search.rs +++ b/src/openhuman/memory/search/tools/vector_search.rs @@ -15,9 +15,9 @@ use crate::openhuman::memory::store::chunks::store::{ get_chunk_embeddings_for_signature_batch, list_chunks, ListChunksQuery, }; use crate::openhuman::memory::store::chunks::types::SourceKind; -use crate::openhuman::memory::store::vectors::cosine_similarity; use crate::openhuman::tools::traits::{Tool, ToolResult}; use tinycortex::memory::retrieval::mmr::{mmr_select, MmrCandidate}; +use tinycortex::memory::store::vectors::cosine_similarity; pub struct MemoryVectorSearchTool; diff --git a/src/openhuman/memory/sources/readers/github.rs b/src/openhuman/memory/sources/readers/github.rs index 1b9b1ec8e3..00921f8678 100644 --- a/src/openhuman/memory/sources/readers/github.rs +++ b/src/openhuman/memory/sources/readers/github.rs @@ -1,278 +1,23 @@ -//! GitHub repo source reader. +//! Product `Config` adapter for the tinycortex GitHub repo reader. //! -//! Pulls **project activity** (commits, issues, PRs) from a GitHub -//! repository — not source code. Uses the `gh` CLI when available for -//! authenticated, higher-rate-limit access; falls back to the public -//! GitHub REST API for unauthenticated reads. +//! The reader itself — commit/issue/PR fetching over `gh`, `git`, and the +//! public REST API — lives in the engine. This module keeps the host-side +//! `SourceReader` shape (`&Config`, `Result<_, String>`) that the sources RPC +//! surface and the sync runner are written against, and re-exports the two +//! coordinate helpers `sources::sync` derives its scopes from. use async_trait::async_trait; -use serde::Deserialize; -use std::collections::HashMap; -use std::path::{Path, PathBuf}; -use std::sync::Mutex; -use std::time::Duration; use crate::openhuman::config::Config; +use crate::openhuman::memory::sources::readers::SourceReader; use crate::openhuman::memory::sources::types::{ - ContentType, MemorySourceEntry, SourceContent, SourceItem, SourceKind, + MemorySourceEntry, SourceContent, SourceItem, SourceKind, }; -use crate::openhuman::memory::store::content::raw::RawKind; -use super::SourceReader; - -const DEFAULT_BRANCH: &str = "main"; - -/// Cache of issue/PR data populated during `list_items` so `read_item` -/// doesn't re-fetch each one individually. The paginated list endpoints -/// already return the full body, state, labels, etc. — caching them -/// halves the API calls (from N individual fetches down to ceil(N/100) -/// paginated pages). -/// -/// Keyed by `"/:"` (e.g. `"org/repo:issue:42"`). -/// Cleared at the start of each `list_items` call for the same repo. -static LIST_CACHE: std::sync::LazyLock>> = - std::sync::LazyLock::new(|| Mutex::new(HashMap::new())); - -enum CachedItem { - Issue(GhIssue), - Pr(GhPr), -} - -/// Default number of items of **each** type (commits, issues, PRs) to pull -/// when the source entry doesn't override it. Tunable per-source via -/// `max_commits` / `max_issues` / `max_prs` on [`MemorySourceEntry`]. -pub(crate) const DEFAULT_GITHUB_ITEM_LIMIT: u32 = 1000; - -/// GitHub REST API maximum page size (`per_page`). -const GH_PAGE_SIZE: u32 = 100; - -/// Hard ceiling on pagination loops so a misbehaving API (always returning a -/// full page) can never spin forever even if `max` is enormous. -const GH_MAX_PAGES: u32 = 1000; +pub use tinycortex::memory::sources::readers::github::{repo_archive_source_id, repo_chunk_scope}; pub struct GithubReader; -/// Parse `owner` and `repo` from a GitHub URL. -/// -/// Accepts only the canonical `https://github.com//[.git][/]` -/// shape — extra segments like `/tree/main` or `/blob/...` are rejected -/// so callers can't accidentally derive the wrong owner/repo from a -/// deep link. -pub(crate) fn parse_github_url(url: &str) -> Result<(String, String), String> { - let trimmed = url.trim(); - let rest = trimmed - .strip_prefix("https://github.com/") - .or_else(|| trimmed.strip_prefix("http://github.com/")) - .or_else(|| trimmed.strip_prefix("git@github.com:")) - .ok_or_else(|| format!("not a GitHub URL: {url}"))?; - let cleaned = rest.trim_end_matches('/').trim_end_matches(".git"); - let parts: Vec<&str> = cleaned.split('/').collect(); - if parts.len() != 2 || parts[0].is_empty() || parts[1].is_empty() { - return Err(format!( - "expected https://github.com//, got: {url}" - )); - } - Ok((parts[0].to_string(), parts[1].to_string())) -} - -fn gh_available() -> bool { - std::process::Command::new("gh") - .arg("--version") - .stdout(std::process::Stdio::null()) - .stderr(std::process::Stdio::null()) - .status() - .map(|s| s.success()) - .unwrap_or(false) -} - -// ── Item types ────────────────────────────────────────────────────── - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum ItemKind { - Commit, - Issue, - PullRequest, -} - -impl ItemKind { - fn prefix(self) -> &'static str { - match self { - ItemKind::Commit => "commit", - ItemKind::Issue => "issue", - ItemKind::PullRequest => "pr", - } - } - - fn from_id(id: &str) -> Option<(Self, &str)> { - if let Some(rest) = id.strip_prefix("commit:") { - Some((ItemKind::Commit, rest)) - } else if let Some(rest) = id.strip_prefix("issue:") { - Some((ItemKind::Issue, rest)) - } else if let Some(rest) = id.strip_prefix("pr:") { - Some((ItemKind::PullRequest, rest)) - } else { - None - } - } -} - -// ── Raw-archive coordinates ───────────────────────────────────────── - -/// Slugifiable raw-archive source id for a repo URL. -/// -/// Returns `github.com//`, which slugifies (via -/// `slugify_source_id`) to `github-com--` so a source's -/// commits/issues/PRs land under -/// `raw/github-com--/{commits,issues,prs}/`. -pub(crate) fn repo_archive_source_id(url: &str) -> Option { - let (owner, repo) = parse_github_url(url).ok()?; - Some(format!("github.com/{owner}/{repo}")) -} - -/// Chunk-store source id for a single repo item (dedup key). -/// -/// `github:/:` keeps per-item uniqueness for the -/// `mem_tree_ingested_sources` dedup table while the separate -/// [`repo_chunk_scope`] drives a shared directory. -pub(crate) fn chunk_source_id(url: &str, item_id: &str) -> Option { - let (owner, repo) = parse_github_url(url).ok()?; - Some(format!("github:{owner}/{repo}:{item_id}")) -} - -/// Repo-scoped chunk path scope so all items from one repo share a -/// single directory in the content store (e.g. `document/github-org-repo/`). -pub(crate) fn repo_chunk_scope(url: &str) -> Option { - let (owner, repo) = parse_github_url(url).ok()?; - Some(format!("github:{owner}/{repo}")) -} - -/// Map a [`SourceItem`] id (`commit:`, `issue:`, `pr:`) to its -/// raw-archive [`RawKind`] and the clean uid used as the filename suffix. -pub(crate) fn raw_archive_coords(item_id: &str) -> Option<(RawKind, String)> { - let (kind, rest) = ItemKind::from_id(item_id)?; - let raw_kind = match kind { - ItemKind::Commit => RawKind::Commit, - ItemKind::Issue => RawKind::Issue, - ItemKind::PullRequest => RawKind::PullRequest, - }; - Some((raw_kind, rest.to_string())) -} - -// ── gh CLI helpers ────────────────────────────────────────────────── - -const GH_CLI_TIMEOUT: Duration = Duration::from_secs(30); - -async fn gh_json(args: &[&str]) -> Result { - let output = tokio::time::timeout( - GH_CLI_TIMEOUT, - tokio::process::Command::new("gh").args(args).output(), - ) - .await - .map_err(|_| format!("gh command timed out after {}s", GH_CLI_TIMEOUT.as_secs()))? - .map_err(|e| format!("gh command failed: {e}"))?; - - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - return Err(format!("gh exited {}: {stderr}", output.status)); - } - - String::from_utf8(output.stdout).map_err(|e| format!("gh output not utf8: {e}")) -} - -// ── API fallback helpers ──────────────────────────────────────────── - -async fn api_get(path: &str) -> Result { - let url = format!("https://api.github.com{path}"); - let client = reqwest::Client::builder() - .timeout(std::time::Duration::from_secs(20)) - .build() - .map_err(|e| format!("failed to build GitHub client: {e}"))?; - let resp = client - .get(&url) - .header("User-Agent", "openhuman") - .header("Accept", "application/vnd.github.v3+json") - .send() - .await - .map_err(|e| format!("GitHub API request failed: {e}"))?; - - if !resp.status().is_success() { - let status = resp.status(); - let body = resp.text().await.unwrap_or_default(); - return Err(format!("GitHub API returned {status}: {body}")); - } - - resp.text() - .await - .map_err(|e| format!("failed to read response: {e}")) -} - -// ── Deserialization types ─────────────────────────────────────────── - -#[derive(Debug, Deserialize)] -struct GhCommit { - sha: String, - commit: GhCommitInner, - /// Top-level GitHub user that authored the commit (distinct from the - /// embedded git author identity). Present when the commit author maps - /// to a GitHub account; absent for unlinked email-only authors. - #[serde(default)] - author: Option, -} - -#[derive(Debug, Deserialize)] -struct GhCommitInner { - message: String, - author: Option, - committer: Option, -} - -#[derive(Debug, Deserialize)] -struct GhAuthor { - name: Option, - email: Option, - date: Option, -} - -#[derive(Debug, Clone, Deserialize)] -struct GhIssue { - number: u64, - title: String, - body: Option, - state: String, - user: Option, - labels: Vec, - created_at: Option, - updated_at: Option, - pull_request: Option, -} - -#[derive(Debug, Clone, Deserialize)] -struct GhUser { - login: String, -} - -#[derive(Debug, Clone, Deserialize)] -struct GhLabel { - name: String, -} - -#[derive(Debug, Clone, Deserialize)] -struct GhPr { - number: u64, - title: String, - body: Option, - state: String, - user: Option, - labels: Vec, - created_at: Option, - updated_at: Option, - merged_at: Option, - #[serde(default)] - comments: u64, -} - -// ── Reader implementation ─────────────────────────────────────────── - #[async_trait] impl SourceReader for GithubReader { fn kind(&self) -> SourceKind { @@ -284,80 +29,16 @@ impl SourceReader for GithubReader { source: &MemorySourceEntry, config: &Config, ) -> Result, String> { - let url = source - .url - .as_deref() - .ok_or("github source requires a url")?; - let (owner, repo) = parse_github_url(url)?; - let use_gh = gh_available(); - - let max_commits = source.max_commits.unwrap_or(DEFAULT_GITHUB_ITEM_LIMIT); - let max_issues = source.max_issues.unwrap_or(DEFAULT_GITHUB_ITEM_LIMIT); - let max_prs = source.max_prs.unwrap_or(DEFAULT_GITHUB_ITEM_LIMIT); - - let cache_dir = git_cache_dir(config, &owner, &repo); - - tracing::debug!( - owner = %owner, - repo = %repo, - use_gh = use_gh, - max_commits, - max_issues, - max_prs, - cache = %cache_dir.display(), - "[memory_sources:github] listing items" - ); - - // Clear the list cache so stale data from a prior sync doesn't - // leak into this run. - if let Ok(mut cache) = LIST_CACHE.lock() { - cache.clear(); - } - - let mut items = Vec::new(); - let mut errors = Vec::new(); - - // Commits via local git (clone/fetch bare repo, then git log) - match list_commits_git(&owner, &repo, max_commits, &cache_dir).await { - Ok(commits) => items.extend(commits), - Err(e) => { - tracing::warn!(error = %e, "[memory_sources:github] git commit list failed, falling back to API"); - match list_commits_api(&owner, &repo, max_commits, use_gh).await { - Ok(commits) => items.extend(commits), - Err(e2) => { - tracing::warn!(error = %e2, "[memory_sources:github] API commit list also failed"); - errors.push(e2); - } - } - } - } - - // Issues and PRs via gh CLI / API (no local equivalent) - match list_issues(&owner, &repo, max_issues, use_gh).await { - Ok(issues) => items.extend(issues), - Err(e) => { - tracing::warn!(error = %e, "[memory_sources:github] failed to list issues"); - errors.push(e); - } - } - - match list_prs(&owner, &repo, max_prs, use_gh).await { - Ok(prs) => items.extend(prs), - Err(e) => { - tracing::warn!(error = %e, "[memory_sources:github] failed to list PRs"); - errors.push(e); - } - } - - if items.is_empty() && !errors.is_empty() { - return Err(format!( - "all GitHub API calls failed: {}", - errors.join("; ") - )); - } - - tracing::debug!(count = items.len(), "[memory_sources:github] found items"); - Ok(items) + tinycortex::memory::sources::SourceReader::list_items( + &tinycortex::memory::sources::readers::github::GithubReader, + source, + &crate::openhuman::memory::tinycortex::memory_config_from( + config, + config.workspace_dir.clone(), + ), + ) + .await + .map_err(|error| error.to_string()) } async fn read_item( @@ -366,816 +47,16 @@ impl SourceReader for GithubReader { item_id: &str, config: &Config, ) -> Result { - let url = source - .url - .as_deref() - .ok_or("github source requires a url")?; - let (owner, repo) = parse_github_url(url)?; - let use_gh = gh_available(); - - let (kind, ref_id) = - ItemKind::from_id(item_id).ok_or_else(|| format!("invalid item id: {item_id}"))?; - - tracing::debug!( - item_id = %item_id, - kind = ?kind, - "[memory_sources:github] reading item" - ); - - match kind { - ItemKind::Commit => { - let cache_dir = git_cache_dir(config, &owner, &repo); - match read_commit_git(&owner, &repo, ref_id, &cache_dir).await { - Ok(content) => Ok(content), - Err(e) => { - tracing::debug!( - sha = %ref_id, - error = %e, - "[memory_sources:github] git read_commit failed, falling back to API" - ); - read_commit_api(&owner, &repo, ref_id, use_gh).await - } - } - } - ItemKind::Issue => { - let num: u64 = ref_id - .parse() - .map_err(|_| format!("invalid issue number: {ref_id}"))?; - read_issue(&owner, &repo, num, use_gh).await - } - ItemKind::PullRequest => { - let num: u64 = ref_id - .parse() - .map_err(|_| format!("invalid PR number: {ref_id}"))?; - read_pr(&owner, &repo, num, use_gh).await - } - } - } -} - -/// Try `gh api` first, fall back to unauthenticated REST API. -async fn fetch_github(api_path: &str, use_gh: bool) -> Result { - if use_gh { - match gh_json(&["api", api_path]).await { - Ok(s) => return Ok(s), - Err(e) => { - tracing::debug!( - error = %e, - path = %api_path, - "[memory_sources:github] gh failed, falling back to API" - ); - } - } - } - api_get(&format!("/{api_path}")).await -} - -// ── List helpers ──────────────────────────────────────────────────── - -/// Fetch up to `max` rows from a paginated GitHub list endpoint. -/// -/// Walks `?per_page=100&page=N` until `max` rows are collected or the API -/// returns a short page (the last page). `extra_query` is appended verbatim -/// (e.g. `"state=all"`). The result is truncated to exactly `max`. -async fn fetch_all_pages( - owner: &str, - repo: &str, - resource: &str, - extra_query: &str, - max: u32, - use_gh: bool, -) -> Result, String> { - let mut out: Vec = Vec::new(); - let mut page = 1u32; - - while (out.len() as u32) < max && page <= GH_MAX_PAGES { - let remaining = max - out.len() as u32; - let per_page = remaining.min(GH_PAGE_SIZE); - let mut path = format!("repos/{owner}/{repo}/{resource}?per_page={per_page}&page={page}"); - if !extra_query.is_empty() { - path.push('&'); - path.push_str(extra_query); - } - - let json_str = fetch_github(&path, use_gh).await?; - let batch: Vec = serde_json::from_str(&json_str) - .map_err(|e| format!("parse {resource} page {page}: {e}"))?; - let got = batch.len(); - out.extend(batch); - - // Short page ⇒ no more rows upstream. - if got < per_page as usize { - break; - } - page += 1; - } - - out.truncate(max as usize); - Ok(out) -} - -// ── Git-based commit helpers ─────────────────────────────────────── - -const GIT_CLONE_TIMEOUT: Duration = Duration::from_secs(120); -const GIT_LOG_TIMEOUT: Duration = Duration::from_secs(30); - -fn git_cache_dir(config: &Config, owner: &str, repo: &str) -> PathBuf { - config - .workspace_dir - .join("git_cache") - .join(owner) - .join(format!("{repo}.git")) -} - -async fn ensure_bare_clone(owner: &str, repo: &str, cache_dir: &Path) -> Result<(), String> { - if cache_dir.join("HEAD").exists() { - tracing::debug!( - cache = %cache_dir.display(), - "[memory_sources:github:git] fetching into existing bare clone" - ); - let output = tokio::time::timeout( - GIT_CLONE_TIMEOUT, - tokio::process::Command::new("git") - .args(["fetch", "--prune", "--quiet"]) - .current_dir(cache_dir) - .output(), + tinycortex::memory::sources::SourceReader::read_item( + &tinycortex::memory::sources::readers::github::GithubReader, + source, + item_id, + &crate::openhuman::memory::tinycortex::memory_config_from( + config, + config.workspace_dir.clone(), + ), ) .await - .map_err(|_| "git fetch timed out".to_string())? - .map_err(|e| format!("git fetch failed: {e}"))?; - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - return Err(format!("git fetch exited {}: {stderr}", output.status)); - } - return Ok(()); - } - - if let Some(parent) = cache_dir.parent() { - std::fs::create_dir_all(parent).map_err(|e| format!("create cache dir: {e}"))?; - } - - let clone_url = format!("https://github.com/{owner}/{repo}.git"); - tracing::info!( - url = %clone_url, - cache = %cache_dir.display(), - "[memory_sources:github:git] cloning bare repo" - ); - - let output = tokio::time::timeout( - GIT_CLONE_TIMEOUT, - tokio::process::Command::new("git") - .args(["clone", "--bare", "--quiet", &clone_url]) - .arg(cache_dir) - .output(), - ) - .await - .map_err(|_| "git clone timed out".to_string())? - .map_err(|e| format!("git clone failed: {e}"))?; - - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - return Err(format!("git clone exited {}: {stderr}", output.status)); - } - - Ok(()) -} - -async fn list_commits_git( - owner: &str, - repo: &str, - max: u32, - cache_dir: &Path, -) -> Result, String> { - ensure_bare_clone(owner, repo, cache_dir).await?; - - // git log with a custom format: sha\tsubject\ttimestamp (ISO 8601) - let output = tokio::time::timeout( - GIT_LOG_TIMEOUT, - tokio::process::Command::new("git") - .args([ - "log", - "--all", - &format!("--max-count={max}"), - "--format=%H\t%s\t%aI", - ]) - .current_dir(cache_dir) - .output(), - ) - .await - .map_err(|_| "git log timed out".to_string())? - .map_err(|e| format!("git log failed: {e}"))?; - - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - return Err(format!("git log exited {}: {stderr}", output.status)); - } - - let stdout = String::from_utf8_lossy(&output.stdout); - let items: Vec = stdout - .lines() - .filter(|line| !line.is_empty()) - .map(|line| { - let parts: Vec<&str> = line.splitn(3, '\t').collect(); - let sha = parts.first().unwrap_or(&""); - let subject = parts.get(1).unwrap_or(&""); - let date = parts.get(2).unwrap_or(&""); - SourceItem { - id: format!("commit:{sha}"), - title: subject.to_string(), - updated_at_ms: parse_iso_ts(date), - } - }) - .collect(); - - tracing::debug!( - count = items.len(), - "[memory_sources:github:git] listed commits via local git" - ); - Ok(items) -} - -async fn read_commit_git( - owner: &str, - repo: &str, - sha: &str, - cache_dir: &Path, -) -> Result { - if !cache_dir.join("HEAD").exists() { - return Err("bare clone not present".to_string()); - } - - // git show with a custom format for author, date, and full message - let output = tokio::time::timeout( - GIT_LOG_TIMEOUT, - tokio::process::Command::new("git") - .args(["show", "--no-patch", "--format=%H%n%aN%n%aE%n%aI%n%B", sha]) - .current_dir(cache_dir) - .output(), - ) - .await - .map_err(|_| "git show timed out".to_string())? - .map_err(|e| format!("git show failed: {e}"))?; - - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - return Err(format!("git show exited {}: {stderr}", output.status)); - } - - let stdout = String::from_utf8_lossy(&output.stdout); - let mut lines = stdout.lines(); - let full_sha = lines.next().unwrap_or(sha); - let author_name = lines.next().unwrap_or("unknown"); - let author_email = lines.next().unwrap_or(""); - let date = lines.next().unwrap_or("unknown"); - let message: String = lines.collect::>().join("\n"); - let message = message.trim(); - - let title = message.lines().next().unwrap_or("").to_string(); - let author = format!("{author_name} <{author_email}>"); - - let body = format!( - "# Commit: {title}\n\n\ - **SHA:** {full_sha}\n\ - **Author:** {author}\n\ - **Date:** {date}\n\n\ - ## Message\n\n\ - {message}", - ); - - Ok(SourceContent { - id: format!("commit:{sha}"), - title, - body, - content_type: ContentType::Markdown, - metadata: serde_json::json!({ - "owner": owner, - "repo": repo, - "sha": full_sha, - "author": author, - }), - }) -} - -// ── API-based commit helpers (fallback) ─────────────────────────── - -async fn list_commits_api( - owner: &str, - repo: &str, - max: u32, - use_gh: bool, -) -> Result, String> { - let commits: Vec = fetch_all_pages(owner, repo, "commits", "", max, use_gh).await?; - - Ok(commits - .into_iter() - .map(|c| { - let title = c.commit.message.lines().next().unwrap_or("").to_string(); - let ts = c - .commit - .committer - .as_ref() - .and_then(|a| a.date.as_deref()) - .and_then(parse_iso_ts); - SourceItem { - id: format!("commit:{}", c.sha), - title, - updated_at_ms: ts, - } - }) - .collect()) -} - -async fn list_issues( - owner: &str, - repo: &str, - max: u32, - use_gh: bool, -) -> Result, String> { - let mut out: Vec = Vec::new(); - let mut page = 1u32; - - while (out.len() as u32) < max && page <= GH_MAX_PAGES { - let path = - format!("repos/{owner}/{repo}/issues?per_page={GH_PAGE_SIZE}&page={page}&state=all"); - let json_str = fetch_github(&path, use_gh).await?; - let batch: Vec = serde_json::from_str(&json_str) - .map_err(|e| format!("parse issues page {page}: {e}"))?; - let got = batch.len(); - - for i in batch { - if i.pull_request.is_some() { - continue; - } - let ts = i.updated_at.as_deref().and_then(parse_iso_ts); - let item_id = format!("issue:{}", i.number); - let cache_key = format!("{owner}/{repo}:{item_id}"); - out.push(SourceItem { - id: item_id, - title: format!("#{} {}", i.number, i.title), - updated_at_ms: ts, - }); - if let Ok(mut cache) = LIST_CACHE.lock() { - cache.insert(cache_key, CachedItem::Issue(i)); - } - if out.len() as u32 >= max { - break; - } - } - - if got < GH_PAGE_SIZE as usize { - break; - } - page += 1; - } - - Ok(out) -} - -async fn list_prs( - owner: &str, - repo: &str, - max: u32, - use_gh: bool, -) -> Result, String> { - let prs: Vec = fetch_all_pages(owner, repo, "pulls", "state=all", max, use_gh).await?; - - let items: Vec = prs - .into_iter() - .map(|p| { - let ts = p.updated_at.as_deref().and_then(parse_iso_ts); - let item_id = format!("pr:{}", p.number); - let cache_key = format!("{owner}/{repo}:{item_id}"); - let item = SourceItem { - id: item_id, - title: format!("PR #{} {}", p.number, p.title), - updated_at_ms: ts, - }; - if let Ok(mut cache) = LIST_CACHE.lock() { - cache.insert(cache_key, CachedItem::Pr(p)); - } - item - }) - .collect(); - - Ok(items) -} - -// ── Read helpers ──────────────────────────────────────────────────── - -async fn read_commit_api( - owner: &str, - repo: &str, - sha: &str, - use_gh: bool, -) -> Result { - let json_str = fetch_github(&format!("repos/{owner}/{repo}/commits/{sha}"), use_gh).await?; - - let commit: GhCommit = - serde_json::from_str(&json_str).map_err(|e| format!("parse commit: {e}"))?; - - let author = commit - .commit - .author - .as_ref() - .map(|a| { - format!( - "{} <{}>", - a.name.as_deref().unwrap_or("unknown"), - a.email.as_deref().unwrap_or("") - ) - }) - .unwrap_or_default(); - - // GitHub login of the committer, rendered as an `@handle` so the - // entity extractor registers it as a `handle:` entity in the memory - // tree (unique committers become first-class entities). - let handle = commit - .author - .as_ref() - .map(|u| format!("@{}", u.login)) - .unwrap_or_default(); - - let date = commit - .commit - .committer - .as_ref() - .and_then(|a| a.date.as_deref()) - .unwrap_or("unknown"); - - let title = commit - .commit - .message - .lines() - .next() - .unwrap_or("") - .to_string(); - - let author_line = if handle.is_empty() { - author.clone() - } else { - format!("{author} ({handle})") - }; - - let body = format!( - "# Commit: {title}\n\n\ - **SHA:** {sha}\n\ - **Author:** {author_line}\n\ - **Date:** {date}\n\n\ - ## Message\n\n\ - {}", - commit.commit.message, - ); - - Ok(SourceContent { - id: format!("commit:{sha}"), - title, - body, - content_type: ContentType::Markdown, - metadata: serde_json::json!({ - "owner": owner, - "repo": repo, - "sha": sha, - "author": author, - "author_handle": commit.author.as_ref().map(|u| u.login.clone()), - }), - }) -} - -async fn read_issue( - owner: &str, - repo: &str, - number: u64, - use_gh: bool, -) -> Result { - let cache_key = format!("{owner}/{repo}:issue:{number}"); - let from_cache = LIST_CACHE - .lock() - .ok() - .and_then(|mut c| c.remove(&cache_key)); - let issue: GhIssue = match from_cache { - Some(CachedItem::Issue(i)) => i, - _ => { - let json_str = - fetch_github(&format!("repos/{owner}/{repo}/issues/{number}"), use_gh).await?; - serde_json::from_str(&json_str).map_err(|e| format!("parse issue: {e}"))? - } - }; - - let author = issue - .user - .as_ref() - .map(|u| u.login.as_str()) - .unwrap_or("unknown"); - let labels: Vec<&str> = issue.labels.iter().map(|l| l.name.as_str()).collect(); - let issue_body = issue.body.as_deref().unwrap_or(""); - - let comments = fetch_issue_comments(owner, repo, number, use_gh).await; - let participants = - unique_handles(std::iter::once(author).chain(comments.iter().map(|c| c.user.as_str()))); - - let mut body = format!( - "# Issue #{number}: {title}\n\n\ - **State:** {state}\n\ - **Author:** @{author}\n\ - **Participants:** {participants}\n\ - **Labels:** {label_str}\n\ - **Created:** {created}\n\ - **Updated:** {updated}\n\n\ - ## Description\n\n\ - {issue_body}", - title = issue.title, - state = issue.state, - label_str = if labels.is_empty() { - "none".to_string() - } else { - labels.join(", ") - }, - created = issue.created_at.as_deref().unwrap_or("unknown"), - updated = issue.updated_at.as_deref().unwrap_or("unknown"), - ); - - if !comments.is_empty() { - body.push_str("\n\n## Comments\n"); - for comment in &comments { - body.push_str(&format!( - "\n### @{} ({})\n\n{}\n", - comment.user, comment.created_at, comment.body - )); - } - } - - Ok(SourceContent { - id: format!("issue:{number}"), - title: format!("#{number} {}", issue.title), - body, - content_type: ContentType::Markdown, - metadata: serde_json::json!({ - "owner": owner, - "repo": repo, - "number": number, - "state": issue.state, - "labels": labels, - }), - }) -} - -async fn read_pr( - owner: &str, - repo: &str, - number: u64, - use_gh: bool, -) -> Result { - let cache_key = format!("{owner}/{repo}:pr:{number}"); - let from_cache = LIST_CACHE - .lock() - .ok() - .and_then(|mut c| c.remove(&cache_key)); - let pr: GhPr = match from_cache { - Some(CachedItem::Pr(p)) => p, - _ => { - let json_str = - fetch_github(&format!("repos/{owner}/{repo}/pulls/{number}"), use_gh).await?; - serde_json::from_str(&json_str).map_err(|e| format!("parse PR: {e}"))? - } - }; - - let author = pr - .user - .as_ref() - .map(|u| u.login.as_str()) - .unwrap_or("unknown"); - let labels: Vec<&str> = pr.labels.iter().map(|l| l.name.as_str()).collect(); - let pr_body = pr.body.as_deref().unwrap_or(""); - - let merged_str = match pr.merged_at.as_deref() { - Some(ts) => format!("merged at {ts}"), - None => "not merged".to_string(), - }; - - let comments = fetch_issue_comments(owner, repo, number, use_gh).await; - let participants = - unique_handles(std::iter::once(author).chain(comments.iter().map(|c| c.user.as_str()))); - - let mut body = format!( - "# PR #{number}: {title}\n\n\ - **State:** {state} ({merged})\n\ - **Author:** @{author}\n\ - **Participants:** {participants}\n\ - **Labels:** {label_str}\n\ - **Created:** {created}\n\ - **Updated:** {updated}\n\n\ - ## Description\n\n\ - {pr_body}", - title = pr.title, - state = pr.state, - merged = merged_str, - label_str = if labels.is_empty() { - "none".to_string() - } else { - labels.join(", ") - }, - created = pr.created_at.as_deref().unwrap_or("unknown"), - updated = pr.updated_at.as_deref().unwrap_or("unknown"), - ); - - if !comments.is_empty() { - body.push_str("\n\n## Comments\n"); - for comment in &comments { - body.push_str(&format!( - "\n### @{} ({})\n\n{}\n", - comment.user, comment.created_at, comment.body - )); - } - } - - Ok(SourceContent { - id: format!("pr:{number}"), - title: format!("PR #{number} {}", pr.title), - body, - content_type: ContentType::Markdown, - metadata: serde_json::json!({ - "owner": owner, - "repo": repo, - "number": number, - "state": pr.state, - "merged": pr.merged_at.is_some(), - "labels": labels, - }), - }) -} - -// ── Comment fetching ──────────────────────────────────────────────── - -struct IssueComment { - user: String, - body: String, - created_at: String, -} - -async fn fetch_issue_comments( - owner: &str, - repo: &str, - number: u64, - use_gh: bool, -) -> Vec { - #[derive(Deserialize)] - struct RawComment { - user: Option, - body: Option, - created_at: Option, - } - - let json_str = fetch_github( - &format!("repos/{owner}/{repo}/issues/{number}/comments?per_page=50"), - use_gh, - ) - .await; - - let Ok(json_str) = json_str else { - return Vec::new(); - }; - - let comments: Vec = serde_json::from_str(&json_str).unwrap_or_default(); - - comments - .into_iter() - .map(|c| IssueComment { - user: c - .user - .as_ref() - .map(|u| u.login.clone()) - .unwrap_or_else(|| "unknown".into()), - body: c.body.unwrap_or_default(), - created_at: c.created_at.unwrap_or_else(|| "unknown".into()), - }) - .collect() -} - -// ── Utilities ─────────────────────────────────────────────────────── - -fn parse_iso_ts(s: &str) -> Option { - chrono::DateTime::parse_from_rfc3339(s) - .ok() - .map(|dt| dt.timestamp_millis()) -} - -/// Render GitHub logins as a deduped, order-preserving, space-separated -/// list of `@handle`s. Empty / `unknown` logins are skipped; an empty -/// result renders as `none`. Used so unique committers/commenters surface -/// as `handle:` entities in the memory tree. -fn unique_handles<'a>(logins: impl Iterator) -> String { - let mut seen = std::collections::HashSet::new(); - let mut out: Vec = Vec::new(); - for login in logins { - let l = login.trim(); - if l.is_empty() || l == "unknown" { - continue; - } - if seen.insert(l.to_string()) { - out.push(format!("@{l}")); - } - } - if out.is_empty() { - "none".to_string() - } else { - out.join(" ") - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn parse_github_url_extracts_owner_and_repo() { - let (owner, repo) = parse_github_url("https://github.com/openai/tiktoken").unwrap(); - assert_eq!(owner, "openai"); - assert_eq!(repo, "tiktoken"); - } - - #[test] - fn parse_github_url_handles_trailing_slash_and_git() { - let (owner, repo) = parse_github_url("https://github.com/org/repo.git/").unwrap(); - assert_eq!(owner, "org"); - assert_eq!(repo, "repo"); - } - - #[test] - fn parse_github_url_rejects_non_repo_paths() { - // Deep links like /tree/main must not silently extract the wrong - // owner/repo. Bare host or non-github URLs also rejected. - assert!(parse_github_url("https://github.com/org/repo/tree/main").is_err()); - assert!(parse_github_url("https://gitlab.com/org/repo").is_err()); - assert!(parse_github_url("https://github.com/org").is_err()); - assert!(parse_github_url("not-a-url").is_err()); - } - - #[test] - fn item_kind_round_trips() { - let cases = [ - ("commit:abc123", ItemKind::Commit, "abc123"), - ("issue:42", ItemKind::Issue, "42"), - ("pr:99", ItemKind::PullRequest, "99"), - ]; - for (id, expected_kind, expected_ref) in cases { - let (kind, ref_id) = ItemKind::from_id(id).unwrap(); - assert_eq!(kind, expected_kind); - assert_eq!(ref_id, expected_ref); - } - } - - #[test] - fn item_kind_rejects_invalid() { - assert!(ItemKind::from_id("unknown:123").is_none()); - assert!(ItemKind::from_id("noprefix").is_none()); - } - - #[test] - fn repo_archive_source_id_slugs_to_repo_folder() { - // `github.com//` → slugify → `github-com--`. - assert_eq!( - repo_archive_source_id("https://github.com/tinyhumansai/openhuman").as_deref(), - Some("github.com/tinyhumansai/openhuman") - ); - assert!(repo_archive_source_id("not-a-url").is_none()); - } - - #[test] - fn chunk_source_id_is_clean_and_per_item() { - assert_eq!( - chunk_source_id("https://github.com/org/repo", "commit:abc123").as_deref(), - Some("github:org/repo:commit:abc123") - ); - assert_eq!( - chunk_source_id("https://github.com/org/repo", "pr:42").as_deref(), - Some("github:org/repo:pr:42") - ); - } - - #[test] - fn unique_handles_dedups_and_skips_unknown() { - assert_eq!( - unique_handles(["alice", "bob", "alice", "unknown", ""].into_iter()), - "@alice @bob" - ); - assert_eq!(unique_handles(["unknown", ""].into_iter()), "none"); - assert_eq!(unique_handles(std::iter::empty()), "none"); - } - - #[test] - fn raw_archive_coords_maps_kind_and_uid() { - assert_eq!( - raw_archive_coords("commit:deadbeef"), - Some((RawKind::Commit, "deadbeef".to_string())) - ); - assert_eq!( - raw_archive_coords("issue:7"), - Some((RawKind::Issue, "7".to_string())) - ); - assert_eq!( - raw_archive_coords("pr:99"), - Some((RawKind::PullRequest, "99".to_string())) - ); - assert!(raw_archive_coords("bogus:1").is_none()); + .map_err(|error| error.to_string()) } } diff --git a/src/openhuman/memory/sources/readers/mod.rs b/src/openhuman/memory/sources/readers/mod.rs index 7d484c6552..1cdc632be5 100644 --- a/src/openhuman/memory/sources/readers/mod.rs +++ b/src/openhuman/memory/sources/readers/mod.rs @@ -40,7 +40,7 @@ pub fn reader_for(kind: &SourceKind) -> Box { SourceKind::Folder => Box::new(folder::FolderReader), SourceKind::GithubRepo => Box::new(github::GithubReader), SourceKind::TwitterQuery => Box::new(twitter::TwitterReader), - SourceKind::RssFeed => Box::new(rss::RssReader), + SourceKind::RssFeed => Box::new(rss::RssReader::new()), SourceKind::WebPage => Box::new(web_page::WebPageReader), } } diff --git a/src/openhuman/memory/sources/readers/rss.rs b/src/openhuman/memory/sources/readers/rss.rs index 526b2912a8..151ae6af60 100644 --- a/src/openhuman/memory/sources/readers/rss.rs +++ b/src/openhuman/memory/sources/readers/rss.rs @@ -1,22 +1,35 @@ -//! RSS/Atom feed source reader. -//! -//! Fetches and parses an RSS or Atom feed, returning entries as -//! source items. Uses a lightweight XML parser (`quick-xml` via -//! manual parsing) to avoid pulling in heavy feed crates. +//! Product `Config` adapter for the tinycortex RSS/Atom feed reader. use async_trait::async_trait; use crate::openhuman::config::Config; +use crate::openhuman::memory::sources::readers::SourceReader; use crate::openhuman::memory::sources::types::{ - ContentType, MemorySourceEntry, SourceContent, SourceItem, SourceKind, + MemorySourceEntry, SourceContent, SourceItem, SourceKind, }; -use super::SourceReader; +/// Product adapter retaining the engine reader for a complete sync pass. +/// +/// The engine reader caches a freshly fetched feed between `list_items` and +/// `read_item`, so constructing it per trait call would turn one sync into +/// N+1 downloads. +pub struct RssReader { + inner: tinycortex::memory::sources::readers::rss::RssReader, +} -const DEFAULT_MAX_ITEMS: u32 = 50; -const MAX_FEED_BYTES: u64 = 5 * 1024 * 1024; // 5 MiB — guards against pathological feeds +impl RssReader { + pub fn new() -> Self { + Self { + inner: tinycortex::memory::sources::readers::rss::RssReader::new(), + } + } +} -pub struct RssReader; +impl Default for RssReader { + fn default() -> Self { + Self::new() + } +} #[async_trait] impl SourceReader for RssReader { @@ -27,328 +40,36 @@ impl SourceReader for RssReader { async fn list_items( &self, source: &MemorySourceEntry, - _config: &Config, + config: &Config, ) -> Result, String> { - let url = source.url.as_deref().ok_or("rss source requires a url")?; - let max_items = source.max_items.unwrap_or(DEFAULT_MAX_ITEMS) as usize; - - tracing::debug!( - host = %url_host(url), - max_items = max_items, - "[memory_sources:rss] listing items" - ); - - let body = fetch_url(url).await?; - let entries = parse_feed(&body, max_items)?; - - tracing::debug!(count = entries.len(), "[memory_sources:rss] parsed entries"); - - Ok(entries) + tinycortex::memory::sources::SourceReader::list_items( + &self.inner, + source, + &crate::openhuman::memory::tinycortex::memory_config_from( + config, + config.workspace_dir.clone(), + ), + ) + .await + .map_err(|error| error.to_string()) } async fn read_item( &self, source: &MemorySourceEntry, item_id: &str, - _config: &Config, + config: &Config, ) -> Result { - let url = source.url.as_deref().ok_or("rss source requires a url")?; - - tracing::debug!( - host = %url_host(url), - item_id = %item_id, - "[memory_sources:rss] reading item" - ); - - let body = fetch_url(url).await?; - let entries = parse_feed_full(&body)?; - - let entry = entries - .into_iter() - .find(|e| e.id == item_id) - .ok_or_else(|| format!("item '{item_id}' not found in feed"))?; - - let content_type = if entry.body.contains('<') { - ContentType::Html - } else { - ContentType::Plaintext - }; - - Ok(SourceContent { - id: entry.id, - title: entry.title, - body: entry.body, - content_type, - metadata: serde_json::json!({ - "link": entry.link, - "published": entry.published, - }), - }) - } -} - -/// Extract just the host portion of a URL for debug-log redaction so we -/// don't leak query params, paths, or embedded credentials. -fn url_host(url: &str) -> String { - let stripped = url - .trim_start_matches("https://") - .trim_start_matches("http://"); - stripped - .split(['/', '?', '#']) - .next() - .unwrap_or(stripped) - .to_string() -} - -async fn fetch_url(url: &str) -> Result { - let client = reqwest::Client::builder() - .timeout(std::time::Duration::from_secs(20)) - .build() - .map_err(|e| format!("failed to build http client: {e}"))?; - let resp = client - .get(url) - .header("User-Agent", "openhuman") - .send() - .await - .map_err(|e| format!("failed to fetch feed: {e}"))?; - - if !resp.status().is_success() { - return Err(format!("feed returned {}", resp.status())); - } - - // Guard against pathologically large feeds before buffering into memory. - if let Some(len) = resp.content_length() { - if len > MAX_FEED_BYTES { - return Err(format!( - "feed body too large: {len} bytes (limit {MAX_FEED_BYTES})" - )); - } - } - - let bytes = resp - .bytes() + tinycortex::memory::sources::SourceReader::read_item( + &self.inner, + source, + item_id, + &crate::openhuman::memory::tinycortex::memory_config_from( + config, + config.workspace_dir.clone(), + ), + ) .await - .map_err(|e| format!("failed to read feed body: {e}"))?; - - if bytes.len() as u64 > MAX_FEED_BYTES { - return Err(format!( - "feed body too large: {} bytes (limit {MAX_FEED_BYTES})", - bytes.len() - )); - } - - String::from_utf8(bytes.to_vec()).map_err(|e| format!("feed body is not valid UTF-8: {e}")) -} - -#[derive(Debug)] -struct FeedEntry { - id: String, - title: String, - body: String, - link: Option, - published: Option, -} - -fn parse_feed(xml: &str, max_items: usize) -> Result, String> { - let entries = parse_feed_full(xml)?; - Ok(entries - .into_iter() - .take(max_items) - .map(|e| SourceItem { - id: e.id, - title: e.title, - updated_at_ms: None, - }) - .collect()) -} - -fn parse_feed_full(xml: &str) -> Result, String> { - // Detect RSS vs Atom by looking for Result, String> { - let mut entries = Vec::new(); - let mut offset = 0; - - while let Some(item_start) = xml[offset..].find("") - .map(|i| abs_start + i + 7) - .unwrap_or(xml.len()); - - let item_xml = &xml[abs_start..item_end]; - let title = extract_tag(item_xml, "title").unwrap_or_default(); - let link = extract_tag(item_xml, "link"); - let guid = extract_tag(item_xml, "guid"); - let description = extract_tag(item_xml, "description") - .or_else(|| extract_cdata(item_xml, "content:encoded")) - .unwrap_or_default(); - let pub_date = extract_tag(item_xml, "pubDate"); - - let id = guid - .or_else(|| link.clone()) - .unwrap_or_else(|| format!("rss-{}", entries.len())); - - entries.push(FeedEntry { - id, - title, - body: description, - link, - published: pub_date, - }); - - offset = item_end; - } - - Ok(entries) -} - -fn parse_atom(xml: &str) -> Result, String> { - let mut entries = Vec::new(); - let mut offset = 0; - - while let Some(entry_start) = xml[offset..].find("") - .map(|i| abs_start + i + 8) - .unwrap_or(xml.len()); - - let entry_xml = &xml[abs_start..entry_end]; - let title = extract_tag(entry_xml, "title").unwrap_or_default(); - let id = extract_tag(entry_xml, "id").unwrap_or_else(|| format!("atom-{}", entries.len())); - let content = extract_tag(entry_xml, "content") - .or_else(|| extract_tag(entry_xml, "summary")) - .unwrap_or_default(); - let link = extract_attr(entry_xml, "link", "href"); - let updated = - extract_tag(entry_xml, "updated").or_else(|| extract_tag(entry_xml, "published")); - - entries.push(FeedEntry { - id, - title, - body: content, - link, - published: updated, - }); - - offset = entry_end; - } - - Ok(entries) -} - -fn extract_tag(xml: &str, tag: &str) -> Option { - let open = format!("<{tag}"); - let close = format!(""); - let start = xml.find(&open)?; - let content_start = xml[start..].find('>')? + start + 1; - let end = xml[content_start..].find(&close)? + content_start; - let content = &xml[content_start..end]; - Some(decode_xml_entities(content.trim())) -} - -fn extract_cdata(xml: &str, tag: &str) -> Option { - let open = format!("<{tag}"); - let close = format!(""); - let start = xml.find(&open)?; - let content_start = xml[start..].find('>')? + start + 1; - let end = xml[content_start..].find(&close)? + content_start; - let content = &xml[content_start..end]; - let cleaned = content - .trim() - .strip_prefix("")) - .unwrap_or(content); - Some(cleaned.trim().to_string()) -} - -fn extract_attr(xml: &str, tag: &str, attr: &str) -> Option { - let open = format!("<{tag} "); - let start = xml.find(&open)?; - let tag_end = xml[start..].find('>')? + start; - let tag_str = &xml[start..tag_end]; - let attr_start = tag_str.find(&format!("{attr}=\""))? + attr.len() + 2; - let attr_end = tag_str[attr_start..].find('"')? + attr_start; - Some(tag_str[attr_start..attr_end].to_string()) -} - -fn decode_xml_entities(s: &str) -> String { - s.replace("&", "&") - .replace("<", "<") - .replace(">", ">") - .replace(""", "\"") - .replace("'", "'") -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn parse_rss_extracts_items() { - let xml = r#" - - - Test Feed - - First post - https://example.com/1 - Body of first post - - - Second post - guid-2 - Body of second - - - "#; - - let entries = parse_rss(xml).unwrap(); - assert_eq!(entries.len(), 2); - assert_eq!(entries[0].title, "First post"); - assert_eq!(entries[0].id, "https://example.com/1"); - assert_eq!(entries[1].id, "guid-2"); - } - - #[test] - fn parse_atom_extracts_entries() { - let xml = r#" - - - Atom entry - urn:entry:1 - Content here - - - "#; - - let entries = parse_atom(xml).unwrap(); - assert_eq!(entries.len(), 1); - assert_eq!(entries[0].title, "Atom entry"); - assert_eq!(entries[0].id, "urn:entry:1"); - assert_eq!( - entries[0].link.as_deref(), - Some("https://example.com/atom/1") - ); - } - - #[test] - fn parse_feed_detects_format() { - let rss = "T"; - assert!(parse_feed(rss, 10).is_ok()); - - let atom = "T1"; - assert!(parse_feed(atom, 10).is_ok()); - - assert!(parse_feed("", 10).is_err()); + .map_err(|error| error.to_string()) } } diff --git a/src/openhuman/memory/sources/readers/web_page.rs b/src/openhuman/memory/sources/readers/web_page.rs index 8117766240..37e8be32fc 100644 --- a/src/openhuman/memory/sources/readers/web_page.rs +++ b/src/openhuman/memory/sources/readers/web_page.rs @@ -1,18 +1,13 @@ -//! Web page source reader. -//! -//! Fetches a single URL and extracts its text content. When a CSS -//! `selector` is configured, only matching elements are included; -//! otherwise the full page body is returned. +//! Product `Config` adapter for the tinycortex single-page web reader. use async_trait::async_trait; use crate::openhuman::config::Config; +use crate::openhuman::memory::sources::readers::SourceReader; use crate::openhuman::memory::sources::types::{ - ContentType, MemorySourceEntry, SourceContent, SourceItem, SourceKind, + MemorySourceEntry, SourceContent, SourceItem, SourceKind, }; -use super::SourceReader; - pub struct WebPageReader; #[async_trait] @@ -24,213 +19,36 @@ impl SourceReader for WebPageReader { async fn list_items( &self, source: &MemorySourceEntry, - _config: &Config, + config: &Config, ) -> Result, String> { - let url = source - .url - .as_deref() - .ok_or("web_page source requires a url")?; - - Ok(vec![SourceItem { - id: url.to_string(), - title: source.label.clone(), - updated_at_ms: None, - }]) + tinycortex::memory::sources::SourceReader::list_items( + &tinycortex::memory::sources::readers::web_page::WebPageReader, + source, + &crate::openhuman::memory::tinycortex::memory_config_from( + config, + config.workspace_dir.clone(), + ), + ) + .await + .map_err(|error| error.to_string()) } async fn read_item( &self, source: &MemorySourceEntry, item_id: &str, - _config: &Config, + config: &Config, ) -> Result { - let url = if item_id.starts_with("http") { - item_id.to_string() - } else { - source.url.clone().ok_or("web_page source requires a url")? - }; - - // SSRF guard: only allow http(s) — reject file://, data://, etc. - if !url.starts_with("http://") && !url.starts_with("https://") { - return Err(format!( - "web_page source requires an http(s) URL, got: {}", - url.chars().take(64).collect::() - )); - } - - tracing::debug!( - host = %url - .trim_start_matches("https://") - .trim_start_matches("http://") - .split(['/', '?', '#']) - .next() - .unwrap_or(""), - selector = ?source.selector, - "[memory_sources:web_page] reading item" - ); - - let client = reqwest::Client::builder() - .timeout(std::time::Duration::from_secs(20)) - .build() - .map_err(|e| format!("failed to build http client: {e}"))?; - let resp = client - .get(&url) - .header("User-Agent", "openhuman") - .send() - .await - .map_err(|e| format!("failed to fetch page: {e}"))?; - - if !resp.status().is_success() { - return Err(format!("page returned {}", resp.status())); - } - - // Cap response body to 10 MiB so a hostile/giant page can't OOM us. - const MAX_BODY_BYTES: u64 = 10 * 1024 * 1024; - if let Some(len) = resp.content_length() { - if len > MAX_BODY_BYTES { - return Err(format!( - "page body exceeds {MAX_BODY_BYTES}-byte limit (Content-Length={len})" - )); - } - } - - let bytes = resp - .bytes() - .await - .map_err(|e| format!("failed to read page body: {e}"))?; - if bytes.len() as u64 > MAX_BODY_BYTES { - return Err(format!( - "page body exceeds {MAX_BODY_BYTES}-byte limit (read {} bytes)", - bytes.len() - )); - } - let body = String::from_utf8_lossy(&bytes).into_owned(); - - let extracted = if let Some(selector) = source.selector.as_deref() { - extract_by_selector(&body, selector) - } else { - strip_html_tags(&body) - }; - - Ok(SourceContent { - id: url.clone(), - title: extract_title(&body).unwrap_or_else(|| url.clone()), - body: extracted, - content_type: ContentType::Plaintext, - metadata: serde_json::json!({ "url": url }), - }) - } -} - -fn extract_title(html: &str) -> Option { - let start = html.find("')? + start + 1; - let end = html[content_start..].find("")? + content_start; - Some(html[content_start..end].trim().to_string()) -} - -fn extract_by_selector(html: &str, selector: &str) -> String { - // Simple tag-name selector support (e.g. "article", "main", "div.content") - // For full CSS selector support, the `scraper` crate would be needed. - // This handles the common case of a single tag name. - let tag = selector.split('.').next().unwrap_or(selector).trim(); - - if tag.is_empty() { - return strip_html_tags(html); - } - - let open = format!("<{tag}"); - let close = format!(""); - - let mut result = String::new(); - let mut offset = 0; - - while let Some(start) = html[offset..].find(&open) { - let abs_start = offset + start; - let content_start = match html[abs_start..].find('>') { - Some(i) => abs_start + i + 1, - None => break, - }; - if let Some(end_offset) = html[content_start..].find(&close) { - let content = &html[content_start..content_start + end_offset]; - if !result.is_empty() { - result.push_str("\n\n"); - } - result.push_str(&strip_html_tags(content)); - offset = content_start + end_offset + close.len(); - } else { - break; - } - } - - if result.is_empty() { - strip_html_tags(html) - } else { - result - } -} - -fn strip_html_tags(html: &str) -> String { - let mut result = String::with_capacity(html.len()); - let mut in_tag = false; - let mut last_was_space = false; - - for ch in html.chars() { - match ch { - '<' => in_tag = true, - '>' => { - in_tag = false; - if !last_was_space && !result.is_empty() { - result.push(' '); - last_was_space = true; - } - } - _ if !in_tag => { - if ch.is_whitespace() { - if !last_was_space { - result.push(' '); - last_was_space = true; - } - } else { - result.push(ch); - last_was_space = false; - } - } - _ => {} - } - } - - result.trim().to_string() -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn strip_html_tags_removes_tags() { - let html = "

Hello world

"; - assert_eq!(strip_html_tags(html), "Hello world"); - } - - #[test] - fn extract_title_finds_title_tag() { - let html = "My Page"; - assert_eq!(extract_title(html).as_deref(), Some("My Page")); - } - - #[test] - fn extract_by_selector_finds_tag_content() { - let html = "

Important content

skip
"; - let result = extract_by_selector(html, "article"); - assert!(result.contains("Important content")); - assert!(!result.contains("skip")); - } - - #[test] - fn extract_by_selector_fallback_on_missing_tag() { - let html = "All the text"; - let result = extract_by_selector(html, "article"); - assert!(result.contains("All the text")); + tinycortex::memory::sources::SourceReader::read_item( + &tinycortex::memory::sources::readers::web_page::WebPageReader, + source, + item_id, + &crate::openhuman::memory::tinycortex::memory_config_from( + config, + config.workspace_dir.clone(), + ), + ) + .await + .map_err(|error| error.to_string()) } } diff --git a/src/openhuman/memory/sources/registry.rs b/src/openhuman/memory/sources/registry.rs index cf26c9115d..9065bb1a5b 100644 --- a/src/openhuman/memory/sources/registry.rs +++ b/src/openhuman/memory/sources/registry.rs @@ -40,6 +40,30 @@ pub async fn get_source(id: &str) -> Result, String> { registry().await?.get(id).map_err(|error| error.to_string()) } +/// [`get_source`] against an **explicit** config rather than the process-global +/// one. +/// +/// [`registry`] resolves its config path through +/// `config_rpc::load_config_with_timeout`, i.e. from the process environment. +/// That is right for RPC handlers, which serve the active user, and wrong for +/// the embedded memory driver +/// ([`crate::openhuman::memory::driver::embedded`]), which is bound to one +/// workspace and holds a `Config` re-anchored to it. Reading the global path +/// there would let a driver bound to workspace B answer with workspace A's +/// sources — the cross-workspace leak the workspace-keyed binding map exists to +/// prevent. +/// +/// Synchronous because the registry read itself is; only the config lookup in +/// [`registry`] was ever async. +pub(crate) fn get_source_in( + config: &crate::openhuman::config::Config, + id: &str, +) -> Result, String> { + tinycortex::memory::sources::SourceRegistry::new(config.config_path.clone()) + .get(id) + .map_err(|error| error.to_string()) +} + pub async fn add_source(entry: MemorySourceEntry) -> Result { let _guard = memory_sources_write_guard().await; log::debug!("[memory_sources] crate add kind={}", entry.kind.as_str()); diff --git a/src/openhuman/memory/sources/sync.rs b/src/openhuman/memory/sources/sync.rs index d252cfc294..ab05182316 100644 --- a/src/openhuman/memory/sources/sync.rs +++ b/src/openhuman/memory/sources/sync.rs @@ -385,3 +385,34 @@ pub(crate) fn derive_scopes(source: &MemorySourceEntry, config: &Config) -> Vec< _ => Vec::new(), } } + +#[cfg(test)] +mod tests { + use super::*; + + /// The two GitHub coordinate helpers are re-exported from tinycortex and + /// they deliberately differ: `tree_scope` slugifies to + /// `github-tinyhumansai-openhuman` while `archive_source_id` slugifies to + /// `github-com-tinyhumansai-openhuman`. Swapping the two still compiles and + /// still type-checks — it just makes reconcile scan an empty directory at + /// runtime. Pin both spellings. + #[test] + fn derive_scopes_keeps_github_tree_and_archive_ids_distinct() { + let source: MemorySourceEntry = serde_json::from_value(serde_json::json!({ + "id": "gh-scope", + "kind": "github_repo", + "label": "Repo", + "url": "https://github.com/tinyhumansai/openhuman", + })) + .expect("github source entry"); + + let scopes = derive_scopes(&source, &Config::default()); + + assert_eq!(scopes.len(), 1); + assert_eq!(scopes[0].tree_scope, "github:tinyhumansai/openhuman"); + assert_eq!( + scopes[0].archive_source_id, + "github.com/tinyhumansai/openhuman" + ); + } +} diff --git a/src/openhuman/memory/store/README.md b/src/openhuman/memory/store/README.md index fe224cdf85..3eff08576e 100644 --- a/src/openhuman/memory/store/README.md +++ b/src/openhuman/memory/store/README.md @@ -9,7 +9,8 @@ chunks/ SQLite chunk rows (metadata + tags + md path pointer + lifecycle status) + the two chunkers that produce them entities/ mem_tree_entity_index — every entity occurrence per node trees/ summary tree persistence (one table, kind-parameterized) -vectors/ local vector DB (cosine, brute-force) +vectors [tinycortex::memory::store::vectors] — local vector DB + (cosine, brute-force), moved into the TinyCortex substrate kv/ global + namespace key-value (kv_global, kv_namespace) contacts/ [removed] facade over people::store (Person/Handle/Interaction) namespace_store/ host-retained namespace documents, graph, episodic/event/ @@ -39,7 +40,7 @@ namespace_store/ host-retained namespace documents, graph, episodic/event/ | [`chunks/`](chunks/) | Full chunk lifecycle. `types.rs` (`Chunk`, `Metadata`, `SourceKind`, `RawRef`, `ListChunksQuery`) + `store.rs` (SQLite persistence + connection cache) + `produce.rs` (source-kind dispatch chunker used by the ingest pipeline) + `semantic.rs` (heading/paragraph-aware chunker). | | [`entities.rs`](entities.rs) | Thin re-export of `memory_tree::score::store` — `index_entity`, `index_entities`, `lookup_entity`, `list_entity_ids_for_node`, `clear_entity_index_for_node`, `count_entity_index`, `EntityHit`. Reads/writes the `mem_tree_entity_index` table. | | [`trees/`](trees/) | `store.rs` (`mem_tree_trees` / `mem_tree_summaries` / `mem_tree_buffers`), `types.rs` (Tree / SummaryNode / TreeKind / TreeStatus / Buffer + topic hotness types), `registry.rs` (kind-parameterized helpers), `hotness.rs` (entity hotness side-table). | -| [`vectors/`](vectors/) | Standalone vector store. `VectorStore` over SQLite, byte-codec for f32 vectors, cosine similarity. | +| `vectors/` | Moved into the TinyCortex substrate (`tinycortex::memory::store::vectors`): standalone vector store, `VectorStore` over SQLite, byte-codec for f32 vectors, cosine similarity. | | [`kv.rs`](kv.rs) | Global + namespace key-value (`kv_global`, `kv_namespace` tables). | | `contacts/` | Removed. Contact access now lives outside `memory_store` via `people::store`. | | [`namespace_store/`](namespace_store/) | Host-retained namespace/document tier over the shared SQLite database: documents, persisted product graph relations, episodic/events, segments, profile facets, and host retrieval policy. TinyCortex owns the generic chunk/vector/tree/queue substrate; this tier remains the stable `Memory` implementation. See [`namespace_store/README.md`](namespace_store/README.md). | diff --git a/src/openhuman/memory/store/chunks/connection.rs b/src/openhuman/memory/store/chunks/connection.rs index 51fd53d0c4..4cbe862892 100644 --- a/src/openhuman/memory/store/chunks/connection.rs +++ b/src/openhuman/memory/store/chunks/connection.rs @@ -4,10 +4,7 @@ use anyhow::Result; use rusqlite::Connection; use crate::openhuman::config::Config; - -fn engine_config(config: &Config) -> tinycortex::memory::MemoryConfig { - crate::openhuman::memory::tinycortex::memory_config_from(config, config.workspace_dir.clone()) -} +use crate::openhuman::memory::tinycortex::engine_config; #[doc(hidden)] pub fn with_connection(config: &Config, f: impl FnOnce(&Connection) -> Result) -> Result { diff --git a/src/openhuman/memory/store/chunks/embeddings.rs b/src/openhuman/memory/store/chunks/embeddings.rs index 6e68471ddf..429f68d44b 100644 --- a/src/openhuman/memory/store/chunks/embeddings.rs +++ b/src/openhuman/memory/store/chunks/embeddings.rs @@ -6,10 +6,7 @@ use anyhow::Result; use rusqlite::{Connection, Transaction}; use crate::openhuman::config::Config; - -fn engine_config(config: &Config) -> tinycortex::memory::MemoryConfig { - crate::openhuman::memory::tinycortex::memory_config_from(config, config.workspace_dir.clone()) -} +use crate::openhuman::memory::tinycortex::engine_config; pub(crate) fn tree_active_signature(config: &Config) -> String { tinycortex::memory::chunks::tree_active_signature(&engine_config(config)) diff --git a/src/openhuman/memory/store/chunks/mod.rs b/src/openhuman/memory/store/chunks/mod.rs index c52cabb96f..e8daa400af 100644 --- a/src/openhuman/memory/store/chunks/mod.rs +++ b/src/openhuman/memory/store/chunks/mod.rs @@ -5,24 +5,22 @@ //! - [`types`] — `Chunk`, `Metadata`, `SourceKind`, `RawRef`, //! `ListChunksQuery`. The persisted shape. //! - [`store`] — SQLite persistence (`chunks` table + connection cache). -//! - [`produce`] — source-kind-dispatch chunker (chat / email / document). -//! Used by the memory ingest pipeline; produces stable -//! per-source sequence numbers and bounded segments. //! - [`semantic`] — heading- and paragraph-aware chunker used by the //! unified memory writer to split large documents into //! LLM-context-sized pieces while preserving heading //! context. //! -//! `produce::chunk_markdown` (the default) and `semantic::chunk_markdown` -//! both yield string-shaped chunks; the store side decides what to do with -//! them. +//! The source-kind-dispatch chunker ([`chunk_markdown`], the default — chat / +//! email / document, with stable per-source sequence numbers and bounded +//! segments) is engine-owned and re-exported straight from `tinycortex`. +//! [`chunk_markdown`] and `semantic::chunk_markdown` both yield string-shaped +//! chunks; the store side decides what to do with them. -pub mod produce; pub mod semantic; pub mod store; pub mod types; -pub use produce::{chunk_markdown, ChunkerInput, ChunkerOptions}; pub use semantic::chunk_markdown as chunk_semantic; pub use store::*; +pub use tinycortex::memory::chunks::{chunk_markdown, ChunkerInput, ChunkerOptions}; pub use types::*; diff --git a/src/openhuman/memory/store/chunks/produce.rs b/src/openhuman/memory/store/chunks/produce.rs deleted file mode 100644 index 6662d2c08f..0000000000 --- a/src/openhuman/memory/store/chunks/produce.rs +++ /dev/null @@ -1,5 +0,0 @@ -//! Compatibility exports for tinycortex's source-aware chunker. - -pub use tinycortex::memory::chunks::{ - chunk_markdown, ChunkerInput, ChunkerOptions, DEFAULT_CHUNK_MAX_TOKENS, -}; diff --git a/src/openhuman/memory/store/chunks/raw_refs.rs b/src/openhuman/memory/store/chunks/raw_refs.rs index e0f87d3550..60769b696c 100644 --- a/src/openhuman/memory/store/chunks/raw_refs.rs +++ b/src/openhuman/memory/store/chunks/raw_refs.rs @@ -16,19 +16,12 @@ use anyhow::Result; use rusqlite::Transaction; use crate::openhuman::config::Config; -use crate::openhuman::memory::tinycortex::memory_config_from; +use crate::openhuman::memory::tinycortex::engine_config; // `RawRef` is re-exported from the crate (identical fields + serde derives), so // every `chunks::RawRef { path, start, end }` construction site keeps compiling. pub use tinycortex::memory::chunks::RawRef; -/// Map the host `Config` to the engine `MemoryConfig` addressing the same -/// `/memory_tree/chunks.db` (only `workspace` is load-bearing for -/// these DB ops). -fn engine_config(config: &Config) -> tinycortex::memory::MemoryConfig { - memory_config_from(config, config.workspace_dir.clone()) -} - /// Stash a list of [`RawRef`] entries on a chunk row. Replaces any previous /// value. pub fn set_chunk_raw_refs(config: &Config, chunk_id: &str, refs: &[RawRef]) -> Result<()> { diff --git a/src/openhuman/memory/store/chunks/store.rs b/src/openhuman/memory/store/chunks/store.rs index 12331ad9a5..e2b429d63e 100644 --- a/src/openhuman/memory/store/chunks/store.rs +++ b/src/openhuman/memory/store/chunks/store.rs @@ -8,16 +8,13 @@ use rusqlite::Transaction; use crate::openhuman::config::Config; use crate::openhuman::memory::store::chunks::types::{Chunk, SourceKind}; use crate::openhuman::memory::store::content::StagedChunk; +use crate::openhuman::memory::tinycortex::engine_config; pub use tinycortex::memory::chunks::{ ListChunksQuery, RawRef, CHUNK_STATUS_ADMITTED, CHUNK_STATUS_BUFFERED, CHUNK_STATUS_DROPPED, CHUNK_STATUS_PENDING_EXTRACTION, CHUNK_STATUS_SEALED, RAW_FILE_GATE_KIND, }; -fn engine_config(config: &Config) -> tinycortex::memory::MemoryConfig { - crate::openhuman::memory::tinycortex::memory_config_from(config, config.workspace_dir.clone()) -} - pub fn upsert_chunks(config: &Config, chunks: &[Chunk]) -> Result { tinycortex::memory::chunks::upsert_chunks(&engine_config(config), chunks) } diff --git a/src/openhuman/memory/store/client.rs b/src/openhuman/memory/store/client.rs index 200bd09345..0b46502615 100644 --- a/src/openhuman/memory/store/client.rs +++ b/src/openhuman/memory/store/client.rs @@ -19,7 +19,8 @@ use crate::openhuman::memory::ingestion::{ }; use crate::openhuman::memory::store::namespace_store::UnifiedMemory; use crate::openhuman::memory::store::types::{ - NamespaceDocumentInput, NamespaceMemoryHit, NamespaceRetrievalContext, + GraphRelationRecord, MemoryKvRecord, NamespaceDocumentInput, NamespaceMemoryHit, + NamespaceRetrievalContext, StoredMemoryDocument, }; /// Reference-counted handle to a `MemoryClient`. @@ -72,7 +73,12 @@ impl MemoryClient { /// want to build on top of the `Memory` trait (e.g. the /// tool-scoped memory layer) without depending on the concrete /// `MemoryClient` type or holding a reference to it. - pub fn memory_handle(&self) -> Arc { + /// + /// Intentionally `pub(crate)` — handing a raw `Arc` to an + /// external consumer bypasses any policy decorator wrapped around the + /// `MemoryClient` API, so the escape hatch stays in-crate. Mirrors + /// [`Self::profile_conn`]. + pub(crate) fn memory_handle(&self) -> Arc { Arc::clone(&self.inner) as Arc } @@ -316,6 +322,21 @@ impl MemoryClient { self.inner.list_documents(namespace).await } + /// Fetch one document by `(namespace, key)`. + /// + /// `pub(crate)` on the same reasoning as [`Self::memory_handle`]: the only + /// in-crate consumer is the embedded memory driver + /// ([`crate::openhuman::memory::driver::embedded`]), which needs a read-one + /// path that [`Self::list_documents`] cannot provide — the latter's SELECT + /// carries no `content` column. + pub(crate) async fn get_document( + &self, + namespace: &str, + key: &str, + ) -> Result, String> { + self.inner.get_document_by_key(namespace, key).await + } + /// List all unique namespaces in the memory store. pub async fn list_namespaces(&self) -> Result, String> { self.inner.list_namespaces().await @@ -446,6 +467,63 @@ impl MemoryClient { } } + /// Typed key/value records for one namespace, or the global slice when + /// `namespace` is `None`. + /// + /// `pub(crate)` for the embedded driver. Distinct from + /// [`Self::kv_list_namespace`], which returns a camelCase + /// `Vec` with no `updated_at` and no global slice — + /// re-parsing that back into [`MemoryKvRecord`] would be lossy new logic. + pub(crate) async fn kv_records( + &self, + namespace: Option<&str>, + ) -> Result, String> { + match namespace { + Some(ns) => self.inner.kv_records_namespace(ns).await, + None => self.inner.kv_records_global().await, + } + } + + /// Typed relation records, filtered by subject/predicate. + /// + /// `namespace: None` spans every namespace *and* the global graph, matching + /// [`Self::graph_query`]'s `None` behaviour. `pub(crate)` for the embedded + /// driver, for the same reason as [`Self::kv_records`]: `graph_query` + /// returns camelCase JSON, these return the record type directly. + /// + /// Inherits the storage layer's hard `LIMIT 300` per SQL statement. + pub(crate) async fn graph_relations( + &self, + namespace: Option<&str>, + subject: Option<&str>, + predicate: Option<&str>, + ) -> Result, String> { + match namespace { + Some(ns) => { + self.inner + .graph_relations_namespace(ns, subject, predicate) + .await + } + None => { + let mut rows = self + .inner + .graph_relations_all_namespaces(subject, predicate) + .await?; + rows.extend( + self.inner + .graph_relations_global(subject, predicate) + .await?, + ); + rows.sort_by(|a, b| { + b.updated_at + .partial_cmp(&a.updated_at) + .unwrap_or(std::cmp::Ordering::Equal) + }); + Ok(rows) + } + } + } + /// List all key-value pairs in a namespace. pub async fn kv_list_namespace( &self, diff --git a/src/openhuman/memory/store/content/README.md b/src/openhuman/memory/store/content/README.md index 2d7d31fb66..6fee154607 100644 --- a/src/openhuman/memory/store/content/README.md +++ b/src/openhuman/memory/store/content/README.md @@ -12,9 +12,9 @@ The body is **immutable** once written — only the YAML front-matter `tags:` bl - `vendor/tinycortex/src/memory/store/content/paths.rs` — path generators. `chunk_rel_path` (`email//.md`, `chat//.md`, `document//.md`); `summary_rel_path` (`summaries/{source,global,topic}/…`). `slugify_source_id` is the canonical filesystem-safe slug. - [`read.rs`](read.rs) — `read_chunk_file` / `read_summary_file` parse front-matter and return body+SHA. `verify_*` compares against an expected SHA. `read_chunk_body` / `read_summary_body` resolve the path via SQLite and verify the integrity hash; this is the authoritative entry-point for callers that need the **full** body (LLM extractor, summariser, embedder, retrieval API). - `vendor/tinycortex/src/memory/store/content/raw.rs` — verbatim source-byte mirror under `/raw/`. Writes the unmodified upstream payload (eml, slack json, raw markdown) so downstream callers can re-canonicalise without re-fetching. -- [`obsidian.rs`](obsidian.rs) + [`obsidian_defaults/`](obsidian_defaults/) — bootstrap an `.obsidian/` config (workspace, graph, app) into the content root on first write so a user opening the vault gets a usable view. +- `vendor/tinycortex/src/memory/store/content/obsidian.rs` + `obsidian_defaults/` (`obsidian` feature) — bootstrap an `.obsidian/` config (workspace, graph, app) into the content root on first write so a user opening the vault gets a usable view. - [`tags.rs`](tags.rs) — post-extraction tag rewrites. `update_chunk_tags` (atomic tempfile rewrite of the `tags:` block) and `update_summary_tags` (fetches entities from `mem_tree_entity_index`, builds Obsidian `kind/Value` tags, rewrites, verifies body SHA is unchanged). `slugify_tag_kind`, `slugify_tag_value`, `entity_tag` build the tag strings. -- [`wiki_git/`](wiki_git/) — initializes `/wiki/.git`, commits only summary-node markdown under `summaries/**` plus the repo `.gitignore`, and stores read high-water marks as lightweight `refs/tags/read/*` pointers. Summary files are staged by `atomic.rs`; seal/ingest callers create descriptive git commits after SQLite persistence succeeds. +- `vendor/tinycortex/src/memory/store/content/wiki_git/` (`wiki-git` feature) — initializes `/wiki/.git`, commits only summary-node markdown under `summaries/**` plus the repo `.gitignore`, and stores read high-water marks as lightweight `refs/tags/read/*` pointers. Summary files are staged by `atomic.rs`; seal/ingest callers create descriptive git commits after SQLite persistence succeeds. ## Integrity contract diff --git a/src/openhuman/memory/store/content/mod.rs b/src/openhuman/memory/store/content/mod.rs index 0032c6be0c..eb17623422 100644 --- a/src/openhuman/memory/store/content/mod.rs +++ b/src/openhuman/memory/store/content/mod.rs @@ -11,16 +11,16 @@ //! - [`atomic`] — tempfile+fsync+rename writes; SHA-256; `stage_summary` //! - [`read`] — read + SHA-256 verification + `split_front_matter`; summary variants //! - [`tags`] — `update_chunk_tags` + `update_summary_tags` + slugifiers +//! - `obsidian` / `obsidian_registry` / `wiki_git` — on-disk content formats, +//! owned by TinyCortex and re-exported here (see the `pub use` below) -pub mod obsidian; -pub mod obsidian_registry; pub mod read; pub mod tags; -pub mod wiki_git; pub use tinycortex::memory::chunks::StagedChunk; pub use tinycortex::memory::store::content::{ - atomic, compose, paths, raw, stage_chunks, StagedSummary, SummaryComposeInput, SummaryTreeKind, + atomic, compose, obsidian, obsidian_registry, paths, raw, stage_chunks, wiki_git, + StagedSummary, SummaryComposeInput, SummaryTreeKind, }; /// Update the `tags:` block in a summary's on-disk `.md` file after an diff --git a/src/openhuman/memory/store/content/obsidian.rs b/src/openhuman/memory/store/content/obsidian.rs deleted file mode 100644 index 0217765e3c..0000000000 --- a/src/openhuman/memory/store/content/obsidian.rs +++ /dev/null @@ -1,143 +0,0 @@ -//! Obsidian vault defaults. -//! -//! When the memory_tree content root is first populated we drop a small -//! `.obsidian/` directory into it so a user opening the vault gets the -//! intended graph-view colour mapping (one colour per summary level) and -//! the front-matter type hints (`time_range_*` as `date`, `sealed_at` as -//! `datetime`) without any manual configuration. -//! -//! The bundled defaults live as static files under `obsidian_defaults/` -//! and are baked into the binary via `include_str!`. We only stage them -//! when the corresponding `.obsidian/` doesn't already exist — -//! never overwrite a file the user has tweaked. -//! -//! Callers should invoke [`ensure_obsidian_defaults`] from any code path -//! that creates files under `content_root` (summary stage, raw write, -//! etc.). The function is idempotent and cheap on the steady-state path -//! (one `Path::exists()` per file). -//! -//! Failure mode: best-effort. A failed stage logs a warn and returns -//! `Ok(())` so seal/raw-write callers don't abort persistence over a -//! cosmetic vault default. - -use std::path::Path; - -use anyhow::Result; - -const GRAPH_JSON: &str = include_str!("obsidian_defaults/graph.json"); -const TYPES_JSON: &str = include_str!("obsidian_defaults/types.json"); - -/// Write the bundled `.obsidian/` defaults into `content_root` if they -/// aren't already there. Idempotent — never overwrites existing files. -pub fn ensure_obsidian_defaults(content_root: &Path) -> Result<()> { - let obsidian_dir = content_root.join(".obsidian"); - if let Err(err) = std::fs::create_dir_all(&obsidian_dir) { - log::warn!( - "[content_store::obsidian] create .obsidian dir failed at {:?}: {err:#} — skipping defaults", - obsidian_dir - ); - return Ok(()); - } - - write_default_if_missing(&obsidian_dir, "graph.json", GRAPH_JSON); - write_default_if_missing(&obsidian_dir, "types.json", TYPES_JSON); - Ok(()) -} - -fn write_default_if_missing(obsidian_dir: &Path, name: &str, body: &str) { - use std::io::{ErrorKind, Write}; - let target = obsidian_dir.join(name); - // `create_new(true)` makes existence-check + create atomic at the - // OS level, so a concurrent staging from another process can't - // race past `target.exists()` and clobber the winner. The - // AlreadyExists branch is the steady-state idempotent no-op. - let mut file = match std::fs::OpenOptions::new() - .write(true) - .create_new(true) - .open(&target) - { - Ok(f) => f, - Err(err) if err.kind() == ErrorKind::AlreadyExists => return, - Err(err) => { - log::warn!( - "[content_store::obsidian] create default {} failed at {:?}: {err:#}", - name, - target - ); - return; - } - }; - match file.write_all(body.as_bytes()) { - Ok(()) => log::info!( - "[content_store::obsidian] staged default {} at {}", - name, - target.display() - ), - Err(err) => { - // `create_new` already produced an empty file at `target`; - // a write_all failure (disk full, transient I/O) leaves a - // truncated remnant. Without cleanup, the next call hits - // the AlreadyExists fast-path and never repairs the bad - // file. Remove it so the next call retries cleanly. - if let Err(cleanup_err) = std::fs::remove_file(&target) { - log::warn!( - "[content_store::obsidian] cleanup partial default {} failed at {:?}: {cleanup_err:#}", - name, - target - ); - } - log::warn!( - "[content_store::obsidian] write default {} failed at {:?}: {err:#}", - name, - target - ); - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - use tempfile::TempDir; - - #[test] - fn stages_defaults_into_fresh_root() { - let tmp = TempDir::new().unwrap(); - ensure_obsidian_defaults(tmp.path()).unwrap(); - let graph = tmp.path().join(".obsidian").join("graph.json"); - let types = tmp.path().join(".obsidian").join("types.json"); - assert!(graph.exists(), "graph.json should be staged"); - assert!(types.exists(), "types.json should be staged"); - // Body must be the bundled content, not empty. - let g = std::fs::read_to_string(&graph).unwrap(); - assert!(g.contains("colorGroups"), "graph.json missing colorGroups"); - } - - #[test] - fn does_not_overwrite_existing_file() { - let tmp = TempDir::new().unwrap(); - let obs = tmp.path().join(".obsidian"); - std::fs::create_dir_all(&obs).unwrap(); - let graph = obs.join("graph.json"); - std::fs::write(&graph, r#"{"user":"custom"}"#).unwrap(); - - ensure_obsidian_defaults(tmp.path()).unwrap(); - - let body = std::fs::read_to_string(&graph).unwrap(); - assert_eq!( - body, r#"{"user":"custom"}"#, - "user-customised graph.json must not be clobbered" - ); - } - - #[test] - fn idempotent_second_call_is_no_op() { - let tmp = TempDir::new().unwrap(); - ensure_obsidian_defaults(tmp.path()).unwrap(); - ensure_obsidian_defaults(tmp.path()).unwrap(); - // Second call must succeed without panicking and must not have - // duplicated or grown the file. - let g = std::fs::read_to_string(tmp.path().join(".obsidian/graph.json")).unwrap(); - assert!(g.contains("colorGroups")); - } -} diff --git a/src/openhuman/memory/store/content/obsidian_defaults/graph.json b/src/openhuman/memory/store/content/obsidian_defaults/graph.json deleted file mode 100644 index a582d66a32..0000000000 --- a/src/openhuman/memory/store/content/obsidian_defaults/graph.json +++ /dev/null @@ -1,65 +0,0 @@ -{ - "collapse-filter": false, - "search": "", - "showTags": false, - "showAttachments": false, - "hideUnresolved": true, - "showOrphans": true, - "collapse-color-groups": false, - "colorGroups": [ - { - "query": "path:L1", - "color": { - "a": 1, - "rgb": 14701138 - } - }, - { - "query": "path:L2", - "color": { - "a": 1, - "rgb": 14725458 - } - }, - { - "query": "path:L3", - "color": { - "a": 1, - "rgb": 11657298 - } - }, - { - "query": "path:L4", - "color": { - "a": 1, - "rgb": 5420768 - } - }, - { - "query": "path:L5", - "color": { - "a": 1, - "rgb": 5431504 - } - }, - { - "query": "path:L6", - "color": { - "a": 1, - "rgb": 14701261 - } - } - ], - "collapse-display": false, - "showArrow": false, - "textFadeMultiplier": 0.9, - "nodeSizeMultiplier": 1.34371527777778, - "lineSizeMultiplier": 1.44048177083333, - "collapse-forces": false, - "centerStrength": 0.493880208333333, - "repelStrength": 10, - "linkStrength": 1, - "linkDistance": 250, - "scale": 0.5443310539518227, - "close": false -} \ No newline at end of file diff --git a/src/openhuman/memory/store/content/obsidian_defaults/types.json b/src/openhuman/memory/store/content/obsidian_defaults/types.json deleted file mode 100644 index 34f567621e..0000000000 --- a/src/openhuman/memory/store/content/obsidian_defaults/types.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "types": { - "aliases": "aliases", - "cssclasses": "multitext", - "tags": "tags", - "time_range_end": "date", - "time_range_start": "date", - "sealed_at": "datetime" - } -} \ No newline at end of file diff --git a/src/openhuman/memory/store/content/obsidian_registry.rs b/src/openhuman/memory/store/content/obsidian_registry.rs deleted file mode 100644 index 2f4555e234..0000000000 --- a/src/openhuman/memory/store/content/obsidian_registry.rs +++ /dev/null @@ -1,321 +0,0 @@ -//! Obsidian vault-*registration* detection. -//! -//! Sibling to [`super::obsidian`] (which writes the `.obsidian/` *defaults* -//! into the content root). This module answers a different question: is the -//! content root actually a vault Obsidian knows about? -//! -//! `obsidian://open?path=` only resolves against vaults already recorded -//! in Obsidian's `obsidian.json` registry — it can **not** register a new -//! vault, and a `.obsidian/` folder on disk is not enough. So before the -//! Memory tab fires that deep link we check whether the content root (or an -//! ancestor) is a registered vault. If it isn't, the UI guides the user to add -//! it once ("Open folder as vault") instead of firing a link Obsidian rejects -//! with *"Unable to find a vault for the URL"*. -//! -//! Detection is **best-effort**: Obsidian can live in non-standard locations -//! (Flatpak, Snap, custom `$XDG_CONFIG_HOME`, portable). A negative result must -//! never block the user — the caller still offers "open anyway" + "reveal -//! folder" + a config-dir override that feeds back in here as `extra`. - -use std::path::{Path, PathBuf}; - -use serde::Deserialize; - -/// Outcome of a registration probe. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct VaultRegistration { - /// `true` when some registered Obsidian vault's path equals or is an - /// ancestor of the content root. - pub registered: bool, - /// `true` when at least one candidate `obsidian.json` was found/read (even - /// if parsing it later fails — see the parse-error branch, which still - /// counts the file as found). Lets the UI distinguish "Obsidian is set up, - /// vault just not added yet" from "couldn't find Obsidian at all" (offer - /// install vs. offer add-as-vault). - pub config_found: bool, -} - -/// Minimal shape of Obsidian's `obsidian.json`. We only need each vault's -/// `path`; `ts`/`open` and any future keys are ignored by `serde`. -#[derive(Debug, Deserialize)] -struct ObsidianConfig { - #[serde(default)] - vaults: std::collections::HashMap, -} - -#[derive(Debug, Deserialize)] -struct VaultEntry { - path: String, -} - -/// Candidate `obsidian.json` locations, in priority order. `extra` (a -/// user-supplied override pointing at Obsidian's *config dir*) is checked -/// first so a power user can correct a non-standard install. -fn candidate_config_files(extra: Option<&Path>) -> Vec { - let mut out = Vec::new(); - - if let Some(dir) = extra { - // Accept either the config dir itself or its parent (users often - // can't tell whether the path should end in `obsidian/`). - out.push(dir.join("obsidian.json")); - out.push(dir.join("obsidian").join("obsidian.json")); - } - - // Standard per-OS config dir: `~/.config` (Linux), `~/Library/Application - // Support` (macOS), `%APPDATA%` (Windows). - if let Some(cfg) = dirs::config_dir() { - out.push(cfg.join("obsidian").join("obsidian.json")); - } - - // Linux sandbox installs keep their own config tree. Harmless to probe on - // other OSes — the paths simply won't exist. - if let Some(home) = dirs::home_dir() { - out.push(home.join(".var/app/md.obsidian.Obsidian/config/obsidian/obsidian.json")); // Flatpak - out.push(home.join("snap/obsidian/current/.config/obsidian/obsidian.json")); - // Snap - } - - out -} - -/// Best-effort: is `content_root` (or an ancestor) a registered Obsidian -/// vault? `extra_config_dir` optionally points at Obsidian's config dir for -/// non-standard installs. Never errors — probe failures report -/// `registered = false`. -pub fn vault_registration_status( - content_root: &Path, - extra_config_dir: Option<&Path>, -) -> VaultRegistration { - registration_in_files(content_root, &candidate_config_files(extra_config_dir)) -} - -/// Core of [`vault_registration_status`], split out so tests can supply an -/// explicit, isolated set of `obsidian.json` paths instead of depending on -/// whatever Obsidian config happens to exist on the host. -fn registration_in_files(content_root: &Path, files: &[PathBuf]) -> VaultRegistration { - let target = lexically_normalize(content_root); - let mut config_found = false; - - for path in files { - let body = match std::fs::read_to_string(path) { - Ok(b) => b, - Err(_) => continue, // missing/unreadable candidate — try the next. - }; - config_found = true; - - let parsed: ObsidianConfig = match serde_json::from_str(&body) { - Ok(p) => p, - Err(err) => { - // Redact the path — it embeds the user's home/username. - log::warn!( - "[content_store::obsidian_registry] parse {} failed: {err} — skipping", - crate::openhuman::memory::util::redact::redact(&path.display().to_string()) - ); - continue; - } - }; - - for entry in parsed.vaults.values() { - let vault = lexically_normalize(Path::new(&entry.path)); - // A malformed/empty vault path normalizes to "" and would otherwise - // match every content root (empty ancestor ⊂ anything) — skip it. - if vault.as_os_str().is_empty() { - continue; - } - if is_ancestor_or_equal(&vault, &target) { - log::debug!( - "[content_store::obsidian_registry] content root is a registered vault \ - (matched in {})", - crate::openhuman::memory::util::redact::redact(&path.display().to_string()) - ); - return VaultRegistration { - registered: true, - config_found: true, - }; - } - } - } - - log::debug!( - "[content_store::obsidian_registry] content root NOT registered (config_found={})", - config_found - ); - VaultRegistration { - registered: false, - config_found, - } -} - -/// Strip trailing separators so `/a/b` and `/a/b/` compare equal. Lexical -/// only — we deliberately do not canonicalize: the vault path may be on an -/// unmounted volume or use a symlink, and canonicalize would error or rewrite -/// it. Both inputs come from trusted local sources, so a textual compare is -/// the safe, dependency-free choice. -fn lexically_normalize(p: &Path) -> PathBuf { - let s = p.to_string_lossy(); - let trimmed = s.trim_end_matches(['/', '\\']); - if trimmed.is_empty() { - // Was a pure root like "/" — keep it. - PathBuf::from(s.as_ref()) - } else { - PathBuf::from(trimmed) - } -} - -/// `true` when `ancestor == descendant`, or `ancestor` is a path-prefix of -/// `descendant` on component boundaries (so `/a/b` contains `/a/b/c` but not -/// `/a/bc`). Case-sensitive — adequate for the Linux target; a false negative -/// on case-insensitive volumes only makes detection conservative (the caller -/// still offers "open anyway"). -fn is_ancestor_or_equal(ancestor: &Path, descendant: &Path) -> bool { - let a: Vec<_> = ancestor.components().collect(); - let d: Vec<_> = descendant.components().collect(); - // An empty ancestor must not match (it would otherwise be a prefix of - // everything); also bail when the ancestor is longer than the descendant. - if a.is_empty() || a.len() > d.len() { - return false; - } - a.iter().zip(d.iter()).all(|(x, y)| x == y) -} - -#[cfg(test)] -mod tests { - use super::*; - use std::io::Write; - - /// Write an `obsidian.json` containing `vault_paths` and return its path. - fn write_config(dir: &Path, vault_paths: &[&str]) -> PathBuf { - let entries: Vec = vault_paths - .iter() - .enumerate() - .map(|(i, p)| { - format!( - "\"id{i}\": {{ \"path\": {}, \"ts\": 1700000000000, \"open\": true }}", - serde_json::to_string(p).unwrap() - ) - }) - .collect(); - let body = format!("{{ \"vaults\": {{ {} }} }}", entries.join(", ")); - let path = dir.join("obsidian.json"); - let mut f = std::fs::File::create(&path).unwrap(); - f.write_all(body.as_bytes()).unwrap(); - path - } - - #[test] - fn exact_match_is_registered() { - let tmp = tempfile::tempdir().unwrap(); - let root = tmp.path().join("memory_tree/content"); - let cfg = write_config(tmp.path(), &[root.to_str().unwrap()]); - let got = registration_in_files(&root, &[cfg]); - assert_eq!( - got, - VaultRegistration { - registered: true, - config_found: true - } - ); - } - - #[test] - fn ancestor_vault_is_registered() { - // A vault rooted at the parent still "contains" the content root. - let tmp = tempfile::tempdir().unwrap(); - let parent = tmp.path().join("workspace"); - let root = parent.join("memory_tree/content"); - let cfg = write_config(tmp.path(), &[parent.to_str().unwrap()]); - assert!(registration_in_files(&root, &[cfg]).registered); - } - - #[test] - fn trailing_slash_does_not_matter() { - let tmp = tempfile::tempdir().unwrap(); - let root = tmp.path().join("memory_tree/content"); - let with_slash = format!("{}/", root.to_str().unwrap()); - let cfg = write_config(tmp.path(), &[&with_slash]); - assert!(registration_in_files(&root, &[cfg]).registered); - } - - #[test] - fn unrelated_vault_is_not_registered_but_config_found() { - let tmp = tempfile::tempdir().unwrap(); - let root = tmp.path().join("memory_tree/content"); - let cfg = write_config(tmp.path(), &["/some/other/vault"]); - let got = registration_in_files(&root, &[cfg]); - assert_eq!( - got, - VaultRegistration { - registered: false, - config_found: true - } - ); - } - - #[test] - fn empty_vault_path_does_not_match_every_root() { - // Regression: a malformed entry with an empty `path` must not - // normalize to "" and match every content root as an ancestor. - let tmp = tempfile::tempdir().unwrap(); - let root = tmp.path().join("memory_tree/content"); - let cfg = write_config(tmp.path(), &[""]); - let got = registration_in_files(&root, &[cfg]); - assert_eq!( - got, - VaultRegistration { - registered: false, - config_found: true - } - ); - } - - #[test] - fn sibling_prefix_is_not_a_false_match() { - // `/a/b/content` must NOT match a vault at `/a/b/content-archive`. - let tmp = tempfile::tempdir().unwrap(); - let root = tmp.path().join("content"); - let decoy = format!("{}-archive", root.to_str().unwrap()); - let cfg = write_config(tmp.path(), &[&decoy]); - assert!(!registration_in_files(&root, &[cfg]).registered); - } - - #[test] - fn missing_config_reports_not_found() { - let tmp = tempfile::tempdir().unwrap(); - let root = tmp.path().join("memory_tree/content"); - let missing = tmp.path().join("does-not-exist.json"); - let got = registration_in_files(&root, &[missing]); - assert_eq!( - got, - VaultRegistration { - registered: false, - config_found: false - } - ); - } - - #[test] - fn malformed_config_is_skipped_not_fatal() { - let tmp = tempfile::tempdir().unwrap(); - let root = tmp.path().join("memory_tree/content"); - let bad = tmp.path().join("obsidian.json"); - std::fs::write(&bad, b"{ this is not json ").unwrap(); - // config_found is true (we read it) but parse fails → not registered. - let got = registration_in_files(&root, &[bad]); - assert_eq!( - got, - VaultRegistration { - registered: false, - config_found: true - } - ); - } - - #[test] - fn second_candidate_wins_when_first_missing() { - let tmp = tempfile::tempdir().unwrap(); - let root = tmp.path().join("memory_tree/content"); - let missing = tmp.path().join("nope.json"); - let real = write_config(tmp.path(), &[root.to_str().unwrap()]); - assert!(registration_in_files(&root, &[missing, real]).registered); - } -} diff --git a/src/openhuman/memory/store/content/read.rs b/src/openhuman/memory/store/content/read.rs index d21de3295f..b17828c629 100644 --- a/src/openhuman/memory/store/content/read.rs +++ b/src/openhuman/memory/store/content/read.rs @@ -1,24 +1,21 @@ //! Product Config adapters over tinycortex content readers. +use crate::openhuman::memory::tinycortex::engine_config; pub use tinycortex::memory::store::content::{ read_chunk_file, read_summary_file, verify_chunk_file, verify_summary_file, ChunkFileContents, VerifyResult, }; -fn memory_config(config: &crate::openhuman::config::Config) -> tinycortex::memory::MemoryConfig { - crate::openhuman::memory::tinycortex::memory_config_from(config, config.workspace_dir.clone()) -} - pub fn read_chunk_body( config: &crate::openhuman::config::Config, chunk_id: &str, ) -> anyhow::Result { - tinycortex::memory::store::content::read_chunk_body(&memory_config(config), chunk_id) + tinycortex::memory::store::content::read_chunk_body(&engine_config(config), chunk_id) } pub fn read_summary_body( config: &crate::openhuman::config::Config, summary_id: &str, ) -> anyhow::Result { - tinycortex::memory::store::content::read_summary_body(&memory_config(config), summary_id) + tinycortex::memory::store::content::read_summary_body(&engine_config(config), summary_id) } diff --git a/src/openhuman/memory/store/content/wiki_git/mod.rs b/src/openhuman/memory/store/content/wiki_git/mod.rs deleted file mode 100644 index 1766784785..0000000000 --- a/src/openhuman/memory/store/content/wiki_git/mod.rs +++ /dev/null @@ -1,364 +0,0 @@ -//! Git history for derived wiki summary nodes. -//! -//! The repository lives at `/wiki/.git` and intentionally tracks -//! only summary-node markdown (`summaries/**`) plus its own restrictive -//! `.gitignore`. Raw source mirrors, chunk intermediates, Obsidian defaults, -//! and future non-summary wiki artifacts are left out of history. - -use std::path::{Path, PathBuf}; -use std::sync::Mutex; - -use anyhow::{Context, Result}; -use chrono::{DateTime, Utc}; -use git2::{ErrorCode, Oid, Repository, RepositoryOpenFlags, Signature}; - -use super::paths::WIKI_PREFIX; - -static WIKI_GIT_LOCK: Mutex<()> = Mutex::new(()); - -const SIG_NAME: &str = "OpenHuman Memory"; -const SIG_EMAIL: &str = "memory-wiki@openhuman.local"; -const GITIGNORE_BODY: &str = "*\n!/.gitignore\n!/summaries/\n!/summaries/**\n"; - -/// Metadata for one summary node included in a wiki git commit. -#[derive(Clone, Debug)] -pub struct SummaryCommitEntry { - pub summary_id: String, - pub content_path: String, - pub level: u32, - pub child_count: usize, - pub token_count: u32, - pub time_range_start: DateTime, - pub time_range_end: DateTime, -} - -/// Metadata for one tree seal represented as a wiki git commit. -#[derive(Clone, Debug)] -pub struct SummaryCommitBatch { - pub reason: String, - pub tree_id: String, - pub tree_scope: String, - pub entries: Vec, -} - -/// Ensure the wiki repository exists and has a commit containing the supplied -/// summary files. Existing non-summary tracked entries are removed from the -/// index so history stays scoped to summary nodes only. -pub fn commit_summaries(content_root: &Path, batch: &SummaryCommitBatch) -> Result<()> { - if batch.entries.is_empty() { - return Ok(()); - } - let summary_repo_paths: Vec = batch - .entries - .iter() - .map(|entry| summary_repo_path(&entry.content_path)) - .collect::>>()?; - let _guard = WIKI_GIT_LOCK.lock().expect("memory wiki git lock poisoned"); - - let repo = open_prepared_repo(content_root)?; - let wiki_root = content_root.join(WIKI_PREFIX); - - let mut index = repo.index().context("open wiki git index")?; - prune_stale_or_non_summary_entries(&mut index, repo.workdir().unwrap_or(&wiki_root))?; - index - .add_path(Path::new(".gitignore")) - .context("stage wiki .gitignore")?; - for path in &summary_repo_paths { - index - .add_path(Path::new(path)) - .with_context(|| format!("stage wiki summary: {path}"))?; - } - stage_existing_summary_paths(&mut index, &wiki_root)?; - index - .write() - .context("write wiki git index after staging summary")?; - - commit_index_if_changed(&repo, batch) -} - -/// Add a timestamped lightweight git tag that represents a reader's high-water -/// mark, and move a stable `latest` alias for quick lookup. -/// -/// This writes `refs/tags/read//` -/// to `target_commit`, or to wiki `HEAD` when `target_commit` is `None`, and -/// also updates `refs/tags/read//latest`. Tags update read -/// state without creating another history commit. -pub fn set_read_pointer_tag( - content_root: &Path, - pointer_id: &str, - target_commit: Option<&str>, -) -> Result { - let _guard = WIKI_GIT_LOCK.lock().expect("memory wiki git lock poisoned"); - let repo = open_prepared_repo(content_root)?; - let oid = match target_commit { - Some(commit) => { - Oid::from_str(commit).with_context(|| format!("bad commit id: {commit}"))? - } - None => repo.head()?.peel_to_commit()?.id(), - }; - let tag_ref = read_pointer_timestamp_ref(pointer_id, Utc::now()); - repo.reference(&tag_ref, oid, true, "advance memory wiki read pointer") - .with_context(|| format!("set wiki read pointer tag: {tag_ref}"))?; - let latest_ref = read_pointer_latest_ref(pointer_id); - repo.reference( - &latest_ref, - oid, - true, - "advance latest memory wiki read pointer", - ) - .with_context(|| format!("set latest wiki read pointer tag: {latest_ref}"))?; - log::debug!( - "[content_store::wiki_git] advanced read pointer tags {} latest={} -> {}", - tag_ref, - latest_ref, - oid - ); - Ok(oid.to_string()) -} - -/// Return the commit id a read-pointer tag currently references. -pub fn get_read_pointer_tag(content_root: &Path, pointer_id: &str) -> Result> { - let _guard = WIKI_GIT_LOCK.lock().expect("memory wiki git lock poisoned"); - let wiki_root = content_root.join(WIKI_PREFIX); - let repo = match open_existing_repo(&wiki_root) { - Ok(repo) => repo, - Err(err) if err.code() == ErrorCode::NotFound => return Ok(None), - Err(err) => return Err(err).context("open wiki git repo for read pointer"), - }; - let tag_ref = read_pointer_latest_ref(pointer_id); - let target = match repo.find_reference(&tag_ref) { - Ok(reference) => Ok(reference.target().map(|oid| oid.to_string())), - Err(err) if err.code() == ErrorCode::NotFound => Ok(None), - Err(err) => Err(err).with_context(|| format!("find wiki read pointer tag: {tag_ref}")), - }; - target -} - -fn open_prepared_repo(content_root: &Path) -> Result { - let wiki_root = content_root.join(WIKI_PREFIX); - std::fs::create_dir_all(&wiki_root) - .with_context(|| format!("create wiki git root: {}", wiki_root.display()))?; - - let repo = open_or_init_repo(&wiki_root)?; - ensure_gitignore(&wiki_root)?; - Ok(repo) -} - -fn open_or_init_repo(wiki_root: &Path) -> Result { - match open_existing_repo(wiki_root) { - Ok(repo) => Ok(repo), - Err(err) if err.code() == ErrorCode::NotFound => { - log::debug!( - "[content_store::wiki_git] initialising summary wiki git repo at {}", - wiki_root.display() - ); - Repository::init(wiki_root) - .with_context(|| format!("init wiki git repo: {}", wiki_root.display())) - } - Err(err) => { - Err(err).with_context(|| format!("open wiki git repo: {}", wiki_root.display())) - } - } -} - -fn open_existing_repo(wiki_root: &Path) -> Result { - Repository::open_ext( - wiki_root, - RepositoryOpenFlags::NO_SEARCH, - &[] as &[&std::ffi::OsStr], - ) -} - -fn ensure_gitignore(wiki_root: &Path) -> Result<()> { - let path = wiki_root.join(".gitignore"); - match std::fs::read_to_string(&path) { - Ok(existing) if existing == GITIGNORE_BODY => Ok(()), - _ => { - std::fs::write(&path, GITIGNORE_BODY) - .with_context(|| format!("write wiki gitignore: {}", path.display()))?; - log::debug!( - "[content_store::wiki_git] wrote summary-only .gitignore at {}", - path.display() - ); - Ok(()) - } - } -} - -fn prune_stale_or_non_summary_entries(index: &mut git2::Index, wiki_root: &Path) -> Result<()> { - let to_remove: Vec = index - .iter() - .filter_map(|entry| { - let path = std::str::from_utf8(&entry.path).ok()?; - if should_keep_index_entry(wiki_root, path) { - None - } else { - Some(PathBuf::from(path)) - } - }) - .collect(); - - for path in to_remove { - index - .remove_path(&path) - .with_context(|| format!("remove non-summary wiki git entry: {}", path.display()))?; - } - Ok(()) -} - -fn should_keep_index_entry(wiki_root: &Path, path: &str) -> bool { - if !is_tracked_wiki_path(path) { - return false; - } - path == ".gitignore" || wiki_root.join(path).exists() -} - -fn is_tracked_wiki_path(path: &str) -> bool { - path == ".gitignore" || path.starts_with("summaries/") -} - -fn stage_existing_summary_paths(index: &mut git2::Index, wiki_root: &Path) -> Result<()> { - let summaries_root = wiki_root.join("summaries"); - if !summaries_root.exists() { - return Ok(()); - } - stage_summary_dir(index, wiki_root, &summaries_root) -} - -fn stage_summary_dir(index: &mut git2::Index, wiki_root: &Path, dir: &Path) -> Result<()> { - for entry in - std::fs::read_dir(dir).with_context(|| format!("read summary dir: {}", dir.display()))? - { - let entry = entry.with_context(|| format!("read summary dir entry: {}", dir.display()))?; - let path = entry.path(); - if path.is_dir() { - stage_summary_dir(index, wiki_root, &path)?; - } else if path.is_file() { - let repo_path = path - .strip_prefix(wiki_root) - .with_context(|| format!("summary path outside wiki root: {}", path.display()))?; - index - .add_path(repo_path) - .with_context(|| format!("stage existing wiki summary: {}", repo_path.display()))?; - } - } - Ok(()) -} - -fn commit_index_if_changed(repo: &Repository, batch: &SummaryCommitBatch) -> Result<()> { - let tree_oid = repo.index()?.write_tree()?; - let tree = repo.find_tree(tree_oid)?; - - let parent_commit = match repo.head() { - Ok(head) => Some(head.peel_to_commit()?), - Err(_) => None, - }; - - if let Some(parent) = &parent_commit { - if parent.tree_id() == tree_oid { - log::debug!( - "[content_store::wiki_git] summary wiki git clean after staging tree_id={} entries={}", - batch.tree_id, - batch.entries.len() - ); - return Ok(()); - } - } - - let sig = Signature::now(SIG_NAME, SIG_EMAIL).context("build wiki git signature")?; - let message = build_commit_message(batch); - let parents: Vec<&git2::Commit> = parent_commit.iter().collect(); - let commit_oid = repo - .commit(Some("HEAD"), &sig, &sig, &message, &tree, &parents) - .context("commit wiki summary update")?; - - log::debug!( - "[content_store::wiki_git] committed summary wiki update commit={} tree_id={} entries={}", - commit_oid, - batch.tree_id, - batch.entries.len() - ); - Ok(()) -} - -fn build_commit_message(batch: &SummaryCommitBatch) -> String { - let mut min_level = u32::MAX; - let mut max_level = 0; - let mut child_count = 0usize; - let mut token_count = 0u32; - let mut start: Option> = None; - let mut end: Option> = None; - - for entry in &batch.entries { - min_level = min_level.min(entry.level); - max_level = max_level.max(entry.level); - child_count = child_count.saturating_add(entry.child_count); - token_count = token_count.saturating_add(entry.token_count); - start = Some(start.map_or(entry.time_range_start, |s| s.min(entry.time_range_start))); - end = Some(end.map_or(entry.time_range_end, |e| e.max(entry.time_range_end))); - } - - let level_label = if min_level == max_level { - format!("L{min_level}") - } else { - format!("L{min_level}-L{max_level}") - }; - let title = format!( - "Seal memory tree {} {} summaries", - batch.tree_scope, level_label - ); - - let mut msg = String::new(); - msg.push_str(&title); - msg.push_str("\n\n"); - msg.push_str(&format!("Reason: {}\n", batch.reason)); - msg.push_str(&format!("Tree-Id: {}\n", batch.tree_id)); - msg.push_str(&format!("Tree-Scope: {}\n", batch.tree_scope)); - msg.push_str(&format!("Summary-Count: {}\n", batch.entries.len())); - msg.push_str(&format!("Level-Range: {level_label}\n")); - msg.push_str(&format!("Child-Count: {child_count}\n")); - msg.push_str(&format!("Token-Count: {token_count}\n")); - if let (Some(start), Some(end)) = (start, end) { - msg.push_str(&format!("Time-Range-Start: {}\n", start.to_rfc3339())); - msg.push_str(&format!("Time-Range-End: {}\n", end.to_rfc3339())); - } - msg.push_str("\nSummaries:\n"); - for entry in &batch.entries { - msg.push_str(&format!( - "- {} L{} children={} tokens={} path={}\n", - entry.summary_id, entry.level, entry.child_count, entry.token_count, entry.content_path - )); - } - msg -} - -fn summary_repo_path(summary_content_path: &str) -> Result { - let prefix = format!("{WIKI_PREFIX}/"); - let Some(repo_path) = summary_content_path.strip_prefix(&prefix) else { - anyhow::bail!( - "summary content path must live under {WIKI_PREFIX}/: {summary_content_path}" - ); - }; - if !repo_path.starts_with("summaries/") { - anyhow::bail!("wiki git only tracks summary nodes: {summary_content_path}"); - } - Ok(repo_path.to_string()) -} - -fn read_pointer_latest_ref(pointer_id: &str) -> String { - format!( - "refs/tags/read/{}/latest", - hex::encode(pointer_id.as_bytes()) - ) -} - -fn read_pointer_timestamp_ref(pointer_id: &str, timestamp: DateTime) -> String { - format!( - "refs/tags/read/{}/{}", - hex::encode(pointer_id.as_bytes()), - timestamp.format("%Y%m%dT%H%M%S%.9fZ") - ) -} - -#[cfg(test)] -mod tests; diff --git a/src/openhuman/memory/store/content/wiki_git/tests.rs b/src/openhuman/memory/store/content/wiki_git/tests.rs deleted file mode 100644 index 5814be21af..0000000000 --- a/src/openhuman/memory/store/content/wiki_git/tests.rs +++ /dev/null @@ -1,328 +0,0 @@ -use super::*; -use git2::IndexAddOption; -use tempfile::TempDir; - -#[test] -fn commit_summary_initializes_repo_and_tracks_only_summaries() { - let dir = TempDir::new().unwrap(); - let wiki = dir.path().join("wiki"); - let summary = wiki.join("summaries/source-slack/L1/summary-1.md"); - let raw = wiki.join("raw/should-not-track.md"); - let note = wiki.join("notes/also-ignored.md"); - std::fs::create_dir_all(summary.parent().unwrap()).unwrap(); - std::fs::create_dir_all(raw.parent().unwrap()).unwrap(); - std::fs::create_dir_all(note.parent().unwrap()).unwrap(); - std::fs::write(&summary, "---\nkind: summary\n---\nbody").unwrap(); - std::fs::write(&raw, "raw").unwrap(); - std::fs::write(¬e, "note").unwrap(); - - commit_summaries( - dir.path(), - &batch( - "queued_seal", - vec![entry( - "summary-1", - "wiki/summaries/source-slack/L1/summary-1.md", - )], - ), - ) - .unwrap(); - - let repo = Repository::open(&wiki).unwrap(); - let head = repo.head().unwrap().peel_to_commit().unwrap(); - let tree = head.tree().unwrap(); - assert!(tree.get_path(Path::new(".gitignore")).is_ok()); - assert!(tree - .get_path(Path::new("summaries/source-slack/L1/summary-1.md")) - .is_ok()); - assert!(tree.get_path(Path::new("raw/should-not-track.md")).is_err()); - assert!(tree.get_path(Path::new("notes/also-ignored.md")).is_err()); -} - -#[test] -fn commit_summary_prunes_existing_non_summary_tracked_entries() { - let dir = TempDir::new().unwrap(); - let wiki = dir.path().join("wiki"); - std::fs::create_dir_all(wiki.join("raw")).unwrap(); - std::fs::create_dir_all(wiki.join("summaries/source/L1")).unwrap(); - std::fs::write(wiki.join("raw/old.md"), "old raw").unwrap(); - std::fs::write(wiki.join("summaries/source/L1/new.md"), "new summary").unwrap(); - - let repo = Repository::init(&wiki).unwrap(); - let mut index = repo.index().unwrap(); - index - .add_all(["*"].iter(), IndexAddOption::DEFAULT, None) - .unwrap(); - index.write().unwrap(); - let tree_oid = index.write_tree().unwrap(); - let tree = repo.find_tree(tree_oid).unwrap(); - let sig = Signature::now(SIG_NAME, SIG_EMAIL).unwrap(); - repo.commit(Some("HEAD"), &sig, &sig, "old mixed commit", &tree, &[]) - .unwrap(); - - commit_summaries( - dir.path(), - &batch( - "queued_seal", - vec![entry("new", "wiki/summaries/source/L1/new.md")], - ), - ) - .unwrap(); - - let head = repo.head().unwrap().peel_to_commit().unwrap(); - let tree = head.tree().unwrap(); - assert!(tree - .get_path(Path::new("summaries/source/L1/new.md")) - .is_ok()); - assert!(tree.get_path(Path::new("raw/old.md")).is_err()); -} - -#[test] -fn commit_summary_opens_only_the_nested_wiki_repo() { - let dir = TempDir::new().unwrap(); - let wiki = dir.path().join("wiki"); - let summary = wiki.join("summaries/source/L1/summary-1.md"); - std::fs::create_dir_all(summary.parent().unwrap()).unwrap(); - std::fs::write(&summary, "summary").unwrap(); - - let parent_repo = Repository::init(dir.path()).unwrap(); - - commit_summaries( - dir.path(), - &batch( - "queued_seal", - vec![entry("summary-1", "wiki/summaries/source/L1/summary-1.md")], - ), - ) - .unwrap(); - - let repo = Repository::open(&wiki).unwrap(); - let tree = repo - .head() - .unwrap() - .peel_to_commit() - .unwrap() - .tree() - .unwrap(); - assert!(tree - .get_path(Path::new("summaries/source/L1/summary-1.md")) - .is_ok()); - assert!( - parent_repo.head().is_err(), - "summary history should not mutate the parent repo" - ); -} - -#[test] -fn commit_summary_drops_deleted_summary_entries_from_the_index() { - let dir = TempDir::new().unwrap(); - let wiki = dir.path().join("wiki"); - let old_summary = wiki.join("summaries/source/L1/old.md"); - let new_summary = wiki.join("summaries/source/L1/new.md"); - std::fs::create_dir_all(old_summary.parent().unwrap()).unwrap(); - std::fs::write(&old_summary, "old summary").unwrap(); - - commit_summaries( - dir.path(), - &batch( - "queued_seal", - vec![entry("old", "wiki/summaries/source/L1/old.md")], - ), - ) - .unwrap(); - - std::fs::remove_file(&old_summary).unwrap(); - std::fs::write(&new_summary, "new summary").unwrap(); - commit_summaries( - dir.path(), - &batch( - "queued_seal", - vec![entry("new", "wiki/summaries/source/L1/new.md")], - ), - ) - .unwrap(); - - let repo = Repository::open(&wiki).unwrap(); - let tree = repo - .head() - .unwrap() - .peel_to_commit() - .unwrap() - .tree() - .unwrap(); - assert!(tree - .get_path(Path::new("summaries/source/L1/new.md")) - .is_ok()); - assert!(tree - .get_path(Path::new("summaries/source/L1/old.md")) - .is_err()); -} - -#[test] -fn commit_summary_recovers_existing_uncommitted_summary_files() { - let dir = TempDir::new().unwrap(); - let wiki = dir.path().join("wiki"); - let missed_summary = wiki.join("summaries/source/L1/missed.md"); - let new_summary = wiki.join("summaries/source/L1/new.md"); - std::fs::create_dir_all(missed_summary.parent().unwrap()).unwrap(); - std::fs::write(&missed_summary, "missed summary").unwrap(); - std::fs::write(&new_summary, "new summary").unwrap(); - - commit_summaries( - dir.path(), - &batch( - "queued_seal", - vec![entry("new", "wiki/summaries/source/L1/new.md")], - ), - ) - .unwrap(); - - let repo = Repository::open(&wiki).unwrap(); - let tree = repo - .head() - .unwrap() - .peel_to_commit() - .unwrap() - .tree() - .unwrap(); - assert!(tree - .get_path(Path::new("summaries/source/L1/new.md")) - .is_ok()); - assert!(tree - .get_path(Path::new("summaries/source/L1/missed.md")) - .is_ok()); -} - -#[test] -fn commit_summary_rejects_non_summary_paths() { - let dir = TempDir::new().unwrap(); - let err = commit_summaries( - dir.path(), - &batch("bad", vec![entry("bad", "wiki/notes/one.md")]), - ) - .unwrap_err(); - assert!(err.to_string().contains("only tracks summary nodes")); -} - -#[test] -fn commit_message_describes_seal_metadata() { - let dir = TempDir::new().unwrap(); - let wiki = dir.path().join("wiki"); - let summary = wiki.join("summaries/source/L2/summary-2.md"); - std::fs::create_dir_all(summary.parent().unwrap()).unwrap(); - std::fs::write(&summary, "---\nkind: summary\n---\nbody").unwrap(); - - commit_summaries( - dir.path(), - &batch( - "sync_cascade", - vec![SummaryCommitEntry { - summary_id: "summary-2".to_string(), - content_path: "wiki/summaries/source/L2/summary-2.md".to_string(), - level: 2, - child_count: 7, - token_count: 123, - time_range_start: ts(1_700_000_000_000), - time_range_end: ts(1_700_003_600_000), - }], - ), - ) - .unwrap(); - - let repo = Repository::open(&wiki).unwrap(); - let head = repo.head().unwrap().peel_to_commit().unwrap(); - let msg = head.message().unwrap(); - assert!(msg.contains("Seal memory tree slack:#eng L2 summaries")); - assert!(msg.contains("Reason: sync_cascade")); - assert!(msg.contains("Summary-Count: 1")); - assert!(msg.contains("Child-Count: 7")); - assert!(msg.contains("Token-Count: 123")); - assert!(msg.contains("summary-2 L2 children=7 tokens=123")); -} - -#[test] -fn read_pointer_tags_are_timestamped_and_move_latest_without_new_commit() { - let dir = TempDir::new().unwrap(); - let wiki = dir.path().join("wiki"); - let summary = wiki.join("summaries/source/L1/summary-1.md"); - std::fs::create_dir_all(summary.parent().unwrap()).unwrap(); - std::fs::write(&summary, "---\nkind: summary\n---\nbody").unwrap(); - commit_summaries( - dir.path(), - &batch( - "queued_seal", - vec![entry("summary-1", "wiki/summaries/source/L1/summary-1.md")], - ), - ) - .unwrap(); - - let repo = Repository::open(&wiki).unwrap(); - let head = repo.head().unwrap().peel_to_commit().unwrap(); - let head_id = head.id().to_string(); - - let tagged = set_read_pointer_tag(dir.path(), "agent:default", None).unwrap(); - assert_eq!(tagged, head_id); - assert_eq!( - get_read_pointer_tag(dir.path(), "agent:default") - .unwrap() - .as_deref(), - Some(head_id.as_str()) - ); - let tag_prefix = format!( - "refs/tags/read/{}/", - hex::encode("agent:default".as_bytes()) - ); - let tags = repo.references().unwrap().fold(Vec::new(), |mut acc, r| { - let r = r.unwrap(); - let name = r.name().unwrap(); - if name.starts_with(&tag_prefix) { - acc.push(name.to_string()); - } - acc - }); - assert!( - tags.iter().any(|name| name.ends_with("/latest")), - "latest read pointer tag should be present: {tags:?}" - ); - assert!( - tags.iter().any(|name| { - let suffix = name.strip_prefix(&tag_prefix).unwrap_or_default(); - suffix.len() == "20260626T045537.123456789Z".len() - && suffix.ends_with('Z') - && suffix.contains('T') - }), - "timestamped read pointer tag should be present: {tags:?}" - ); - let mut walk = repo.revwalk().unwrap(); - walk.push_head().unwrap(); - assert_eq!( - walk.count(), - 1, - "moving the read pointer must not create commits" - ); -} - -fn batch(reason: &str, entries: Vec) -> SummaryCommitBatch { - SummaryCommitBatch { - reason: reason.to_string(), - tree_id: "tree-1".to_string(), - tree_scope: "slack:#eng".to_string(), - entries, - } -} - -fn entry(summary_id: &str, content_path: &str) -> SummaryCommitEntry { - SummaryCommitEntry { - summary_id: summary_id.to_string(), - content_path: content_path.to_string(), - level: 1, - child_count: 2, - token_count: 10, - time_range_start: ts(1_700_000_000_000), - time_range_end: ts(1_700_000_001_000), - } -} - -fn ts(ms: i64) -> DateTime { - DateTime::::from_timestamp_millis(ms).unwrap() -} diff --git a/src/openhuman/memory/store/mod.rs b/src/openhuman/memory/store/mod.rs index efa65c84f0..73fa06fbbf 100644 --- a/src/openhuman/memory/store/mod.rs +++ b/src/openhuman/memory/store/mod.rs @@ -29,7 +29,6 @@ pub mod tools; pub mod traits; pub mod trees; pub mod types; -pub mod vectors; mod client; pub mod factories; diff --git a/src/openhuman/memory/store/namespace_store/documents.rs b/src/openhuman/memory/store/namespace_store/documents.rs index 6920d0967b..bebf3fdebc 100644 --- a/src/openhuman/memory/store/namespace_store/documents.rs +++ b/src/openhuman/memory/store/namespace_store/documents.rs @@ -383,6 +383,79 @@ impl UnifiedMemory { Ok(document_id) } + /// Fetch a single document by `(namespace, key)`. + /// + /// The same SELECT as [`Self::load_documents_for_scope`] with a `key` + /// predicate bolted on — deliberately *not* implemented as + /// `load_documents_for_scope(ns).find(…)`, which would load every document + /// body in the namespace to return one. + /// + /// `key` goes through [`safety::canonical_document_key`], the exact + /// transform `upsert_document` applies before writing the column. Reading + /// the raw key here would reproduce #5164: the lookup misses, the caller + /// treats the row as absent, and writes it again. + pub(crate) async fn get_document_by_key( + &self, + namespace: &str, + key: &str, + ) -> Result, String> { + let conn = self.conn.lock(); + let ns = Self::sanitize_namespace(namespace); + let key = safety::canonical_document_key(key); + let mut stmt = conn + .prepare( + "SELECT + document_id, + namespace, + key, + title, + content, + source_type, + priority, + tags_json, + metadata_json, + category, + session_id, + created_at, + updated_at, + markdown_rel_path, + taint + FROM memory_docs + WHERE namespace = ?1 AND key = ?2 + LIMIT 1", + ) + .map_err(|e| format!("prepare get_document_by_key: {e}"))?; + let mut rows = stmt + .query(params![ns, key]) + .map_err(|e| format!("query get_document_by_key: {e}"))?; + let Some(row) = rows + .next() + .map_err(|e| format!("row get_document_by_key: {e}"))? + else { + return Ok(None); + }; + let tags_json: String = row.get(7).map_err(|e| e.to_string())?; + let metadata_json: String = row.get(8).map_err(|e| e.to_string())?; + let taint_str: String = row.get(14).map_err(|e| e.to_string())?; + Ok(Some(StoredMemoryDocument { + document_id: row.get(0).map_err(|e| e.to_string())?, + namespace: row.get(1).map_err(|e| e.to_string())?, + key: row.get(2).map_err(|e| e.to_string())?, + title: row.get(3).map_err(|e| e.to_string())?, + content: row.get(4).map_err(|e| e.to_string())?, + source_type: row.get(5).map_err(|e| e.to_string())?, + priority: row.get(6).map_err(|e| e.to_string())?, + tags: serde_json::from_str(&tags_json).unwrap_or_default(), + metadata: serde_json::from_str(&metadata_json).unwrap_or_else(|_| json!({})), + category: row.get(9).map_err(|e| e.to_string())?, + session_id: row.get(10).map_err(|e| e.to_string())?, + created_at: row.get(11).map_err(|e| e.to_string())?, + updated_at: row.get(12).map_err(|e| e.to_string())?, + markdown_rel_path: row.get(13).map_err(|e| e.to_string())?, + taint: crate::openhuman::memory::MemoryTaint::from_db_str(&taint_str), + })) + } + pub(crate) async fn load_documents_for_scope( &self, namespace: &str, diff --git a/src/openhuman/memory/store/trees/hotness.rs b/src/openhuman/memory/store/trees/hotness.rs index f5b4990a46..c04867da7f 100644 --- a/src/openhuman/memory/store/trees/hotness.rs +++ b/src/openhuman/memory/store/trees/hotness.rs @@ -4,10 +4,7 @@ use anyhow::Result; use crate::openhuman::config::Config; use crate::openhuman::memory::store::trees::types::HotnessCounters; - -fn engine_config(config: &Config) -> tinycortex::memory::MemoryConfig { - crate::openhuman::memory::tinycortex::memory_config_from(config, config.workspace_dir.clone()) -} +use crate::openhuman::memory::tinycortex::engine_config; pub fn get(config: &Config, entity_id: &str) -> Result> { tinycortex::memory::tree::store::hotness::get(&engine_config(config), entity_id) diff --git a/src/openhuman/memory/store/trees/registry.rs b/src/openhuman/memory/store/trees/registry.rs index 8c842ec678..116cd2f1ae 100644 --- a/src/openhuman/memory/store/trees/registry.rs +++ b/src/openhuman/memory/store/trees/registry.rs @@ -4,10 +4,7 @@ use anyhow::Result; use crate::openhuman::config::Config; use crate::openhuman::memory::store::trees::types::{Tree, TreeKind}; - -fn engine_config(config: &Config) -> tinycortex::memory::MemoryConfig { - crate::openhuman::memory::tinycortex::memory_config_from(config, config.workspace_dir.clone()) -} +use crate::openhuman::memory::tinycortex::engine_config; pub fn list_trees_by_kind(config: &Config, kind: TreeKind) -> Result> { tinycortex::memory::tree::store::list_trees_by_kind(&engine_config(config), kind) diff --git a/src/openhuman/memory/store/trees/store.rs b/src/openhuman/memory/store/trees/store.rs index 5888beb778..aaa6897ba5 100644 --- a/src/openhuman/memory/store/trees/store.rs +++ b/src/openhuman/memory/store/trees/store.rs @@ -9,13 +9,10 @@ use rusqlite::{Connection, Transaction}; use crate::openhuman::config::Config; use crate::openhuman::memory::store::content::StagedSummary; use crate::openhuman::memory::store::trees::types::{Buffer, SummaryNode, Tree, TreeKind}; +use crate::openhuman::memory::tinycortex::engine_config; pub(crate) use tinycortex::memory::tree::store::TreeCascadeDeletion; -fn engine_config(config: &Config) -> tinycortex::memory::MemoryConfig { - crate::openhuman::memory::tinycortex::memory_config_from(config, config.workspace_dir.clone()) -} - pub fn insert_tree(config: &Config, tree: &Tree) -> Result<()> { tinycortex::memory::tree::store::insert_tree(&engine_config(config), tree) } diff --git a/src/openhuman/memory/store/vectors/mod.rs b/src/openhuman/memory/store/vectors/mod.rs deleted file mode 100644 index d22ac368ef..0000000000 --- a/src/openhuman/memory/store/vectors/mod.rs +++ /dev/null @@ -1,6 +0,0 @@ -//! TinyCortex-owned local vector storage. - -pub use tinycortex::memory::store::vectors::{ - bytes_to_vec, cosine_similarity, format_embedding_signature, vec_to_bytes, EmbeddingBackend, - InertEmbedding, SearchResult, VectorStore, -}; diff --git a/src/openhuman/memory/sync/composio/providers/clickup/mod.rs b/src/openhuman/memory/sync/composio/providers/clickup/mod.rs index 3a9bd416eb..9f528be5dc 100644 --- a/src/openhuman/memory/sync/composio/providers/clickup/mod.rs +++ b/src/openhuman/memory/sync/composio/providers/clickup/mod.rs @@ -6,14 +6,17 @@ //! re-learning a new shape: //! //! - `provider.rs` — `impl ComposioProvider for ClickUpProvider` -//! - `normalization.rs` — payload-shape helpers (results extraction, title) +//! - `normalization` — payload-shape helpers, now `tinycortex::…::normalize::clickup` //! - `ingest.rs` — memory_tree document ingest (issue #2885) //! - `tools.rs` — `CLICKUP_CURATED` whitelist of Composio actions //! - `tests.rs` — unit tests for the helpers + trait metadata //! //! Issue: #2288 (introduction); #2885 (memory_tree migration). -mod normalization; +// The payload normalisers moved to tinycortex (they are pure Value +// transforms, i.e. driver-side). Aliased under the old module name so +// every `normalization::extract_*` call site below stays unchanged. +use tinycortex::memory::sync::composio::providers::normalize::clickup as normalization; mod provider; #[cfg(test)] mod tests; diff --git a/src/openhuman/memory/sync/composio/providers/clickup/normalization.rs b/src/openhuman/memory/sync/composio/providers/clickup/normalization.rs deleted file mode 100644 index 2a543af8fc..0000000000 --- a/src/openhuman/memory/sync/composio/providers/clickup/normalization.rs +++ /dev/null @@ -1,229 +0,0 @@ -//! ClickUp host normalization helpers — result extraction, task-title extraction, -//! and time utilities. -//! -//! ClickUp's REST API (and therefore Composio's wrapping of it) returns -//! task lists in a small handful of shapes depending on which endpoint -//! is called. The functions here walk the union of common shapes so the -//! provider doesn't have to branch per Composio envelope variant. - -use serde_json::Value; - -use crate::openhuman::memory::sync::composio::providers::pick_str; - -/// Walk the Composio response envelope for ClickUp task list results. -/// -/// ClickUp's "filtered team tasks" endpoint returns `{ "tasks": [...] }` -/// at the top level; Composio re-wraps the upstream payload under -/// `data` or `data.data` depending on the action. We probe each shape -/// in order and return the first array we find. -pub(crate) fn extract_tasks(data: &Value) -> Vec { - let candidates = [ - data.pointer("/data/tasks"), - data.pointer("/tasks"), - data.pointer("/data/data/tasks"), - data.pointer("/data/results"), - data.pointer("/results"), - data.pointer("/data/items"), - data.pointer("/items"), - ]; - for cand in candidates.into_iter().flatten() { - if let Some(arr) = cand.as_array() { - return arr.clone(); - } - } - Vec::new() -} - -/// Extract a human-readable title from a ClickUp task object. -/// -/// ClickUp tasks store the name at `name` (or `data.name` after Composio -/// envelope wrapping). When the name is missing we fall back to the -/// task ID so chunks remain identifiable. -pub(crate) fn extract_task_name(task: &Value) -> Option { - pick_str(task, &["name", "data.name", "title", "data.title"]) -} - -/// Extract a stable cursor timestamp (milliseconds since epoch as a -/// string) from a ClickUp task object. -/// -/// The ClickUp API returns `date_updated` as a stringified epoch ms -/// (e.g. `"1733412345678"`); we keep it as a string so lexicographic -/// comparison against the stored cursor remains valid as long as the -/// length doesn't change (it won't until year 33658). -pub(crate) fn extract_task_updated(task: &Value) -> Option { - pick_str( - task, - &[ - "date_updated", - "data.date_updated", - "updated_at", - "data.updated_at", - "dateUpdated", - "data.dateUpdated", - ], - ) -} - -/// Current wall-clock time in milliseconds since the UNIX epoch. -pub(crate) fn now_ms() -> u64 { - use std::time::{SystemTime, UNIX_EPOCH}; - SystemTime::now() - .duration_since(UNIX_EPOCH) - .map(|d| d.as_millis() as u64) - .unwrap_or(0) -} - -/// Extract the authorized user's numeric ID from the -/// `CLICKUP_GET_AUTHORIZED_USER` response. -/// -/// Composio wraps the upstream `{"user": {"id": …}}` shape; this walker -/// is defensive against both raw and wrapped payloads. Returns the ID -/// as a string because `CLICKUP_GET_FILTERED_TEAM_TASKS` accepts the -/// `assignees` filter as a string array. -pub(crate) fn extract_user_id(data: &Value) -> Option { - let candidates = [ - data.pointer("/user/id"), - data.pointer("/data/user/id"), - data.pointer("/id"), - data.pointer("/data/id"), - ]; - for cand in candidates.into_iter().flatten() { - if let Some(n) = cand.as_u64() { - return Some(n.to_string()); - } - if let Some(n) = cand.as_i64() { - return Some(n.to_string()); - } - if let Some(s) = cand.as_str() { - let trimmed = s.trim(); - if !trimmed.is_empty() { - return Some(trimmed.to_string()); - } - } - } - None -} - -/// Extract a list of workspace (team) IDs from the -/// `CLICKUP_GET_AUTHORIZED_TEAMS_WORKSPACES` response. -/// -/// ClickUp returns `{"teams": [{"id": "...", "name": "..."}, …]}`. We -/// keep the IDs as strings — `CLICKUP_GET_FILTERED_TEAM_TASKS` requires -/// a `team_id` (string) argument. -pub(crate) fn extract_workspace_ids(data: &Value) -> Vec { - let candidates = [ - data.pointer("/teams"), - data.pointer("/data/teams"), - data.pointer("/workspaces"), - data.pointer("/data/workspaces"), - ]; - for cand in candidates.into_iter().flatten() { - if let Some(arr) = cand.as_array() { - return arr - .iter() - .filter_map(|t| pick_str(t, &["id", "team_id", "workspace_id"])) - .collect(); - } - } - Vec::new() -} - -#[cfg(test)] -mod tests { - use super::*; - use serde_json::json; - - #[test] - fn extract_tasks_from_data_tasks() { - let data = json!({ "data": { "tasks": [{"id": "t1"}] } }); - assert_eq!(extract_tasks(&data).len(), 1); - } - - #[test] - fn extract_tasks_from_top_level_tasks() { - let data = json!({ "tasks": [{"id": "a"}, {"id": "b"}] }); - assert_eq!(extract_tasks(&data).len(), 2); - } - - #[test] - fn extract_tasks_empty_when_missing() { - let data = json!({ "foo": "bar" }); - assert!(extract_tasks(&data).is_empty()); - } - - #[test] - fn extract_task_name_from_top_level() { - let task = json!({ "id": "t1", "name": "Build feature X" }); - assert_eq!(extract_task_name(&task), Some("Build feature X".into())); - } - - #[test] - fn extract_task_name_falls_back_to_data_name() { - let task = json!({ "data": { "name": "Wrapped" } }); - assert_eq!(extract_task_name(&task), Some("Wrapped".into())); - } - - #[test] - fn extract_task_name_none_when_missing() { - let task = json!({ "id": "t1" }); - assert!(extract_task_name(&task).is_none()); - } - - #[test] - fn extract_task_updated_handles_string_form() { - let task = json!({ "date_updated": "1733412345678" }); - assert_eq!( - extract_task_updated(&task), - Some("1733412345678".to_string()) - ); - } - - #[test] - fn extract_task_updated_handles_nested_data() { - let task = json!({ "data": { "dateUpdated": "1700000000000" } }); - assert_eq!( - extract_task_updated(&task), - Some("1700000000000".to_string()) - ); - } - - #[test] - fn extract_user_id_handles_numeric_id() { - let data = json!({ "user": { "id": 12345 } }); - assert_eq!(extract_user_id(&data), Some("12345".to_string())); - } - - #[test] - fn extract_user_id_handles_wrapped_payload() { - let data = json!({ "data": { "user": { "id": "777" } } }); - assert_eq!(extract_user_id(&data), Some("777".to_string())); - } - - #[test] - fn extract_user_id_none_when_missing() { - let data = json!({ "foo": "bar" }); - assert!(extract_user_id(&data).is_none()); - } - - #[test] - fn extract_workspace_ids_from_teams_array() { - let data = json!({ - "teams": [ - { "id": "ws1", "name": "Personal" }, - { "id": "ws2", "name": "Acme" }, - ] - }); - assert_eq!(extract_workspace_ids(&data), vec!["ws1", "ws2"]); - } - - #[test] - fn extract_workspace_ids_empty_when_no_teams() { - let data = json!({ "foo": "bar" }); - assert!(extract_workspace_ids(&data).is_empty()); - } - - #[test] - fn now_ms_returns_nonzero() { - assert!(now_ms() > 0); - } -} diff --git a/src/openhuman/memory/sync/composio/providers/github/mod.rs b/src/openhuman/memory/sync/composio/providers/github/mod.rs index 4e01482604..cfd72a36e4 100644 --- a/src/openhuman/memory/sync/composio/providers/github/mod.rs +++ b/src/openhuman/memory/sync/composio/providers/github/mod.rs @@ -6,13 +6,16 @@ //! re-learning a new shape: //! //! - `provider.rs` — `impl ComposioProvider for GitHubProvider` -//! - `normalization.rs` — payload-shape helpers (result extraction, title, cursor) +//! - `normalization` — payload-shape helpers, now `tinycortex::…::normalize::github` //! - `tools.rs` — `GITHUB_CURATED` whitelist of Composio actions //! - `tests.rs` — unit tests for the helpers + trait metadata //! //! Issue: #2408. -mod normalization; +// The payload normalisers moved to tinycortex (they are pure Value +// transforms, i.e. driver-side). Aliased under the old module name so +// every `normalization::extract_*` call site below stays unchanged. +use tinycortex::memory::sync::composio::providers::normalize::github as normalization; mod provider; #[cfg(test)] mod tests; diff --git a/src/openhuman/memory/sync/composio/providers/github/normalization.rs b/src/openhuman/memory/sync/composio/providers/github/normalization.rs deleted file mode 100644 index ae7274fc13..0000000000 --- a/src/openhuman/memory/sync/composio/providers/github/normalization.rs +++ /dev/null @@ -1,248 +0,0 @@ -//! GitHub host normalization helpers — result extraction, identity helpers, and time utilities. -//! -//! GitHub's REST API (proxied through Composio) returns search results and -//! authenticated-user payloads in a small number of shapes. The functions here -//! walk the union of common Composio envelope variants so the provider stays -//! clean and branch-free. - -use serde_json::Value; - -use crate::openhuman::memory::sync::composio::providers::pick_str; - -/// Walk the Composio response envelope for GitHub search issue results. -/// -/// `GITHUB_SEARCH_ISSUES_AND_PULL_REQUESTS` wraps GitHub's `GET /search/issues` response, which -/// returns `{"total_count": N, "items": [...]}`. Composio may re-wrap this under -/// `data` or `data.data`; we probe each shape in order. -pub(crate) fn extract_issues(data: &Value) -> Vec { - let candidates = [ - data.pointer("/data/items"), - data.pointer("/items"), - data.pointer("/data/data/items"), - data.pointer("/data/results"), - data.pointer("/results"), - ]; - for cand in candidates.into_iter().flatten() { - if let Some(arr) = cand.as_array() { - return arr.clone(); - } - } - Vec::new() -} - -/// Extract a stable, globally unique identifier for a GitHub issue or PR. -/// -/// GitHub's internal `id` field is a large integer unique across all issues -/// and PRs on github.com. We convert it to a string for use as a sync key. -/// Falls back to composing from `html_url` path if `id` is absent. -pub(crate) fn extract_issue_id(issue: &Value) -> Option { - // Primary: numeric internal GitHub ID. - if let Some(id) = issue.get("id").or_else(|| issue.pointer("/data/id")) { - if let Some(n) = id.as_u64() { - return Some(n.to_string()); - } - if let Some(s) = id.as_str() { - let trimmed = s.trim(); - if !trimmed.is_empty() { - return Some(trimmed.to_string()); - } - } - } - // Fallback: parse owner/repo/number from html_url path segments. - // URL shape: https://github.com/{owner}/{repo}/issues/{number} - if let Some(url) = pick_str(issue, &["html_url", "data.html_url", "url", "data.url"]) { - if let Some(slug) = github_url_to_slug(&url) { - return Some(slug); - } - } - None -} - -/// Build a human-readable document title for a GitHub issue/PR. -/// -/// Format: `GitHub: {owner}/{repo}#{number}: {title}`. -/// Falls back to just the title or a placeholder when fields are missing. -pub(crate) fn extract_issue_title(issue: &Value) -> Option { - let title = pick_str(issue, &["title", "data.title"])?; - - // Best-effort: extract owner/repo#N from html_url for the prefix. - let prefix = pick_str(issue, &["html_url", "data.html_url"]) - .and_then(|url| github_url_to_slug(&url)) - .unwrap_or_default(); - - if prefix.is_empty() { - Some(title) - } else { - Some(format!("GitHub: {prefix}: {title}")) - } -} - -/// Parse `https://github.com/{owner}/{repo}/issues/{number}` (or `/pull/`) -/// into `"{owner}/{repo}#{number}"`. Returns `None` for unrecognised shapes. -fn github_url_to_slug(url: &str) -> Option { - let segs: Vec<&str> = url.trim_end_matches('/').split('/').collect(); - // Minimum: ["https:", "", "github.com", owner, repo, "issues", number] - if segs.len() >= 7 { - let number = segs[segs.len() - 1]; - let _kind = segs[segs.len() - 2]; // "issues" or "pull" — ignored - let repo = segs[segs.len() - 3]; - let owner = segs[segs.len() - 4]; - if !owner.is_empty() && !repo.is_empty() && !number.is_empty() { - return Some(format!("{owner}/{repo}#{number}")); - } - } - None -} - -/// Extract the `updated_at` ISO 8601 timestamp from a GitHub issue. -/// -/// GitHub returns `updated_at` as `"2024-05-21T15:30:00Z"`. ISO 8601 strings -/// sort lexicographically, so we use them directly as the sync cursor. -pub(crate) fn extract_issue_updated_at(issue: &Value) -> Option { - pick_str( - issue, - &[ - "updated_at", - "data.updated_at", - "updatedAt", - "data.updatedAt", - ], - ) -} - -/// Extract the authenticated user's login handle from a -/// `GITHUB_GET_THE_AUTHENTICATED_USER` response. -pub(crate) fn extract_user_login(data: &Value) -> Option { - pick_str(data, &["login", "data.login"]) -} - -/// Current wall-clock time in milliseconds since the UNIX epoch. -pub(crate) fn now_ms() -> u64 { - use std::time::{SystemTime, UNIX_EPOCH}; - SystemTime::now() - .duration_since(UNIX_EPOCH) - .map(|d| d.as_millis() as u64) - .unwrap_or(0) -} - -#[cfg(test)] -mod tests { - use super::*; - use serde_json::json; - - #[test] - fn extract_issues_from_data_items() { - let data = json!({ "data": { "items": [{"id": 1}] } }); - assert_eq!(extract_issues(&data).len(), 1); - } - - #[test] - fn extract_issues_from_top_level_items() { - let data = json!({ "items": [{"id": 1}, {"id": 2}] }); - assert_eq!(extract_issues(&data).len(), 2); - } - - #[test] - fn extract_issues_empty_when_missing() { - let data = json!({ "foo": "bar" }); - assert!(extract_issues(&data).is_empty()); - } - - #[test] - fn extract_issue_id_from_numeric_field() { - let issue = json!({ "id": 123456789u64, "title": "Fix bug" }); - assert_eq!(extract_issue_id(&issue), Some("123456789".to_string())); - } - - #[test] - fn extract_issue_id_from_wrapped_data() { - let issue = json!({ "data": { "id": 99u64 } }); - assert_eq!(extract_issue_id(&issue), Some("99".to_string())); - } - - #[test] - fn extract_issue_id_falls_back_to_html_url() { - let issue = json!({ - "html_url": "https://github.com/owner/repo/issues/42" - }); - assert_eq!(extract_issue_id(&issue), Some("owner/repo#42".to_string())); - } - - #[test] - fn extract_issue_id_none_when_missing() { - let issue = json!({ "title": "No ID here" }); - assert!(extract_issue_id(&issue).is_none()); - } - - #[test] - fn extract_issue_title_builds_prefixed_title() { - let issue = json!({ - "id": 1u64, - "title": "Fix race condition", - "html_url": "https://github.com/acme/core/issues/99" - }); - assert_eq!( - extract_issue_title(&issue), - Some("GitHub: acme/core#99: Fix race condition".to_string()) - ); - } - - #[test] - fn extract_issue_title_returns_raw_title_when_no_url() { - let issue = json!({ "title": "Bare title" }); - assert_eq!(extract_issue_title(&issue), Some("Bare title".to_string())); - } - - #[test] - fn extract_issue_title_none_when_missing() { - let issue = json!({ "id": 1u64 }); - assert!(extract_issue_title(&issue).is_none()); - } - - #[test] - fn extract_issue_updated_at_from_top_level() { - let issue = json!({ "updated_at": "2024-05-21T15:30:00Z" }); - assert_eq!( - extract_issue_updated_at(&issue), - Some("2024-05-21T15:30:00Z".to_string()) - ); - } - - #[test] - fn extract_issue_updated_at_from_data_wrapper() { - let issue = json!({ "data": { "updated_at": "2023-01-01T00:00:00Z" } }); - assert_eq!( - extract_issue_updated_at(&issue), - Some("2023-01-01T00:00:00Z".to_string()) - ); - } - - #[test] - fn extract_issue_updated_at_none_when_missing() { - let issue = json!({ "id": 1u64 }); - assert!(extract_issue_updated_at(&issue).is_none()); - } - - #[test] - fn extract_user_login_from_top_level() { - let data = json!({ "login": "octocat" }); - assert_eq!(extract_user_login(&data), Some("octocat".to_string())); - } - - #[test] - fn extract_user_login_from_data_wrapper() { - let data = json!({ "data": { "login": "monalisa" } }); - assert_eq!(extract_user_login(&data), Some("monalisa".to_string())); - } - - #[test] - fn extract_user_login_none_when_missing() { - let data = json!({ "id": 1u64 }); - assert!(extract_user_login(&data).is_none()); - } - - #[test] - fn now_ms_returns_nonzero() { - assert!(now_ms() > 0); - } -} diff --git a/src/openhuman/memory/sync/composio/providers/gmail/mod.rs b/src/openhuman/memory/sync/composio/providers/gmail/mod.rs index 391f9b5c68..fcdc23ebb1 100644 --- a/src/openhuman/memory/sync/composio/providers/gmail/mod.rs +++ b/src/openhuman/memory/sync/composio/providers/gmail/mod.rs @@ -1,4 +1,7 @@ -mod post_process; +// The Gmail post-processor moved to tinycortex (a pure Value transform, i.e. +// driver-side). Aliased under the old module name so the single call site in +// `provider.rs` stays unchanged. +use tinycortex::memory::sync::composio::providers::normalize::gmail_post_process as post_process; mod provider; #[cfg(test)] mod tests; diff --git a/src/openhuman/memory/sync/composio/providers/gmail/post_process.rs b/src/openhuman/memory/sync/composio/providers/gmail/post_process.rs deleted file mode 100644 index 1e3494ba89..0000000000 --- a/src/openhuman/memory/sync/composio/providers/gmail/post_process.rs +++ /dev/null @@ -1,492 +0,0 @@ -//! Gmail-specific post-processing of Composio action responses. -//! -//! The upstream `GMAIL_FETCH_EMAILS` payload is extremely verbose -//! (full MIME tree under `payload.parts[]`, 50+ `Received:` headers, -//! display-layer noise the model never uses). This module rewrites -//! it into a slim envelope per message: -//! -//! ```json -//! { -//! "messages": [ -//! { -//! "id": "…", -//! "threadId": "…", -//! "subject": "…", -//! "from": "…", -//! "to": "…", -//! "date": "…", -//! "labels": ["INBOX", "UNREAD"], -//! "markdown": "…body…", -//! "attachments": [ { "filename": "...", "mimeType": "..." } ] -//! } -//! ], -//! "nextPageToken": "…", -//! "resultSizeEstimate": 201 -//! } -//! ``` -//! -//! ## Body source -//! -//! Composio's backend ships a -//! `markdownFormatted` field on the response envelope — one string -//! per tool call, pre-rendered with HTML stripped, URLs shortened, -//! footers removed, whitespace normalised. We split it per message -//! along `\n---\n` boundaries (with `## ` heading fallbacks) and -//! pin each slice to the corresponding entry in `messages[]` via -//! [`apply_response_level_markdown`]. The reshape's -//! [`extract_markdown_body`] then prefers that pinned field over -//! falling back to the upstream `messageText`. -//! -//! No in-house HTML→markdown conversion lives here anymore — the -//! backend does the cleaning. If `markdownFormatted` is absent for -//! a given response we fall through to whatever plain text the -//! upstream provided in `messageText`. -//! -//! Callers that need the raw Composio shape can pass `raw_html: -//! true` (or `rawHtml: true`) in the action arguments — this -//! short-circuits the reshape entirely. -//! -//! Only `GMAIL_FETCH_EMAILS` is reshaped today; other Gmail action -//! responses are passed through unchanged. When we add envelopes for -//! more slugs they should live in this file, branched from -//! [`post_process`]. - -use serde_json::{json, Map, Value}; - -/// Entry point called from `GmailProvider::post_process_action_result`. -/// -/// Dispatches on the Composio action slug. Unknown Gmail slugs fall -/// through to a no-op. -pub fn post_process(slug: &str, arguments: Option<&Value>, data: &mut Value) { - if is_raw_html_flag_set(arguments) { - tracing::debug!( - slug, - "[composio:gmail][post-process] raw_html flag set, passing through" - ); - return; - } - if slug == "GMAIL_FETCH_EMAILS" { - reshape_fetch_emails(data) - } -} - -/// Stash per-message slices of the response-level `markdownFormatted` -/// onto the corresponding entries inside `data.messages[]`. -/// -/// The Composio backend (tinyhumansai/backend#683) ships ONE -/// `markdownFormatted` string per tool call covering all messages — -/// already URL-shortened, footer-stripped, and whitespace-normalised. -/// To get per-email files in the raw archive we split that string -/// along section boundaries (`## ` headings or `---` rules) and pin -/// each slice to the message at the same index. `extract_markdown_body` -/// then prefers `msg.markdownFormatted` over re-decoding the MIME -/// tree. -/// -/// **Must be called BEFORE [`post_process`]** because `post_process` -/// reshapes `data` into the slim envelope; once `messages[]` carries -/// our slim shape the upstream message ordering is already locked in -/// but we may have lost original ordering signals if any. -/// -/// No-op when the slice count doesn't match `messages.len()` — we -/// can't safely align segments to messages without an exact match, -/// so we let `extract_markdown_body` fall through to its MIME path. -pub fn apply_response_level_markdown(data: &mut Value, top_md: &str) { - let trimmed = top_md.trim(); - if trimmed.is_empty() { - return; - } - let container = match data.get_mut("messages") { - Some(_) => data, - None => match data.get_mut("data").and_then(|v| v.as_object_mut()) { - Some(_) => data.get_mut("data").unwrap(), - None => { - tracing::debug!( - "[composio:gmail][post-process] apply_response_level_markdown: \ - no messages container in response — skipping" - ); - return; - } - }, - }; - let Some(messages) = container.get_mut("messages").and_then(|v| v.as_array_mut()) else { - return; - }; - let count = messages.len(); - if count == 0 { - return; - } - // Clone hints out of the messages array so the slice borrows - // don't conflict with the upcoming `messages.iter_mut()` mutation. - let hints: Vec = messages.clone(); - let Some(slices) = split_response_markdown_per_message_with_hint(trimmed, count, Some(&hints)) - else { - tracing::debug!( - messages = count, - md_len = trimmed.len(), - "[composio:gmail][post-process] could not split response-level markdownFormatted \ - into {count} slices — falling back to per-message MIME decode" - ); - return; - }; - for (msg, slice) in messages.iter_mut().zip(slices) { - if let Some(obj) = msg.as_object_mut() { - obj.insert("markdownFormatted".to_string(), Value::String(slice)); - } - } - tracing::debug!( - messages = count, - "[composio:gmail][post-process] stashed per-message markdownFormatted slices" - ); -} - -/// Split a top-level `markdownFormatted` string into per-message -/// segments. Returns `Some(slices)` only when the split yields -/// exactly `expected_count` entries — otherwise the format isn't one -/// of the patterns we know about and we let the caller fall back. -/// -/// Primary boundary is the `\n---\n` horizontal rule the backend -/// emits between messages (confirmed against real -/// `GMAIL_FETCH_EMAILS` output). H2/H3 headings are kept as -/// fallbacks for older renderings. The preamble (`# Inbox (N -/// messages)`-style intro, if present) is dropped — we accept -/// either `expected` parts (no preamble) or `expected + 1` -/// (preamble + N messages). -/// -/// `messages_hint` is the slim message array from the same response -/// — when present we use the per-message `subject` field to verify -/// each segment really does belong to the message at the same index. -/// Mismatches force a fallback so we never write a wrong-message body -/// to the raw archive. -pub(crate) fn split_response_markdown_per_message( - md: &str, - expected_count: usize, -) -> Option> { - split_response_markdown_per_message_with_hint(md, expected_count, None) -} - -pub(crate) fn split_response_markdown_per_message_with_hint( - md: &str, - expected_count: usize, - messages_hint: Option<&[Value]>, -) -> Option> { - if expected_count == 0 { - return None; - } - if expected_count == 1 { - return Some(vec![md.to_string()]); - } - - // Boundary patterns to try, in priority order. `\n---\n` is the - // confirmed marker; the heading variants stay as belt-and-braces - // for older / variant backend renderings. - let candidates: &[(&str, &str)] = &[ - ("\n---\n", "---\n"), - ("\n\n## ", "## "), - ("\n\n### ", "### "), - ("\n\n# ", "# "), - ("\n***\n", "***\n"), - ]; - - for (sep, prefix) in candidates { - let parts: Vec<&str> = md.split(sep).collect(); - let (drop_preamble, prepend_first) = if parts.len() == expected_count { - (false, false) // no preamble; first segment had no prefix - } else if parts.len() == expected_count + 1 { - (true, true) // preamble dropped; every kept segment had a prefix - } else { - continue; - }; - let segments: Vec = parts - .into_iter() - .skip(if drop_preamble { 1 } else { 0 }) - .enumerate() - .map(|(i, s)| { - if i == 0 && !prepend_first { - s.to_string() - } else { - format!("{prefix}{s}") - } - }) - .collect(); - - // Validate alignment against the JSON message array: every - // segment whose corresponding message has a non-empty subject - // must mention that subject somewhere in its body. If a single - // pair fails, we treat the split as unreliable and try the - // next pattern. Empty / null subjects skip validation (e.g. - // notification mails where the subject is ""). - if let Some(hints) = messages_hint { - if !validate_segments_against_hints(&segments, hints) { - tracing::debug!( - expected = expected_count, - sep = sep, - "[composio:gmail][post-process] split candidate failed subject check" - ); - continue; - } - } - return Some(segments); - } - None -} - -/// True if every (segment, message) pair where the message has a -/// non-empty subject contains that subject somewhere in the segment -/// (case-insensitive substring match — a defensive heuristic, not a -/// strict equality check, since the backend may format subjects -/// inside markdown links or with surrounding decoration). -fn validate_segments_against_hints(segments: &[String], hints: &[Value]) -> bool { - if segments.len() != hints.len() { - return false; - } - for (seg, hint) in segments.iter().zip(hints.iter()) { - let subject = hint - .get("subject") - .and_then(|v| v.as_str()) - .unwrap_or("") - .trim(); - if subject.is_empty() { - continue; - } - if !seg - .to_ascii_lowercase() - .contains(&subject.to_ascii_lowercase()) - { - return false; - } - } - true -} - -/// Returns true when the caller explicitly set `raw_html: true` (or the -/// camelCase `rawHtml: true`) in the `arguments` object. -fn is_raw_html_flag_set(arguments: Option<&Value>) -> bool { - let Some(obj) = arguments.and_then(|v| v.as_object()) else { - return false; - }; - obj.get("raw_html") - .or_else(|| obj.get("rawHtml")) - .and_then(|v| v.as_bool()) - .unwrap_or(false) -} - -/// Rewrite a `GMAIL_FETCH_EMAILS` `data` object in place into the slim -/// envelope documented at the module level. -/// -/// The Composio response can be shaped either as `{ messages, nextPageToken, ... }` -/// directly, or wrapped one level deeper under `{ data: { messages: … } }` -/// depending on backend version; we handle both. -fn reshape_fetch_emails(data: &mut Value) { - // Unwrap an optional `data:` envelope so downstream logic only has - // to deal with one shape. - let container = match data.get_mut("messages") { - Some(_) => data, - None => match data.get_mut("data").and_then(|v| v.as_object_mut()) { - Some(_) => data.get_mut("data").unwrap(), - None => return, - }, - }; - - let Some(obj) = container.as_object_mut() else { - return; - }; - - let raw_messages = obj - .remove("messages") - .and_then(|v| match v { - Value::Array(arr) => Some(arr), - _ => None, - }) - .unwrap_or_default(); - let next_page_token = obj.remove("nextPageToken").unwrap_or(Value::Null); - let result_size_estimate = obj.remove("resultSizeEstimate").unwrap_or(Value::Null); - - let messages: Vec = raw_messages.into_iter().map(reshape_message).collect(); - - let mut envelope = Map::new(); - envelope.insert("messages".into(), Value::Array(messages)); - if !next_page_token.is_null() { - envelope.insert("nextPageToken".into(), next_page_token); - } - if !result_size_estimate.is_null() { - envelope.insert("resultSizeEstimate".into(), result_size_estimate); - } - - *container = Value::Object(envelope); -} - -/// Parse an RFC 3339 or RFC 2822 date string into a UTC `DateTime`. -pub(crate) fn parse_email_date(date_str: &str) -> Option> { - date_str - .parse::>() - .or_else(|_| { - chrono::DateTime::parse_from_rfc2822(date_str).map(|d| d.with_timezone(&chrono::Utc)) - }) - .ok() -} - -const EMAIL_LOCAL_TIME_FMT: &str = "%Y-%m-%d %I:%M %p %:z"; - -/// Format a UTC `DateTime` in the given timezone. Returns `None` when the -/// formatted result is identical to the UTC rendering (no-op for UTC hosts). -pub(crate) fn format_at_tz( - utc: chrono::DateTime, - tz: &Tz, -) -> Option -where - Tz::Offset: std::fmt::Display, -{ - let local_dt = utc.with_timezone(tz); - let formatted = local_dt.format(EMAIL_LOCAL_TIME_FMT).to_string(); - - let utc_formatted = utc.format(EMAIL_LOCAL_TIME_FMT).to_string(); - if formatted == utc_formatted { - return None; - } - Some(formatted) -} - -/// Convert a UTC email timestamp string to a human-readable local-time string. -/// -/// Accepts RFC 3339 (`"2026-05-31T10:33:00Z"`) or RFC 2822 -/// (`"Sat, 31 May 2026 10:33:00 +0000"`) input. Returns a formatted string -/// in the host's local timezone, e.g. `"2026-05-31 05:33 AM -05:00"`, -/// so the agent can present local times without UTC arithmetic. -/// -/// The raw `date` field is always preserved alongside this field so -/// internal sorting, deduplication, and debugging remain UTC-based. -/// -/// Returns `None` when the input cannot be parsed or the output format -/// would be identical to the UTC input (no-op for UTC hosts). -pub(crate) fn format_email_local_time(date_str: &str) -> Option { - let utc = parse_email_date(date_str)?; - format_at_tz(utc, &chrono::Local) -} - -/// Map one raw Composio message object to its slim counterpart. -/// -/// Body source picked by [`extract_markdown_body`]: -/// 1. The per-message `markdownFormatted` slice pinned by -/// [`apply_response_level_markdown`] (preferred — backend-rendered). -/// 2. The upstream `messageText` plaintext (fallback). -/// 3. Empty string. -fn reshape_message(raw: Value) -> Value { - let Value::Object(obj) = raw else { - return raw; - }; - - let id = obj.get("messageId").cloned().unwrap_or(Value::Null); - let thread_id = obj.get("threadId").cloned().unwrap_or(Value::Null); - let subject = obj.get("subject").cloned().unwrap_or(Value::Null); - let sender = obj.get("sender").cloned().unwrap_or(Value::Null); - let to = obj.get("to").cloned().unwrap_or(Value::Null); - let date = obj - .get("messageTimestamp") - .cloned() - .or_else(|| pick_header(&obj, "Date")) - .unwrap_or(Value::Null); - let labels = obj - .get("labelIds") - .cloned() - .unwrap_or_else(|| Value::Array(Vec::new())); - let list_unsubscribe = pick_header(&obj, "List-Unsubscribe").unwrap_or(Value::Null); - - let markdown = extract_markdown_body(&obj); - let attachments = extract_attachments(&obj); - - // Compute a local-time representation of the UTC `date` so the agent - // presents times in the user's timezone rather than quoting raw UTC. - let date_local = date.as_str().and_then(format_email_local_time); - - let mut out = Map::new(); - out.insert("id".into(), id); - out.insert("threadId".into(), thread_id); - out.insert("subject".into(), subject); - out.insert("from".into(), sender); - out.insert("to".into(), to); - out.insert("date".into(), date); - if let Some(local) = date_local { - out.insert("date_local".into(), Value::String(local)); - } - out.insert("labels".into(), labels); - if !list_unsubscribe.is_null() { - out.insert("list_unsubscribe".into(), list_unsubscribe); - } - out.insert("markdown".into(), Value::String(markdown)); - if !attachments.is_empty() { - out.insert("attachments".into(), Value::Array(attachments)); - } - Value::Object(out) -} - -/// Find a header value by (case-insensitive) name in the Composio -/// `payload.headers[]` array. Returns `Some(Value::String)` on hit. -fn pick_header(msg: &Map, name: &str) -> Option { - let headers = msg.get("payload")?.get("headers")?.as_array()?; - for h in headers { - let hn = h.get("name").and_then(|v| v.as_str()).unwrap_or(""); - if hn.eq_ignore_ascii_case(name) { - if let Some(v) = h.get("value").and_then(|v| v.as_str()) { - return Some(Value::String(v.to_string())); - } - } - } - None -} - -/// Pick a body for the slim envelope. -/// -/// We trust the Composio backend's pre-rendered `markdownFormatted` -/// (set per-message by [`apply_response_level_markdown`] from the -/// response-level field). When that's absent we fall back to the -/// upstream's plain-text `messageText` verbatim — no in-house -/// HTML→markdown decoding lives here anymore. The backend already -/// strips HTML, shortens URLs, and normalises whitespace; running -/// our own pipeline on top duplicated work and corrupted some -/// renderings. -fn extract_markdown_body(msg: &Map) -> String { - if let Some(formatted) = msg - .get("markdownFormatted") - .or_else(|| msg.get("markdown_formatted")) - .and_then(|v| v.as_str()) - .map(str::trim) - .filter(|s| !s.is_empty()) - { - return formatted.to_string(); - } - if let Some(text) = msg - .get("messageText") - .and_then(|v| v.as_str()) - .map(str::trim) - .filter(|s| !s.is_empty()) - { - return text.to_string(); - } - String::new() -} - -/// Pull a minimal attachments descriptor from the Composio -/// `attachmentList` array. -fn extract_attachments(msg: &Map) -> Vec { - if let Some(list) = msg.get("attachmentList").and_then(|v| v.as_array()) { - return list - .iter() - .filter_map(|a| { - let filename = a.get("filename").and_then(|v| v.as_str())?; - if filename.is_empty() { - return None; - } - let mime = a - .get("mimeType") - .and_then(|v| v.as_str()) - .unwrap_or_default(); - Some(json!({ "filename": filename, "mimeType": mime })) - }) - .collect(); - } - Vec::new() -} - -#[cfg(test)] -#[path = "post_process_tests.rs"] -mod tests; diff --git a/src/openhuman/memory/sync/composio/providers/gmail/post_process_tests.rs b/src/openhuman/memory/sync/composio/providers/gmail/post_process_tests.rs deleted file mode 100644 index a143e95abf..0000000000 --- a/src/openhuman/memory/sync/composio/providers/gmail/post_process_tests.rs +++ /dev/null @@ -1,354 +0,0 @@ -use super::*; -use serde_json::json; - -fn fixture_with_backend_markdown() -> Value { - json!({ - "messages": [ - { - "messageId": "m1", - "threadId": "t1", - "subject": "Hello", - "sender": "a@x.com", - "to": "b@y.com", - "messageTimestamp": "2026-04-17T12:00:00Z", - "labelIds": ["INBOX", "UNREAD"], - // Pre-rendered slice (set by `apply_response_level_markdown` - // in production; inline here for the reshape test). - "markdownFormatted": "# Hello\n\nbody copy", - "messageText": "fallback should not be used", - "display_url": "ignore-me", - "preview": { "body": "Hi plain", "subject": "Hello" }, - "attachmentList": [ - { "filename": "report.pdf", "mimeType": "application/pdf", "size": 12345 }, - { "filename": "", "mimeType": "text/html" } - ], - "payload": {} - } - ], - "nextPageToken": "tok-1", - "resultSizeEstimate": 42 - }) -} - -#[test] -fn reshape_emits_slim_envelope() { - let mut v = fixture_with_backend_markdown(); - post_process("GMAIL_FETCH_EMAILS", None, &mut v); - - assert_eq!(v["nextPageToken"], "tok-1"); - assert_eq!(v["resultSizeEstimate"], 42); - - let msgs = v["messages"].as_array().unwrap(); - assert_eq!(msgs.len(), 1); - let m = &msgs[0]; - - assert_eq!(m["id"], "m1"); - assert_eq!(m["threadId"], "t1"); - assert_eq!(m["subject"], "Hello"); - assert_eq!(m["from"], "a@x.com"); - assert_eq!(m["to"], "b@y.com"); - assert_eq!(m["date"], "2026-04-17T12:00:00Z"); - assert_eq!(m["labels"], json!(["INBOX", "UNREAD"])); - - let md = m["markdown"].as_str().unwrap(); - assert_eq!(md, "# Hello\n\nbody copy"); - - // Noise fields removed. - assert!(m.get("display_url").is_none()); - assert!(m.get("preview").is_none()); - assert!(m.get("payload").is_none()); - assert!(m.get("messageText").is_none()); - - // Attachments: empty filename entry is filtered. - let atts = m["attachments"].as_array().unwrap(); - assert_eq!(atts.len(), 1); - assert_eq!(atts[0]["filename"], "report.pdf"); - assert_eq!(atts[0]["mimeType"], "application/pdf"); -} - -#[test] -fn raw_html_flag_passes_through_unchanged() { - let mut v = fixture_with_backend_markdown(); - let original = v.clone(); - let args = json!({ "raw_html": true }); - post_process("GMAIL_FETCH_EMAILS", Some(&args), &mut v); - assert_eq!( - v, original, - "raw_html=true must preserve the Composio shape" - ); -} - -#[test] -fn camel_case_raw_html_also_recognized() { - let mut v = fixture_with_backend_markdown(); - let original = v.clone(); - let args = json!({ "rawHtml": true }); - post_process("GMAIL_FETCH_EMAILS", Some(&args), &mut v); - assert_eq!(v, original); -} - -#[test] -fn falls_back_to_message_text_when_no_backend_markdown() { - let mut v = json!({ - "messages": [{ - "messageId": "m1", - "threadId": "t1", - "subject": "s", - "sender": "a@x.com", - "to": "b@y.com", - "messageTimestamp": "2026-04-17", - "labelIds": [], - "messageText": " plain body text ", - "payload": {} - }], - "nextPageToken": null - }); - post_process("GMAIL_FETCH_EMAILS", None, &mut v); - let md = v["messages"][0]["markdown"].as_str().unwrap(); - assert_eq!(md, "plain body text"); - assert!(v.get("nextPageToken").is_none(), "null tokens dropped"); -} - -#[test] -fn unwraps_data_envelope() { - let mut v = json!({ - "data": { - "messages": [{ - "messageId": "m1", - "threadId": "t1", - "subject": "s", - "sender": "a@x.com", - "to": "b@y.com", - "messageTimestamp": "2026-04-17", - "labelIds": [], - "messageText": "body", - "payload": {} - }] - } - }); - post_process("GMAIL_FETCH_EMAILS", None, &mut v); - // Reshape writes into `data` in place. - let msgs = v["data"]["messages"].as_array().unwrap(); - assert_eq!(msgs.len(), 1); - assert_eq!(msgs[0]["markdown"], "body"); -} - -#[test] -fn non_fetch_slug_is_noop() { - let mut v = json!({ "messages": [{ "messageId": "m1", "messageText": "x" }] }); - let original = v.clone(); - post_process("GMAIL_SEND_EMAIL", None, &mut v); - assert_eq!(v, original); -} - -#[test] -fn prefers_backend_markdown_formatted_when_present() { - // Composio backend (tinyhumansai/backend#683 +) ships - // `markdownFormatted` already URL-shortened + footer-stripped - // per message (after `apply_response_level_markdown` slices the - // response-level field). When present, our post-processor must - // use it verbatim instead of falling back to `messageText`. - let mut v = json!({ - "messages": [{ - "messageId": "m1", - "threadId": "t1", - "subject": "s", - "sender": "a@x.com", - "to": "b@y.com", - "messageTimestamp": "2026-04-17", - "labelIds": [], - "markdownFormatted": "# Already nice\n\nShort URL: https://gh.io/abc", - "messageText": "fallback should not be used", - "payload": {} - }] - }); - post_process("GMAIL_FETCH_EMAILS", None, &mut v); - let md = v["messages"][0]["markdown"].as_str().unwrap(); - assert_eq!(md, "# Already nice\n\nShort URL: https://gh.io/abc"); -} - -#[test] -fn empty_markdown_formatted_falls_through_to_message_text() { - let mut v = json!({ - "messages": [{ - "messageId": "m1", - "threadId": "t1", - "subject": "s", - "sender": "a@x.com", - "to": "b@y.com", - "messageTimestamp": "2026-04-17", - "labelIds": [], - "markdownFormatted": " \n \n", - "messageText": "real body", - "payload": {} - }] - }); - post_process("GMAIL_FETCH_EMAILS", None, &mut v); - let md = v["messages"][0]["markdown"].as_str().unwrap(); - assert!(md.contains("real body")); -} - -// ── split_response_markdown_per_message ───────────────────────────────── - -#[test] -fn split_response_markdown_uses_horizontal_rule_marker() { - // The confirmed backend marker is `\n---\n`. Three messages → - // expect three slices when there's no preamble. - let md = "## Alice's update\n\nbody A with https://gh.io/abc\n---\n## Bob's reply\n\nbody B\n---\n## Carol\n\nbody C"; - let slices = super::split_response_markdown_per_message(md, 3).unwrap(); - assert_eq!(slices.len(), 3); - assert!(slices[0].contains("Alice's update")); - assert!(slices[1].contains("Bob's reply")); - assert!(slices[2].contains("Carol")); - // The `---\n` prefix is preserved on every-but-the-first segment - // so the section break survives the round-trip. - assert!(slices[1].starts_with("---\n")); - assert!(slices[2].starts_with("---\n")); -} - -#[test] -fn split_response_markdown_drops_preamble() { - // When a preamble like `# Inbox` precedes the first marker, we - // see N+1 parts after split — the preamble must be dropped. - let md = "# Inbox (2 messages)\n---\n## A\n\nbody A\n---\n## B\n\nbody B"; - let slices = super::split_response_markdown_per_message(md, 2).unwrap(); - assert_eq!(slices.len(), 2); - assert!(slices[0].contains("body A")); - assert!(slices[1].contains("body B")); - // Both segments should carry the prefix when preamble was dropped. - assert!(slices[0].starts_with("---\n")); - assert!(slices[1].starts_with("---\n")); -} - -#[test] -fn split_response_markdown_falls_back_to_h2_marker() { - // No `---` rules — backend used h2 headings as boundaries. - let md = "## Alice\n\nbody A\n\n## Bob\n\nbody B"; - let slices = super::split_response_markdown_per_message(md, 2).unwrap(); - assert_eq!(slices.len(), 2); - assert!(slices[0].contains("body A")); - assert!(slices[1].contains("body B")); -} - -#[test] -fn split_response_markdown_returns_none_on_count_mismatch() { - let md = "## only one section here"; - assert!(super::split_response_markdown_per_message(md, 3).is_none()); -} - -#[test] -fn split_response_markdown_single_message_returns_whole_input() { - let md = "## solo\n\nthe whole body"; - let slices = super::split_response_markdown_per_message(md, 1).unwrap(); - assert_eq!(slices, vec![md.to_string()]); -} - -#[test] -fn split_with_hint_rejects_when_subjects_dont_match() { - let md = "## Foo\nbody1\n---\n## Bar\nbody2"; - let hints = vec![ - json!({"subject": "Completely different subject A"}), - json!({"subject": "Completely different subject B"}), - ]; - let out = super::split_response_markdown_per_message_with_hint(md, 2, Some(&hints)); - assert!(out.is_none(), "subject mismatch must force fallback"); -} - -#[test] -fn split_with_hint_accepts_when_subjects_match() { - let md = "## Welcome to Gmail\nbody1\n---\n## Your invoice\nbody2"; - let hints = vec![ - json!({"subject": "Welcome to Gmail"}), - json!({"subject": "Your invoice"}), - ]; - let slices = super::split_response_markdown_per_message_with_hint(md, 2, Some(&hints)).unwrap(); - assert_eq!(slices.len(), 2); - assert!(slices[0].contains("Welcome to Gmail")); - assert!(slices[1].contains("Your invoice")); -} - -#[test] -fn split_with_hint_skips_messages_with_blank_subject() { - let md = "## A\nbody1\n---\n## B\nbody2"; - let hints = vec![json!({"subject": "A"}), json!({"subject": ""})]; - let slices = super::split_response_markdown_per_message_with_hint(md, 2, Some(&hints)).unwrap(); - assert_eq!(slices.len(), 2); -} - -// ── format_email_local_time ────────────────────────────────────────────────── - -#[test] -fn format_email_local_time_returns_none_for_unparseable_date() { - assert!(super::format_email_local_time("not-a-date").is_none()); - assert!(super::format_email_local_time("").is_none()); -} - -#[test] -fn format_email_local_time_preserves_utc_raw_date_in_reshape() { - let mut v = json!({ - "messages": [{ - "messageId": "m1", - "threadId": "t1", - "subject": "Test", - "sender": "a@example.com", - "to": "b@example.com", - "messageTimestamp": "2026-05-31T10:33:00Z", - "labelIds": [], - "messageText": "body", - "payload": {} - }] - }); - post_process("GMAIL_FETCH_EMAILS", None, &mut v); - let msg = &v["messages"][0]; - assert_eq!(msg["date"], "2026-05-31T10:33:00Z"); -} - -#[test] -fn parse_email_date_accepts_rfc3339_and_rfc2822() { - assert!(super::parse_email_date("2026-05-31T10:33:00Z").is_some()); - assert!(super::parse_email_date("Sun, 31 May 2026 10:33:00 +0000").is_some()); - assert!(super::parse_email_date("not-a-date").is_none()); -} - -#[test] -fn format_at_tz_deterministic_with_fixed_offset() { - use chrono::FixedOffset; - - let utc = super::parse_email_date("2026-05-31T10:33:00Z").unwrap(); - - let est = FixedOffset::west_opt(5 * 3600).unwrap(); - let result = super::format_at_tz(utc, &est).unwrap(); - assert_eq!(result, "2026-05-31 05:33 AM -05:00"); - - let ist = FixedOffset::east_opt(5 * 3600 + 1800).unwrap(); - let result = super::format_at_tz(utc, &ist).unwrap(); - assert_eq!(result, "2026-05-31 04:03 PM +05:30"); -} - -#[test] -fn format_at_tz_returns_none_for_utc() { - let utc = super::parse_email_date("2026-05-31T10:33:00Z").unwrap(); - let utc_tz = chrono::FixedOffset::east_opt(0).unwrap(); - assert!(super::format_at_tz(utc, &utc_tz).is_none()); -} - -#[test] -fn apply_response_level_markdown_stashes_per_message_field() { - let mut data = json!({ - "messages": [ - {"messageId": "m1", "subject": "Hello"}, - {"messageId": "m2", "subject": "World"}, - ] - }); - let top_md = "## Hello\nbody A — link https://gh.io/abc\n---\n## World\nbody B"; - super::apply_response_level_markdown(&mut data, top_md); - let m1 = data["messages"][0]["markdownFormatted"].as_str().unwrap(); - let m2 = data["messages"][1]["markdownFormatted"].as_str().unwrap(); - assert!(m1.contains("Hello")); - assert!( - m1.contains("https://gh.io/abc"), - "shortened URL must survive" - ); - assert!(m2.contains("World")); - assert!(!m1.contains("World"), "no cross-message bleed"); -} diff --git a/src/openhuman/memory/sync/composio/providers/helpers.rs b/src/openhuman/memory/sync/composio/providers/helpers.rs index 92c3ccf9bf..8c909ceec1 100644 --- a/src/openhuman/memory/sync/composio/providers/helpers.rs +++ b/src/openhuman/memory/sync/composio/providers/helpers.rs @@ -1,36 +1,11 @@ //! Shared helpers for Composio provider implementations. +//! +//! `pick_str` used to live here. It is a provider payload normaliser, so it +//! moved to `tinycortex::memory::sync::composio::providers::normalize::helpers` +//! and is re-exported from this module's parent. The helpers that remain are +//! request-building rather than normalisation, and stay host-side. -/// Helper used by every provider's `fetch_user_profile` impl. -/// -/// Walks a JSON object using a list of dotted-path candidates and -/// returns the first non-empty string match. Keeps each provider's -/// extraction code free of repetitive `as_object().and_then(...)` -/// chains. -pub(crate) fn pick_str(value: &serde_json::Value, paths: &[&str]) -> Option { - for path in paths { - let mut cur = value; - let mut ok = true; - for segment in path.split('.') { - match cur.get(segment) { - Some(next) => cur = next, - None => { - ok = false; - break; - } - } - } - if !ok { - continue; - } - if let Some(s) = cur.as_str() { - let trimmed = s.trim(); - if !trimmed.is_empty() { - return Some(trimmed.to_string()); - } - } - } - None -} +use tinycortex::memory::sync::composio::providers::normalize::helpers::pick_str; /// Shallow-merge an `extra` JSON object into a (mutable) action-args /// object. Only object-typed extras are merged; non-object `extra` diff --git a/src/openhuman/memory/sync/composio/providers/linear/mod.rs b/src/openhuman/memory/sync/composio/providers/linear/mod.rs index 971e9e4dba..27ff7b736d 100644 --- a/src/openhuman/memory/sync/composio/providers/linear/mod.rs +++ b/src/openhuman/memory/sync/composio/providers/linear/mod.rs @@ -3,7 +3,10 @@ //! //! Issue: #2400. -mod normalization; +// The payload normalisers moved to tinycortex (they are pure Value +// transforms, i.e. driver-side). Aliased under the old module name so +// every `normalization::extract_*` call site below stays unchanged. +use tinycortex::memory::sync::composio::providers::normalize::linear as normalization; mod provider; #[cfg(test)] mod tests; diff --git a/src/openhuman/memory/sync/composio/providers/linear/normalization.rs b/src/openhuman/memory/sync/composio/providers/linear/normalization.rs deleted file mode 100644 index b1b33b2bab..0000000000 --- a/src/openhuman/memory/sync/composio/providers/linear/normalization.rs +++ /dev/null @@ -1,300 +0,0 @@ -//! Linear host normalization helpers — result extraction, issue-title extraction, -//! viewer identity, cursor extraction, and time utilities. -//! -//! Linear's GraphQL API (and therefore Composio's wrapping of it) returns -//! connection-style lists (`{ nodes: [...], pageInfo: {...} }`) at the top -//! level or nested under `data`. The functions here walk the union of -//! common shapes so the provider does not have to branch per Composio -//! envelope variant. - -use serde_json::Value; - -use crate::openhuman::memory::sync::composio::providers::pick_str; - -/// Walk the Composio response envelope for Linear issue list results. -/// -/// Linear's list endpoints return `{ nodes: [...] }` or -/// `{ issues: { nodes: [...] } }` shapes; Composio may re-wrap the -/// upstream payload under `data` or `data.data`. We probe each shape -/// in order and return the first array we find. -pub(crate) fn extract_issues(data: &Value) -> Vec { - let candidates = [ - data.pointer("/data/nodes"), - data.pointer("/nodes"), - data.pointer("/data/data/nodes"), - data.pointer("/data/issues/nodes"), - data.pointer("/data/results"), - data.pointer("/results"), - data.pointer("/data/items"), - data.pointer("/items"), - ]; - for cand in candidates.into_iter().flatten() { - if let Some(arr) = cand.as_array() { - return arr.clone(); - } - } - Vec::new() -} - -/// Extract a human-readable title from a Linear issue object. -/// -/// Linear issues store the name at `title` (or `data.title` after -/// Composio envelope wrapping). Falls back to `name` / `identifier` -/// so the chunk remains identifiable even for unusual response shapes. -pub(crate) fn extract_issue_title(issue: &Value) -> Option { - pick_str( - issue, - &[ - "title", - "data.title", - "name", - "data.name", - "identifier", - "data.identifier", - ], - ) -} - -/// Extract a stable cursor timestamp from a Linear issue object. -/// -/// Linear uses ISO-8601 strings for timestamps (`updatedAt`). We keep -/// the value as a string so lexicographic comparison against the stored -/// cursor is valid. -pub(crate) fn extract_issue_updated(issue: &Value) -> Option { - pick_str( - issue, - &[ - "updatedAt", - "data.updatedAt", - "updated_at", - "data.updated_at", - ], - ) -} - -/// Extract the viewer (authenticated user) object from a -/// `LINEAR_LIST_LINEAR_USERS { isMe: true }` response. -/// -/// Linear's GraphQL viewer endpoint returns `{ nodes: [{ id, email, … }] }`. -/// Composio may wrap this under `data` or `data.data`. We probe each -/// shape and return the first element of the nodes array, falling back -/// to the payload itself if it looks like a direct user object (has -/// `id` or `email`). -pub(crate) fn extract_viewer(data: &Value) -> Option { - let array_candidates = [ - data.pointer("/data/nodes"), - data.pointer("/nodes"), - data.pointer("/data/data/nodes"), - data.pointer("/data/users/nodes"), - ]; - for cand in array_candidates.into_iter().flatten() { - if let Some(arr) = cand.as_array() { - if let Some(first) = arr.first() { - return Some(first.clone()); - } - } - } - // Fallback: if the payload itself looks like a user object, return it. - if data.get("id").is_some() || data.get("email").is_some() { - return Some(data.clone()); - } - None -} - -/// Extract the viewer's ID string from a `LINEAR_LIST_LINEAR_USERS` -/// response. Returns `None` if the payload does not contain a -/// recognizable user ID. -pub(crate) fn extract_viewer_id(data: &Value) -> Option { - let viewer = extract_viewer(data)?; - pick_str(&viewer, &["id", "data.id"]) -} - -/// Extract a pagination cursor from a Linear connection `pageInfo` block. -/// -/// Returns `Some(endCursor)` only when `hasNextPage` is `true`; -/// `None` when the last page has been reached or when the envelope does -/// not carry `pageInfo` at all. -pub(crate) fn extract_pagination_cursor(data: &Value) -> Option { - let page_info_candidates = [ - data.pointer("/data/pageInfo"), - data.pointer("/pageInfo"), - data.pointer("/data/data/pageInfo"), - data.pointer("/data/issues/pageInfo"), - ]; - for cand in page_info_candidates.into_iter().flatten() { - let has_next = cand - .get("hasNextPage") - .and_then(|v| v.as_bool()) - .unwrap_or(false); - if has_next { - if let Some(cursor) = cand.get("endCursor").and_then(|v| v.as_str()) { - let trimmed = cursor.trim(); - if !trimmed.is_empty() { - return Some(trimmed.to_string()); - } - } - } - } - None -} - -/// Current wall-clock time in milliseconds since the UNIX epoch. -pub(crate) fn now_ms() -> u64 { - use std::time::{SystemTime, UNIX_EPOCH}; - SystemTime::now() - .duration_since(UNIX_EPOCH) - .map(|d| d.as_millis() as u64) - .unwrap_or(0) -} - -#[cfg(test)] -mod tests { - use super::*; - use serde_json::json; - - // ── extract_issues ─────────────────────────────────────────────── - - #[test] - fn extract_issues_from_data_nodes() { - let data = json!({ "data": { "nodes": [{"id": "i1"}, {"id": "i2"}] } }); - assert_eq!(extract_issues(&data).len(), 2); - } - - #[test] - fn extract_issues_from_top_level_nodes() { - let data = json!({ "nodes": [{"id": "i3"}] }); - assert_eq!(extract_issues(&data).len(), 1); - } - - #[test] - fn extract_issues_from_data_issues_nodes() { - let data = json!({ "data": { "issues": { "nodes": [{"id": "i4"}, {"id": "i5"}, {"id": "i6"}] } } }); - assert_eq!(extract_issues(&data).len(), 3); - } - - #[test] - fn extract_issues_from_results() { - let data = json!({ "results": [{"id": "i7"}] }); - assert_eq!(extract_issues(&data).len(), 1); - } - - #[test] - fn extract_issues_empty_when_missing() { - let data = json!({ "foo": "bar" }); - assert!(extract_issues(&data).is_empty()); - } - - // ── extract_issue_title ────────────────────────────────────────── - - #[test] - fn extract_issue_title_from_title_field() { - let issue = json!({ "id": "i1", "title": "Fix the login bug" }); - assert_eq!( - extract_issue_title(&issue), - Some("Fix the login bug".into()) - ); - } - - #[test] - fn extract_issue_title_falls_back_to_wrapped_data() { - let issue = json!({ "data": { "title": "Wrapped issue" } }); - assert_eq!(extract_issue_title(&issue), Some("Wrapped issue".into())); - } - - #[test] - fn extract_issue_title_falls_back_to_identifier() { - let issue = json!({ "identifier": "ENG-42" }); - assert_eq!(extract_issue_title(&issue), Some("ENG-42".into())); - } - - // ── extract_issue_updated ──────────────────────────────────────── - - #[test] - fn extract_issue_updated_from_updated_at() { - let issue = json!({ "updatedAt": "2026-03-01T12:00:00.000Z" }); - assert_eq!( - extract_issue_updated(&issue), - Some("2026-03-01T12:00:00.000Z".to_string()) - ); - } - - #[test] - fn extract_issue_updated_falls_back_to_snake_case() { - let issue = json!({ "data": { "updated_at": "2026-01-15T08:30:00.000Z" } }); - assert_eq!( - extract_issue_updated(&issue), - Some("2026-01-15T08:30:00.000Z".to_string()) - ); - } - - // ── extract_viewer ─────────────────────────────────────────────── - - #[test] - fn extract_viewer_from_data_nodes() { - let data = json!({ "data": { "nodes": [{ "id": "usr_1", "email": "a@b.com" }] } }); - let v = extract_viewer(&data).expect("should find viewer"); - assert_eq!(v["id"], "usr_1"); - } - - #[test] - fn extract_viewer_from_top_level_nodes() { - let data = json!({ "nodes": [{ "id": "usr_2" }] }); - let v = extract_viewer(&data).expect("should find viewer"); - assert_eq!(v["id"], "usr_2"); - } - - #[test] - fn extract_viewer_fallback_direct_object() { - let data = json!({ "id": "usr_direct", "name": "Direct User" }); - let v = extract_viewer(&data).expect("should return direct object"); - assert_eq!(v["id"], "usr_direct"); - } - - #[test] - fn extract_viewer_returns_none_when_absent() { - let data = json!({ "foo": "bar" }); - assert!(extract_viewer(&data).is_none()); - } - - // ── extract_pagination_cursor ──────────────────────────────────── - - #[test] - fn extract_pagination_cursor_returns_cursor_when_has_next_page() { - let data = json!({ - "data": { - "pageInfo": { - "hasNextPage": true, - "endCursor": "cursor_abc" - } - } - }); - assert_eq!( - extract_pagination_cursor(&data), - Some("cursor_abc".to_string()) - ); - } - - #[test] - fn extract_pagination_cursor_returns_none_when_last_page() { - let data = json!({ - "pageInfo": { - "hasNextPage": false, - "endCursor": "cursor_xyz" - } - }); - assert!(extract_pagination_cursor(&data).is_none()); - } - - #[test] - fn extract_pagination_cursor_returns_none_when_absent() { - let data = json!({ "nodes": [{"id": "i1"}] }); - assert!(extract_pagination_cursor(&data).is_none()); - } - - // ── now_ms ─────────────────────────────────────────────────────── - - #[test] - fn now_ms_returns_nonzero() { - assert!(now_ms() > 0); - } -} diff --git a/src/openhuman/memory/sync/composio/providers/mod.rs b/src/openhuman/memory/sync/composio/providers/mod.rs index cefe9ae178..f8406cd8c1 100644 --- a/src/openhuman/memory/sync/composio/providers/mod.rs +++ b/src/openhuman/memory/sync/composio/providers/mod.rs @@ -276,11 +276,16 @@ pub fn agent_ready_toolkits() -> Vec<&'static str> { } pub use descriptions::toolkit_description; -pub(crate) use helpers::{first_array_str, merge_extra, pick_str}; +pub(crate) use helpers::{first_array_str, merge_extra}; +// `pick_str` is a provider payload normaliser and lives in tinycortex; it is +// re-exported here so the ~40 in-tree call sites keep resolving unchanged. +// Note this is deliberately NOT `providers::common::pick_str`, which coerces +// numbers to strings — see the doc comments on both definitions. pub use registry::{ all_providers, get_provider, init_default_providers, register_provider, ProviderArc, }; pub use scope_lookup::{curated_scope_for, toolkit_has_scope}; +pub(crate) use tinycortex::memory::sync::composio::providers::normalize::helpers::pick_str; pub use tool_scope::{classify_unknown, find_curated, toolkit_from_slug, CuratedTool, ToolScope}; pub use traits::{resolve_sync_interval_secs, sync_interval_env_var, ComposioProvider}; pub use types::{ diff --git a/src/openhuman/memory/sync/composio/providers/notion/mod.rs b/src/openhuman/memory/sync/composio/providers/notion/mod.rs index 07d2a73e3b..5613ed0bd3 100644 --- a/src/openhuman/memory/sync/composio/providers/notion/mod.rs +++ b/src/openhuman/memory/sync/composio/providers/notion/mod.rs @@ -1,4 +1,7 @@ -mod normalization; +// The payload normalisers moved to tinycortex (they are pure Value +// transforms, i.e. driver-side). Aliased under the old module name so +// every `normalization::extract_*` call site below stays unchanged. +use tinycortex::memory::sync::composio::providers::normalize::notion as normalization; mod provider; #[cfg(test)] mod tests; diff --git a/src/openhuman/memory/sync/composio/providers/notion/normalization.rs b/src/openhuman/memory/sync/composio/providers/notion/normalization.rs deleted file mode 100644 index d225af6a52..0000000000 --- a/src/openhuman/memory/sync/composio/providers/notion/normalization.rs +++ /dev/null @@ -1,252 +0,0 @@ -//! Notion host normalization helpers — result extraction, pagination cursor, -//! page title extraction, and time utilities. - -use serde_json::Value; - -use crate::openhuman::memory::sync::composio::providers::pick_str; - -/// Walk the Composio response envelope for Notion page results. -pub(crate) fn extract_results(data: &Value) -> Vec { - let candidates = [ - data.pointer("/data/results"), - data.pointer("/results"), - data.pointer("/data/data/results"), - data.pointer("/data/items"), - data.pointer("/items"), - ]; - for cand in candidates.into_iter().flatten() { - if let Some(arr) = cand.as_array() { - return arr.clone(); - } - } - Vec::new() -} - -/// Extract the rendered page body markdown from a `NOTION_GET_PAGE_MARKDOWN` -/// response. Composio wraps action output in varying envelope shapes, so we -/// try the common locations tolerantly and return the first non-empty string. -/// Returns `None` if no markdown field is found (caller falls back to the -/// metadata-only body and logs the raw shape for diagnosis). -pub(crate) fn extract_page_markdown(data: &Value) -> Option { - const PATHS: &[&str] = &[ - "/markdown", - "/data/markdown", - "/data/response_data/markdown", - "/response_data/markdown", - "/data/content", - "/content", - "/data/markdown_content", - "/markdown_content", - "/text", - "/data/text", - ]; - for p in PATHS { - if let Some(s) = data.pointer(p).and_then(Value::as_str) { - if !s.trim().is_empty() { - return Some(s.to_string()); - } - } - } - None -} - -/// Extract the Notion pagination cursor (for `start_cursor` on the -/// next request). -pub(crate) fn extract_notion_cursor(data: &Value) -> Option { - let candidates = [ - data.pointer("/data/next_cursor"), - data.pointer("/next_cursor"), - data.pointer("/data/data/next_cursor"), - ]; - for cand in candidates.into_iter().flatten() { - if let Some(s) = cand.as_str() { - let trimmed = s.trim(); - if !trimmed.is_empty() { - return Some(trimmed.to_string()); - } - } - } - None -} - -/// Try to extract a human-readable title from a Notion page object. -/// -/// Notion pages store the title in `properties.title` or -/// `properties.Name.title[0].plain_text`. We try several shapes. -pub(crate) fn extract_page_title(page: &Value) -> Option { - // Try the common `properties.title.title[0].plain_text` shape. - let props = page - .get("properties") - .or_else(|| page.get("data")?.get("properties")); - if let Some(props) = props { - // Walk all properties looking for a "title" type field. - if let Some(obj) = props.as_object() { - for (_key, val) in obj { - if val.get("type").and_then(Value::as_str) == Some("title") { - if let Some(arr) = val.get("title").and_then(Value::as_array) { - let text: String = arr - .iter() - .filter_map(|t| t.get("plain_text").and_then(Value::as_str)) - .collect::>() - .join(""); - if !text.is_empty() { - return Some(text); - } - } - } - } - } - } - - // Fallback: top-level "title" field (some Composio shapes). - pick_str(page, &["title", "data.title", "name", "data.name"]) -} - -pub(crate) fn now_ms() -> u64 { - use std::time::{SystemTime, UNIX_EPOCH}; - SystemTime::now() - .duration_since(UNIX_EPOCH) - .map(|d| d.as_millis() as u64) - .unwrap_or(0) -} - -#[cfg(test)] -mod tests { - use super::*; - use serde_json::json; - - #[test] - fn extract_results_from_data_results() { - let data = json!({"data": {"results": [{"id": "page1"}]}}); - let results = extract_results(&data); - assert_eq!(results.len(), 1); - } - - #[test] - fn extract_page_markdown_reads_top_level_field() { - // Matches the live GET_PAGE_MARKDOWN envelope observed empirically: - // {id, markdown, object, request_id, truncated, unknown_block_ids}. - let data = json!({ - "id": "p1", - "markdown": "# Heading\n\nbody text", - "object": "page", - "truncated": false, - }); - assert_eq!( - extract_page_markdown(&data).as_deref(), - Some("# Heading\n\nbody text") - ); - } - - #[test] - fn extract_page_markdown_reads_nested_envelope() { - let data = json!({ "data": { "markdown": "nested body" } }); - assert_eq!(extract_page_markdown(&data).as_deref(), Some("nested body")); - } - - #[test] - fn extract_page_markdown_none_for_empty_or_missing() { - // Empty markdown (a DB row with no body blocks) → None → metadata-only. - assert_eq!(extract_page_markdown(&json!({ "markdown": "" })), None); - assert_eq!(extract_page_markdown(&json!({ "markdown": " " })), None); - // No markdown field at all → None. - assert_eq!(extract_page_markdown(&json!({ "id": "p1" })), None); - } - - #[test] - fn extract_results_from_top_level() { - let data = json!({"results": [{"id": "a"}, {"id": "b"}]}); - let results = extract_results(&data); - assert_eq!(results.len(), 2); - } - - #[test] - fn extract_results_from_data_items() { - let data = json!({"data": {"items": [{"id": "x"}]}}); - let results = extract_results(&data); - assert_eq!(results.len(), 1); - } - - #[test] - fn extract_results_empty_when_no_match() { - let data = json!({"foo": "bar"}); - assert!(extract_results(&data).is_empty()); - } - - #[test] - fn extract_notion_cursor_from_data() { - let data = json!({"data": {"next_cursor": "cur123"}}); - assert_eq!(extract_notion_cursor(&data), Some("cur123".into())); - } - - #[test] - fn extract_notion_cursor_from_top_level() { - let data = json!({"next_cursor": "abc"}); - assert_eq!(extract_notion_cursor(&data), Some("abc".into())); - } - - #[test] - fn extract_notion_cursor_none_when_empty() { - let data = json!({"data": {"next_cursor": " "}}); - assert_eq!(extract_notion_cursor(&data), None); - } - - #[test] - fn extract_notion_cursor_none_when_missing() { - assert_eq!(extract_notion_cursor(&json!({})), None); - } - - #[test] - fn extract_page_title_from_properties_title_type() { - let page = json!({ - "properties": { - "Name": { - "type": "title", - "title": [{"plain_text": "Hello"}, {"plain_text": " World"}] - } - } - }); - assert_eq!(extract_page_title(&page), Some("Hello World".into())); - } - - #[test] - fn extract_page_title_from_nested_data_properties() { - let page = json!({ - "data": { - "properties": { - "Title": { - "type": "title", - "title": [{"plain_text": "My Page"}] - } - } - } - }); - assert_eq!(extract_page_title(&page), Some("My Page".into())); - } - - #[test] - fn extract_page_title_fallback_to_top_level_title() { - let page = json!({"title": "Fallback Title"}); - assert_eq!(extract_page_title(&page), Some("Fallback Title".into())); - } - - #[test] - fn extract_page_title_none_when_empty() { - let page = json!({"properties": {"Name": {"type": "title", "title": []}}}); - // Empty title array means no text - assert!( - extract_page_title(&page).is_none() || extract_page_title(&page) == Some(String::new()) - ); - } - - #[test] - fn extract_page_title_none_when_no_title_field() { - let page = json!({"id": "123"}); - assert!(extract_page_title(&page).is_none()); - } - - #[test] - fn now_ms_returns_nonzero() { - assert!(now_ms() > 0); - } -} diff --git a/src/openhuman/memory/sync/composio/providers/slack/mod.rs b/src/openhuman/memory/sync/composio/providers/slack/mod.rs index acb10a6f5b..e796b55b13 100644 --- a/src/openhuman/memory/sync/composio/providers/slack/mod.rs +++ b/src/openhuman/memory/sync/composio/providers/slack/mod.rs @@ -6,7 +6,11 @@ //! against the user's Composio-authorized Slack connection. The reusable //! synchronization and ingestion engine is owned by tinycortex. -pub mod post_process; +// The Slack post-processor moved to tinycortex (a pure Value transform, i.e. +// driver-side). Re-exported under the old module name — `pub`, not a plain +// `use`, because `tests/raw_coverage/memory_threads_raw_coverage_e2e.rs` +// imports this path directly. +pub use tinycortex::memory::sync::composio::providers::normalize::slack_post_process as post_process; pub mod rpc; pub mod schemas; pub mod types; diff --git a/src/openhuman/memory/sync/composio/providers/slack/post_process.rs b/src/openhuman/memory/sync/composio/providers/slack/post_process.rs deleted file mode 100644 index 06a6a6e986..0000000000 --- a/src/openhuman/memory/sync/composio/providers/slack/post_process.rs +++ /dev/null @@ -1,248 +0,0 @@ -//! Slack-specific post-processing of Composio action responses. -//! -//! Composio's Slack responses are verbose API envelopes. This module -//! rewrites each supported action's response into a slim, stable shape -//! that the ingest pipeline and enrichers can consume without walking -//! Composio's unstable nested envelopes. -//! -//! ## Supported slugs -//! -//! - `SLACK_FETCH_CONVERSATION_HISTORY` — reshapes into top-level -//! `messages[]` with `{ ts, user, text, thread_ts, channel_id }`. -//! Empty-text messages are dropped. `channel_id` is absent here (it's -//! in the request, not the response); the caller injects it via the -//! enricher in [`super::sync`]. -//! -//! - `SLACK_LIST_CONVERSATIONS` — reshapes into top-level `channels[]` -//! with `{ id, name, is_private }` per channel. Entries with an empty -//! id are dropped. -//! -//! - `SLACK_SEARCH_MESSAGES` — reshapes `messages.matches[]` (possibly -//! nested) into top-level `messages[]` with `{ ts, user, text, -//! thread_ts, channel_id }`. `channel_id` is pulled from each match's -//! `channel.id` field. `paging.pages` is preserved at top-level for -//! caller pagination. -//! -//! ## Design note: user-id resolution is NOT here -//! -//! `SlackUsers` is a per-sync cache built from a separate API call — -//! not a function of any individual response. Resolving user ids -//! happens in [`super::sync`] (the enricher layer), keeping this module -//! purely data-shape–oriented. This matches Gmail's pattern of -//! "post_process is data-only". -//! -//! Unknown slugs are silently no-ops so new Composio actions don't -//! break the provider. - -use serde_json::{Map, Value}; - -/// Entry point called from `SlackProvider::post_process_action_result`. -/// -/// Dispatches on the Composio action slug and rewrites `data` in place. -/// Unknown slugs are silently ignored. -pub fn post_process(slug: &str, _arguments: Option<&Value>, data: &mut Value) { - log::debug!("[composio:slack][post-process] slug={slug}"); - match slug { - "SLACK_FETCH_CONVERSATION_HISTORY" => reshape_fetch_history(data), - "SLACK_LIST_CONVERSATIONS" => reshape_list_conversations(data), - "SLACK_SEARCH_MESSAGES" => reshape_search_messages(data), - _ => { - log::debug!("[composio:slack][post-process] unknown slug={slug}, passing through"); - } - } -} - -// ─── SLACK_FETCH_CONVERSATION_HISTORY ────────────────────────────────────── - -/// Rewrite a `SLACK_FETCH_CONVERSATION_HISTORY` response in place. -/// -/// Walks possible nested envelopes (`/data/messages`, `/messages`, -/// `/data/data/messages`) to find the raw messages array, drops messages -/// with empty `text`, and emits a slim `{ ts, user, text, thread_ts }` -/// shape under a top-level `messages[]` key. The caller injects -/// `channel_id` via [`super::sync::extract_messages`]. -fn reshape_fetch_history(data: &mut Value) { - let arr = extract_messages_array(data); - let slim: Vec = arr.into_iter().filter_map(slim_history_message).collect(); - let obj = ensure_object(data); - obj.insert("messages".to_string(), Value::Array(slim)); - log::debug!("[composio:slack][post-process] SLACK_FETCH_CONVERSATION_HISTORY reshaped"); -} - -fn slim_history_message(raw: Value) -> Option { - let text = raw - .get("text") - .and_then(|v| v.as_str()) - .unwrap_or("") - .trim(); - if text.is_empty() { - return None; - } - let mut out = Map::new(); - if let Some(ts) = raw.get("ts") { - out.insert("ts".into(), ts.clone()); - } else { - return None; // ts is required — no ts means we can't cursor or archive - } - if let Some(user) = raw.get("user").or_else(|| raw.get("bot_id")) { - out.insert("user".into(), user.clone()); - } - out.insert("text".into(), Value::String(text.to_string())); - if let Some(thread_ts) = raw.get("thread_ts") { - out.insert("thread_ts".into(), thread_ts.clone()); - } - if let Some(permalink) = raw.get("permalink") { - out.insert("permalink".into(), permalink.clone()); - } - Some(Value::Object(out)) -} - -/// Walk possible nested envelopes to find a messages array. Tries -/// `/data/messages`, `/messages`, then `/data/data/messages` in order. -fn extract_messages_array(data: &Value) -> Vec { - let candidates = [ - data.pointer("/data/messages"), - data.pointer("/messages"), - data.pointer("/data/data/messages"), - ]; - candidates - .into_iter() - .flatten() - .find_map(|v| v.as_array().cloned()) - .unwrap_or_default() -} - -// ─── SLACK_LIST_CONVERSATIONS ─────────────────────────────────────────────── - -/// Rewrite a `SLACK_LIST_CONVERSATIONS` response in place. -/// -/// Reshapes into a top-level `channels[]` with `{ id, name, is_private }` -/// per channel; entries with an empty id are dropped. -fn reshape_list_conversations(data: &mut Value) { - let candidates = [ - data.pointer("/data/channels"), - data.pointer("/channels"), - data.pointer("/data/data/channels"), - data.pointer("/data/conversations"), - data.pointer("/conversations"), - ]; - let arr: Vec = candidates - .into_iter() - .flatten() - .find_map(|v| v.as_array().cloned()) - .unwrap_or_default(); - - let slim: Vec = arr.into_iter().filter_map(slim_channel).collect(); - let obj = ensure_object(data); - obj.insert("channels".to_string(), Value::Array(slim)); - log::debug!("[composio:slack][post-process] SLACK_LIST_CONVERSATIONS reshaped"); -} - -fn slim_channel(raw: Value) -> Option { - let id = raw.get("id").and_then(|v| v.as_str()).unwrap_or("").trim(); - if id.is_empty() { - return None; - } - let name = raw - .get("name") - .and_then(|v| v.as_str()) - .unwrap_or(id) - .trim(); - let is_private = raw - .get("is_private") - .and_then(|v| v.as_bool()) - .unwrap_or(false); - Some(Value::Object({ - let mut m = Map::new(); - m.insert("id".into(), Value::String(id.to_string())); - m.insert("name".into(), Value::String(name.to_string())); - m.insert("is_private".into(), Value::Bool(is_private)); - m - })) -} - -// ─── SLACK_SEARCH_MESSAGES ────────────────────────────────────────────────── - -/// Rewrite a `SLACK_SEARCH_MESSAGES` response in place. -/// -/// Reshapes `messages.matches[]` (possibly nested under one or two -/// `data` envelopes) into top-level `messages[]`. `channel_id` is pulled -/// from each match's `channel.id` field. `paging.pages` is preserved at -/// top-level under `pages` for the caller to drive pagination. -fn reshape_search_messages(data: &mut Value) { - let candidates = [ - data.pointer("/data/messages/matches"), - data.pointer("/messages/matches"), - data.pointer("/data/data/messages/matches"), - ]; - let arr: Vec = candidates - .into_iter() - .flatten() - .find_map(|v| v.as_array().cloned()) - .unwrap_or_default(); - - // Preserve paging info before mutating data. - let pages = [ - data.pointer("/data/messages/paging/pages"), - data.pointer("/messages/paging/pages"), - ] - .into_iter() - .flatten() - .find_map(|v| v.as_u64()) - .unwrap_or(1); - - let slim: Vec = arr.into_iter().filter_map(slim_search_match).collect(); - let obj = ensure_object(data); - obj.insert("messages".to_string(), Value::Array(slim)); - obj.insert("pages".to_string(), Value::Number(pages.into())); - log::debug!("[composio:slack][post-process] SLACK_SEARCH_MESSAGES reshaped"); -} - -fn slim_search_match(raw: Value) -> Option { - let text = raw - .get("text") - .and_then(|v| v.as_str()) - .unwrap_or("") - .trim(); - if text.is_empty() { - return None; - } - let ts = raw.get("ts")?; - let channel_id = raw - .pointer("/channel/id") - .and_then(|v| v.as_str()) - .unwrap_or("") - .trim(); - - let mut out = Map::new(); - out.insert("ts".into(), ts.clone()); - if let Some(user) = raw.get("user").or_else(|| raw.get("bot_id")) { - out.insert("user".into(), user.clone()); - } - out.insert("text".into(), Value::String(text.to_string())); - if let Some(thread_ts) = raw.get("thread_ts") { - out.insert("thread_ts".into(), thread_ts.clone()); - } - if !channel_id.is_empty() { - out.insert("channel_id".into(), Value::String(channel_id.to_string())); - } - if let Some(permalink) = raw.get("permalink") { - out.insert("permalink".into(), permalink.clone()); - } - Some(Value::Object(out)) -} - -// ─── Helpers ──────────────────────────────────────────────────────────────── - -/// Ensure `data` is a JSON object, replacing it with an empty object if -/// not. Returns a mutable ref to the inner map. -fn ensure_object(data: &mut Value) -> &mut Map { - if !data.is_object() { - *data = Value::Object(Map::new()); - } - data.as_object_mut().unwrap() -} - -#[cfg(test)] -#[path = "post_process_tests.rs"] -mod tests; diff --git a/src/openhuman/memory/sync/composio/providers/slack/post_process_tests.rs b/src/openhuman/memory/sync/composio/providers/slack/post_process_tests.rs deleted file mode 100644 index 7b48189c64..0000000000 --- a/src/openhuman/memory/sync/composio/providers/slack/post_process_tests.rs +++ /dev/null @@ -1,180 +0,0 @@ -use super::*; -use serde_json::json; - -// ─── SLACK_FETCH_CONVERSATION_HISTORY ───────────────────────────────────── - -#[test] -fn history_reshapes_top_level_messages() { - let mut data = json!({ - "messages": [ - { "ts": "1714003200.000100", "user": "U1", "text": "hello" }, - { "ts": "1714003300.000200", "user": "U2", "text": "world", "thread_ts": "1714003200.0" }, - { "ts": "1714003400.000300", "user": "U3", "text": " " }, // dropped: empty text - ], - "response_metadata": { "next_cursor": "abc" } - }); - post_process("SLACK_FETCH_CONVERSATION_HISTORY", None, &mut data); - - let msgs = data["messages"].as_array().unwrap(); - assert_eq!(msgs.len(), 2, "empty-text message must be dropped"); - assert_eq!(msgs[0]["ts"], "1714003200.000100"); - assert_eq!(msgs[0]["user"], "U1"); - assert_eq!(msgs[0]["text"], "hello"); - assert!(msgs[0].get("thread_ts").is_none()); - assert_eq!(msgs[1]["thread_ts"], "1714003200.0"); -} - -#[test] -fn history_reshapes_nested_data_envelope() { - let mut data = json!({ - "data": { - "messages": [ - { "ts": "1714003200.0", "user": "U1", "text": "hi" } - ] - } - }); - post_process("SLACK_FETCH_CONVERSATION_HISTORY", None, &mut data); - let msgs = data["messages"].as_array().unwrap(); - assert_eq!(msgs.len(), 1); - assert_eq!(msgs[0]["text"], "hi"); -} - -#[test] -fn history_reshapes_doubly_nested_envelope() { - let mut data = json!({ - "data": { - "data": { - "messages": [ - { "ts": "1714003200.0", "user": "U1", "text": "deep" } - ] - } - } - }); - post_process("SLACK_FETCH_CONVERSATION_HISTORY", None, &mut data); - let msgs = data["messages"].as_array().unwrap(); - assert_eq!(msgs.len(), 1); - assert_eq!(msgs[0]["text"], "deep"); -} - -#[test] -fn history_drops_message_without_ts() { - let mut data = json!({ - "messages": [ - { "user": "U1", "text": "no timestamp" }, - { "ts": "1714003200.0", "user": "U2", "text": "has ts" }, - ] - }); - post_process("SLACK_FETCH_CONVERSATION_HISTORY", None, &mut data); - let msgs = data["messages"].as_array().unwrap(); - assert_eq!(msgs.len(), 1); - assert_eq!(msgs[0]["text"], "has ts"); -} - -// ─── SLACK_LIST_CONVERSATIONS ───────────────────────────────────────────── - -#[test] -fn list_conversations_reshapes_channels() { - let mut data = json!({ - "data": { - "channels": [ - { "id": "C1", "name": "eng", "is_private": false, "extra": "noise" }, - { "id": "G1", "name": "ops", "is_private": true }, - { "id": "", "name": "empty-id" }, // dropped - ] - } - }); - post_process("SLACK_LIST_CONVERSATIONS", None, &mut data); - let channels = data["channels"].as_array().unwrap(); - assert_eq!(channels.len(), 2, "empty-id entry must be dropped"); - assert_eq!(channels[0]["id"], "C1"); - assert_eq!(channels[0]["name"], "eng"); - assert_eq!(channels[0]["is_private"], false); - assert!( - channels[0].get("extra").is_none(), - "noise fields must be removed" - ); - assert_eq!(channels[1]["id"], "G1"); - assert_eq!(channels[1]["is_private"], true); -} - -#[test] -fn list_conversations_falls_back_to_conversations_key() { - let mut data = json!({ - "conversations": [ - { "id": "C2", "name": "dev", "is_private": false } - ] - }); - post_process("SLACK_LIST_CONVERSATIONS", None, &mut data); - let channels = data["channels"].as_array().unwrap(); - assert_eq!(channels.len(), 1); - assert_eq!(channels[0]["id"], "C2"); -} - -// ─── SLACK_SEARCH_MESSAGES ──────────────────────────────────────────────── - -#[test] -fn search_messages_reshapes_matches() { - let mut data = json!({ - "messages": { - "matches": [ - { - "ts": "1714003200.0", - "user": "U1", - "text": "hello from search", - "channel": { "id": "C1" } - }, - { - "ts": "1714003300.0", - "user": "U2", - "text": " ", // dropped: whitespace only - "channel": { "id": "C1" } - }, - ], - "paging": { "pages": 3 } - } - }); - post_process("SLACK_SEARCH_MESSAGES", None, &mut data); - let msgs = data["messages"].as_array().unwrap(); - assert_eq!(msgs.len(), 1, "empty-text match must be dropped"); - assert_eq!(msgs[0]["ts"], "1714003200.0"); - assert_eq!(msgs[0]["text"], "hello from search"); - assert_eq!(msgs[0]["channel_id"], "C1"); - assert_eq!(data["pages"], 3, "paging.pages must be preserved"); -} - -#[test] -fn search_messages_nested_data_envelope() { - let mut data = json!({ - "data": { - "messages": { - "matches": [ - { "ts": "1714003200.0", "user": "U1", "text": "nested", "channel": { "id": "C2" } } - ], - "paging": { "pages": 1 } - } - } - }); - post_process("SLACK_SEARCH_MESSAGES", None, &mut data); - let msgs = data["messages"].as_array().unwrap(); - assert_eq!(msgs.len(), 1); - assert_eq!(msgs[0]["channel_id"], "C2"); - assert_eq!(data["pages"], 1_u64); -} - -#[test] -fn search_messages_no_matches_emits_empty_array() { - let mut data = json!({ "messages": { "matches": [] } }); - post_process("SLACK_SEARCH_MESSAGES", None, &mut data); - let msgs = data["messages"].as_array().unwrap(); - assert!(msgs.is_empty()); -} - -// ─── Unknown slug ───────────────────────────────────────────────────────── - -#[test] -fn unknown_slug_is_noop() { - let mut data = json!({ "foo": "bar" }); - let original = data.clone(); - post_process("SLACK_SEND_MESSAGE", None, &mut data); - assert_eq!(data, original, "unknown slug must not mutate data"); -} diff --git a/src/openhuman/memory/sync/sync_status/mod.rs b/src/openhuman/memory/sync/sync_status/mod.rs index 38f7a642d0..d8e931c14a 100644 --- a/src/openhuman/memory/sync/sync_status/mod.rs +++ b/src/openhuman/memory/sync/sync_status/mod.rs @@ -15,10 +15,9 @@ pub mod rpc; pub mod schemas; -pub mod types; pub use schemas::{ all_controller_schemas as all_memory_sync_status_controller_schemas, all_registered_controllers as all_memory_sync_status_registered_controllers, }; -pub use types::{FreshnessLabel, MemorySyncStatus}; +pub use tinycortex::memory::sync::{FreshnessLabel, MemorySyncStatus}; diff --git a/src/openhuman/memory/sync/sync_status/rpc.rs b/src/openhuman/memory/sync/sync_status/rpc.rs index f27dd20adb..1fe635e561 100644 --- a/src/openhuman/memory/sync/sync_status/rpc.rs +++ b/src/openhuman/memory/sync/sync_status/rpc.rs @@ -3,7 +3,7 @@ use crate::openhuman::config::Config; use crate::rpc::RpcOutcome; -use super::types::StatusListResponse; +use tinycortex::memory::sync::StatusListResponse; pub async fn status_list_rpc(config: &Config) -> Result, String> { tracing::debug!("[memory_sync_status][rpc] status_list via tinycortex"); diff --git a/src/openhuman/memory/sync/sync_status/schemas.rs b/src/openhuman/memory/sync/sync_status/schemas.rs index c0a4c1733e..6bc55abc49 100644 --- a/src/openhuman/memory/sync/sync_status/schemas.rs +++ b/src/openhuman/memory/sync/sync_status/schemas.rs @@ -1,8 +1,9 @@ //! Controller-registry schemas for `openhuman.memory_sync_status_list`. //! //! Wired into `src/core/all.rs` via the `all_memory_sync_status_*` -//! re-exports in `super::mod`. Single method now — see `rpc.rs` and -//! `types.rs` for the simplified design (#1136 rewrite). +//! re-exports in `super::mod`. Single method now — see `rpc.rs` for the +//! simplified design (#1136 rewrite). The wire types are engine-owned +//! (`tinycortex::memory::sync`). use serde_json::{Map, Value}; diff --git a/src/openhuman/memory/sync/sync_status/types.rs b/src/openhuman/memory/sync/sync_status/types.rs deleted file mode 100644 index deb4d29178..0000000000 --- a/src/openhuman/memory/sync/sync_status/types.rs +++ /dev/null @@ -1,3 +0,0 @@ -//! Sync status wire types owned by tinycortex. - -pub use tinycortex::memory::sync::{FreshnessLabel, MemorySyncStatus, StatusListResponse}; diff --git a/src/openhuman/memory/tinycortex/config.rs b/src/openhuman/memory/tinycortex/config.rs index 7e30d40bc9..36b99555c9 100644 --- a/src/openhuman/memory/tinycortex/config.rs +++ b/src/openhuman/memory/tinycortex/config.rs @@ -44,6 +44,17 @@ pub fn memory_config_from(config: &Config, workspace: PathBuf) -> MemoryConfig { mc } +/// Build a [`MemoryConfig`] rooted at the host's own `workspace_dir`. +/// +/// This is the shape ~15 `memory/**` adapter modules each used to re-declare as +/// a private `fn engine_config` / `fn memory_config` / `fn config`; they were +/// byte-identical, so they now all call this. Use [`memory_config_from`] +/// directly only when the workspace root is *not* `config.workspace_dir` (the +/// sync/rebuild paths that address an alternate root). +pub fn engine_config(config: &Config) -> MemoryConfig { + memory_config_from(config, config.workspace_dir.clone()) +} + #[cfg(test)] mod tests { use super::*; @@ -76,4 +87,23 @@ mod tests { assert_eq!(mc.tree.summary_fanout, 10); assert_eq!(mc.tree.flush_age_secs, 604_800); } + + #[test] + fn engine_config_roots_at_host_workspace_dir() { + // Pins the wrapper's only behavioural claim: identical to + // `memory_config_from(config, config.workspace_dir.clone())`. + let mut config = Config::default(); + config.memory.embedding_dimensions = 768; + config.memory_tree.embedding_strict = true; + + let via_wrapper = engine_config(&config); + let via_explicit = memory_config_from(&config, config.workspace_dir.clone()); + + assert_eq!(via_wrapper.workspace, config.workspace_dir); + assert_eq!(via_wrapper.workspace, via_explicit.workspace); + assert_eq!(via_wrapper.content_root, via_explicit.content_root); + assert_eq!(via_wrapper.embedding.dim, via_explicit.embedding.dim); + assert_eq!(via_wrapper.embedding.model, via_explicit.embedding.model); + assert_eq!(via_wrapper.embedding.strict, via_explicit.embedding.strict); + } } diff --git a/src/openhuman/memory/tinycortex/mod.rs b/src/openhuman/memory/tinycortex/mod.rs index ae666cd2e2..45489a59a9 100644 --- a/src/openhuman/memory/tinycortex/mod.rs +++ b/src/openhuman/memory/tinycortex/mod.rs @@ -47,7 +47,7 @@ mod summariser; mod sync; pub use chat::{build_chat_provider, SeamChatProvider}; -pub use config::memory_config_from; +pub use config::{engine_config, memory_config_from}; pub use embeddings::SeamEmbedder; pub use ingest::{context as ingest_context, HostTreeJobSink}; pub use persona::{ diff --git a/src/openhuman/memory/tree/README.md b/src/openhuman/memory/tree/README.md index 9bc90b16cc..22bbd000ce 100644 --- a/src/openhuman/memory/tree/README.md +++ b/src/openhuman/memory/tree/README.md @@ -16,8 +16,8 @@ memory_tree (this module — generic mechanics) ├── retrieval/ agent-facing read tools (walk, drill, fetch) ├── score/ scoring, embedding, entity extraction ├── tools.rs re-exports from memory::query - └── io.rs canonical Tree{Write,Read}{Request,Outcome,Result} - │ + └── mod.rs re-exports the canonical Tree{Write,Read}{Request,Outcome,Result} + │ contract types from tinycortex::memory::tree ▼ memory_store::trees (persistence: one Tree table, one schema) ``` @@ -26,8 +26,7 @@ memory_store::trees (persistence: one Tree table, one schema) | Path | Role | | --- | --- | -| [`mod.rs`](mod.rs) | Re-exports `io::*` and the controller-schema registries hosted in `memory`. Re-exports `memory::tree_global` + `memory::tree_topic` under the legacy `memory_tree::tree_{global,topic}` paths. | -| [`io.rs`](io.rs) | Canonical contract types: `TreeWriteRequest`/`TreeWriteOutcome`, `TreeReadRequest`/`TreeReadHit`/`TreeReadResult`, `TreeLeafPayload`, `TreeLabelStrategy`. Pure types, no IO. | +| [`mod.rs`](mod.rs) | Re-exports the canonical contract types from `tinycortex::memory::tree` (`TreeWriteRequest`/`TreeWriteOutcome`, `TreeReadRequest`/`TreeReadHit`/`TreeReadResult`, `TreeLeafPayload`, `TreeLabelStrategy` — pure types, no IO) and the controller-schema registries hosted in `memory`. Re-exports `memory::tree_global` + `memory::tree_topic` under the legacy `memory_tree::tree_{global,topic}` paths. | | [`tree/`](tree/) | `bucket_seal` (append leaf + cascade seal), `flush` (time-based partial seal), `registry` (kind-parameterized `get_or_create_tree` with UNIQUE-race recovery), `mod.rs` (re-exports + `memory_store::trees` shims for legacy paths). | | [`summarise.rs`](summarise.rs) | One function: produce the next-level summary text for a bucket. Wraps the chat model with a fixed prompt and token budget. | | [`retrieval/`](retrieval/) | Agent-facing tools. Read: `walk` (agentic), `drill_down`, `fetch_leaves`, `query_{source,global,topic}`, `search_entities`. Write: `ingest_document` (orchestrator-facing). | diff --git a/src/openhuman/memory/tree/graph/store.rs b/src/openhuman/memory/tree/graph/store.rs index fbc8882b74..dac8417e7c 100644 --- a/src/openhuman/memory/tree/graph/store.rs +++ b/src/openhuman/memory/tree/graph/store.rs @@ -4,13 +4,10 @@ use anyhow::Result; use rusqlite::Transaction; use crate::openhuman::config::Config; +use crate::openhuman::memory::tinycortex::engine_config; pub use tinycortex::memory::graph::pairs_from_entities; -fn engine_config(config: &Config) -> tinycortex::memory::MemoryConfig { - crate::openhuman::memory::tinycortex::memory_config_from(config, config.workspace_dir.clone()) -} - pub fn upsert_edges_tx( transaction: &Transaction<'_>, pairs: &[(String, String)], diff --git a/src/openhuman/memory/tree/health/mod.rs b/src/openhuman/memory/tree/health/mod.rs index dd99ac8429..05bee36b8c 100644 --- a/src/openhuman/memory/tree/health/mod.rs +++ b/src/openhuman/memory/tree/health/mod.rs @@ -1,28 +1,22 @@ -//! Typed failure + degradation model for the memory pipeline. +//! Host-side surface of the memory pipeline's failure + degradation model. //! -//! The chunk→wiki pipeline and the time-tree summarizer fail in several -//! distinct ways (budget exhausted, missing/invalid key, missing local -//! model, dimension mismatch, extraction timeout, transient network). -//! Historically these all collapsed into an opaque error string and were -//! retried identically — so a hard "Insufficient budget" 4xx burned the -//! retry budget and the user saw a generic `error: N failed jobs`. +//! The **taxonomy itself** — [`FailureCode`], [`FailureClass`], +//! [`PipelineFailure`], [`DegradedState`], and the `classify_embed_error` +//! classifier — now lives in the engine crate at +//! `tinycortex::memory::health`, and is re-exported below so every existing +//! `memory::tree::health::…` path keeps resolving. It moved because a build +//! whose only driver was a third-party external backend would have no use for +//! the engine's private failure vocabulary. //! -//! This module is the single source of truth that fixes that: +//! What stays here is everything that is *host* surface rather than engine +//! vocabulary: //! -//! - [`FailureCode`] enumerates every distinguishable cause. -//! - Each code maps to a [`FailureClass`] (`Transient` ⇒ retry with -//! backoff, `Unrecoverable` ⇒ fail fast) and a stable i18n -//! `remediation_key` so the status surface / doctor / job row all show -//! consistent, actionable text. Embeddings remediation leads with the -//! local-Ollama path (the steered primary fix), with BYO key secondary. -//! - [`PipelineFailure`] is a `std::error::Error`, so it can be wrapped in -//! `anyhow` and propagated up through the job processor, then downcast in -//! the queue worker to decide retry-vs-fail. -//! - [`DegradedState`] captures "the pipeline ran but recall/structure is -//! reduced" — surfaced so degraded output is never presented as success. - -use serde::{Deserialize, Serialize}; -use std::fmt; +//! - the **process-visible degradation flags** (`mark_*` / `clear_*` / +//! [`current_degraded_state`]) — set by host plumbing deep in the job worker, +//! read by the `pipeline_status` RPC, and coupled to a socket broadcast; +//! - [`doctor`] — the health report, which reads the host's scheduler-gate +//! config; +//! - `user_error` — whose `kind` string is a pinned contract with the frontend. pub mod doctor; pub use doctor::{async_run_doctor, run_doctor, DoctorCounters, DoctorReport, StageHealth}; @@ -30,394 +24,13 @@ pub use doctor::{async_run_doctor, run_doctor, DoctorCounters, DoctorReport, Sta pub(crate) mod user_error; pub(crate) use user_error::publish_local_model_unavailable_user_error; -/// Whether a failure should be retried (`Transient`) or fail fast -/// (`Unrecoverable`). -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum FailureClass { - /// Retry with backoff up to `max_attempts` (network 5xx, timeouts, - /// truncated streams). - Transient, - /// Stop immediately — retrying the same input cannot succeed (budget - /// exhausted, bad/missing key, missing local model, dim mismatch). - Unrecoverable, -} - -impl FailureClass { - pub fn as_str(self) -> &'static str { - match self { - Self::Transient => "transient", - Self::Unrecoverable => "unrecoverable", - } - } -} - -/// A distinguishable pipeline failure cause. Each variant carries a fixed -/// [`FailureClass`] and i18n remediation key. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum FailureCode { - /// Managed embeddings route returned an out-of-budget error (4xx). - BudgetExhausted, - /// No auth/session available for the embeddings provider. - AuthMissing, - /// Auth present but rejected (expired/invalid key or JWT). - AuthInvalid, - /// No embeddings provider is configured at all. - EmbeddingsUnconfigured, - /// Provider returned vectors of an unexpected dimensionality. - EmbeddingDimMismatch, - /// A required local model (Ollama) is not available. - LocalModelUnavailable, - /// The extraction model timed out / exhausted retries. - ExtractionTimeout, - /// No summarization provider could be resolved for "Build Summary Trees" - /// — neither local AI nor a configured cloud chat provider. Distinct from - /// [`LocalModelUnavailable`](Self::LocalModelUnavailable), which implies the - /// local path was selected; this covers the cloud-only setup whose provider - /// failed to resolve, so the remediation names both paths. - SummarizerUnavailable, - /// The embedding provider refused an empty/whitespace input at the - /// pre-flight guard (#13021). Unrecoverable per-row: the offending row - /// will never become embeddable, so the worker must tombstone it instead - /// of retrying. Bail wording for both `OpenAiEmbedding::embed` and - /// `OpenHumanCloudEmbedding::embed` starts with - /// `" embed: refusing empty/whitespace input ..."`. - EmptyInputRefused, - /// The host filesystem cannot service the memory_tree path — `create_dir` - /// / DB open returned a persistent OS-level I/O error (EIO `5`, ENOSPC - /// `28`, EROFS `30`), e.g. a failing/disconnected SD card or a volume the - /// kernel remounted read-only. Unrecoverable from inside the app: only the - /// user can reseat/replace/free the storage. Distinct from the embeddings - /// provider faults above and from the SQLite-level `SQLITE_FULL` / - /// `SQLITE_CORRUPT` handled in the queue worker — this is the - /// directory/DB-init layer below them. - StorageUnavailable, - /// Catch-all transient failure (network 5xx, timeout, truncated JSON). - Transient, -} - -impl FailureCode { - /// Stable wire string. - pub fn as_str(self) -> &'static str { - match self { - Self::BudgetExhausted => "budget_exhausted", - Self::AuthMissing => "auth_missing", - Self::AuthInvalid => "auth_invalid", - Self::EmbeddingsUnconfigured => "embeddings_unconfigured", - Self::EmbeddingDimMismatch => "embedding_dim_mismatch", - Self::LocalModelUnavailable => "local_model_unavailable", - Self::ExtractionTimeout => "extraction_timeout", - Self::SummarizerUnavailable => "summarizer_unavailable", - Self::EmptyInputRefused => "empty_input_refused", - Self::StorageUnavailable => "storage_unavailable", - Self::Transient => "transient", - } - } - - pub fn from_str(s: &str) -> Option { - Some(match s { - "budget_exhausted" => Self::BudgetExhausted, - "auth_missing" => Self::AuthMissing, - "auth_invalid" => Self::AuthInvalid, - "embeddings_unconfigured" => Self::EmbeddingsUnconfigured, - "embedding_dim_mismatch" => Self::EmbeddingDimMismatch, - "local_model_unavailable" => Self::LocalModelUnavailable, - "extraction_timeout" => Self::ExtractionTimeout, - "summarizer_unavailable" => Self::SummarizerUnavailable, - "empty_input_refused" => Self::EmptyInputRefused, - "storage_unavailable" => Self::StorageUnavailable, - "transient" => Self::Transient, - _ => return None, - }) - } - - /// Retry policy for this cause. - /// - /// [`LocalModelUnavailable`](Self::LocalModelUnavailable) is deliberately - /// **transient** even though the user has to act: the condition (Ollama - /// daemon stopped, model not pulled) clears from outside the app, and only - /// transient rows are picked up by `requeue_transient_failed` — the - /// automatic self-healing requeue. Classifying it unrecoverable would park - /// every affected job until someone clicks "Retry failed" by hand, so a - /// user who simply restarts Ollama would never see ingestion resume. - pub fn class(self) -> FailureClass { - match self { - Self::Transient | Self::ExtractionTimeout | Self::LocalModelUnavailable => { - FailureClass::Transient - } - _ => FailureClass::Unrecoverable, - } - } - - /// i18n key for the user-facing remediation. Embeddings causes lead - /// with the local-Ollama path (the steered primary fix per spec FR-015). - pub fn remediation_key(self) -> &'static str { - match self { - Self::BudgetExhausted => "memory.health.remediation.budget_exhausted", - Self::AuthMissing => "memory.health.remediation.auth_missing", - Self::AuthInvalid => "memory.health.remediation.auth_invalid", - Self::EmbeddingsUnconfigured => "memory.health.remediation.embeddings_unconfigured", - Self::EmbeddingDimMismatch => "memory.health.remediation.embedding_dim_mismatch", - Self::LocalModelUnavailable => "memory.health.remediation.local_model_unavailable", - Self::ExtractionTimeout => "memory.health.remediation.extraction_timeout", - Self::SummarizerUnavailable => "memory.health.remediation.summarizer_unavailable", - Self::EmptyInputRefused => "memory.health.remediation.empty_input_refused", - Self::StorageUnavailable => "memory.health.remediation.storage_unavailable", - Self::Transient => "memory.health.remediation.transient", - } - } -} - -/// A typed pipeline failure: a [`FailureCode`] plus the derived class + -/// remediation key (carried on the wire so the frontend stays -/// presentational) and an optional human-readable detail for logs/diagnosis. -/// -/// Implements [`std::error::Error`] so it can be `anyhow`-wrapped at the -/// embed/extract/summarize boundary, propagated through the job processor, -/// and downcast in the queue worker to drive retry-vs-fail. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct PipelineFailure { - pub code: FailureCode, - pub class: FailureClass, - /// i18n key — the frontend resolves this to localized remediation text. - pub remediation_key: String, - /// Optional non-localized detail for logs/diagnosis (never a secret). - #[serde(default, skip_serializing_if = "Option::is_none")] - pub detail: Option, -} - -impl PipelineFailure { - /// Build a failure from a code, deriving class + remediation key. - pub fn new(code: FailureCode) -> Self { - Self { - code, - class: code.class(), - remediation_key: code.remediation_key().to_string(), - detail: None, - } - } - - /// Attach a non-localized detail string (truncated by callers; never - /// log secrets). - pub fn with_detail(mut self, detail: impl Into) -> Self { - self.detail = Some(detail.into()); - self - } - - /// True when this failure should fail fast (no retry budget). - pub fn is_unrecoverable(&self) -> bool { - self.class == FailureClass::Unrecoverable - } -} - -/// Classify an embedding-stage error into a typed [`PipelineFailure`]. -/// -/// The embed path bottoms out in `embeddings::openai::OpenAiEmbedding::embed`, -/// which on a non-2xx response bails with the message -/// `"Embedding API error (): "` (status is reqwest's -/// `StatusCode` Display, e.g. `402 Payment Required`). Dimension mismatches -/// surface from the memory-tree `CloudEmbedder`/trait validator as -/// `"... returned N dims, expected M"` or `"... dims, expected ..."`. We -/// parse those shapes to decide retry-vs-fail: -/// -/// - `401` / `403` → `auth_invalid` (a bearer was sent but rejected). -/// - `402` / `429` / a body mentioning budget/quota/insufficient → -/// `budget_exhausted` (the managed Voyage route is out of budget; the -/// user must bring their own key or top up — retrying won't help). -/// - dimension-mismatch text → `embedding_dim_mismatch`. -/// - Ollama daemon-unreachable / model-not-pulled text → -/// `local_model_unavailable`, so the panel names the local-runtime fix. -/// - everything else (5xx, timeouts, transport, unparseable) → `transient`, -/// so the worker's existing retry-with-backoff still applies. -/// -/// Operates on the flattened `anyhow` chain (`{err:#}`) so it still matches -/// when the embed error has been `.context()`-wrapped on the way up. -pub fn classify_embed_error(err: &anyhow::Error) -> PipelineFailure { - let msg = format!("{err:#}"); - classify_embed_error_str(&msg) -} - -/// String-level core of [`classify_embed_error`], split out so unit tests can -/// exercise the mapping without constructing reqwest errors. -pub fn classify_embed_error_str(msg: &str) -> PipelineFailure { - let lower = msg.to_ascii_lowercase(); - - // #13021: client-side refusal from the provider pre-flight guard fires - // *before* any HTTP round-trip, so it carries no `Embedding API error - // ()` shape. Without an explicit match it would fall through to - // `Transient` and the `reembed_backfill` worker would retry the same - // un-embeddable row forever (and eventually fail the whole job). - // Classify as unrecoverable per-row so the worker tombstones the chunk / - // summary instead. Both `OpenAiEmbedding::embed` and - // `OpenHumanCloudEmbedding::embed` use the literal phrase - // "refusing empty/whitespace". - if lower.contains("refusing empty/whitespace") { - return PipelineFailure::new(FailureCode::EmptyInputRefused) - .with_detail(truncate_detail(msg)); - } - - // Sibling of the #13021 case above: `OpenHumanCloudEmbedding::resolve_bearer` - // bails *before any HTTP round-trip* when the desktop/backend session - // bearer is absent (user signed out), with the literal phrase - // "No backend session for cloud embeddings ..." (see - // `src/openhuman/inference/embeddings/cloud.rs`). Being a client-side bail it carries - // no `Embedding API error ()` shape, so without this match it falls - // through to `Transient` — the Memory Tree then shows "temporary error… - // will retry automatically" and the worker retries an auth failure that a - // retry can never fix. Classify as `AuthMissing` so the health banner - // surfaces the "log in to OpenHuman" remediation and the job fails fast. - if lower.contains("no backend session") { - return PipelineFailure::new(FailureCode::AuthMissing).with_detail(truncate_detail(msg)); - } - - // #5354 — the local Ollama runtime is not usable: the daemon is not - // listening, or the configured embedding model was never pulled. Both are - // emitted by `tinyagents::harness::embeddings::ollama` with the fix already - // in the text: - // - // "ollama embed request failed (is Ollama running at ?): …" - // "Ollama embedding model `` is not installed at . Run `ollama pull ` …" - // - // Neither carries an `Embedding API error ()` shape — the first is a - // transport bail, the second a rewritten 404 — so both used to fall through - // to `Transient` and surface as "a temporary error … will retry - // automatically". That is the wrong remediation: retrying cannot start a - // daemon or pull a model, and the user was never told what to do. Match the - // two shapes explicitly so the status panel renders the - // `local_model_unavailable` remediation instead. The class stays transient - // (see `FailureCode::class`) so jobs auto-resume once Ollama is back. - // - // Anchored on Ollama-specific wording so a generic cloud-embedder transport - // failure ("error sending request for url …") keeps its `Transient` code. - if lower.contains("is ollama running at") - || (lower.contains("ollama embedding model") && lower.contains("is not installed at")) - { - return PipelineFailure::new(FailureCode::LocalModelUnavailable) - .with_detail(truncate_detail(msg)); - } - - // Dimension mismatch — the trait validator / CloudEmbedder rejects a - // vector whose length isn't EMBEDDING_DIM. Check before status parsing: - // it's a 2xx-but-wrong-shape case with no HTTP status to match. - if lower.contains("dims, expected") || lower.contains("dimensions, expected") { - return PipelineFailure::new(FailureCode::EmbeddingDimMismatch) - .with_detail(truncate_detail(msg)); - } - - // Budget/quota wording wins regardless of the numeric status — the - // managed backend may surface budget exhaustion as 4xx with an explicit - // body, and we always want the BYO-key remediation here. - if lower.contains("insufficient budget") - || lower.contains("budget") - || lower.contains("quota") - || lower.contains("payment required") - { - return PipelineFailure::new(FailureCode::BudgetExhausted) - .with_detail(truncate_detail(msg)); - } - - // Parse the HTTP status out of the `Embedding API error (): ...` - // shape. reqwest renders e.g. `402 Payment Required`, so the first - // 3-digit run after the opening paren is the code. - if let Some(code) = parse_http_status(msg) { - return match code { - 401 | 403 => { - PipelineFailure::new(FailureCode::AuthInvalid).with_detail(truncate_detail(msg)) - } - 402 => { - PipelineFailure::new(FailureCode::BudgetExhausted).with_detail(truncate_detail(msg)) - } - 429 => PipelineFailure::new(FailureCode::Transient).with_detail(truncate_detail(msg)), - // 4xx other than the above is a hard client error retrying won't - // fix (malformed request, model not found); fail fast but tag it - // generically as auth_invalid's sibling — use Transient only for - // 5xx/unknown. We treat unknown 4xx as unrecoverable via - // budget? No — be conservative: only the known codes above are - // unrecoverable; other 4xx fall through to transient so we don't - // wedge on a transient 408/425. - 500..=599 => { - PipelineFailure::new(FailureCode::Transient).with_detail(truncate_detail(msg)) - } - _ => PipelineFailure::new(FailureCode::Transient).with_detail(truncate_detail(msg)), - }; - } - - // No recognizable status — transport error, timeout, connection reset, - // or an unparseable message. Treat as transient so retry/backoff applies. - PipelineFailure::new(FailureCode::Transient).with_detail(truncate_detail(msg)) -} - -/// Extract the first HTTP status code from an `Embedding API error ()` -/// message. Returns the leading 3-digit number inside the first parenthesised -/// group, if present. -fn parse_http_status(msg: &str) -> Option { - let open = msg.find('(')?; - let rest = &msg[open + 1..]; - let digits: String = rest - .trim_start() - .chars() - .take_while(|c| c.is_ascii_digit()) - .collect(); - if digits.len() == 3 { - digits.parse().ok() - } else { - None - } -} - -/// Cap a detail string so we never balloon logs / wire payloads with a full -/// provider response body. Never contains a secret (it's an error body), but -/// keep it short anyway. -fn truncate_detail(s: &str) -> String { - const MAX: usize = 200; - if s.chars().count() <= MAX { - return s.to_string(); - } - let truncated: String = s.chars().take(MAX).collect(); - format!("{truncated}…") -} - -impl fmt::Display for PipelineFailure { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{} ({})", self.code.as_str(), self.class.as_str())?; - if let Some(detail) = &self.detail { - write!(f, ": {detail}")?; - } - Ok(()) - } -} - -impl std::error::Error for PipelineFailure {} - -/// "The pipeline ran, but output quality is reduced." Surfaced so degraded -/// results are never presented as success. -#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] -pub struct DegradedState { - /// True when embeddings were skipped (no usable provider) so semantic - /// recall falls back to recency-only. - pub semantic_recall: bool, - /// True when extraction yielded empty across the board so the wiki has - /// no entity/topic structure. - pub structure: bool, - /// True when the memory_tree's own storage path is unusable — the host - /// filesystem returned a persistent I/O error on dir-create / DB open - /// (EIO/ENOSPC/EROFS). This is the most severe degradation: the pipeline - /// can't even open its DB, so nothing else runs. `#[serde(default)]` keeps - /// the wire format backward-compatible (older clients omit it → `false`). - #[serde(default)] - pub storage: bool, - /// The cause of the most significant degradation, when known. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub cause: Option, -} - -impl DegradedState { - /// True when any degradation is present. - pub fn is_degraded(&self) -> bool { - self.semantic_recall || self.structure || self.storage - } -} +/// The failure taxonomy proper. Re-exported (rather than re-declared) so the +/// ~30 `crate::openhuman::memory::tree::health::{…}` call sites across the host +/// are unaffected by the move, and so there is exactly one definition. +pub use tinycortex::memory::health::{ + classify_embed_error, classify_embed_error_str, DegradedState, FailureClass, FailureCode, + PipelineFailure, +}; // ── Process-visible degradation flags ──────────────────────────────────── // @@ -669,348 +282,6 @@ mod tests { use super::*; use user_error::LOCAL_MODEL_UNAVAILABLE_KIND; - const ALL_CODES: [FailureCode; 11] = [ - FailureCode::BudgetExhausted, - FailureCode::AuthMissing, - FailureCode::AuthInvalid, - FailureCode::EmbeddingsUnconfigured, - FailureCode::EmbeddingDimMismatch, - FailureCode::LocalModelUnavailable, - FailureCode::ExtractionTimeout, - FailureCode::SummarizerUnavailable, - FailureCode::EmptyInputRefused, - FailureCode::StorageUnavailable, - FailureCode::Transient, - ]; - - #[test] - fn every_code_has_class_and_nonempty_remediation_key() { - for code in ALL_CODES { - let key = code.remediation_key(); - assert!( - !key.is_empty(), - "{} has empty remediation key", - code.as_str() - ); - assert!( - key.starts_with("memory.health.remediation."), - "{} remediation key has unexpected prefix: {key}", - code.as_str() - ); - // class() must be total (no panic); Transient, ExtractionTimeout - // and LocalModelUnavailable are retryable, everything else is - // unrecoverable. - let class = code.class(); - match code { - FailureCode::Transient - | FailureCode::ExtractionTimeout - | FailureCode::LocalModelUnavailable => { - assert_eq!( - class, - FailureClass::Transient, - "{} should be transient", - code.as_str() - ); - } - _ => { - assert_eq!( - class, - FailureClass::Unrecoverable, - "{} should be unrecoverable", - code.as_str() - ); - } - } - } - } - - #[test] - fn code_str_roundtrips() { - for code in ALL_CODES { - assert_eq!(FailureCode::from_str(code.as_str()), Some(code)); - } - assert_eq!(FailureCode::from_str("nonsense"), None); - } - - #[test] - fn new_fills_class_and_remediation_from_code() { - let f = PipelineFailure::new(FailureCode::BudgetExhausted); - assert_eq!(f.code, FailureCode::BudgetExhausted); - assert_eq!(f.class, FailureClass::Unrecoverable); - assert_eq!( - f.remediation_key, - "memory.health.remediation.budget_exhausted" - ); - assert!(f.detail.is_none()); - assert!(f.is_unrecoverable()); - } - - #[test] - fn with_detail_and_display() { - let f = PipelineFailure::new(FailureCode::Transient).with_detail("HTTP 503"); - assert_eq!(f.detail.as_deref(), Some("HTTP 503")); - assert!(!f.is_unrecoverable()); - assert_eq!(f.to_string(), "transient (transient): HTTP 503"); - } - - #[test] - fn pipeline_failure_serde_roundtrips() { - let f = PipelineFailure::new(FailureCode::EmbeddingDimMismatch).with_detail("got 3072"); - let json = serde_json::to_string(&f).unwrap(); - let back: PipelineFailure = serde_json::from_str(&json).unwrap(); - assert_eq!(f, back); - // detail omitted when None. - let none = PipelineFailure::new(FailureCode::AuthMissing); - assert!(!serde_json::to_string(&none).unwrap().contains("detail")); - } - - #[test] - fn degraded_state_default_is_healthy() { - let d = DegradedState::default(); - assert!(!d.is_degraded()); - let d2 = DegradedState { - structure: true, - ..Default::default() - }; - assert!(d2.is_degraded()); - } - - #[test] - fn pipeline_failure_is_error_and_downcasts_from_anyhow() { - let err: anyhow::Error = - anyhow::Error::new(PipelineFailure::new(FailureCode::BudgetExhausted)); - let downcast = err.downcast_ref::(); - assert!(downcast.is_some()); - assert!(downcast.unwrap().is_unrecoverable()); - } - - // ── classify_embed_error (T008) ────────────────────────────────────── - - #[test] - fn classify_budget_from_body_wording() { - // The managed Voyage route surfaces budget exhaustion in the body. - let f = classify_embed_error_str( - "Embedding API error (400 Bad Request): {\"error\":\"Insufficient budget\"}", - ); - assert_eq!(f.code, FailureCode::BudgetExhausted); - assert!(f.is_unrecoverable()); - } - - #[test] - fn classify_budget_from_402() { - let f = classify_embed_error_str("Embedding API error (402 Payment Required): nope"); - assert_eq!(f.code, FailureCode::BudgetExhausted); - assert!(f.is_unrecoverable()); - } - - #[test] - fn classify_429_rate_limit_as_transient() { - let f = classify_embed_error_str("Embedding API error (429 Too Many Requests): nope"); - assert_eq!(f.code, FailureCode::Transient); - assert!(!f.is_unrecoverable()); - } - - #[test] - fn classify_auth_from_401_403() { - for status in ["401 Unauthorized", "403 Forbidden"] { - let f = classify_embed_error_str(&format!("Embedding API error ({status}): denied")); - assert_eq!(f.code, FailureCode::AuthInvalid, "status {status}"); - assert!(f.is_unrecoverable()); - } - } - - #[test] - fn classify_dim_mismatch() { - let f = classify_embed_error_str("cloud embedder returned 3072 dims, expected 1024"); - assert_eq!(f.code, FailureCode::EmbeddingDimMismatch); - assert!(f.is_unrecoverable()); - } - - /// #13021: the provider pre-flight bail wording from both OpenAI and the - /// cloud wrapper must classify as `EmptyInputRefused` (unrecoverable) so - /// `reembed_backfill` tombstones the offending row instead of retrying - /// the same blank input forever and eventually failing the job. - #[test] - fn classify_empty_input_refusal_as_unrecoverable() { - for msg in [ - "openai embed: refusing empty/whitespace input at index 0 of 1 (model=text-embedding-3-small)", - "cloud embed: refusing empty/whitespace input at index 2 of 5 (model=embedding-v1)", - ] { - let f = classify_embed_error_str(msg); - assert_eq!( - f.code, - FailureCode::EmptyInputRefused, - "expected EmptyInputRefused for {msg:?}" - ); - assert!( - f.is_unrecoverable(), - "EmptyInputRefused must be unrecoverable for {msg:?}" - ); - } - } - - /// The refusal must out-rank the dim-mismatch and budget rules even when - /// the wrapped error happens to contain those tokens — the refusal phrase - /// is the most specific signal and the only one that means "this row is - /// permanently un-embeddable", not "the provider is misbehaving". - #[test] - fn classify_empty_input_refusal_through_anyhow_context_chain() { - let base = anyhow::anyhow!( - "openai embed: refusing empty/whitespace input at index 0 of 1 (model=embedding-v1)" - ); - let wrapped = base - .context("embed summary during seal tree_id=t level=0") - .context("reembed_backfill chunk_id=c"); - let f = classify_embed_error(&wrapped); - assert_eq!(f.code, FailureCode::EmptyInputRefused); - assert!(f.is_unrecoverable()); - } - - /// #4359: `OpenHumanCloudEmbedding::resolve_bearer` bails with "No backend - /// session for cloud embeddings ..." *before any HTTP call* when the user - /// is signed out. This must classify as `AuthMissing` (unrecoverable, "log - /// in to OpenHuman" remediation) rather than falling through to `Transient` - /// ("will retry automatically" — a loop that an auth failure can never win). - #[test] - fn classify_no_backend_session_as_auth_missing() { - let msg = "No backend session for cloud embeddings: log in to OpenHuman, or set \ - memory.embedding_provider to \"ollama\" / \"none\" in config.toml"; - let f = classify_embed_error_str(msg); - assert_eq!( - f.code, - FailureCode::AuthMissing, - "expected AuthMissing for {msg:?}" - ); - assert_eq!(f.class, FailureClass::Unrecoverable); - assert_eq!(f.remediation_key, "memory.health.remediation.auth_missing"); - assert!(f.is_unrecoverable()); - } - - /// The match must be case-insensitive and survive `anyhow` context wrapping: - /// the bail is `.context()`-wrapped on its way up through the embed pipeline - /// (e.g. `embed_each_via_provider` adds "cloud embeddings failed"), and - /// `classify_embed_error` flattens the chain via `{err:#}`. - #[test] - fn classify_no_backend_session_through_anyhow_context_chain() { - let base = anyhow::anyhow!( - "No backend session for cloud embeddings: log in to OpenHuman, or set \ - memory.embedding_provider to \"ollama\" / \"none\" in config.toml" - ); - let wrapped = base - .context("cloud embeddings failed") - .context("reembed_backfill chunk_id=c"); - let f = classify_embed_error(&wrapped); - assert_eq!(f.code, FailureCode::AuthMissing); - assert!(f.is_unrecoverable()); - } - - #[test] - fn classify_5xx_is_transient() { - let f = classify_embed_error_str("Embedding API error (503 Service Unavailable): retry"); - assert_eq!(f.code, FailureCode::Transient); - assert!(!f.is_unrecoverable()); - } - - #[test] - fn classify_transport_error_is_transient() { - let f = classify_embed_error_str("error sending request for url (...): connection reset"); - assert_eq!(f.code, FailureCode::Transient); - assert!(!f.is_unrecoverable()); - } - - /// #5354 — the Ollama daemon is not listening. Verbatim wording from - /// `tinyagents::harness::embeddings::ollama::OllamaEmbeddingModel::request`. - /// Note the parenthesised hint: `parse_http_status` reads the first `(`, so - /// without an explicit match this fell through to `Transient` and the panel - /// told the user to wait for a retry that can never start their daemon. - #[test] - fn classify_ollama_daemon_down_as_local_model_unavailable() { - let f = classify_embed_error_str( - "ollama embed request failed (is Ollama running at http://localhost:11434?): \ - error sending request for url (http://localhost:11434/api/embed)", - ); - assert_eq!(f.code, FailureCode::LocalModelUnavailable); - assert_eq!( - f.remediation_key, - "memory.health.remediation.local_model_unavailable" - ); - // Transient so `requeue_transient_failed` resumes ingestion by itself - // once the user starts Ollama again. - assert!(!f.is_unrecoverable()); - } - - /// #5354 — the model was never pulled. `ollama_http_error` rewrites the - /// 404 into remediation prose, so the `Embedding API error ()` - /// shape the status parser looks for is gone. - #[test] - fn classify_ollama_model_not_pulled_as_local_model_unavailable() { - let f = classify_embed_error_str( - "Ollama embedding model `bge-m3` is not installed at http://localhost:11434. \ - Run `ollama pull bge-m3` or choose an installed embedding model", - ); - assert_eq!(f.code, FailureCode::LocalModelUnavailable); - assert!(!f.is_unrecoverable()); - } - - /// The real call path wraps the provider error twice (`ProviderEmbedder` - /// adds "ollama embeddings failed", then the seal/reembed site adds its - /// own context), so the matcher must survive the flattened chain. - #[test] - fn classify_ollama_daemon_down_through_anyhow_context_chain() { - let base = anyhow::anyhow!( - "ollama embed request failed (is Ollama running at http://127.0.0.1:11434?): \ - tcp connect error: Connection refused (os error 61)" - ); - let wrapped = base - .context("ollama embeddings failed") - .context("seal embedding failed"); - let f = classify_embed_error(&wrapped); - assert_eq!(f.code, FailureCode::LocalModelUnavailable); - } - - /// Regression guard for the matcher's blast radius: a cloud-embedder - /// transport failure carries no Ollama wording and must keep its generic - /// `Transient` code, or every network blip would start telling users to - /// install Ollama. - #[test] - fn classify_non_ollama_transport_error_stays_transient() { - let f = classify_embed_error_str( - "cloud embeddings failed: error sending request for url \ - (https://api.tinyhumans.ai/openai/v1/embeddings): connection reset", - ); - assert_eq!(f.code, FailureCode::Transient); - } - - #[test] - fn classify_through_anyhow_context_chain() { - // The embed error is commonly `.context()`-wrapped on the way up; - // the flattened `{err:#}` must still classify. - let base = anyhow::anyhow!("Embedding API error (402 Payment Required): out of budget"); - let wrapped = base - .context("cloud embeddings failed") - .context("seal embed"); - let f = classify_embed_error(&wrapped); - assert_eq!(f.code, FailureCode::BudgetExhausted); - } - - #[test] - fn parse_http_status_extracts_leading_code() { - assert_eq!( - parse_http_status("Embedding API error (402 Payment Required): x"), - Some(402) - ); - assert_eq!(parse_http_status("no parens here"), None); - assert_eq!(parse_http_status("(not a status): x"), None); - } - - #[test] - fn truncate_detail_caps_length() { - let long = "x".repeat(500); - let out = truncate_detail(&long); - assert!(out.chars().count() <= 201, "got {}", out.chars().count()); - assert!(out.ends_with('…')); - } - /// #5354 — a classified local-runtime failure flips the recall flag with /// its own cause, so the panel names the Ollama fix from the first failed /// embed instead of waiting out the retry budget. diff --git a/src/openhuman/memory/tree/io.rs b/src/openhuman/memory/tree/io.rs deleted file mode 100644 index b6b92f6e4b..0000000000 --- a/src/openhuman/memory/tree/io.rs +++ /dev/null @@ -1,6 +0,0 @@ -//! Stable host path for tinycortex-owned tree I/O contracts. - -pub use tinycortex::memory::tree::{ - TreeLabelStrategy, TreeLeafPayload, TreeReadHit, TreeReadRequest, TreeReadResult, - TreeWriteOutcome, TreeWriteRequest, -}; diff --git a/src/openhuman/memory/tree/mod.rs b/src/openhuman/memory/tree/mod.rs index c65970a422..4d60bd79e8 100644 --- a/src/openhuman/memory/tree/mod.rs +++ b/src/openhuman/memory/tree/mod.rs @@ -8,7 +8,6 @@ pub mod graph; pub mod health; pub mod ingest; -pub mod io; pub mod nlp; pub mod retrieval; pub mod score; @@ -21,7 +20,8 @@ pub mod summarise; pub mod tree; pub mod tree_runtime; -pub use io::{ +// Tree I/O contracts are engine-owned. +pub use tinycortex::memory::tree::{ TreeLabelStrategy, TreeLeafPayload, TreeReadHit, TreeReadRequest, TreeReadResult, TreeWriteOutcome, TreeWriteRequest, }; diff --git a/src/openhuman/memory/tree/retrieval/cover.rs b/src/openhuman/memory/tree/retrieval/cover.rs index a9b8113e8f..bfef7c47dd 100644 --- a/src/openhuman/memory/tree/retrieval/cover.rs +++ b/src/openhuman/memory/tree/retrieval/cover.rs @@ -3,7 +3,7 @@ use anyhow::Result; use crate::openhuman::config::Config; use crate::openhuman::memory::source_scope::current_source_scope; use crate::openhuman::memory::store::chunks::types::SourceKind; -use crate::openhuman::memory::tree::retrieval::engine::config as engine_config; +use crate::openhuman::memory::tinycortex::engine_config; use crate::openhuman::memory::tree::retrieval::types::QueryResponse; const DEFAULT_LIMIT: usize = 200; diff --git a/src/openhuman/memory/tree/retrieval/drill_down.rs b/src/openhuman/memory/tree/retrieval/drill_down.rs index 36101dcf41..97648c2024 100644 --- a/src/openhuman/memory/tree/retrieval/drill_down.rs +++ b/src/openhuman/memory/tree/retrieval/drill_down.rs @@ -2,7 +2,8 @@ use anyhow::Result; use crate::openhuman::config::Config; use crate::openhuman::memory::source_scope::current_source_scope; -use crate::openhuman::memory::tree::retrieval::engine::{config as engine_config, EmbedderBridge}; +use crate::openhuman::memory::tinycortex::engine_config; +use crate::openhuman::memory::tree::retrieval::engine::EmbedderBridge; use crate::openhuman::memory::tree::retrieval::types::RetrievalHit; use crate::openhuman::memory::tree::score::embed::{build_embedder_from_config, InertEmbedder}; diff --git a/src/openhuman/memory/tree/retrieval/engine.rs b/src/openhuman/memory/tree/retrieval/engine.rs index b77a96a486..2e40fe7e71 100644 --- a/src/openhuman/memory/tree/retrieval/engine.rs +++ b/src/openhuman/memory/tree/retrieval/engine.rs @@ -1,13 +1,8 @@ use anyhow::Result; use async_trait::async_trait; -use crate::openhuman::config::Config; use crate::openhuman::memory::tree::score::embed::Embedder as HostEmbedder; -pub(super) fn config(config: &Config) -> tinycortex::memory::MemoryConfig { - crate::openhuman::memory::tinycortex::memory_config_from(config, config.workspace_dir.clone()) -} - pub(super) struct EmbedderBridge<'a>(pub &'a dyn HostEmbedder); #[async_trait] diff --git a/src/openhuman/memory/tree/retrieval/fast.rs b/src/openhuman/memory/tree/retrieval/fast.rs index 53c625f6df..a2f3495ce2 100644 --- a/src/openhuman/memory/tree/retrieval/fast.rs +++ b/src/openhuman/memory/tree/retrieval/fast.rs @@ -4,8 +4,9 @@ use anyhow::Result; use crate::openhuman::config::Config; use crate::openhuman::memory::source_scope::current_source_scope; +use crate::openhuman::memory::tinycortex::engine_config; use crate::openhuman::memory::tree::nlp; -use crate::openhuman::memory::tree::retrieval::engine::{config as engine_config, EmbedderBridge}; +use crate::openhuman::memory::tree::retrieval::engine::EmbedderBridge; use crate::openhuman::memory::tree::retrieval::types::QueryResponse; use crate::openhuman::memory::tree::score::embed::build_embedder_from_config; diff --git a/src/openhuman/memory/tree/retrieval/fetch.rs b/src/openhuman/memory/tree/retrieval/fetch.rs index 209dc75cbc..9bd0ccc715 100644 --- a/src/openhuman/memory/tree/retrieval/fetch.rs +++ b/src/openhuman/memory/tree/retrieval/fetch.rs @@ -4,7 +4,7 @@ use crate::openhuman::config::Config; use crate::openhuman::memory::source_scope::chunk_source_allowed_in; use crate::openhuman::memory::source_scope::current_source_scope; use crate::openhuman::memory::store::chunks::store::get_chunks_batch; -use crate::openhuman::memory::tree::retrieval::engine::config as engine_config; +use crate::openhuman::memory::tinycortex::engine_config; use crate::openhuman::memory::tree::retrieval::types::RetrievalHit; pub use tinycortex::memory::retrieval::MAX_BATCH; diff --git a/src/openhuman/memory/tree/retrieval/mod.rs b/src/openhuman/memory/tree/retrieval/mod.rs index 41e207a468..2508b56327 100644 --- a/src/openhuman/memory/tree/retrieval/mod.rs +++ b/src/openhuman/memory/tree/retrieval/mod.rs @@ -32,6 +32,8 @@ pub mod types; mod benchmarks; #[cfg(test)] mod integration_tests; +#[cfg(test)] +mod source_scope_tests; pub use cover::cover_window; pub use drill_down::drill_down; diff --git a/src/openhuman/memory/tree/retrieval/search.rs b/src/openhuman/memory/tree/retrieval/search.rs index 9f970c4839..6d4c214619 100644 --- a/src/openhuman/memory/tree/retrieval/search.rs +++ b/src/openhuman/memory/tree/retrieval/search.rs @@ -1,7 +1,7 @@ use anyhow::Result; use crate::openhuman::config::Config; -use crate::openhuman::memory::tree::retrieval::engine::config as engine_config; +use crate::openhuman::memory::tinycortex::engine_config; use crate::openhuman::memory::tree::retrieval::types::EntityMatch; use crate::openhuman::memory::tree::score::extract::EntityKind; diff --git a/src/openhuman/memory/tree/retrieval/source.rs b/src/openhuman/memory/tree/retrieval/source.rs index 7af57a605d..de70b1d6a0 100644 --- a/src/openhuman/memory/tree/retrieval/source.rs +++ b/src/openhuman/memory/tree/retrieval/source.rs @@ -3,7 +3,8 @@ use anyhow::Result; use crate::openhuman::config::Config; use crate::openhuman::memory::source_scope::current_source_scope; use crate::openhuman::memory::store::chunks::types::SourceKind; -use crate::openhuman::memory::tree::retrieval::engine::{config as engine_config, EmbedderBridge}; +use crate::openhuman::memory::tinycortex::engine_config; +use crate::openhuman::memory::tree::retrieval::engine::EmbedderBridge; use crate::openhuman::memory::tree::retrieval::types::QueryResponse; use crate::openhuman::memory::tree::score::embed::build_embedder_from_config; diff --git a/src/openhuman/memory/tree/retrieval/source_scope_tests.rs b/src/openhuman/memory/tree/retrieval/source_scope_tests.rs new file mode 100644 index 0000000000..87ba75647b --- /dev/null +++ b/src/openhuman/memory/tree/retrieval/source_scope_tests.rs @@ -0,0 +1,664 @@ +//! Characterization tests for the THREE distinct `source_scope` predicates. +//! +//! These pin **current** behaviour — including behaviour that looks wrong. Do +//! not "fix" anything asserted here: a failure means a refactor changed one of +//! the predicates, which is exactly what these tests exist to catch. +//! +//! 1. `fetch.rs` → `source_scope::chunk_source_allowed_in`: fail-OPEN for +//! chunks without the `memory_sources` tag, otherwise equality on +//! `source_id` OR the `mem_src:{id}:` composite rule via +//! `sync_events::extract_mem_src_id` (which returns `None` for an EMPTY +//! item id, so `mem_src:src-abc:` is BLOCKED host-side). +//! 2. `source.rs` / `drill_down.rs` → `hits.retain(|h| set.contains(&h.tree_scope))`: +//! PLAIN EQUALITY on a DIFFERENT field. No tag fail-open, no `mem_src:` +//! prefix rule. For leaf hits `tree_scope` *is* the chunk's `source_id` +//! (`tinycortex` `retrieval::{fetch,drill_down}`), so on leaves this is +//! strictly narrower than predicate 1. `source.rs` / `cover.rs` additionally +//! carry a *pre-filter* short circuit on the explicit `source_id` argument. +//! 3. `tinycortex::memory::chunks::store_list::append_source_scope` — a SQL +//! predicate applied BEFORE `LIMIT`. Reached via `cover_window_scoped` and +//! the raw `list_chunks` callers. It admits `mem_src:src-abc:` (empty item +//! id), diverging from predicate 1. +//! +//! Note: `fast_retrieve` does NOT reach predicate 3 — it threads the scope into +//! `resolve_local` / `dense`, which apply the predicate-2 `tree_scope` retain. + +#![cfg(test)] + +use std::collections::HashSet; + +use chrono::{TimeZone, Utc}; +use tempfile::TempDir; + +use crate::openhuman::config::Config; +use crate::openhuman::memory::source_scope::{chunk_source_allowed_in, with_source_scope}; +use crate::openhuman::memory::store::chunks::store::{ + list_chunks, upsert_chunks, upsert_staged_chunks_tx, with_connection, ListChunksQuery, +}; +use crate::openhuman::memory::store::chunks::types::{ + chunk_id, Chunk, Metadata, SourceKind, SourceRef, +}; +use crate::openhuman::memory::store::content as content_store; +use crate::openhuman::memory::store::trees::store::{insert_summary_tx, insert_tree}; +use crate::openhuman::memory::store::trees::types::{SummaryNode, Tree, TreeKind, TreeStatus}; +use crate::openhuman::memory::tree::retrieval::{ + cover_window, drill_down, fetch_leaves, query_source, +}; + +const BASE_MS: i64 = 1_700_000_000_000; +const MEMORY_SOURCES: &str = "memory_sources"; + +// ── fixtures ───────────────────────────────────────────────────────────── + +fn test_config() -> (TempDir, Config) { + let tmp = TempDir::new().unwrap(); + let mut cfg = Config::default(); + cfg.workspace_dir = tmp.path().to_path_buf(); + // Inert embedder keeps these deterministic and avoids any real provider + // call. Every retrieval call below passes `query: None`, so no embedder is + // ever built. + cfg.memory_tree.embedding_endpoint = None; + cfg.memory_tree.embedding_model = None; + cfg.memory_tree.embedding_strict = false; + (tmp, cfg) +} + +/// A chunk in `source`, tagged with `tags`, timestamped `ts_ms`. +fn src_chunk(source: &str, seq: u32, tags: &[&str], ts_ms: i64) -> Chunk { + let ts = Utc.timestamp_millis_opt(ts_ms).unwrap(); + Chunk { + id: chunk_id(SourceKind::Chat, source, seq, "test-content"), + content: format!("content-{source}-{seq}"), + metadata: Metadata { + source_kind: SourceKind::Chat, + source_id: source.into(), + owner: "alice".into(), + timestamp: ts, + time_range: (ts, ts), + tags: tags.iter().map(|t| (*t).to_string()).collect(), + source_ref: Some(SourceRef::new(format!("slack://{source}/{seq}"))), + path_scope: None, + }, + token_count: 20, + seq_in_source: seq, + created_at: ts, + partial_message: false, + } +} + +/// Persist chunk rows AND their staged content bodies, mirroring `rpc.rs`. +fn seed_chunks(cfg: &Config, chunks: &[Chunk]) { + upsert_chunks(cfg, chunks).expect("upsert_chunks"); + let content_root = cfg.memory_tree_content_root(); + std::fs::create_dir_all(&content_root).expect("create content_root for test"); + let staged = content_store::stage_chunks(&content_root, chunks).expect("stage_chunks"); + with_connection(cfg, |conn| { + let tx = conn.unchecked_transaction()?; + upsert_staged_chunks_tx(&tx, &staged)?; + tx.commit()?; + Ok(()) + }) + .expect("persist staged chunk pointers"); +} + +fn seed_tree(cfg: &Config, id: &str, scope: &str, root_id: &str, max_level: u32) { + let ts = Utc.timestamp_millis_opt(BASE_MS).unwrap(); + let tree = Tree { + id: id.to_string(), + kind: TreeKind::Source, + scope: scope.to_string(), + ask: None, + root_id: Some(root_id.to_string()), + max_level, + status: TreeStatus::Active, + created_at: ts, + last_sealed_at: Some(ts), + }; + insert_tree(cfg, &tree).expect("insert_tree"); +} + +fn seed_summary(cfg: &Config, id: &str, tree_id: &str, level: u32, children: &[&str]) { + let ts = Utc.timestamp_millis_opt(BASE_MS).unwrap(); + let node = SummaryNode { + id: id.to_string(), + tree_id: tree_id.to_string(), + tree_kind: TreeKind::Source, + level, + parent_id: None, + child_ids: children.iter().map(|c| (*c).to_string()).collect(), + content: format!("seal-{id}"), + token_count: 100, + entities: vec![], + topics: vec![], + time_range_start: ts, + time_range_end: ts, + score: 0.5, + sealed_at: ts, + deleted: false, + embedding: None, + doc_id: None, + version_ms: None, + }; + with_connection(cfg, |conn| { + let tx = conn.unchecked_transaction()?; + insert_summary_tx(&tx, &node, None, "test")?; + tx.commit()?; + Ok(()) + }) + .expect("insert summary"); +} + +fn set_of(items: &[&str]) -> HashSet { + items.iter().map(|s| (*s).to_string()).collect() +} + +fn scoped_query(scope: Option<&[&str]>) -> ListChunksQuery { + ListChunksQuery { + source_scope: scope.map(set_of), + exclude_dropped: false, + ..Default::default() + } +} + +fn ids_of(chunks: &[Chunk]) -> Vec { + chunks.iter().map(|c| c.id.clone()).collect() +} + +// ═════════════════════════════════════════════════════════════════════════ +// Group 1 — predicate 1: `chunk_source_allowed_in`, via `fetch_leaves`. +// ═════════════════════════════════════════════════════════════════════════ + +/// Every group-1 fixture at once: one chunk per interesting source shape. +fn group1_chunks() -> Vec { + vec![ + // Untagged → fail-open under predicate 1. + src_chunk("gmail:alice", 0, &[], BASE_MS), + // Tagged, exact source-id match. + src_chunk("slack:#eng", 1, &[MEMORY_SOURCES], BASE_MS + 1_000), + // Tagged, `mem_src:` composite with a non-empty item id. + src_chunk( + "mem_src:src-abc:item-1", + 2, + &[MEMORY_SOURCES], + BASE_MS + 2_000, + ), + // Tagged, longer registry id — must NOT be smeared into by `src-abc`. + src_chunk( + "mem_src:src-abcdef:item-1", + 3, + &[MEMORY_SOURCES], + BASE_MS + 3_000, + ), + // Tagged, EMPTY item id — `extract_mem_src_id` returns None here. + src_chunk("mem_src:src-abc:", 4, &[MEMORY_SOURCES], BASE_MS + 4_000), + ] +} + +async fn fetch_ids_under( + cfg: &Config, + chunks: &[Chunk], + scope: Option>, +) -> Vec { + let ids = ids_of(chunks); + let hits = with_source_scope(scope, async { fetch_leaves(cfg, &ids).await }) + .await + .expect("fetch_leaves"); + hits.into_iter().map(|h| h.node_id).collect() +} + +#[tokio::test] +async fn fetch_leaves_fails_open_for_untagged_chunk() { + let (_tmp, cfg) = test_config(); + let chunks = group1_chunks(); + seed_chunks(&cfg, &chunks); + + let got = fetch_ids_under(&cfg, &chunks, Some(vec!["src-abc".into()])).await; + assert!( + got.contains(&chunks[0].id), + "untagged chunk must fail OPEN through predicate 1: {got:?}" + ); +} + +#[tokio::test] +async fn fetch_leaves_allows_exact_source_id_match() { + let (_tmp, cfg) = test_config(); + let chunks = group1_chunks(); + seed_chunks(&cfg, &chunks); + + let got = fetch_ids_under(&cfg, &chunks, Some(vec!["slack:#eng".into()])).await; + assert!( + got.contains(&chunks[1].id), + "exact source_id match: {got:?}" + ); +} + +#[tokio::test] +async fn fetch_leaves_allows_mem_src_prefix_match() { + let (_tmp, cfg) = test_config(); + let chunks = group1_chunks(); + seed_chunks(&cfg, &chunks); + + let got = fetch_ids_under(&cfg, &chunks, Some(vec!["src-abc".into()])).await; + assert!( + got.contains(&chunks[2].id), + "mem_src:src-abc:item-1 must resolve to registry id src-abc: {got:?}" + ); +} + +#[tokio::test] +async fn fetch_leaves_prefix_does_not_smear_to_longer_source_id() { + let (_tmp, cfg) = test_config(); + let chunks = group1_chunks(); + seed_chunks(&cfg, &chunks); + + let got = fetch_ids_under(&cfg, &chunks, Some(vec!["src-abc".into()])).await; + assert!( + !got.contains(&chunks[3].id), + "src-abc must not smear into src-abcdef: {got:?}" + ); +} + +#[tokio::test] +async fn fetch_leaves_blocks_mem_src_with_empty_item_id() { + let (_tmp, cfg) = test_config(); + let chunks = group1_chunks(); + seed_chunks(&cfg, &chunks); + + // `extract_mem_src_id` bails when nothing follows the registry-id colon + // (`colon_pos + 1 >= rest.len()`), so the composite never resolves and the + // tagged chunk is blocked — even though the SQL predicate admits it (see + // `list_chunks_scope_admits_empty_item_id_unlike_the_host_predicate`). + let set = set_of(&["src-abc"]); + let tags = vec![MEMORY_SOURCES.to_string()]; + assert!(!chunk_source_allowed_in(&set, &tags, "mem_src:src-abc:")); + + let got = fetch_ids_under(&cfg, &chunks, Some(vec!["src-abc".into()])).await; + assert!( + !got.contains(&chunks[4].id), + "empty-item-id composite must be blocked host-side: {got:?}" + ); +} + +#[tokio::test] +async fn fetch_leaves_empty_allowlist_blocks_tagged_but_not_untagged() { + let (_tmp, cfg) = test_config(); + let chunks = group1_chunks(); + seed_chunks(&cfg, &chunks); + + let got = fetch_ids_under(&cfg, &chunks, Some(vec![])).await; + assert_eq!( + got, + vec![chunks[0].id.clone()], + "an empty allowlist keeps only the fail-open untagged chunk: {got:?}" + ); +} + +#[tokio::test] +async fn fetch_leaves_without_scope_returns_every_chunk() { + let (_tmp, cfg) = test_config(); + let chunks = group1_chunks(); + seed_chunks(&cfg, &chunks); + + let ids = ids_of(&chunks); + let hits = fetch_leaves(&cfg, &ids).await.expect("fetch_leaves"); + assert_eq!(hits.len(), chunks.len(), "absent scope is unrestricted"); +} + +// ═════════════════════════════════════════════════════════════════════════ +// Group 2 — predicate 2: plain equality on `tree_scope`. +// ═════════════════════════════════════════════════════════════════════════ + +#[tokio::test] +async fn query_source_retains_only_exact_tree_scope_matches() { + let (_tmp, cfg) = test_config(); + seed_tree(&cfg, "tree-eng", "slack:#eng", "s-eng", 1); + seed_tree(&cfg, "tree-secret", "slack:#secret", "s-secret", 1); + seed_summary(&cfg, "s-eng", "tree-eng", 1, &["leaf-a"]); + seed_summary(&cfg, "s-secret", "tree-secret", 1, &["leaf-b"]); + + let resp = with_source_scope(Some(vec!["slack:#eng".into()]), async { + query_source(&cfg, None, None, None, None, 10).await + }) + .await + .expect("query_source"); + + assert_eq!(resp.hits.len(), 1, "hits: {:?}", resp.hits); + assert_eq!(resp.hits[0].tree_scope, "slack:#eng"); + assert_eq!(resp.hits[0].node_id, "s-eng"); +} + +#[tokio::test] +async fn query_source_tree_scope_filter_has_no_mem_src_prefix_rule() { + let (_tmp, cfg) = test_config(); + seed_tree(&cfg, "tree-m", "mem_src:src-abc:item-1", "s-m", 1); + seed_summary(&cfg, "s-m", "tree-m", 1, &["leaf-a"]); + + // Predicate 1 WOULD admit this identifier… + let set = set_of(&["src-abc"]); + let tags = vec![MEMORY_SOURCES.to_string()]; + assert!(chunk_source_allowed_in( + &set, + &tags, + "mem_src:src-abc:item-1" + )); + + // …but predicate 2 is plain equality on `tree_scope`, so it does not. + let resp = with_source_scope(Some(vec!["src-abc".into()]), async { + query_source(&cfg, None, None, None, None, 10).await + }) + .await + .expect("query_source"); + assert!( + resp.hits.is_empty(), + "tree_scope retain has no mem_src rule: {:?}", + resp.hits + ); +} + +#[tokio::test] +async fn query_source_empty_allowlist_returns_no_hits() { + let (_tmp, cfg) = test_config(); + seed_tree(&cfg, "tree-eng", "slack:#eng", "s-eng", 1); + seed_summary(&cfg, "s-eng", "tree-eng", 1, &["leaf-a"]); + + let resp = with_source_scope(Some(vec![]), async { + query_source(&cfg, None, None, None, None, 10).await + }) + .await + .expect("query_source"); + assert!(resp.hits.is_empty()); + assert_eq!(resp.total, 0); +} + +#[tokio::test] +async fn query_source_without_scope_returns_every_tree() { + let (_tmp, cfg) = test_config(); + seed_tree(&cfg, "tree-eng", "slack:#eng", "s-eng", 1); + seed_tree(&cfg, "tree-secret", "slack:#secret", "s-secret", 1); + seed_summary(&cfg, "s-eng", "tree-eng", 1, &["leaf-a"]); + seed_summary(&cfg, "s-secret", "tree-secret", 1, &["leaf-b"]); + + let resp = query_source(&cfg, None, None, None, None, 10) + .await + .expect("query_source"); + assert_eq!(resp.hits.len(), 2, "absent scope is unrestricted"); +} + +#[tokio::test] +async fn query_source_explicit_source_id_outside_scope_short_circuits() { + let (_tmp, cfg) = test_config(); + seed_tree(&cfg, "tree-secret", "slack:#secret", "s-secret", 1); + seed_summary(&cfg, "s-secret", "tree-secret", 1, &["leaf-b"]); + + // The `source.rs` PRE-filter: plain equality on the request argument, + // returning `QueryResponse::empty()` before the engine is even called. + // This is a fourth predicate, distinct from the post-filter retain. + let resp = with_source_scope(Some(vec!["slack:#eng".into()]), async { + query_source(&cfg, Some("slack:#secret"), None, None, None, 10).await + }) + .await + .expect("query_source"); + assert!(resp.hits.is_empty()); + assert_eq!(resp.total, 0); + assert!(!resp.truncated); +} + +#[tokio::test] +async fn drill_down_retains_only_exact_tree_scope_matches() { + let (_tmp, cfg) = test_config(); + seed_tree(&cfg, "tree-eng", "slack:#eng", "s-root", 2); + seed_tree(&cfg, "tree-secret", "slack:#secret", "s-b", 1); + seed_summary(&cfg, "s-root", "tree-eng", 2, &["s-a", "s-b"]); + seed_summary(&cfg, "s-a", "tree-eng", 1, &["leaf-a"]); + seed_summary(&cfg, "s-b", "tree-secret", 1, &["leaf-b"]); + + let hits = with_source_scope(Some(vec!["slack:#eng".into()]), async { + drill_down(&cfg, "s-root", 1, None, None).await + }) + .await + .expect("drill_down"); + + let ids: Vec<&str> = hits.iter().map(|h| h.node_id.as_str()).collect(); + assert_eq!(ids, vec!["s-a"], "hits: {ids:?}"); +} + +#[tokio::test] +async fn drill_down_without_scope_keeps_every_hit() { + let (_tmp, cfg) = test_config(); + seed_tree(&cfg, "tree-eng", "slack:#eng", "s-root", 2); + seed_tree(&cfg, "tree-secret", "slack:#secret", "s-b", 1); + seed_summary(&cfg, "s-root", "tree-eng", 2, &["s-a", "s-b"]); + seed_summary(&cfg, "s-a", "tree-eng", 1, &["leaf-a"]); + seed_summary(&cfg, "s-b", "tree-secret", 1, &["leaf-b"]); + + let hits = drill_down(&cfg, "s-root", 1, None, None) + .await + .expect("drill_down"); + let ids: Vec<&str> = hits.iter().map(|h| h.node_id.as_str()).collect(); + assert_eq!(ids, vec!["s-a", "s-b"], "hits: {ids:?}"); +} + +#[tokio::test] +async fn drill_down_chunk_leaves_are_scoped_by_source_id_not_by_tag() { + let (_tmp, cfg) = test_config(); + // An UNTAGGED chunk and a TAGGED `mem_src:` chunk hanging off one L1 node. + let untagged = src_chunk("gmail:alice", 0, &[], BASE_MS); + let tagged = src_chunk( + "mem_src:src-abc:item-1", + 1, + &[MEMORY_SOURCES], + BASE_MS + 1_000, + ); + seed_chunks(&cfg, &[untagged.clone(), tagged.clone()]); + seed_tree(&cfg, "tree-eng", "slack:#eng", "s-leaves", 1); + seed_summary( + &cfg, + "s-leaves", + "tree-eng", + 1, + &[untagged.id.as_str(), tagged.id.as_str()], + ); + + // Leaves carry `tree_scope = chunk.metadata.source_id`, so an allowlist + // naming that source id keeps the chunk — with NO tag fail-open for the + // untagged one, which is why the tagged sibling drops out here. + let hits = with_source_scope(Some(vec!["gmail:alice".into()]), async { + drill_down(&cfg, "s-leaves", 1, None, None).await + }) + .await + .expect("drill_down"); + let ids: Vec<&str> = hits.iter().map(|h| h.node_id.as_str()).collect(); + assert_eq!(ids, vec![untagged.id.as_str()], "hits: {ids:?}"); + + // And the `mem_src:` prefix rule does NOT apply on this path either: + // predicate 1 would admit `mem_src:src-abc:item-1` under `src-abc`. + let hits = with_source_scope(Some(vec!["src-abc".into()]), async { + drill_down(&cfg, "s-leaves", 1, None, None).await + }) + .await + .expect("drill_down"); + assert!( + hits.is_empty(), + "leaf retain is plain equality on source_id: {hits:?}" + ); +} + +#[tokio::test] +async fn drill_down_scope_widens_engine_limit_so_a_blocked_prefix_cannot_starve_results() { + let (_tmp, cfg) = test_config(); + seed_tree(&cfg, "tree-eng", "slack:#eng", "s-root", 2); + seed_tree(&cfg, "tree-secret", "slack:#secret", "s-b1", 1); + // BFS order puts the two blocked children FIRST. + seed_summary(&cfg, "s-root", "tree-eng", 2, &["s-b1", "s-b2", "s-a"]); + seed_summary(&cfg, "s-b1", "tree-secret", 1, &["leaf-1"]); + seed_summary(&cfg, "s-b2", "tree-secret", 1, &["leaf-2"]); + seed_summary(&cfg, "s-a", "tree-eng", 1, &["leaf-3"]); + + // `drill_down.rs` forces the ENGINE limit to `None` whenever a scope is + // active, then applies the caller's limit after the retain. Without that, + // the engine would return only `s-b1` and the retain would empty it. + let hits = with_source_scope(Some(vec!["slack:#eng".into()]), async { + drill_down(&cfg, "s-root", 1, None, Some(1)).await + }) + .await + .expect("drill_down"); + let ids: Vec<&str> = hits.iter().map(|h| h.node_id.as_str()).collect(); + assert_eq!(ids, vec!["s-a"], "hits: {ids:?}"); +} + +// ═════════════════════════════════════════════════════════════════════════ +// Group 3 — predicate 3: the SQL `append_source_scope`, applied before LIMIT. +// ═════════════════════════════════════════════════════════════════════════ + +#[tokio::test] +async fn list_chunks_scope_fails_open_for_untagged_chunk() { + let (_tmp, cfg) = test_config(); + let chunks = group1_chunks(); + seed_chunks(&cfg, &chunks); + + let got = list_chunks(&cfg, &scoped_query(Some(&["src-abc"]))).expect("list_chunks"); + let ids: Vec<&str> = got.iter().map(|c| c.id.as_str()).collect(); + assert!( + ids.contains(&chunks[0].id.as_str()), + "SQL `NOT EXISTS json_each(...)` fail-open: {ids:?}" + ); +} + +#[tokio::test] +async fn list_chunks_scope_matches_exact_source_id() { + let (_tmp, cfg) = test_config(); + let chunks = group1_chunks(); + seed_chunks(&cfg, &chunks); + + let got = list_chunks(&cfg, &scoped_query(Some(&["slack:#eng"]))).expect("list_chunks"); + let ids: Vec<&str> = got.iter().map(|c| c.id.as_str()).collect(); + assert!(ids.contains(&chunks[1].id.as_str()), "ids: {ids:?}"); +} + +#[tokio::test] +async fn list_chunks_scope_matches_mem_src_prefix() { + let (_tmp, cfg) = test_config(); + let chunks = group1_chunks(); + seed_chunks(&cfg, &chunks); + + let got = list_chunks(&cfg, &scoped_query(Some(&["src-abc"]))).expect("list_chunks"); + let ids: Vec<&str> = got.iter().map(|c| c.id.as_str()).collect(); + assert!(ids.contains(&chunks[2].id.as_str()), "ids: {ids:?}"); +} + +#[tokio::test] +async fn list_chunks_scope_does_not_smear_prefix() { + let (_tmp, cfg) = test_config(); + let chunks = group1_chunks(); + seed_chunks(&cfg, &chunks); + + let got = list_chunks(&cfg, &scoped_query(Some(&["src-abc"]))).expect("list_chunks"); + let ids: Vec<&str> = got.iter().map(|c| c.id.as_str()).collect(); + assert!( + !ids.contains(&chunks[3].id.as_str()), + "substr(source_id, 1, length('mem_src:src-abc:')) must not match \ + mem_src:src-abcdef:item-1: {ids:?}" + ); +} + +#[tokio::test] +async fn list_chunks_scope_admits_empty_item_id_unlike_the_host_predicate() { + let (_tmp, cfg) = test_config(); + let chunks = group1_chunks(); + seed_chunks(&cfg, &chunks); + + // THE headline divergence, characterized as observed (not endorsed): + // the SQL prefix test is a pure `substr` compare with no "item id must be + // non-empty" rule, so `mem_src:src-abc:` passes here… + let got = list_chunks(&cfg, &scoped_query(Some(&["src-abc"]))).expect("list_chunks"); + let ids: Vec<&str> = got.iter().map(|c| c.id.as_str()).collect(); + assert!( + ids.contains(&chunks[4].id.as_str()), + "SQL admits mem_src:src-abc: : {ids:?}" + ); + + // …while the host predicate blocks the very same source_id. + let set = set_of(&["src-abc"]); + let tags = vec![MEMORY_SOURCES.to_string()]; + assert!(!chunk_source_allowed_in(&set, &tags, "mem_src:src-abc:")); +} + +#[tokio::test] +async fn list_chunks_empty_allowlist_keeps_only_untagged_chunks() { + let (_tmp, cfg) = test_config(); + let chunks = group1_chunks(); + seed_chunks(&cfg, &chunks); + + let got = list_chunks(&cfg, &scoped_query(Some(&[]))).expect("list_chunks"); + let ids: Vec<&str> = got.iter().map(|c| c.id.as_str()).collect(); + assert_eq!(ids, vec![chunks[0].id.as_str()], "ids: {ids:?}"); +} + +#[tokio::test] +async fn list_chunks_absent_scope_returns_everything() { + let (_tmp, cfg) = test_config(); + let chunks = group1_chunks(); + seed_chunks(&cfg, &chunks); + + let got = list_chunks(&cfg, &scoped_query(None)).expect("list_chunks"); + assert_eq!(got.len(), chunks.len()); +} + +#[tokio::test] +async fn list_chunks_scope_is_applied_before_limit() { + let (_tmp, cfg) = test_config(); + // Three blocked chunks NEWER than the single allowed one. Ordering is + // `timestamp_ms DESC`, so a post-filter with LIMIT 1 would return nothing. + let blocked: Vec = (0..3) + .map(|i| { + src_chunk( + "slack:#secret", + i, + &[MEMORY_SOURCES], + BASE_MS + 10_000 + i64::from(i) * 1_000, + ) + }) + .collect(); + let allowed = src_chunk("slack:#eng", 9, &[MEMORY_SOURCES], BASE_MS); + let mut all = blocked.clone(); + all.push(allowed.clone()); + seed_chunks(&cfg, &all); + + let got = list_chunks( + &cfg, + &ListChunksQuery { + source_scope: Some(set_of(&["slack:#eng"])), + limit: Some(1), + exclude_dropped: false, + ..Default::default() + }, + ) + .expect("list_chunks"); + let ids: Vec<&str> = got.iter().map(|c| c.id.as_str()).collect(); + assert_eq!(ids, vec![allowed.id.as_str()], "ids: {ids:?}"); +} + +#[tokio::test] +async fn cover_window_scope_matches_mem_src_prefix() { + let (_tmp, cfg) = test_config(); + let allowed = src_chunk("mem_src:src-abc:item-1", 0, &[MEMORY_SOURCES], BASE_MS); + let blocked = src_chunk( + "mem_src:src-zzz:item-1", + 1, + &[MEMORY_SOURCES], + BASE_MS + 1_000, + ); + seed_chunks(&cfg, &[allowed.clone(), blocked.clone()]); + + // `cover_window` hands the allowlist straight to `cover_window_scoped`, + // which applies predicate 3 in SQL — so the `mem_src:` prefix rule holds + // here, unlike on the `tree_scope` paths above. + let resp = with_source_scope(Some(vec!["src-abc".into()]), async { + cover_window(&cfg, 0, 4_000_000_000_000, None, None, 0).await + }) + .await + .expect("cover_window"); + let ids: Vec<&str> = resp.hits.iter().map(|h| h.node_id.as_str()).collect(); + assert!(ids.contains(&allowed.id.as_str()), "ids: {ids:?}"); + assert!(!ids.contains(&blocked.id.as_str()), "ids: {ids:?}"); +} diff --git a/src/openhuman/memory/tree/score/store.rs b/src/openhuman/memory/tree/score/store.rs index df7613e456..17d8bd91f8 100644 --- a/src/openhuman/memory/tree/score/store.rs +++ b/src/openhuman/memory/tree/score/store.rs @@ -6,15 +6,12 @@ use anyhow::Result; use rusqlite::Transaction; use crate::openhuman::config::Config; +use crate::openhuman::memory::tinycortex::engine_config; pub use tinycortex::memory::score::store::{EntityHit, ScoreRow}; -fn memory_config(config: &Config) -> tinycortex::memory::MemoryConfig { - crate::openhuman::memory::tinycortex::memory_config_from(config, config.workspace_dir.clone()) -} - pub fn upsert_score(config: &Config, row: &ScoreRow) -> Result<()> { - tinycortex::memory::score::store::upsert_score(&memory_config(config), row) + tinycortex::memory::score::store::upsert_score(&engine_config(config), row) } pub(crate) fn upsert_score_tx(tx: &Transaction<'_>, row: &ScoreRow) -> Result<()> { @@ -22,11 +19,11 @@ pub(crate) fn upsert_score_tx(tx: &Transaction<'_>, row: &ScoreRow) -> Result<() } pub fn get_score(config: &Config, chunk_id: &str) -> Result> { - tinycortex::memory::score::store::get_score(&memory_config(config), chunk_id) + tinycortex::memory::score::store::get_score(&engine_config(config), chunk_id) } pub fn get_scores_batch(config: &Config, chunk_ids: &[String]) -> Result> { - tinycortex::memory::score::store::get_scores_batch(&memory_config(config), chunk_ids) + tinycortex::memory::score::store::get_scores_batch(&engine_config(config), chunk_ids) } pub use crate::openhuman::memory::store::entities::{ @@ -137,5 +134,5 @@ fn to_store_entity( } pub fn count_scores(config: &Config) -> Result { - tinycortex::memory::score::store::count_scores(&memory_config(config)) + tinycortex::memory::score::store::count_scores(&engine_config(config)) } diff --git a/src/openhuman/memory/tree/tree/bucket_seal.rs b/src/openhuman/memory/tree/tree/bucket_seal.rs index bc18e287c5..c08ca2d333 100644 --- a/src/openhuman/memory/tree/tree/bucket_seal.rs +++ b/src/openhuman/memory/tree/tree/bucket_seal.rs @@ -5,13 +5,10 @@ use chrono::{DateTime, Utc}; use crate::openhuman::config::Config; use crate::openhuman::memory::store::trees::types::{Buffer, Tree}; +use crate::openhuman::memory::tinycortex::engine_config; pub use tinycortex::memory::tree::{LabelStrategy, LeafRef, MERGE_LEVEL_BASE}; -fn engine_config(config: &Config) -> tinycortex::memory::MemoryConfig { - crate::openhuman::memory::tinycortex::memory_config_from(config, config.workspace_dir.clone()) -} - pub async fn append_leaf( config: &Config, tree: &Tree, diff --git a/src/openhuman/memory/tree/tree_runtime/engine.rs b/src/openhuman/memory/tree/tree_runtime/engine.rs index c8538e3cd5..9e1b693a15 100644 --- a/src/openhuman/memory/tree/tree_runtime/engine.rs +++ b/src/openhuman/memory/tree/tree_runtime/engine.rs @@ -14,13 +14,10 @@ use tinycortex::memory::tree::runtime::{ use crate::core::bus::BUS; use crate::core::events::DomainEvent; use crate::openhuman::config::Config; +use crate::openhuman::memory::tinycortex::engine_config; const SUMMARIZATION_TEMP: f64 = 0.3; -fn engine_config(config: &Config) -> tinycortex::memory::MemoryConfig { - crate::openhuman::memory::tinycortex::memory_config_from(config, config.workspace_dir.clone()) -} - struct ChatSummariser<'a>(&'a dyn ChatModel<()>); #[async_trait] diff --git a/src/openhuman/memory/tree/tree_runtime/mod.rs b/src/openhuman/memory/tree/tree_runtime/mod.rs index 13532937a6..ad97c7a5ea 100644 --- a/src/openhuman/memory/tree/tree_runtime/mod.rs +++ b/src/openhuman/memory/tree/tree_runtime/mod.rs @@ -15,7 +15,6 @@ pub(crate) mod cli; pub mod engine; pub mod ops; pub mod store; -pub mod types; mod schemas; @@ -24,4 +23,5 @@ pub use schemas::{ all_controller_schemas as all_tree_summarizer_controller_schemas, all_registered_controllers as all_tree_summarizer_registered_controllers, }; -pub use types::*; +// Runtime tree types are engine-owned. +pub use tinycortex::memory::tree::runtime::*; diff --git a/src/openhuman/memory/tree/tree_runtime/ops.rs b/src/openhuman/memory/tree/tree_runtime/ops.rs index 96e065c4e7..1e32d3fab8 100644 --- a/src/openhuman/memory/tree/tree_runtime/ops.rs +++ b/src/openhuman/memory/tree/tree_runtime/ops.rs @@ -4,8 +4,9 @@ use chrono::{DateTime, Utc}; use serde_json::{json, Value}; use crate::openhuman::config::Config; -use crate::openhuman::memory::tree::tree_runtime::{engine, store, types::*}; +use crate::openhuman::memory::tree::tree_runtime::{engine, store}; use crate::rpc::RpcOutcome; +use tinycortex::memory::tree::runtime::*; /// Append raw content to the ingestion buffer. pub async fn tree_summarizer_ingest( @@ -163,7 +164,14 @@ pub async fn tree_summarizer_rebuild( /// 3. Error otherwise — "Build Summary Trees" is local-only by default; /// the user must opt in to cloud summarization via the /// `memory_tree.cloud_summarization_opt_in` setting. -fn create_provider( +/// +/// Visibility note: `pub(crate)` so the embedded memory driver's +/// [`MemoryTree`](tinycortex_api::provider::MemoryTree) `seal`/`cascade` reach +/// the **same** resolver the RPC path uses. Duplicating the local-AI / +/// cloud-opt-in precedence in the driver would be new policy logic, and the +/// `summarizer_available` doc below is explicit that this function is the +/// single source of truth. +pub(crate) fn create_provider( config: &Config, ) -> Result< ( diff --git a/src/openhuman/memory/tree/tree_runtime/store.rs b/src/openhuman/memory/tree/tree_runtime/store.rs index ad2f5ebebe..5ffea5df87 100644 --- a/src/openhuman/memory/tree/tree_runtime/store.rs +++ b/src/openhuman/memory/tree/tree_runtime/store.rs @@ -7,11 +7,8 @@ use chrono::{DateTime, Utc}; use serde_json::Value; use crate::openhuman::config::Config; -use crate::openhuman::memory::tree::tree_runtime::types::{TreeNode, TreeStatus}; - -fn engine_config(config: &Config) -> tinycortex::memory::MemoryConfig { - crate::openhuman::memory::tinycortex::memory_config_from(config, config.workspace_dir.clone()) -} +use crate::openhuman::memory::tinycortex::engine_config; +use tinycortex::memory::tree::runtime::{TreeNode, TreeStatus}; pub fn tree_dir(config: &Config, namespace: &str) -> PathBuf { tinycortex::memory::tree::runtime::store::tree_dir(&engine_config(config), namespace) diff --git a/src/openhuman/memory/tree/tree_runtime/types.rs b/src/openhuman/memory/tree/tree_runtime/types.rs deleted file mode 100644 index dbf693ba8e..0000000000 --- a/src/openhuman/memory/tree/tree_runtime/types.rs +++ /dev/null @@ -1,6 +0,0 @@ -//! Stable host path for tinycortex-owned markdown tree runtime types. - -pub use tinycortex::memory::tree::runtime::{ - derive_node_ids, derive_parent_id, estimate_tokens, level_from_node_id, node_id_to_path, - IngestRequest, NodeLevel, QueryResult, TreeNode, TreeStatus, -}; diff --git a/src/openhuman/security/credentials/ops.rs b/src/openhuman/security/credentials/ops.rs index 5f1d1fc102..47e83f9105 100644 --- a/src/openhuman/security/credentials/ops.rs +++ b/src/openhuman/security/credentials/ops.rs @@ -555,8 +555,9 @@ async fn store_session_inner( logs.push(format!("memory client bind warning: {e}")); } } - match crate::core::runtime::context::CoreContext::rebind_default_workspace_dir( + match crate::core::runtime::context::CoreContext::rebind_default_workspace( &effective_config.workspace_dir, + effective_config.subsystems.memory.clone(), ) { Ok(_) => logs.push(format!( "core context bound to workspace {}", @@ -811,9 +812,10 @@ pub async fn clear_session(config: &Config) -> Result Result<(), String> { + if !self.can_act() { + log::warn!( + "[openhuman:policy] Operation '{}' blocked: read-only mode", + operation_name + ); + return Err(format!( + "{POLICY_BLOCKED_MARKER} Security policy: read-only mode, cannot perform \ + '{operation_name}'. Do not retry; this tier blocks all write actions." + )); + } + Ok(()) + } + /// Enforce policy for a tool operation. /// /// Read operations are always allowed by autonomy/rate gates. @@ -25,16 +50,7 @@ impl SecurityPolicy { match operation { ToolOperation::Read => Ok(()), ToolOperation::Act => { - if !self.can_act() { - log::warn!( - "[openhuman:policy] Operation '{}' blocked: read-only mode", - operation_name - ); - return Err(format!( - "{POLICY_BLOCKED_MARKER} Security policy: read-only mode, cannot perform \ - '{operation_name}'. Do not retry; this tier blocks all write actions." - )); - } + self.enforce_write_tier(operation_name)?; if !self.record_action() { log::warn!( diff --git a/src/openhuman/skills/e2e_plumbing_tests.rs b/src/openhuman/skills/e2e_plumbing_tests.rs index 5f23f4e080..2f82806de1 100644 --- a/src/openhuman/skills/e2e_plumbing_tests.rs +++ b/src/openhuman/skills/e2e_plumbing_tests.rs @@ -76,6 +76,7 @@ fn tool_call(id: &str, name: &str, args: serde_json::Value) -> ModelResponse { raw: None, resolved_model: None, continue_turn: None, + served_from_cache: false, } } diff --git a/src/openhuman/skills/e2e_run_tests.rs b/src/openhuman/skills/e2e_run_tests.rs index 2d6a480294..c7681edfe7 100644 --- a/src/openhuman/skills/e2e_run_tests.rs +++ b/src/openhuman/skills/e2e_run_tests.rs @@ -108,6 +108,7 @@ fn tool_call_resp(id: &str, name: &str, args: serde_json::Value) -> ModelRespons raw: None, resolved_model: None, continue_turn: None, + served_from_cache: false, } } diff --git a/src/openhuman/tools/ops.rs b/src/openhuman/tools/ops.rs index aabf19cd06..a8e01fd68b 100644 --- a/src/openhuman/tools/ops.rs +++ b/src/openhuman/tools/ops.rs @@ -1229,29 +1229,46 @@ pub fn all_tools_with_runtime( "[rhai_workflows] rhai_workflows tool not registered — flows feature disabled at compile time" ); - // DomainSet post-filter (#4796): drop tools whose DomainGroup is disabled - // under the ambient CoreContext. With no active context, or under - // `DomainSet::full()`, every tool is kept (byte-identical). Under - // `harness()` the gate-family tools (web3/mcp/skills/flows/media/voice/meet) - // are dropped so agent turns can't call a domain that isn't live; only the - // memory + threads tools survive (the mapped harness families) — see - // `tool_group` for the classification and its Platform-default caveat. + // Two INDEPENDENT post-filters over the assembled list (kernel.md §3.7's + // separate axes — a narrowed DomainSet must not narrow capabilities, and + // vice versa): + // + // 1. DomainSet (#4796): drop tools whose DomainGroup is disabled under the + // ambient CoreContext. With no active context, or under + // `DomainSet::full()`, every tool is kept (byte-identical). Under + // `harness()` the gate-family tools (web3/mcp/skills/flows/media/voice/ + // meet) are dropped so agent turns can't call a domain that isn't live; + // only the memory + threads tools survive (the mapped harness families) + // — see `tool_group` for the classification and its Platform-default + // caveat. + // 2. Memory capability (M5.3): drop tools whose memory family the bound + // driver does not advertise — see `tool_capability`. + // + // Both default OPEN: with no ambient context and with nothing bound the + // list is unchanged. Absence beats a stub that errors — a + // registered-but-failing memory tool teaches the model the capability + // exists and makes it retry (the `flows` compile-gate's reasoning). + let before = tools.len(); let domains = crate::core::runtime::context::CoreContext::current().map(|c| c.domains()); - if let Some(set) = domains { - let before = tools.len(); - let filtered: Vec> = tools + let mut tools: Vec> = if let Some(set) = domains { + tools .into_iter() .filter(|t| set.allows(tool_group(t.name()))) - .collect(); - log::debug!( - "[tools::ops][domain-filter] ambient DomainSet active — {} of {before} tools retained", - filtered.len() - ); - filtered + .collect() } else { - // No ambient context (unit tests / pre-boot) ⇒ no filtering. + // No ambient context (unit tests / pre-boot) ⇒ no domain filtering. tools - } + }; + let after_domains = tools.len(); + + tools.retain(|t| crate::core::all::capability_allowed(tool_capability(t.name()))); + + log::debug!( + "[tools::ops][post-filter] {before} assembled → {after_domains} after DomainSet → {} after \ + memory capabilities", + tools.len() + ); + tools } /// Classify an agent tool into its [`DomainGroup`](crate::core::all::DomainGroup) @@ -1501,6 +1518,99 @@ fn tool_group(name: &str) -> crate::core::all::DomainGroup { DomainGroup::Platform } +/// Classify an agent tool into the memory capability family its surface +/// requires, so [`all_tools_with_runtime`] can drop tools the bound memory +/// driver does not advertise (`docs/specs/kernel.md` §3.3). +/// +/// `None` means "not backed by the memory driver" — a workspace file +/// (`update_memory_md`), the per-workspace people SQLite store, pure +/// introspection (`memory_store_kinds`), or a flow-sandboxed namespace +/// (`flow_memory_*`, already `DomainGroup::Flows`). Such a tool is never +/// filtered on the capability axis. `None` here is a *decision*, not a default: +/// `every_memory_tool_has_an_explicit_capability_or_is_core` forces every +/// memory-family tool through this function so a new one cannot land in the +/// always-present bucket by accident. +/// +/// The mandatory families ([`Capability::Core`], [`Capability::Recall`]) are +/// returned explicitly rather than folded into `None`. Against a *driver's* +/// advertised set the filter is a no-op for them by construction (a bindable +/// driver always advertises `Capability::MANDATORY`) — but it is load-bearing +/// for one host decision below the driver: `CoreContext::memory_capabilities` +/// answers with the empty set for a deliberate `[subsystems.memory] driver = +/// "null"`, and that is what drops `memory_store` / `memory_forget` / the +/// recall tools when an operator turns memory off. Folding them into `None` +/// would leave an agent able to persist, expose or delete memory through the +/// session builder's own `Arc` in exactly that configuration. +/// +/// **The `memory_` prefix is deliberately NOT a catch-all here.** [`tool_group`] +/// can prefix-match because every `memory_*` tool is one family on the +/// *DomainSet* axis; on the capability axis the family differs per tool, and a +/// wrong default is worse than no rule. Hence enumeration plus two narrow +/// prefix rules, backed by the drift guard. +/// +/// ## Honesty clause — three assignments run ahead of the plumbing +/// +/// `goals_*` is filesystem-backed today (`memory::goals::store`), not +/// `MemoryGoals`; `tool_stats` reads the legacy `Arc` plus +/// `agent::learning::tool_tracker`, not `MemoryToolMemory`; `memory_diff` reads +/// `memory::diff::ops`, not `MemoryDiff`. Filtering them on the driver's +/// advertised set is nevertheless the correct M5 behaviour: §3.3 is a contract +/// about what the *model is told exists*, and the later re-point onto +/// `MemoryGuard` must not change the advertised surface. Assigning them `None` +/// to dodge the mismatch would bake the wrong contract in. +fn tool_capability(name: &str) -> Option { + use tinycortex_api::capabilities::Capability; + + // Not driver-backed. Each entry is an argued exception, not a fallthrough. + if name == "update_memory_md" // writes the workspace `MEMORY.md` file directly + || name == "memory_store_kinds" // enumerates `MemoryKind` constants; no store access + || name.starts_with("people_") // per-workspace people SQLite store, not the driver + || name.starts_with("flow_memory_") + // flow-sandboxed; DomainGroup::Flows + { + return None; + } + + let capability = match name { + // ── Mandatory families: always advertised, listed for the record ── + "memory_store" | "memory_forget" | "remember_preference" | "save_preference" => { + Capability::Core + } + // Chunk/recall retrieval surface. NOT `Tree` — these read chunk + // embeddings and chunk rows, never the summary tree. + "memory_recall" + | "memory_vector_search" + | "memory_chunk_context" + | "memory_hybrid_search" + | "memory_store_raw_chunks" => Capability::Recall, + + // ── Optional families: absence means the tool disappears ── + // The one registered tree tool (`MemoryQueryTool` is an alias of + // `MemoryTreeTool`, `memory/query/mod.rs`) plus the compiled persona + // flavour reader, which reads a flavoured summary-tree root. + "memory_tree" | "memory_flavour" => Capability::Tree, + // Free-text search over the canonical *entity* index + // (`memory::tree::retrieval::search::search_entities`). + "memory_store_raw_search" => Capability::Entities, + "memory_diff" => Capability::Diff, + "memory_doctor" => Capability::Maintenance, + "tool_stats" => Capability::ToolMemory, + + // Prefix rules, so a NEW tool in one of these families auto-gates + // instead of silently landing in the un-filtered bucket — the same + // reasoning as `tool_group`'s prefix families (#4808 review). Ordered + // after the exact arms so `memory_tree` is not swallowed by + // `memory_tree_`. The underscore in `goals_` is load-bearing: the + // per-thread `goal_get`/`goal_set`/`goal_complete` tools are + // `DomainGroup::Threads` and must not be caught. + n if n.starts_with("goals_") => Capability::Goals, + n if n.starts_with("memory_tree_") => Capability::Tree, + + _ => return None, + }; + Some(capability) +} + #[cfg(test)] #[path = "ops_tests.rs"] mod tests; diff --git a/src/openhuman/tools/ops_tests.rs b/src/openhuman/tools/ops_tests.rs index 8700c7b2a2..bc99323639 100644 --- a/src/openhuman/tools/ops_tests.rs +++ b/src/openhuman/tools/ops_tests.rs @@ -2764,3 +2764,274 @@ const TOOL_LESS: &[crate::core::all::DomainGroup] = { use crate::core::all::DomainGroup as G; &[G::Config, G::Security, G::Meet, G::Medulla] }; + +// ---- tool_capability() drift guard (M5.3) ---------------------------------- + +/// Driver-backed memory tools and the capability each requires. +const MEMORY_TOOL_CAPABILITIES: &[(&str, tinycortex_api::capabilities::Capability)] = { + use tinycortex_api::capabilities::Capability as C; + &[ + ("memory_store", C::Core), + ("memory_forget", C::Core), + ("remember_preference", C::Core), + ("save_preference", C::Core), + ("memory_recall", C::Recall), + ("memory_vector_search", C::Recall), + ("memory_chunk_context", C::Recall), + ("memory_hybrid_search", C::Recall), + ("memory_store_raw_chunks", C::Recall), + ("memory_tree", C::Tree), + ("memory_flavour", C::Tree), + ("memory_store_raw_search", C::Entities), + ("memory_diff", C::Diff), + ("memory_doctor", C::Maintenance), + ("tool_stats", C::ToolMemory), + ("goals_list", C::Goals), + ("goals_add", C::Goals), + ("goals_edit", C::Goals), + ("goals_delete", C::Goals), + ] +}; + +/// Memory-family tools that are deliberately NOT driver-backed. Each entry is +/// an argument, not an omission — see `tool_capability`. +const MEMORY_TOOLS_NOT_DRIVER_BACKED: &[&str] = &[ + "update_memory_md", + "memory_store_kinds", + "people_list", + "people_resolve", + "people_score", + "people_get", + "people_add_alias", + "people_record_interaction", + "people_refresh_address_book", +]; + +/// Every `DomainGroup::Memory` tool must be a deliberate decision in +/// [`tool_capability`]: either it maps to a capability, or it is listed as +/// explicitly not driver-backed. +/// +/// The failure this prevents is silent and one-directional. A new memory tool +/// with no `tool_capability` rule returns `None`, which the post-filter reads as +/// "never filter" — so it stays advertised to the model under a driver that +/// cannot serve it, which is exactly the registered-but-failing surface +/// `kernel.md` §3.3 exists to prevent. +/// +/// Deliberately tests the FUNCTION, not a built registry, for the same reason +/// `every_domain_group_is_accounted_for_in_tool_group` does: which tools a +/// registry contains depends on config flags, security tier and enabled +/// integrations. `tool_stats` is the live example — it is only registered when +/// `learning.enabled && learning.tool_tracking_enabled`. +#[test] +fn every_memory_tool_has_an_explicit_capability_or_is_core() { + use crate::core::all::DomainGroup; + + for name in MEMORY_TOOLS_NOT_DRIVER_BACKED { + assert_eq!( + tool_group(name), + DomainGroup::Memory, + "`{name}` is no longer a Memory-family tool — this table is stale" + ); + assert!( + tool_capability(name).is_none(), + "`{name}` is listed as not driver-backed but now maps to a capability" + ); + } + for (name, want) in MEMORY_TOOL_CAPABILITIES { + assert_eq!( + tool_group(name), + DomainGroup::Memory, + "`{name}` is no longer a Memory-family tool — this table is stale" + ); + assert_eq!( + tool_capability(name), + Some(*want), + "`{name}` must map to {want:?}; if it moved, the rule has drifted" + ); + } +} + +/// A new tool in a prefix-gated memory family must NOT fall through to `None` +/// (the never-filtered bucket). Synthetic names matching only the prefix. +#[test] +fn no_prefix_family_memory_tool_silently_defaults_to_uncapped() { + use tinycortex_api::capabilities::Capability; + for (name, want) in [ + ("goals_new_thing", Capability::Goals), + ("memory_tree_new_thing", Capability::Tree), + ] { + assert_eq!(tool_capability(name), Some(want), "`{name}` must auto-gate"); + } + // …and the `goals_` prefix must not swallow the per-thread goal tools, + // which are `DomainGroup::Threads` and not memory-driver-backed at all. + for name in ["goal_get", "goal_set", "goal_complete"] { + assert_eq!(tool_capability(name), None, "`{name}` is a Threads tool"); + } +} + +/// Neither table may rot into names no tool answers to. +#[test] +fn memory_capability_table_names_are_real() { + let tmp = TempDir::new().unwrap(); + let names = tool_names(&expansion_tools_for(&tmp)); + for name in MEMORY_TOOL_CAPABILITIES + .iter() + .map(|(n, _)| *n) + .chain(MEMORY_TOOLS_NOT_DRIVER_BACKED.iter().copied()) + // `tool_stats` is registered only when `learning.tool_tracking_enabled`, + // so it is config-dependent and asserted by the function-level guard + // above instead. + .filter(|n| *n != "tool_stats") + { + assert!( + names.iter().any(|n| n == name), + "`{name}` is not a real registered tool; got: {names:?}" + ); + } +} + +// ---- both-ways: the capability post-filter (M5.3) -------------------------- +// +// The ABSENT half is the one that proves the filter removes anything. + +/// A distinct workspace per test: the memory binding cache is keyed by +/// workspace dir, so sharing one path between an ON and an OFF test would make +/// one of them silently assert the other's driver (the `caps_ws` convention +/// from `core::all_tests`). +fn caps_tools_ws(name: &str) -> std::path::PathBuf { + std::env::temp_dir().join(format!("oh-m53-tools-{name}")) +} + +/// `[subsystems.memory] driver = "null"` — `NullMemoryProvider` advertises +/// exactly `Capability::MANDATORY` = {core, recall, portability}, so every +/// optional family is OFF at once. An operator who wrote `driver = "null"` is +/// honoured rather than falling back (`memory::binding`). +fn null_driver_memory_cfg() -> crate::openhuman::config::schema::MemorySubsystemConfig { + crate::openhuman::config::schema::MemorySubsystemConfig { + driver: "null".into(), + ..Default::default() + } +} + +/// The optional-family tools that must vanish under a driver advertising +/// nothing optional. +const OPTIONAL_FAMILY_MEMORY_TOOLS: &[&str] = &[ + "memory_tree", + "memory_flavour", + "memory_store_raw_search", + "memory_diff", + "memory_doctor", + "goals_list", + "goals_add", + "goals_edit", + "goals_delete", +]; + +/// Tools that must survive any driver: the mandatory families plus the +/// host-owned surface that never touches the driver. +const ALWAYS_PRESENT_MEMORY_TOOLS: &[&str] = &[ + "memory_store", + "memory_recall", + "memory_forget", + "remember_preference", + "save_preference", + "update_memory_md", + "memory_store_kinds", + "memory_vector_search", + "memory_chunk_context", + "memory_hybrid_search", + "memory_store_raw_chunks", + "people_list", +]; + +/// The ~4000-pre-boot-test default-open property, asserted once directly: with +/// no ambient context at all the capability filter removes nothing. +#[test] +fn memory_tools_all_present_with_no_ambient_context() { + let tmp = TempDir::new().unwrap(); + let names = tool_names(&expansion_tools_for(&tmp)); + for name in OPTIONAL_FAMILY_MEMORY_TOOLS + .iter() + .chain(ALWAYS_PRESENT_MEMORY_TOOLS.iter()) + { + assert!( + names.iter().any(|n| n == name), + "`{name}` must be present with no ambient context; got: {names:?}" + ); + } +} + +/// Under the default (`driver = "tinycortex"`) binding the embedded driver +/// advertises all thirteen families, so the list is byte-identical to today. +#[tokio::test] +async fn memory_tools_all_present_under_the_embedded_driver() { + use crate::core::runtime::context::CoreContext; + use crate::core::runtime::DomainSet; + + let tmp = TempDir::new().unwrap(); + let ctx = CoreContext::for_test( + DomainSet::full(), + Some(caps_tools_ws("embedded")), + Some(crate::openhuman::config::schema::MemorySubsystemConfig::default()), + ); + let names = CoreContext::scope(ctx, async { tool_names(&expansion_tools_for(&tmp)) }).await; + for name in OPTIONAL_FAMILY_MEMORY_TOOLS + .iter() + .chain(ALWAYS_PRESENT_MEMORY_TOOLS.iter()) + { + assert!( + names.iter().any(|n| n == name), + "`{name}` must survive the embedded driver; got: {names:?}" + ); + } +} + +/// The half that proves the filter removes anything. +#[tokio::test] +async fn optional_family_memory_tools_absent_under_the_null_driver() { + use crate::core::runtime::context::CoreContext; + use crate::core::runtime::DomainSet; + + let tmp = TempDir::new().unwrap(); + let ctx = CoreContext::for_test( + DomainSet::full(), + Some(caps_tools_ws("null")), + Some(null_driver_memory_cfg()), + ); + let names = CoreContext::scope(ctx, async { tool_names(&expansion_tools_for(&tmp)) }).await; + + for absent in OPTIONAL_FAMILY_MEMORY_TOOLS { + assert!( + !names.iter().any(|n| n == absent), + "`{absent}` must be ABSENT under the null driver; got: {names:?}" + ); + } + for present in ALWAYS_PRESENT_MEMORY_TOOLS { + assert!( + names.iter().any(|n| n == present), + "`{present}` is mandatory or host-owned and must survive the null driver" + ); + } +} + +/// The two post-filters are independent axes (kernel.md §3.7): a narrowed +/// capability set must not narrow the DomainSet axis. +#[tokio::test] +async fn narrow_capabilities_do_not_narrow_the_domain_axis() { + use crate::core::runtime::context::CoreContext; + use crate::core::runtime::DomainSet; + + let tmp = TempDir::new().unwrap(); + let ctx = CoreContext::for_test( + DomainSet::full(), + Some(caps_tools_ws("axes")), + Some(null_driver_memory_cfg()), + ); + let names = CoreContext::scope(ctx, async { tool_names(&expansion_tools_for(&tmp)) }).await; + for name in ["shell", "file_read", "file_write", "thread_list"] { + assert!( + names.iter().any(|n| n == name), + "a narrowed memory capability set must not remove `{name}`" + ); + } +} diff --git a/tests/agent_harness_e2e.rs b/tests/agent_harness_e2e.rs index f188d376b9..770b088266 100644 --- a/tests/agent_harness_e2e.rs +++ b/tests/agent_harness_e2e.rs @@ -2478,6 +2478,7 @@ mod streaming_support { raw: None, resolved_model: None, continue_turn: None, + served_from_cache: false, } } diff --git a/tests/calendar_grounding_e2e.rs b/tests/calendar_grounding_e2e.rs index 70db274e89..0eb305ef57 100644 --- a/tests/calendar_grounding_e2e.rs +++ b/tests/calendar_grounding_e2e.rs @@ -56,6 +56,7 @@ impl ChatModel<()> for MockCalendarModel { raw: None, resolved_model: None, continue_turn: None, + served_from_cache: false, }) } else { // End the loop diff --git a/tests/composio_list_tools_stack_overflow_regression.rs b/tests/composio_list_tools_stack_overflow_regression.rs index 6142dc73b1..b52fc1eb33 100644 --- a/tests/composio_list_tools_stack_overflow_regression.rs +++ b/tests/composio_list_tools_stack_overflow_regression.rs @@ -226,6 +226,7 @@ impl ChatModel<()> for StubModel { raw: None, resolved_model: None, continue_turn: None, + served_from_cache: false, }) } else { Ok(ModelResponse::assistant("done")) diff --git a/tests/monitor_agent_e2e.rs b/tests/monitor_agent_e2e.rs index 995322211c..d264817fa0 100644 --- a/tests/monitor_agent_e2e.rs +++ b/tests/monitor_agent_e2e.rs @@ -207,6 +207,7 @@ fn tool_response(id: &str, name: &str, arguments: serde_json::Value) -> ModelRes raw: None, resolved_model: None, continue_turn: None, + served_from_cache: false, } } diff --git a/tests/raw_coverage/agent_archivist_debug_round21_raw_coverage_e2e.rs b/tests/raw_coverage/agent_archivist_debug_round21_raw_coverage_e2e.rs index e6bdca545c..2ff44dc70a 100644 --- a/tests/raw_coverage/agent_archivist_debug_round21_raw_coverage_e2e.rs +++ b/tests/raw_coverage/agent_archivist_debug_round21_raw_coverage_e2e.rs @@ -217,6 +217,7 @@ fn tool_response(name: &str, arguments: serde_json::Value) -> ModelResponse { raw: None, resolved_model: None, continue_turn: None, + served_from_cache: false, } } diff --git a/tests/raw_coverage/agent_harness_leftovers_raw_coverage_e2e.rs b/tests/raw_coverage/agent_harness_leftovers_raw_coverage_e2e.rs index f4736fe2cb..1de7bdc191 100644 --- a/tests/raw_coverage/agent_harness_leftovers_raw_coverage_e2e.rs +++ b/tests/raw_coverage/agent_harness_leftovers_raw_coverage_e2e.rs @@ -248,6 +248,7 @@ fn tool_response(id: &str, name: &str, arguments: serde_json::Value) -> ModelRes raw: None, resolved_model: None, continue_turn: None, + served_from_cache: false, } } diff --git a/tests/raw_coverage/agent_harness_raw_coverage_e2e.rs b/tests/raw_coverage/agent_harness_raw_coverage_e2e.rs index cbde0549c6..aebf99867e 100644 --- a/tests/raw_coverage/agent_harness_raw_coverage_e2e.rs +++ b/tests/raw_coverage/agent_harness_raw_coverage_e2e.rs @@ -191,6 +191,7 @@ fn response( raw: None, resolved_model: None, continue_turn: None, + served_from_cache: false, } } @@ -216,6 +217,7 @@ fn response_with_cached( raw: None, resolved_model: None, continue_turn: None, + served_from_cache: false, } } diff --git a/tests/raw_coverage/agent_large_round25_raw_coverage_e2e.rs b/tests/raw_coverage/agent_large_round25_raw_coverage_e2e.rs index a64767e922..52e90fa159 100644 --- a/tests/raw_coverage/agent_large_round25_raw_coverage_e2e.rs +++ b/tests/raw_coverage/agent_large_round25_raw_coverage_e2e.rs @@ -271,6 +271,7 @@ fn tool_response(name: &str, args: serde_json::Value) -> ModelResponse { raw: None, resolved_model: None, continue_turn: None, + served_from_cache: false, } } diff --git a/tests/raw_coverage/agent_prompts_subagent_raw_coverage_e2e.rs b/tests/raw_coverage/agent_prompts_subagent_raw_coverage_e2e.rs index c9756eadfb..990fab326e 100644 --- a/tests/raw_coverage/agent_prompts_subagent_raw_coverage_e2e.rs +++ b/tests/raw_coverage/agent_prompts_subagent_raw_coverage_e2e.rs @@ -229,6 +229,7 @@ fn tool_response(name: &str, arguments: serde_json::Value) -> ModelResponse { raw: None, resolved_model: None, continue_turn: None, + served_from_cache: false, } } diff --git a/tests/raw_coverage/agent_session_round24_raw_coverage_e2e.rs b/tests/raw_coverage/agent_session_round24_raw_coverage_e2e.rs index 9cbf913de0..662cedff74 100644 --- a/tests/raw_coverage/agent_session_round24_raw_coverage_e2e.rs +++ b/tests/raw_coverage/agent_session_round24_raw_coverage_e2e.rs @@ -352,6 +352,7 @@ fn tool_response(value: &str) -> ModelResponse { raw: None, resolved_model: None, continue_turn: None, + served_from_cache: false, } } diff --git a/tests/raw_coverage/agent_session_turn_raw_coverage_e2e.rs b/tests/raw_coverage/agent_session_turn_raw_coverage_e2e.rs index 4786847b07..9e0699fc95 100644 --- a/tests/raw_coverage/agent_session_turn_raw_coverage_e2e.rs +++ b/tests/raw_coverage/agent_session_turn_raw_coverage_e2e.rs @@ -497,6 +497,7 @@ fn reasoning_text_response(text: &str, reasoning: &str, usage: Usage) -> ModelRe raw: None, resolved_model: None, continue_turn: None, + served_from_cache: false, } } @@ -517,6 +518,7 @@ fn tool_response(id: &str, name: &str, args: serde_json::Value) -> ModelResponse raw: None, resolved_model: None, continue_turn: None, + served_from_cache: false, } } diff --git a/tests/raw_coverage/agent_tool_loop_raw_coverage_e2e.rs b/tests/raw_coverage/agent_tool_loop_raw_coverage_e2e.rs index 5ae879ef4a..a9846c789b 100644 --- a/tests/raw_coverage/agent_tool_loop_raw_coverage_e2e.rs +++ b/tests/raw_coverage/agent_tool_loop_raw_coverage_e2e.rs @@ -345,6 +345,7 @@ fn tool_response(name: &str, arguments: serde_json::Value) -> ModelResponse { raw: None, resolved_model: None, continue_turn: None, + served_from_cache: false, } } diff --git a/tests/raw_coverage/agent_turn_builder_leftovers_raw_coverage_e2e.rs b/tests/raw_coverage/agent_turn_builder_leftovers_raw_coverage_e2e.rs index 814644c314..8a991971db 100644 --- a/tests/raw_coverage/agent_turn_builder_leftovers_raw_coverage_e2e.rs +++ b/tests/raw_coverage/agent_turn_builder_leftovers_raw_coverage_e2e.rs @@ -286,6 +286,7 @@ fn tool_response(name: &str, arguments: serde_json::Value, usage: Usage) -> Mode raw: None, resolved_model: None, continue_turn: None, + served_from_cache: false, } } diff --git a/tests/raw_coverage/agent_turn_toolloop_round22_raw_coverage_e2e.rs b/tests/raw_coverage/agent_turn_toolloop_round22_raw_coverage_e2e.rs index a0d55d588b..e265ae6c4d 100644 --- a/tests/raw_coverage/agent_turn_toolloop_round22_raw_coverage_e2e.rs +++ b/tests/raw_coverage/agent_turn_toolloop_round22_raw_coverage_e2e.rs @@ -185,6 +185,7 @@ fn tool_response(name: &str, args: serde_json::Value) -> ModelResponse { raw: None, resolved_model: None, continue_turn: None, + served_from_cache: false, } } diff --git a/tests/raw_coverage/app_credentials_threads_memory_sources_raw_coverage_e2e.rs b/tests/raw_coverage/app_credentials_threads_memory_sources_raw_coverage_e2e.rs index f4ec48fdaa..fa04a497e7 100644 --- a/tests/raw_coverage/app_credentials_threads_memory_sources_raw_coverage_e2e.rs +++ b/tests/raw_coverage/app_credentials_threads_memory_sources_raw_coverage_e2e.rs @@ -765,7 +765,7 @@ async fn round19_memory_sources_registry_readers_sync_and_reconcile_edges() { max_items: Some(1), ..source_entry("src-rss", SourceKind::RssFeed, "Feed") }; - let rss_reader = openhuman_core::openhuman::memory::sources::readers::rss::RssReader; + let rss_reader = openhuman_core::openhuman::memory::sources::readers::rss::RssReader::new(); let feed_items = rss_reader .list_items(&rss, &config) .await diff --git a/tests/raw_coverage/memory_sources_readers_round21_raw_coverage_e2e.rs b/tests/raw_coverage/memory_sources_readers_round21_raw_coverage_e2e.rs index 186ec7a05c..c2fa43ff33 100644 --- a/tests/raw_coverage/memory_sources_readers_round21_raw_coverage_e2e.rs +++ b/tests/raw_coverage/memory_sources_readers_round21_raw_coverage_e2e.rs @@ -118,7 +118,7 @@ async fn round21_rss_reader_covers_http_body_guards_and_invalid_utf8() { let _lock = env_lock(); let tmp = tempdir(); let config = config(&tmp); - let reader = openhuman_core::openhuman::memory::sources::readers::rss::RssReader; + let reader = openhuman_core::openhuman::memory::sources::readers::rss::RssReader::new(); let (status_url, status_server) = one_response_server("503 Service Unavailable", "", b"down".to_vec()).await; diff --git a/tests/raw_coverage/memory_sync_sources_raw_coverage_e2e.rs b/tests/raw_coverage/memory_sync_sources_raw_coverage_e2e.rs index bf7f8ecb14..86be0187d4 100644 --- a/tests/raw_coverage/memory_sync_sources_raw_coverage_e2e.rs +++ b/tests/raw_coverage/memory_sync_sources_raw_coverage_e2e.rs @@ -332,7 +332,7 @@ async fn rss_reader_lists_reads_and_reports_feed_errors_from_loopback() { ); let (base, server) = loopback_router(router).await; - let reader = openhuman_core::openhuman::memory::sources::readers::rss::RssReader; + let reader = openhuman_core::openhuman::memory::sources::readers::rss::RssReader::new(); let mut entry = source(SourceKind::RssFeed, "rss-round15"); entry.url = Some(format!("{base}/rss")); entry.max_items = Some(1); diff --git a/tests/raw_coverage/memory_threads_raw_coverage_e2e.rs b/tests/raw_coverage/memory_threads_raw_coverage_e2e.rs index 0feffa064c..d61df11454 100644 --- a/tests/raw_coverage/memory_threads_raw_coverage_e2e.rs +++ b/tests/raw_coverage/memory_threads_raw_coverage_e2e.rs @@ -2664,7 +2664,7 @@ async fn memory_source_sync_entrypoint_rejects_disabled_and_ingests_folder_items #[test] fn memory_tree_io_contract_types_round_trip_leaf_read_and_write_shapes() { let now = Utc.with_ymd_and_hms(2026, 5, 29, 16, 0, 0).unwrap(); - let payload = openhuman_core::openhuman::memory::tree::io::TreeLeafPayload { + let payload = openhuman_core::openhuman::memory::tree::TreeLeafPayload { chunk_id: "chunk-contract-1".into(), token_count: 42, timestamp: now, @@ -2677,12 +2677,12 @@ fn memory_tree_io_contract_types_round_trip_leaf_read_and_write_shapes() { assert_eq!(leaf_ref.chunk_id, payload.chunk_id); assert_eq!(leaf_ref.entities, payload.entities); let round_trip = - openhuman_core::openhuman::memory::tree::io::TreeLeafPayload::from(leaf_ref.clone()); + openhuman_core::openhuman::memory::tree::TreeLeafPayload::from(leaf_ref.clone()); assert_eq!(round_trip.content, payload.content); assert_eq!(round_trip.score, payload.score); let write_default_json = serde_json::to_value( - openhuman_core::openhuman::memory::tree::io::TreeWriteRequest { + openhuman_core::openhuman::memory::tree::TreeWriteRequest { tree_id: "tree-contract".into(), tree_kind: TreeKind::Source, leaf: round_trip.clone(), @@ -2694,7 +2694,7 @@ fn memory_tree_io_contract_types_round_trip_leaf_read_and_write_shapes() { assert_eq!(write_default_json["label_strategy"], "inherit"); assert_eq!(write_default_json["deferred"], false); - let decoded_write: openhuman_core::openhuman::memory::tree::io::TreeWriteRequest = + let decoded_write: openhuman_core::openhuman::memory::tree::TreeWriteRequest = serde_json::from_value(json!({ "tree_id": "tree-contract", "tree_kind": "global", @@ -2711,12 +2711,12 @@ fn memory_tree_io_contract_types_round_trip_leaf_read_and_write_shapes() { assert_eq!(decoded_write.tree_kind, TreeKind::Global); assert_eq!( decoded_write.label_strategy, - openhuman_core::openhuman::memory::tree::io::TreeLabelStrategy::Empty + openhuman_core::openhuman::memory::tree::TreeLabelStrategy::Empty ); assert!(decoded_write.leaf.entities.is_empty()); assert!(decoded_write.deferred); - let outcome = openhuman_core::openhuman::memory::tree::io::TreeWriteOutcome { + let outcome = openhuman_core::openhuman::memory::tree::TreeWriteOutcome { new_summary_ids: vec!["summary-1".into()], seal_pending: true, }; @@ -2724,7 +2724,7 @@ fn memory_tree_io_contract_types_round_trip_leaf_read_and_write_shapes() { assert_eq!(outcome_json["new_summary_ids"][0], "summary-1"); assert_eq!(outcome_json["seal_pending"], true); - let read_request: openhuman_core::openhuman::memory::tree::io::TreeReadRequest = + let read_request: openhuman_core::openhuman::memory::tree::TreeReadRequest = serde_json::from_value(json!({ "tree_id": "tree-contract", "max_depth": 2, @@ -2736,14 +2736,14 @@ fn memory_tree_io_contract_types_round_trip_leaf_read_and_write_shapes() { assert_eq!(read_request.max_depth, 2); assert_eq!(read_request.limit, Some(3)); - let hit = openhuman_core::openhuman::memory::tree::io::TreeReadHit { + let hit = openhuman_core::openhuman::memory::tree::TreeReadHit { node_id: "summary-1".into(), node_kind: "summary".into(), level: 1, content: "Summary text".into(), score: 0.42, }; - let result = openhuman_core::openhuman::memory::tree::io::TreeReadResult { + let result = openhuman_core::openhuman::memory::tree::TreeReadResult { hits: vec![hit], total: 4, tree_id: "tree-contract".into(), @@ -2763,7 +2763,7 @@ fn memory_tree_io_contract_types_round_trip_leaf_read_and_write_shapes() { created_at: now, last_sealed_at: None, }; - let empty = openhuman_core::openhuman::memory::tree::io::TreeReadResult::empty(&tree); + let empty = openhuman_core::openhuman::memory::tree::TreeReadResult::empty(&tree); assert_eq!(empty.tree_id, "empty-tree"); assert!(empty.hits.is_empty()); } @@ -3310,24 +3310,24 @@ fn memory_sync_profile_markdown_and_status_helpers_are_idempotent() { let now = 1_700_000_000_000_i64; assert_eq!( - openhuman_core::openhuman::memory::sync::sync_status::types::FreshnessLabel::from_age_ms( + openhuman_core::openhuman::memory::sync::sync_status::FreshnessLabel::from_age_ms( Some(now - 30_000), now ), - openhuman_core::openhuman::memory::sync::sync_status::types::FreshnessLabel::Active + openhuman_core::openhuman::memory::sync::sync_status::FreshnessLabel::Active ); assert_eq!( - openhuman_core::openhuman::memory::sync::sync_status::types::FreshnessLabel::from_age_ms( + openhuman_core::openhuman::memory::sync::sync_status::FreshnessLabel::from_age_ms( Some(now - 30_001), now ), - openhuman_core::openhuman::memory::sync::sync_status::types::FreshnessLabel::Recent + openhuman_core::openhuman::memory::sync::sync_status::FreshnessLabel::Recent ); assert_eq!( - openhuman_core::openhuman::memory::sync::sync_status::types::FreshnessLabel::from_age_ms( + openhuman_core::openhuman::memory::sync::sync_status::FreshnessLabel::from_age_ms( None, now ), - openhuman_core::openhuman::memory::sync::sync_status::types::FreshnessLabel::Idle + openhuman_core::openhuman::memory::sync::sync_status::FreshnessLabel::Idle ); } diff --git a/tests/raw_coverage/near90_closure_raw_coverage_e2e.rs b/tests/raw_coverage/near90_closure_raw_coverage_e2e.rs index 31d8824e53..bce795cbbd 100644 --- a/tests/raw_coverage/near90_closure_raw_coverage_e2e.rs +++ b/tests/raw_coverage/near90_closure_raw_coverage_e2e.rs @@ -397,7 +397,7 @@ async fn round20_memory_sources_readers_and_sync_cover_error_edges_without_netwo let harness = setup("http://127.0.0.1:9"); let config = harness.config().await; - let rss = openhuman_core::openhuman::memory::sources::readers::rss::RssReader; + let rss = openhuman_core::openhuman::memory::sources::readers::rss::RssReader::new(); let mut missing_url = source_entry("rss-missing-url", SourceKind::RssFeed); assert_eq!( rss.list_items(&missing_url, &config) diff --git a/tests/raw_coverage/tools_agent_credentials_state_raw_coverage_e2e.rs b/tests/raw_coverage/tools_agent_credentials_state_raw_coverage_e2e.rs index 5b86e4cd9f..30c6071684 100644 --- a/tests/raw_coverage/tools_agent_credentials_state_raw_coverage_e2e.rs +++ b/tests/raw_coverage/tools_agent_credentials_state_raw_coverage_e2e.rs @@ -342,6 +342,7 @@ fn response(text: Option<&str>, tool_calls: Vec) -> ModelResponse { raw: None, resolved_model: None, continue_turn: None, + served_from_cache: false, } } diff --git a/tests/subconscious_fullstack_e2e.rs b/tests/subconscious_fullstack_e2e.rs index 2e5101978f..73ef0d431c 100644 --- a/tests/subconscious_fullstack_e2e.rs +++ b/tests/subconscious_fullstack_e2e.rs @@ -123,6 +123,7 @@ impl MockLlm { raw: None, resolved_model: None, continue_turn: None, + served_from_cache: false, }; } else { "Mock orchestrator handled the promoted trigger.".to_string() diff --git a/vendor/tinyagents b/vendor/tinyagents index 107a515d23..27a3f39dc6 160000 --- a/vendor/tinyagents +++ b/vendor/tinyagents @@ -1 +1 @@ -Subproject commit 107a515d2385686167931423b7dc8be53b14be15 +Subproject commit 27a3f39dc6d7db676efe58d0f7b89752a8ab4746 diff --git a/vendor/tinycortex b/vendor/tinycortex index 5fabcf18d9..ce98837b50 160000 --- a/vendor/tinycortex +++ b/vendor/tinycortex @@ -1 +1 @@ -Subproject commit 5fabcf18d9e3907d6b26b59528ad49cebfc1c271 +Subproject commit ce98837b50178ec7db23571064360f0258a2d429