diff --git a/crates/tinymemory-api/src/provider/content.rs b/crates/tinymemory-api/src/provider/content.rs index 4a76f275..079e76a5 100644 --- a/crates/tinymemory-api/src/provider/content.rs +++ b/crates/tinymemory-api/src/provider/content.rs @@ -17,12 +17,17 @@ //! the families that looked most config-dependent turned out not to need any. use async_trait::async_trait; +// Named through the wire crate rather than as a dependency of this one: the +// contract crate is deliberately dependency-light, and `tinymemory-bus` +// re-exports the chrono it serializes with precisely so a signature here names +// the same crate a frame decodes into. +use tinymemory_bus::chrono::{DateTime, Utc}; use crate::capabilities::Capability; use crate::chunks::Chunk; use crate::error::MemoryError; use crate::provider::types::{IngestItem, IngestOutcome, SourceScope}; -use crate::tree::{IngestRequest, QueryResult, SummaryForest, TreeLeaf, TreeStatus}; +use crate::tree::{IngestRequest, QueryResult, SummaryForest, TreeLeaf, TreeNode, TreeStatus}; use crate::types::{NamespaceDocumentInput, NamespaceRetrievalContext, StoredMemoryDocument}; // The value types the summariser door exchanges. They are defined in @@ -537,4 +542,205 @@ pub trait MemoryTree: Send + Sync { ) -> Result, MemoryError> { Err(MemoryError::unsupported(Capability::Tree)) } + + // ── The runtime-tree doors ─────────────────────────────────────────── + // + // The six members below are the markdown time tree addressed node by + // node — the doors under the host's `tree_summarizer_*` RPC surface, which + // reported the engine's answers verbatim and therefore needs the engine's + // exact shapes: the landing path of a buffered write, a single node as an + // `Option`, a child list, a status row, and the node/status a + // summarisation pass produced. [`Self::append`], [`Self::drill_down`], + // [`Self::seal`] and [`Self::cascade`] are the same tree at a coarser + // grain, and each one folds away a piece of the reply the RPCs carry — + // which is why migrating that surface onto them would have changed its + // wire format, and a door that changes what the host reports is not a + // door, it is a new surface. + + /// Buffer raw content for the markdown time tree, answering with the path + /// it landed at. + /// + /// The finer-grained sibling of [`Self::append`], which is this write with + /// both ends trimmed off: `append` defaults a missing timestamp to the + /// driver's "now" and discards the landing path. Here the timestamp is + /// **required** — the caller's reply echoes the instant it filed the + /// content under, and a timestamp resolved driver-side would disagree with + /// the one the caller reports by however long the call took to cross — + /// and the path comes back, because the reply names it. + /// + /// The path is a *report*, not an invitation: it is the driver's own + /// spelling of the buffer file it wrote, inside the driver's workspace, + /// and the next seal consumes it. A caller displays it; nothing should + /// dereference it. + /// + /// `metadata` rides into the buffer entry's frontmatter when present, and + /// the produced node carries it onward — the same field + /// [`IngestRequest::metadata`] documents. + /// + /// # Errors + /// + /// [`MemoryError::Invalid`] for a rejected namespace or content that is + /// blank after trimming, otherwise backend failures. + async fn runtime_buffer_write( + &self, + _namespace: &str, + _content: &str, + _timestamp: DateTime, + _metadata: Option, + ) -> Result { + Err(MemoryError::unsupported(Capability::Tree)) + } + + /// One time-tree node, or `None` when nothing sits at `node_id`. + /// + /// Half of [`Self::drill_down`], unbundled — and the unbundling is the + /// point, twice over. `drill_down` folds "no such node" into + /// [`MemoryError::NotFound`] with its own message, which is right for + /// navigation and wrong for the caller here, which shapes its own miss and + /// treats "does the root exist yet" as a probe rather than a fault. And it + /// always pays for the child read, which a caller reporting one node did + /// not ask for. + /// + /// # Errors + /// + /// [`MemoryError::Invalid`] for a rejected namespace or a `node_id` that + /// is not `root` / `YYYY[/MM[/DD[/HH]]]`-shaped, otherwise backend + /// failures. Absence is `Ok(None)`, never an error. + async fn runtime_read_node( + &self, + _namespace: &str, + _node_id: &str, + ) -> Result, MemoryError> { + Err(MemoryError::unsupported(Capability::Tree)) + } + + /// The direct children of one time-tree node, in the store's own order. + /// + /// The other half of [`Self::drill_down`], on the same terms as + /// [`Self::runtime_read_node`]. A parent that does not exist has no + /// children: an empty vector, not an error — the same answer an hour leaf + /// gives, because a leaf has nothing under it by construction. + /// + /// # Errors + /// + /// [`MemoryError::Invalid`] on the same terms as + /// [`Self::runtime_read_node`], otherwise backend failures. + async fn runtime_read_children( + &self, + _namespace: &str, + _parent_id: &str, + ) -> Result, MemoryError> { + Err(MemoryError::unsupported(Capability::Tree)) + } + + /// One namespace's time-tree shape: node count, depth, and coverage. + /// + /// The read [`Self::seal`] and [`Self::cascade`] already answer with — + /// exposed on its own so a status panel can poll it without running the + /// pass it describes. A namespace with no tree yet is the all-empty + /// status, not an error: zero nodes, zero depth, every timestamp `None`. + /// + /// # Errors + /// + /// [`MemoryError::Invalid`] for a rejected namespace, otherwise backend + /// failures. + async fn runtime_tree_status(&self, _namespace: &str) -> Result { + Err(MemoryError::unsupported(Capability::Tree)) + } + + /// Drain the namespace's buffer into hour leaves and propagate upward, + /// answering with the last hour node written. + /// + /// [`Self::seal`] runs the same pass and answers with tree *state*; this + /// answers with the *work* — the node the pass produced, which is what the + /// triggering surface reports ("node `2024/03/15/09`, 340 tokens") and + /// what a status row cannot be unfolded into. `Ok(None)` is a pass that + /// found nothing buffered, which is a successful no-op on the same terms + /// as `seal`'s idempotence. + /// + /// The fold runs on the driver's own chat provider, built the way every + /// scheduled seal builds it — see [`Self::summarise`] for why the provider + /// cannot be the caller's, and note the consequence is the same here: the + /// spend happens driver-side, under the driver's configuration. + /// + /// `timestamp` is the instant the pass files freshly-drained content + /// under, supplied by the caller for the reason + /// [`Self::runtime_buffer_write`] gives. + /// + /// # Errors + /// + /// [`MemoryError::Invalid`] for a rejected namespace — checked before the + /// provider is built, so a bad namespace answers the same with or without + /// one configured. A driver whose summarisation provider cannot be + /// resolved fails even when the buffer is empty: the caller offered a + /// "run now" control, and "nothing to do" from a runner that could not + /// have run is the lie a disabled control exists to avoid. Otherwise + /// backend failures, which include the provider call itself. + async fn runtime_summarize( + &self, + _namespace: &str, + _timestamp: DateTime, + ) -> Result, MemoryError> { + Err(MemoryError::unsupported(Capability::Tree)) + } + + /// Rebuild the whole time tree above the hour leaves, answering with the + /// resulting status. + /// + /// [`Self::cascade`] with one behavioural difference, preserved on + /// purpose: `cascade` short-circuits an empty tree without touching the + /// provider, because a scheduler calls it unconditionally. This member is + /// a person's explicit "rebuild now", and it resolves the provider + /// *first* — on the terms [`Self::runtime_summarize`] gives — so a setup + /// that could never rebuild says so instead of reporting an empty rebuild + /// that ran nothing. + /// + /// # Errors + /// + /// As [`Self::runtime_summarize`]. + async fn runtime_rebuild(&self, _namespace: &str) -> Result { + Err(MemoryError::unsupported(Capability::Tree)) + } + + /// The compiled flavoured-root profile for one tree scope, front-matter + /// included — or `None` while nothing has been distilled for it. + /// + /// The read behind a persona tool: flavoured trees are sealed under a + /// standing ask (see [`SummaryContext::ask`]), and their root compiles to + /// a small fixed-path markdown artifact. This member collapses the whole + /// lookup the host used to run against the engine directly — try the + /// compiled artifact, fall back to the tree, recompile its root — behind + /// one scope-shaped question. + /// + /// # The split with the caller + /// + /// The driver owns the lookup and the built/not-built verdict; the caller + /// owns the vocabulary and the presentation. Scope strings like + /// `persona/communication` are the caller's naming scheme — the driver + /// matches them literally and validates nothing about their shape beyond + /// non-emptiness, so an unknown scope is indistinguishable from an unbuilt + /// one, deliberately: both answer `Ok(None)`, and only the caller knows + /// which scopes it ever writes. The returned markdown is the **full + /// compiled artifact including its front-matter**, because the front + /// matter is part of what was compiled; a caller that wants only the prose + /// strips it, and that choice stays on the caller's side of the wire. + /// + /// # `None` versus empty + /// + /// `Ok(None)` means *not built*: no flavoured tree for the scope, or a + /// tree whose compiled root has an empty body — a tree that exists but has + /// never sealed compiles to front-matter over nothing, and a profile with + /// no prose is not a profile. A driver must never answer `Ok(Some)` with a + /// body-less artifact, because the caller hands the body to a model and an + /// empty string reads as "this person has no communication style". + /// + /// # Errors + /// + /// [`MemoryError::Invalid`] for a blank scope. Otherwise backend failures + /// — the tree lookup or the compile step failing is an error, distinct + /// from `None`, because "could not read the store" reported as "not built + /// yet" tells a user to re-run an ingestion that already worked. + async fn flavour_profile(&self, _scope: &str) -> Result, MemoryError> { + Err(MemoryError::unsupported(Capability::Tree)) + } } diff --git a/crates/tinymemory-bus/README.md b/crates/tinymemory-bus/README.md index c28f5fa8..720f57a1 100644 --- a/crates/tinymemory-bus/README.md +++ b/crates/tinymemory-bus/README.md @@ -4,9 +4,10 @@ Every type that crosses the TinyMemory `TinyBus` boundary, and the names of the members that carry them. TinyMemory ships as a loadable module so a host does not compile the engine: -`crates/tinymemory-module` exports one object with 120 members on it, built as a -`cdylib`. A host can load that binary but cannot `use` anything out of it, so -the payload vocabulary has to be published as an ordinary library. This is it. +`crates/tinymemory-module` exports one object with `METHODS.len()` members on +it, built as a `cdylib`. A host can load that binary but cannot `use` anything +out of it, so the payload vocabulary has to be published as an ordinary +library. This is it. | module | what it holds | | ---------------------------------------------------------------- | ---------------------------------------------- | diff --git a/crates/tinymemory-bus/src/lib.rs b/crates/tinymemory-bus/src/lib.rs index 4e0c2e4a..14927f5a 100644 --- a/crates/tinymemory-bus/src/lib.rs +++ b/crates/tinymemory-bus/src/lib.rs @@ -99,3 +99,15 @@ pub mod wire; pub use names::{BUS_NAME, METHODS, OBJECT_PATH}; pub use version::{is_compatible, CONTRACT_VERSION}; + +/// The one foreign crate in the wire vocabulary, re-exported by name. +/// +/// Timestamps cross this boundary as `chrono::DateTime` — `tree::TreeNode` +/// and its siblings have always carried them in their fields, and the +/// runtime-tree members take one as a bare argument. A crate that spells those +/// signatures has to spell *this* chrono: `tinymemory-api` is deliberately +/// dependency-light and adds no crate of its own, so it names the type through +/// here, and anything else that does the same is guaranteed the exact crate +/// this one serializes with rather than whichever `chrono = "0.4"` its own +/// lockfile happened to resolve. +pub use chrono; diff --git a/crates/tinymemory-bus/src/names.rs b/crates/tinymemory-bus/src/names.rs index e5b4f7c5..c8020fd7 100644 --- a/crates/tinymemory-bus/src/names.rs +++ b/crates/tinymemory-bus/src/names.rs @@ -103,6 +103,29 @@ pub mod methods { /// `RootSummaries` — every namespace's root summary, capped. pub const ROOT_SUMMARIES: &str = "RootSummaries"; + // The runtime (markdown time) tree, node by node. `Seal` and `Cascade` + // above run whole passes and answer with tree state; these are the doors + // under the host's tree-summarizer RPC surface — the buffered write that + // reports where it landed, the two structural reads and the status read + // unbundled from `DrillDown`, and the two provider-backed passes with the + // shapes the RPC replies actually carry. + /// `RuntimeBufferWrite` — buffer raw content for the time tree, answering + /// with the path it landed at. + pub const RUNTIME_BUFFER_WRITE: &str = "RuntimeBufferWrite"; + /// `RuntimeReadNode` — one time-tree node, or none. + pub const RUNTIME_READ_NODE: &str = "RuntimeReadNode"; + /// `RuntimeReadChildren` — a time-tree node's direct children. + pub const RUNTIME_READ_CHILDREN: &str = "RuntimeReadChildren"; + /// `RuntimeTreeStatus` — one namespace's time-tree shape and coverage. + pub const RUNTIME_TREE_STATUS: &str = "RuntimeTreeStatus"; + /// `RuntimeSummarize` — drain the buffer into the tree on the driver's + /// provider, answering with the last hour node it wrote. + pub const RUNTIME_SUMMARIZE: &str = "RuntimeSummarize"; + /// `RuntimeRebuild` — rebuild the whole time tree from its hour leaves. + pub const RUNTIME_REBUILD: &str = "RuntimeRebuild"; + /// `FlavourProfile` — the compiled flavoured-root profile for one scope. + pub const FLAVOUR_PROFILE: &str = "FlavourProfile"; + // Entities, relations and the namespaced key/value store. /// `Entities` — entities. pub const ENTITIES: &str = "Entities"; @@ -341,7 +364,7 @@ pub mod methods { /// The order matters: `tinybus`'s `Interface::members()` returns declaration /// order, and the module compares the two sequences directly rather than as /// sets, so a reordering is caught alongside an addition or a removal. -pub const METHODS: [&str; 131] = [ +pub const METHODS: [&str; 138] = [ methods::DRIVER_ID, methods::CAPABILITIES, methods::HEALTH, @@ -473,6 +496,13 @@ pub const METHODS: [&str; 131] = [ methods::DEGRADED_STATE, methods::CHUNK_SCORE, methods::SOURCE_INGEST_STATUS, + methods::RUNTIME_BUFFER_WRITE, + methods::RUNTIME_READ_NODE, + methods::RUNTIME_READ_CHILDREN, + methods::RUNTIME_TREE_STATUS, + methods::RUNTIME_SUMMARIZE, + methods::RUNTIME_REBUILD, + methods::FLAVOUR_PROFILE, ]; #[cfg(test)] diff --git a/crates/tinymemory-bus/src/names_tests.rs b/crates/tinymemory-bus/src/names_tests.rs index b0ebb212..4fab5ba9 100644 --- a/crates/tinymemory-bus/src/names_tests.rs +++ b/crates/tinymemory-bus/src/names_tests.rs @@ -87,16 +87,18 @@ fn the_newest_members_are_appended_rather_than_filed_with_their_family() { // against this table as a *sequence*, so a new member filed beside its // family renumbers every member after it. That is invisible here and shows // up as the wrong method being invoked on a host built against a different - // release, which is why the tail order is asserted and not just membership. - let tail = &METHODS[METHODS.len() - 3..]; - assert_eq!( - tail, - [ - methods::DEGRADED_STATE, - methods::CHUNK_SCORE, - methods::SOURCE_INGEST_STATUS, - ] - ); + // release, which is why the positions are asserted and not just + // membership. + // + // By absolute index since the runtime-tree round: this began as a + // tail-of-table assertion, which fires on every append and is then + // re-pointed at the new tail — doing the drift-witness job once, at the + // cost of re-stating it each round. The absolute slots are the released + // wire positions themselves, which is the stronger pin and the one the + // summariser-door test below already argues for. + assert_eq!(METHODS[128], methods::DEGRADED_STATE); + assert_eq!(METHODS[129], methods::CHUNK_SCORE); + assert_eq!(METHODS[130], methods::SOURCE_INGEST_STATUS); } #[test] @@ -116,3 +118,33 @@ fn the_summariser_door_holds_the_wire_slots_it_was_released_in() { assert_eq!(METHODS[126], methods::SUMMARISE); assert_eq!(METHODS[127], methods::ROOT_SUMMARIES); } + +#[test] +fn the_runtime_tree_doors_hold_the_wire_slots_they_were_released_in() { + // The final seven doors of the engine shed: the members the host's + // tree-summarizer RPC surface and the flavour tool stand on once nothing + // in the host links the engine. Their spellings are pinned here as well as + // read off the table for the reason every earlier round gives — a typo is + // an `UnknownMethod` the first time a released module is asked, never a + // compile error on either side. + assert_eq!(methods::RUNTIME_BUFFER_WRITE, "RuntimeBufferWrite"); + assert_eq!(methods::RUNTIME_READ_NODE, "RuntimeReadNode"); + assert_eq!(methods::RUNTIME_READ_CHILDREN, "RuntimeReadChildren"); + assert_eq!(methods::RUNTIME_TREE_STATUS, "RuntimeTreeStatus"); + assert_eq!(methods::RUNTIME_SUMMARIZE, "RuntimeSummarize"); + assert_eq!(methods::RUNTIME_REBUILD, "RuntimeRebuild"); + assert_eq!(methods::FLAVOUR_PROFILE, "FlavourProfile"); + + // Their positions are pinned by absolute index, not from the tail, for the + // reason the summariser-door test above gives: member order is wire order, + // and an assertion measured from the end moves silently under the next + // append — which is exactly the edit this exists to catch. + assert_eq!(METHODS.len(), 138); + assert_eq!(METHODS[131], methods::RUNTIME_BUFFER_WRITE); + assert_eq!(METHODS[132], methods::RUNTIME_READ_NODE); + assert_eq!(METHODS[133], methods::RUNTIME_READ_CHILDREN); + assert_eq!(METHODS[134], methods::RUNTIME_TREE_STATUS); + assert_eq!(METHODS[135], methods::RUNTIME_SUMMARIZE); + assert_eq!(METHODS[136], methods::RUNTIME_REBUILD); + assert_eq!(METHODS[137], methods::FLAVOUR_PROFILE); +} diff --git a/crates/tinymemory-bus/src/tree_tests.rs b/crates/tinymemory-bus/src/tree_tests.rs index 67b5c684..46eb209c 100644 --- a/crates/tinymemory-bus/src/tree_tests.rs +++ b/crates/tinymemory-bus/src/tree_tests.rs @@ -214,3 +214,77 @@ fn a_root_summary_travels_by_name_so_its_two_strings_cannot_be_swapped() { let decoded: RootSummary = serde_json::from_value(payload).unwrap(); assert_eq!(decoded, summary); } + +#[test] +fn a_tree_node_round_trips_with_its_level_spelled_as_the_files_spell_it() { + // `TreeNode` predates the runtime-tree members but never crossed a frame + // as a *response* until `RuntimeReadNode`/`RuntimeReadChildren`/ + // `RuntimeSummarize` — this pins the shape those members now serve. The + // level's wire string matters doubly: it is also the spelling the engine's + // markdown frontmatter uses, so a rename here would not just break decode, + // it would disagree with every node already on disk. + let node = TreeNode { + node_id: "2024/03/15/09".to_string(), + namespace: "team".to_string(), + level: NodeLevel::Hour, + parent_id: Some("2024/03/15".to_string()), + summary: "the morning standup, folded".to_string(), + token_count: 7, + child_count: 0, + created_at: Utc.with_ymd_and_hms(2024, 3, 15, 9, 0, 0).unwrap(), + updated_at: Utc.with_ymd_and_hms(2024, 3, 15, 9, 30, 0).unwrap(), + metadata: None, + }; + let payload = serde_json::to_value(&node).unwrap(); + assert_eq!(payload["level"], "hour"); + assert_eq!(payload["node_id"], "2024/03/15/09"); + assert!( + payload.get("metadata").is_none(), + "absent metadata is omitted, not serialized as null" + ); + let decoded: TreeNode = serde_json::from_value(payload).unwrap(); + assert_eq!(decoded.node_id, node.node_id); + assert_eq!(decoded.level, node.level); + assert_eq!(decoded.parent_id, node.parent_id); + assert_eq!(decoded.summary, node.summary); + assert_eq!(decoded.updated_at, node.updated_at); + assert_eq!(decoded.metadata, None); +} + +#[test] +fn a_tree_status_keeps_its_absent_timestamps_absent() { + // The status of a namespace that has never been sealed is all-`None`, and + // `RuntimeTreeStatus` serves exactly that on a fresh workspace. The three + // options must decode back to `None` rather than to an epoch, because a + // dashboard renders `oldest_entry` as coverage and an epoch reads as + // "since 1970". + let empty = TreeStatus { + namespace: "team".to_string(), + total_nodes: 0, + depth: 0, + oldest_entry: None, + newest_entry: None, + last_run_at: None, + }; + let decoded: TreeStatus = + serde_json::from_str(&serde_json::to_string(&empty).unwrap()).unwrap(); + assert_eq!(decoded.namespace, "team"); + assert_eq!(decoded.total_nodes, 0); + assert_eq!(decoded.oldest_entry, None); + assert_eq!(decoded.last_run_at, None); + + let run_at = Utc.with_ymd_and_hms(2024, 3, 15, 10, 0, 0).unwrap(); + let populated = TreeStatus { + namespace: "team".to_string(), + total_nodes: 12, + depth: 5, + oldest_entry: Some(Utc.with_ymd_and_hms(2024, 1, 1, 0, 0, 0).unwrap()), + newest_entry: Some(run_at), + last_run_at: Some(run_at), + }; + let decoded: TreeStatus = + serde_json::from_str(&serde_json::to_string(&populated).unwrap()).unwrap(); + assert_eq!(decoded.total_nodes, 12); + assert_eq!(decoded.depth, 5); + assert_eq!(decoded.newest_entry, Some(run_at)); +} diff --git a/crates/tinymemory-bus/src/version.rs b/crates/tinymemory-bus/src/version.rs index 7a666402..b6f6232d 100644 --- a/crates/tinymemory-bus/src/version.rs +++ b/crates/tinymemory-bus/src/version.rs @@ -60,7 +60,7 @@ /// added to a family a driver may already advertise** (negotiation is /// family-granular, not method-granular, so that case cannot be made minor-safe /// by negotiation alone). -pub const CONTRACT_VERSION: (u16, u16) = (3, 0); +pub const CONTRACT_VERSION: (u16, u16) = (4, 0); /// Whether a driver speaking `remote` can be bound against this build. /// diff --git a/crates/tinymemory-bus/src/version_tests.rs b/crates/tinymemory-bus/src/version_tests.rs index c334d793..866d7b0f 100644 --- a/crates/tinymemory-bus/src/version_tests.rs +++ b/crates/tinymemory-bus/src/version_tests.rs @@ -7,20 +7,28 @@ use super::*; #[test] -fn contract_version_is_three_zero() { - // (3, 0): `count_chunks`, the three entity-occurrence members and the two - // tree-forest members were added to families a driver may ALREADY - // advertise. The rule makes that a major bump and not a minor one, and the - // reason is the whole point of the rule: negotiation is family-granular, - // so a driver advertising `Chunks` at (2, 2) would be bound and then asked - // for a method it has never heard of. The major half is what refuses that - // bind instead of discovering it at the call. +fn contract_version_is_four_zero() { + // (4, 0): the six runtime-tree members and `flavour_profile` were added to + // `Tree` — a family a driver may ALREADY advertise. The rule makes that a + // major bump and not a minor one, and the reason is the whole point of the + // rule: negotiation is family-granular, so a driver advertising `Tree` at + // (3, 0) would be bound and then asked for a method it has never heard of. + // The major half is what refuses that bind instead of discovering it at + // the call. + // + // (3, 0) was the same shape one round earlier: `count_chunks`, the three + // entity-occurrence members and the two tree-forest members, all onto + // already-advertised families. // // Note for anyone reading the history: #85/#86/#89/#90 also added methods - // to advertised families and stayed on the minor half. That was wrong by - // this rule; those releases and their hosts moved in lockstep so nothing - // was bound across the gap, but it is drift, not precedent. - assert_eq!(CONTRACT_VERSION, (3, 0)); + // to advertised families and stayed on the minor half, and so did #122 — + // the first round of the openhuman engine shed, which put `Summarise`, + // `RootSummaries`, `ChunkScore`, `DegradedState` and `SourceIngestStatus` + // onto existing families and shipped as v1.13.5 without touching this + // constant. All of that was wrong by this rule; those releases and their + // hosts moved in lockstep so nothing was bound across the gap, but it is + // drift, not precedent. This round declines to extend it. + assert_eq!(CONTRACT_VERSION, (4, 0)); } #[test] diff --git a/crates/tinymemory-module/src/lib.rs b/crates/tinymemory-module/src/lib.rs index 9808732d..5fb4693b 100644 --- a/crates/tinymemory-module/src/lib.rs +++ b/crates/tinymemory-module/src/lib.rs @@ -683,6 +683,16 @@ mod exports { "DegradedState", "ChunkScore", "SourceIngestStatus", + // The final round of the shed: the markdown time tree node by + // node — the shapes the host's tree-summarizer RPCs report — and + // the compiled flavoured-root profile read. + "RuntimeBufferWrite", + "RuntimeReadNode", + "RuntimeReadChildren", + "RuntimeTreeStatus", + "RuntimeSummarize", + "RuntimeRebuild", + "FlavourProfile", ], signals = [], // The host's embedder is deliberately NOT declared as `requires`. That diff --git a/crates/tinymemory-module/src/service/mod.rs b/crates/tinymemory-module/src/service/mod.rs index a8d5082c..1d713d13 100644 --- a/crates/tinymemory-module/src/service/mod.rs +++ b/crates/tinymemory-module/src/service/mod.rs @@ -57,6 +57,13 @@ //! RecentLeaves(limit, scope) -> [TreeLeaf] //! Summarise(inputs, context) -> SummaryOutput //! RootSummaries(per_namespace_cap, total_cap) -> [RootSummary] +//! RuntimeBufferWrite(ns, content, ts, metadata) -> String +//! RuntimeReadNode(ns, node_id) -> Option +//! RuntimeReadChildren(ns, parent_id) -> [TreeNode] +//! RuntimeTreeStatus(ns) -> TreeStatus +//! RuntimeSummarize(ns, ts) -> Option +//! RuntimeRebuild(ns) -> TreeStatus +//! FlavourProfile(scope) -> Option //! //! TopEntities(kind, limit) -> [EntityOccurrence] //! ChunkEntities(chunk_ids, kinds) -> [ChunkEntityOccurrence] @@ -152,6 +159,7 @@ use std::sync::Arc; // be held across. use tokio::sync::Mutex; +use chrono::{DateTime, Utc}; use tinybus::{Connection, Error as BusError, Result as BusResult}; use tinymemory_api::capabilities::{Capabilities, Capability}; use tinymemory_api::chunks::Chunk; @@ -194,7 +202,7 @@ use tinymemory_api::recall::OwnedRecallOpts; use tinymemory_api::tool_memory::ToolMemoryRule; use tinymemory_api::tree::{ IngestRequest, QueryResult, RootSummary, SummaryContext, SummaryForest, SummaryInput, - SummaryOutput, TreeLeaf, TreeStatus, + SummaryOutput, TreeLeaf, TreeNode, TreeStatus, }; use tinymemory_api::types::{ GraphRelationRecord, MemoryCategory, MemoryEntry, MemoryKvRecord, MemoryTaint, @@ -2054,6 +2062,131 @@ impl MemoryService { ensure_response_fits(&rows, "SourceIngestStatus")?; Ok(rows) } + + /// Buffer raw content for the markdown time tree, answering with where it + /// landed. + /// + /// Appended here rather than filed beside `Append` for the reason + /// `count_chunks` gives above: member order is wire order. + /// + /// Not size-checked. The response is one path string; the *request* + /// carries the content, and that bound is the caller's, exactly as it is + /// for `Summarise`. + async fn runtime_buffer_write( + &self, + namespace: String, + content: String, + timestamp: DateTime, + metadata: Option, + ) -> BusResult { + require_family!(self, as_tree, Capability::Tree) + .runtime_buffer_write(&namespace, &content, timestamp, metadata) + .await + .map_err(|error| into_bus_error(&error)) + } + + /// One time-tree node, or none — appended here for the reason above. + /// + /// Size-checked, unlike `DrillDown`. The level budget bounds a node's + /// *summary* and nothing else: `token_count` is documented as the count of + /// `summary`, and the fold applies `NodeLevel::max_tokens` when it + /// summarises the body. `TreeNode::metadata` is outside it — an + /// `Option` the engine fills with a serialized pending-fold + /// receipt whose `buffer_filenames` holds one name per buffered entry in + /// the hour, so it grows with how much was buffered rather than with any + /// level's budget. Without the check an oversized node fails during frame + /// encoding; with it the caller gets `BUDGET_EXCEEDED` and a reason. + async fn runtime_read_node( + &self, + namespace: String, + node_id: String, + ) -> BusResult> { + let node = require_family!(self, as_tree, Capability::Tree) + .runtime_read_node(&namespace, &node_id) + .await + .map_err(|error| into_bus_error(&error))?; + ensure_response_fits(&node, "RuntimeReadNode")?; + Ok(node) + } + + /// A time-tree node's direct children — appended here for the reason + /// above. + /// + /// Size-checked on `RuntimeReadNode`'s reasoning, which applies harder + /// here: the calendar bounds the fanout to at most 31 children, but 31 + /// unbounded metadata blobs is still unbounded. + async fn runtime_read_children( + &self, + namespace: String, + parent_id: String, + ) -> BusResult> { + let children = require_family!(self, as_tree, Capability::Tree) + .runtime_read_children(&namespace, &parent_id) + .await + .map_err(|error| into_bus_error(&error))?; + ensure_response_fits(&children, "RuntimeReadChildren")?; + Ok(children) + } + + /// One namespace's time-tree shape and coverage — appended here for the + /// reason above. Not size-checked: counts and timestamps. + async fn runtime_tree_status(&self, namespace: String) -> BusResult { + require_family!(self, as_tree, Capability::Tree) + .runtime_tree_status(&namespace) + .await + .map_err(|error| into_bus_error(&error)) + } + + /// Drain the buffer into the tree on the driver's provider — appended here + /// for the reason above. + /// + /// Long-running on `Summarise`'s terms: provider calls, over the network, + /// priced at the driver's rate — one per hour group drained plus the + /// propagation above them. + /// + /// Size-checked on `RuntimeReadNode`'s reasoning. The node this answers + /// with is the one the pass just wrote, so its receipt names every buffer + /// file the pass drained — the largest metadata blob in the tree is the + /// one returned here. + async fn runtime_summarize( + &self, + namespace: String, + timestamp: DateTime, + ) -> BusResult> { + let node = require_family!(self, as_tree, Capability::Tree) + .runtime_summarize(&namespace, timestamp) + .await + .map_err(|error| into_bus_error(&error))?; + ensure_response_fits(&node, "RuntimeSummarize")?; + Ok(node) + } + + /// Rebuild the whole time tree from its hour leaves — appended here for + /// the reason above. Long-running on `RuntimeSummarize`'s terms; the + /// answer is one status row. + async fn runtime_rebuild(&self, namespace: String) -> BusResult { + require_family!(self, as_tree, Capability::Tree) + .runtime_rebuild(&namespace) + .await + .map_err(|error| into_bus_error(&error)) + } + + /// The compiled flavoured-root profile for one scope — appended here for + /// the reason above. + /// + /// Not size-checked: the body is clamped driver-side to the flavoured + /// root's own token budget at compile time, so no scope can make the + /// artifact outgrow a frame. + /// + /// The scope is not logged — today's scopes are facet names, but the + /// vocabulary is the caller's and nothing here may assume it stays free of + /// user data. + async fn flavour_profile(&self, scope: String) -> BusResult> { + require_family!(self, as_tree, Capability::Tree) + .flavour_profile(&scope) + .await + .map_err(|error| into_bus_error(&error)) + } } /// The response-size ceiling for a method that returns a list of entries. diff --git a/crates/tinymemory-module/src/service/test.rs b/crates/tinymemory-module/src/service/test.rs index 0ce829d7..f48fb053 100644 --- a/crates/tinymemory-module/src/service/test.rs +++ b/crates/tinymemory-module/src/service/test.rs @@ -14,6 +14,7 @@ use tinybus::Error as BusError; use tinymemory_api::error::MemoryError; +use tinymemory_api::tree::{NodeLevel, TreeNode}; use tinymemory_api::wire; use super::into_bus_error; @@ -719,3 +720,277 @@ async fn the_two_new_families_are_gated_on_their_own_capability() { .expect_err("a driver without the maintenance family must refuse"); assert_eq!(refusal(error), wire::UNSUPPORTED); } + +#[tokio::test] +async fn the_runtime_tree_doors_refuse_through_the_tree_gate() { + // The seven members of the shed's second round, reached on a driver that + // does not serve their family: `test_provider` advertises Core/Recall/ + // Portability and nothing else, so every one of them must refuse. + // + // What this pins is narrower than the family wiring, and worth stating so + // the next reader does not credit it with more: a door gated on the *wrong* + // family cannot be caught here, because `require_family!` names the + // accessor, and an accessor whose trait lacks the method is a compile + // error rather than a test failure. What it does catch is the shape of the + // refusal — a member that answers `Ok` with a default instead of refusing, + // one that panics or hangs on a family it cannot serve, and one whose error + // leaves under a wire name other than the contract's `UNSUPPORTED`. Each of + // those is a live-at-runtime bug with no compile error anywhere, which is + // the same reason the round before this one asserted it. + let service = super::MemoryService::new(test_provider()); + + let refusal = |error: BusError| match error { + BusError::MethodFailed { name, .. } => name, + other => panic!("expected a named MethodFailed, got {other:?}"), + }; + + let at = chrono::DateTime::from_timestamp(1_700_000_000, 0).expect("timestamp"); + + let error = service + .runtime_buffer_write("team".to_string(), "standup".to_string(), at, None) + .await + .expect_err("a driver without the tree family must refuse"); + assert_eq!(refusal(error), wire::UNSUPPORTED); + + let error = service + .runtime_read_node("team".to_string(), "root".to_string()) + .await + .expect_err("a driver without the tree family must refuse"); + assert_eq!(refusal(error), wire::UNSUPPORTED); + + let error = service + .runtime_read_children("team".to_string(), "root".to_string()) + .await + .expect_err("a driver without the tree family must refuse"); + assert_eq!(refusal(error), wire::UNSUPPORTED); + + let error = service + .runtime_tree_status("team".to_string()) + .await + .expect_err("a driver without the tree family must refuse"); + assert_eq!(refusal(error), wire::UNSUPPORTED); + + let error = service + .runtime_summarize("team".to_string(), at) + .await + .expect_err("a driver without the tree family must refuse"); + assert_eq!(refusal(error), wire::UNSUPPORTED); + + let error = service + .runtime_rebuild("team".to_string()) + .await + .expect_err("a driver without the tree family must refuse"); + assert_eq!(refusal(error), wire::UNSUPPORTED); + + let error = service + .flavour_profile("persona/communication".to_string()) + .await + .expect_err("a driver without the tree family must refuse"); + assert_eq!(refusal(error), wire::UNSUPPORTED); +} + +#[tokio::test] +async fn the_runtime_tree_doors_carry_the_engine_answers_back_through_the_port() { + // The other half of the pair above: the same seven members on a driver that + // *does* serve Tree, so the delegation past the gate is what runs. The + // conformance suite pins these shapes at the engine; what is pinned here is + // that this port carries them out unchanged — an absent node still arrives + // as `None` and not as a refusal, a fresh namespace still has a status, and + // a buffered write still answers the path it landed at. + let workspace = tempfile::tempdir().expect("tempdir"); + let connection = test_connection().await; + let config = test_config(workspace.path()); + // Opening the store requires the process-global host to be installed, even + // though nothing below embeds anything: the guard is what makes that safe + // to do from a test, and it restores the previous host on the way out. + let _embedding_host = EmbeddingHostRestore::install(connection, &config); + let client = std::sync::Arc::new( + tinymemory_core::store::MemoryClient::from_workspace_dir(workspace.path().to_path_buf()) + .expect("open the workspace store"), + ); + let service = super::MemoryService::new(std::sync::Arc::new(crate::provider::provider( + &config, client, + ))); + + let at = chrono::DateTime::from_timestamp(1_700_000_000, 0).expect("timestamp"); + + // Absence is data, not a refusal — the distinction the host's RPC surface + // depends on, and the one a gate-only test cannot see. + assert!(service + .runtime_read_node("team".to_string(), "root".to_string()) + .await + .expect("an absent node is not an error") + .is_none()); + assert!(service + .runtime_read_children("team".to_string(), "root".to_string()) + .await + .expect("an absent parent has no children") + .is_empty()); + + let status = service + .runtime_tree_status("team".to_string()) + .await + .expect("a namespace with no tree still has a status"); + assert_eq!(status.namespace, "team"); + assert_eq!(status.total_nodes, 0); + assert_eq!(status.last_run_at, None); + + // Nothing has been distilled, so the profile is not built — `None` rather + // than an empty string, which is what stops a caller handing a model a + // blank persona. + assert_eq!( + service + .flavour_profile("persona/communication".to_string()) + .await + .expect("an unbuilt profile is not an error"), + None + ); + + // The write answers a path that names a real file inside the workspace the + // module was given, which is the reply the host reports verbatim. + let path = service + .runtime_buffer_write("team".to_string(), "standup".to_string(), at, None) + .await + .expect("a buffered write answers its landing path"); + let landed = std::path::Path::new(&path); + assert!(landed.is_file(), "the reported path names a real file"); + assert!( + landed.starts_with(workspace.path()), + "the buffer file lands inside the module's own workspace" + ); + + // A bad namespace is refused by the engine and leaves under the contract's + // name for it, not the family gate's. + let error = service + .runtime_buffer_write("../escape".to_string(), "x".to_string(), at, None) + .await + .expect_err("a traversal namespace is refused"); + assert_eq!( + match error { + BusError::MethodFailed { name, .. } => name, + other => panic!("expected a named MethodFailed, got {other:?}"), + }, + wire::INVALID + ); + + // No summariser is configured here, so both provider-backed passes must + // fail rather than report a run that never happened. + service + .runtime_summarize("team".to_string(), at) + .await + .expect_err("an unresolvable summariser is a failure, not an empty pass"); + service + .runtime_rebuild("team".to_string()) + .await + .expect_err("an unresolvable summariser fails a rebuild"); + + // The budget check is actually wired, not merely present as a helper. + // Seeded through the engine's own writer so the node comes back out of a + // real read: a summary well inside the hour budget, and a metadata blob + // over the response ceiling — the shape a drained hour with a large + // pending-fold receipt produces. Deleting the `ensure_response_fits` call + // from `runtime_read_node` makes this fail, which is the point of asserting + // it here rather than only against the helper. + let engine_config = + tinymemory_tinycortex::engine::EngineRuntimeConfig::from(&test_config(workspace.path())); + tinymemory_core::tree::tree_runtime::store::write_node( + &engine_config, + &tinymemory_api::tree::TreeNode { + node_id: "2024/03/15/09".to_string(), + namespace: "team".to_string(), + level: NodeLevel::Hour, + parent_id: Some("2024/03/15".to_string()), + summary: "a summary comfortably inside the hour budget".to_string(), + token_count: 9, + child_count: 0, + created_at: at, + updated_at: at, + metadata: Some("m".repeat(super::MAX_RESPONSE_BYTES)), + }, + ) + .expect("seed an oversized node"); + + let error = service + .runtime_read_node("team".to_string(), "2024/03/15/09".to_string()) + .await + .expect_err("a node over the response ceiling must be refused, not encoded"); + assert_eq!( + match error { + BusError::MethodFailed { name, .. } => name, + other => panic!("expected a named MethodFailed, got {other:?}"), + }, + wire::BUDGET_EXCEEDED + ); +} + +#[test] +fn a_tree_node_within_its_level_budget_can_still_overrun_the_response_ceiling() { + // The reason `RuntimeReadNode`/`RuntimeReadChildren`/`RuntimeSummarize` + // are size-checked at all, pinned as a fact rather than left to prose. + // + // A level's `max_tokens` bounds the node's *summary* — `token_count` is + // documented as the count of `summary`, and the fold passes + // `NodeLevel::max_tokens` to the summariser for the body alone. It says + // nothing about `metadata`, which the engine fills with a serialized + // pending-fold receipt naming every buffer file the pass drained. That + // list grows with how much was buffered into the hour, not with any + // level's budget. + // + // So a node can sit comfortably inside the hour budget and still be too + // large to cross a frame. Constructed here rather than driven through the + // engine because the point is the *shape* being possible: reaching it via + // a real fold would mean buffering megabytes of entries, which is slow and + // would pass for the wrong reason if the receipt format ever changed. + let summary = "x".repeat(NodeLevel::Hour.max_tokens() as usize); + assert!( + summary.len() < super::MAX_RESPONSE_BYTES, + "the summary alone must be nowhere near the ceiling, or this proves nothing" + ); + + let node = TreeNode { + node_id: "2024/03/15/09".to_string(), + namespace: "team".to_string(), + level: NodeLevel::Hour, + parent_id: Some("2024/03/15".to_string()), + summary, + token_count: NodeLevel::Hour.max_tokens(), + child_count: 0, + created_at: chrono::DateTime::from_timestamp(1_700_000_000, 0).expect("timestamp"), + updated_at: chrono::DateTime::from_timestamp(1_700_000_000, 0).expect("timestamp"), + metadata: Some("m".repeat(super::MAX_RESPONSE_BYTES)), + }; + + let error = super::ensure_response_fits(&Some(node), "RuntimeReadNode") + .expect_err("a node whose metadata overruns the ceiling must be refused"); + match error { + BusError::MethodFailed { name, message } => { + assert_eq!(name, wire::BUDGET_EXCEEDED); + assert!(message.contains("RuntimeReadNode"), "{message}"); + } + other => panic!("expected MethodFailed, got {other:?}"), + } +} + +#[test] +fn an_ordinary_tree_node_read_is_not_refused() { + // The other side of the ceiling: the check must not fire on the shape the + // host actually reads back, or every tree read becomes a budget error. + let node = TreeNode { + node_id: "2024/03/15/09".to_string(), + namespace: "team".to_string(), + level: NodeLevel::Hour, + parent_id: Some("2024/03/15".to_string()), + summary: "the morning standup, folded".to_string(), + token_count: 7, + child_count: 0, + created_at: chrono::DateTime::from_timestamp(1_700_000_000, 0).expect("timestamp"), + updated_at: chrono::DateTime::from_timestamp(1_700_000_000, 0).expect("timestamp"), + metadata: Some(r#"{"buffer_filenames":["1700000000000_a.md"]}"#.to_string()), + }; + + assert!(super::ensure_response_fits(&Some(node.clone()), "RuntimeReadNode").is_ok()); + // A full calendar month of children is the realistic worst case for the + // child read, and it must pass. + let children: Vec = (0..31).map(|_| node.clone()).collect(); + assert!(super::ensure_response_fits(&children, "RuntimeReadChildren").is_ok()); +} diff --git a/crates/tinymemory-module/tests/module_e2e.rs b/crates/tinymemory-module/tests/module_e2e.rs index ef0e7106..766f1fe4 100644 --- a/crates/tinymemory-module/tests/module_e2e.rs +++ b/crates/tinymemory-module/tests/module_e2e.rs @@ -725,6 +725,15 @@ const EXPECTED_METHODS: &[&str] = &[ "DegradedState", "ChunkScore", "SourceIngestStatus", + // The runtime-tree doors and the flavoured-root profile read — the final + // round of the engine shed. + "RuntimeBufferWrite", + "RuntimeReadNode", + "RuntimeReadChildren", + "RuntimeTreeStatus", + "RuntimeSummarize", + "RuntimeRebuild", + "FlavourProfile", ]; #[tokio::test] diff --git a/crates/tinymemory-tinycortex/src/engine/mod.rs b/crates/tinymemory-tinycortex/src/engine/mod.rs index 1c5f4f6c..95a2e5ab 100644 --- a/crates/tinymemory-tinycortex/src/engine/mod.rs +++ b/crates/tinymemory-tinycortex/src/engine/mod.rs @@ -20,7 +20,7 @@ use std::sync::Arc; use crate::TinycortexMemory; use async_trait::async_trait; -use chrono::Utc; +use chrono::{DateTime, Utc}; use tinymemory_api::capabilities::Capabilities; use tinymemory_api::chunks::Chunk; use tinymemory_api::error::MemoryError; @@ -60,7 +60,7 @@ use tinymemory_api::recall::OwnedRecallOpts; use tinymemory_api::tool_memory::ToolMemoryRule; use tinymemory_api::tree::{ IngestRequest, QueryResult, RootSummary, SummaryContext, SummaryForest, SummaryInput, - SummaryOutput, TreeLeaf, TreeStatus, TreeSummary, + SummaryOutput, TreeLeaf, TreeNode, TreeStatus, TreeSummary, }; use tinymemory_api::types::{ GraphRelationRecord, MemoryCategory, MemoryEntry, MemoryKvRecord, MemoryTaint, @@ -1425,6 +1425,221 @@ impl MemoryTree for TinycortexProvider { }) .collect()) } + + async fn runtime_buffer_write( + &self, + namespace: &str, + content: &str, + timestamp: DateTime, + metadata: Option, + ) -> Result { + // The same two refusals `append` makes, in the same order, so the two + // writes cannot disagree about what a writable request is. The + // timestamp is not defaulted: the contract makes it required so the + // caller's reply and the buffer file agree on the instant — see the + // trait. + tinycortex::memory::tree::runtime::store::validate_namespace(namespace) + .map_err(MemoryError::Invalid)?; + if content.trim().is_empty() { + return Err(MemoryError::Invalid( + "content must not be empty".to_string(), + )); + } + let namespace = namespace.trim().to_string(); + let content = content.to_string(); + let path = blocking(self.config.clone(), "buffer tree content", move |config| { + tinymemory_core::tree::tree_runtime::store::buffer_write( + config, + &namespace, + &content, + ×tamp, + metadata.as_ref(), + ) + }) + .await?; + // `display()` is exactly what the host printed when it held the + // `PathBuf` itself, so the string a caller reports does not change + // with the seam. The components are engine-generated ASCII under the + // module's own workspace root; nothing here invites non-UTF-8. + Ok(path.display().to_string()) + } + + async fn runtime_read_node( + &self, + namespace: &str, + node_id: &str, + ) -> Result, MemoryError> { + tinycortex::memory::tree::runtime::store::validate_namespace(namespace) + .map_err(MemoryError::Invalid)?; + tinycortex::memory::tree::runtime::store::validate_node_id(node_id) + .map_err(MemoryError::Invalid)?; + let namespace = namespace.trim().to_string(); + let node_id = node_id.to_string(); + // Returned as the store hands it back: the engine's `TreeNode` *is* + // the contract's — `tinycortex-api` re-exports `tinymemory-api`'s + // tree module, unified by the workspace patch table — so unlike + // `drill_down`'s historical `cross`, there is nothing to convert and + // the compiler proves it. + blocking(self.config.clone(), "read tree node", move |config| { + tinymemory_core::tree::tree_runtime::store::read_node(config, &namespace, &node_id) + }) + .await + } + + async fn runtime_read_children( + &self, + namespace: &str, + parent_id: &str, + ) -> Result, MemoryError> { + tinycortex::memory::tree::runtime::store::validate_namespace(namespace) + .map_err(MemoryError::Invalid)?; + tinycortex::memory::tree::runtime::store::validate_node_id(parent_id) + .map_err(MemoryError::Invalid)?; + let namespace = namespace.trim().to_string(); + let parent_id = parent_id.to_string(); + blocking(self.config.clone(), "read tree children", move |config| { + tinymemory_core::tree::tree_runtime::store::read_children( + config, &namespace, &parent_id, + ) + }) + .await + } + + async fn runtime_tree_status(&self, namespace: &str) -> Result { + tinycortex::memory::tree::runtime::store::validate_namespace(namespace) + .map_err(MemoryError::Invalid)?; + let namespace = namespace.trim().to_string(); + blocking(self.config.clone(), "read tree status", move |config| { + tinymemory_core::tree::tree_runtime::store::get_tree_status(config, &namespace) + }) + .await + } + + async fn runtime_summarize( + &self, + namespace: &str, + timestamp: DateTime, + ) -> Result, MemoryError> { + tinycortex::memory::tree::runtime::store::validate_namespace(namespace) + .map_err(MemoryError::Invalid)?; + // The provider is resolved before the engine is asked anything — + // ahead of even the "is there work" check the engine makes — because + // this is a caller's explicit run: a setup that cannot summarise must + // say so rather than answer `None` as if it had looked. Built through + // the same seam `seal` uses, so the module's chat host carries the + // call back to the host and the routing policy stays where it always + // was. + let (model, _) = tinymemory_core::chat_host::create_chat_model_with_model_id( + "summarization", + &self.config, + self.config.default_temperature, + ) + .map_err(|error| Self::other("create summarizer", error))?; + tinymemory_core::tree::tree_runtime::engine::run_summarization( + &self.config, + model.as_ref(), + namespace.trim(), + timestamp, + ) + .await + // `{:#}` keeps the cause chain the way the host's own RPC reported + // it: the top context alone says "summarization failed" and drops the + // provider's actual complaint, which is the actionable half. + .map_err(|error| Self::other("run tree summarization", format!("{error:#}"))) + } + + async fn runtime_rebuild(&self, namespace: &str) -> Result { + tinycortex::memory::tree::runtime::store::validate_namespace(namespace) + .map_err(MemoryError::Invalid)?; + // No empty-tree short-circuit, unlike `cascade`: the provider + // resolves first, on the terms `runtime_summarize` gives. + let (model, _) = tinymemory_core::chat_host::create_chat_model_with_model_id( + "summarization", + &self.config, + self.config.default_temperature, + ) + .map_err(|error| Self::other("create summarizer", error))?; + tinymemory_core::tree::tree_runtime::engine::rebuild_tree( + &self.config, + model.as_ref(), + namespace.trim(), + ) + .await + .map_err(|error| Self::other("rebuild tree", format!("{error:#}"))) + } + + async fn flavour_profile(&self, scope: &str) -> Result, MemoryError> { + if scope.trim().is_empty() { + return Err(MemoryError::Invalid("scope must not be empty".to_string())); + } + // Matched literally from here on — the scope is the caller's naming + // scheme (`persona/` today) and the tree row's key, and the + // driver has no vocabulary of its own to normalise it against. + let scope = scope.to_string(); + blocking(self.config.clone(), "read flavour profile", move |config| { + let mc = tinymemory_core::engine::engine_config(config); + + // Fast path: the compiled artifact already on disk with a + // non-empty body — read it without touching the tree store. An + // unreadable or body-less file falls through to the recompile + // rather than failing, exactly as the host's lookup did: the + // artifact is a staged projection, and the tree is the truth. + let compiled = tinycortex::memory::tree::flavoured_root_abs_path(&mc, &scope); + if compiled.is_file() { + if let Ok(markdown) = std::fs::read_to_string(&compiled) { + if !body_after_front_matter(&markdown).trim().is_empty() { + return Ok(Some(markdown)); + } + } + } + + // Slow path: look the flavoured tree up and (re)compile its root. + // No tree is `None` — not built is an answer, not a fault — while + // a lookup or compile *failure* propagates: reporting a broken + // store as "not built yet" tells the user to re-run an ingestion + // that already worked. + let Some(tree) = tinycortex::memory::tree::store::get_tree_by_scope( + &mc, + TreeKind::Flavoured, + &scope, + )? + else { + return Ok(None); + }; + let markdown = tinycortex::memory::tree::compile_flavoured_root(&mc, &tree.id)?; + // A tree that exists but has never sealed compiles to front-matter + // over an empty body; the contract says that is still "not built". + if body_after_front_matter(&markdown).trim().is_empty() { + return Ok(None); + } + Ok(Some(markdown)) + }) + .await + } +} + +/// Strip the YAML front matter `compile_flavoured_root` writes +/// (`---\n…\n---\n`) and return just the body. +/// +/// The driver strips only to *decide*, never to serve: the full artifact, +/// front-matter included, is what `MemoryTree::flavour_profile` returns, and +/// presentation stays the caller's. What is settled here is built-versus-not — +/// an artifact whose body is blank after this strip is a tree that has never +/// sealed, and the door answers `None` for it. +/// +/// Front-matter field values are single-line (the engine's `yaml_quote` +/// collapses interior newlines), so the first `\n---\n` after the opening +/// delimiter is always the closing one. An opener with no closer falls back to +/// everything after the opener, so the delimiter itself is never mistaken for +/// prose. +fn body_after_front_matter(content: &str) -> &str { + match content.strip_prefix("---\n") { + Some(rest) => match rest.find("\n---\n") { + Some(pos) => &rest[pos + "\n---\n".len()..], + None => rest, + }, + None => content, + } } /// Validate entity-kind wire strings and re-emit them in the index's spelling. diff --git a/crates/tinymemory-tinycortex/src/engine/test.rs b/crates/tinymemory-tinycortex/src/engine/test.rs index ee753928..edc244c6 100644 --- a/crates/tinymemory-tinycortex/src/engine/test.rs +++ b/crates/tinymemory-tinycortex/src/engine/test.rs @@ -23,10 +23,10 @@ use tinymemory_api::provider::types::IngestItem; use tinymemory_api::types::MemoryTaint; use super::{ - advertised_capabilities, audit_entry, degraded_capabilities, diagnosis_failure, - facet_type_to_engine, handle_to_contract, handle_to_engine, like_prefix_pattern, - parse_person_id, refuse_composio_dispatch, scope_to_engine, validate_ingest_item, - EngineRuntimeConfig, + advertised_capabilities, audit_entry, body_after_front_matter, degraded_capabilities, + diagnosis_failure, facet_type_to_engine, handle_to_contract, handle_to_engine, + like_prefix_pattern, parse_person_id, refuse_composio_dispatch, scope_to_engine, + validate_ingest_item, EngineRuntimeConfig, }; fn ingest_item(content: &str, mime: Option<&str>, taint: MemoryTaint) -> IngestItem { @@ -576,3 +576,32 @@ fn a_chunk_id_prefix_is_matched_literally() { "an empty prefix matches everything, which is what an empty prefix means" ); } + +#[test] +fn the_front_matter_strip_decides_built_versus_not_the_way_the_host_did() { + // The strip exists for one verdict — is there prose under the compiled + // artifact's front-matter — and these are the host's own decision points, + // reproduced: a well-formed artifact yields its body, a body of pure + // whitespace reads as unbuilt, an opener with no closer never leaks the + // delimiter as prose, and content with no front-matter at all is already + // the body. + assert_eq!( + body_after_front_matter("---\nscope: persona/communication\n---\nShort sentences.\n"), + "Short sentences.\n" + ); + assert!( + body_after_front_matter("---\nscope: x\n---\n \n\t") + .trim() + .is_empty(), + "front-matter over whitespace is not a profile" + ); + assert_eq!( + body_after_front_matter("---\nscope: x\nno closer follows"), + "scope: x\nno closer follows", + "a malformed opener falls back to everything after it, not to the raw artifact" + ); + assert_eq!( + body_after_front_matter("plain body, no front matter"), + "plain body, no front matter" + ); +} diff --git a/crates/tinymemory-tinycortex/tests/full_provider_conformance.rs b/crates/tinymemory-tinycortex/tests/full_provider_conformance.rs index 825f0149..9b35af83 100644 --- a/crates/tinymemory-tinycortex/tests/full_provider_conformance.rs +++ b/crates/tinymemory-tinycortex/tests/full_provider_conformance.rs @@ -3556,3 +3556,296 @@ async fn the_root_summary_read_caps_each_namespace_and_then_the_whole_block() { assert_eq!(bounded.len(), 1); assert_eq!(bounded[0].namespace, "alpha"); } + +/// The four runtime-tree store doors, over a real workspace. +/// +/// These are the reads and the write under the host's `tree_summarizer_*` RPC +/// surface, and what is pinned is the shape that surface reports verbatim: the +/// landing path of a buffered write, `None` for an absent node rather than an +/// error, an empty child list for an absent parent, and the all-empty status of +/// a namespace that has never sealed. +#[tokio::test(flavor = "multi_thread")] +async fn the_runtime_tree_store_doors_answer_the_shapes_the_rpc_surface_reports() { + use tinymemory_api::error::MemoryError; + use tinymemory_api::provider::MemoryProvider; + use tinymemory_api::tree::NodeLevel; + use tinymemory_core::tree::tree_runtime::{ + derive_parent_id, estimate_tokens, level_from_node_id, TreeNode, + }; + + let workspace = tempfile::tempdir().expect("workspace"); + let provider = provider_over(workspace.path()); + let config = provider_config(workspace.path(), serde_json::Value::Null); + let tree = provider.as_tree().expect("Tree"); + let at = chrono::DateTime::from_timestamp(1_700_000_000, 0).expect("timestamp"); + + // A rejected namespace is `Invalid` on every door, and a rejected node id + // on both node-addressed reads — the same refusals the host's RPC layer + // made, now made where the store is. + for error in [ + tree.runtime_buffer_write("../escape", "content", at, None) + .await + .expect_err("a traversal namespace is refused"), + tree.runtime_read_node("../escape", "root") + .await + .expect_err("a traversal namespace is refused"), + tree.runtime_read_children("../escape", "root") + .await + .expect_err("a traversal namespace is refused"), + tree.runtime_tree_status("../escape") + .await + .expect_err("a traversal namespace is refused"), + tree.runtime_read_node("team", "2024/../2025") + .await + .expect_err("a traversal node id is refused"), + tree.runtime_read_children("team", "not-a-node") + .await + .expect_err("a malformed parent id is refused"), + tree.runtime_buffer_write("team", " \n\t ", at, None) + .await + .expect_err("blank content is refused"), + ] { + assert!( + matches!(error, MemoryError::Invalid(_)), + "expected Invalid, got {error:?}" + ); + } + + // Absence is data on a fresh workspace: no root yet, no children, and the + // all-`None` status — not one of them an error. + assert!(tree + .runtime_read_node("team", "root") + .await + .expect("an absent node is not an error") + .is_none()); + assert!(tree + .runtime_read_children("team", "root") + .await + .expect("an absent parent has no children") + .is_empty()); + let empty = tree + .runtime_tree_status("team") + .await + .expect("a namespace with no tree still has a status"); + assert_eq!(empty.namespace, "team"); + assert_eq!(empty.total_nodes, 0); + assert_eq!(empty.oldest_entry, None); + assert_eq!(empty.last_run_at, None); + + // The buffered write answers with the engine's own path, exactly the + // string the host printed when it held the `PathBuf` itself: a real file, + // filed by the caller's timestamp, with the metadata staged in + // front-matter for the seal to carry onward. + let path = tree + .runtime_buffer_write( + " team ", + "standup notes", + at, + Some(serde_json::json!({"origin": "conformance"})), + ) + .await + .expect("a buffered write answers its landing path"); + let on_disk = std::path::Path::new(&path); + assert!(on_disk.is_file(), "the reported path names a real file"); + assert!( + on_disk.starts_with(workspace.path()), + "the buffer file lands inside the driver's workspace" + ); + let staged = std::fs::read_to_string(on_disk).expect("read the buffer entry"); + assert!(staged.contains("standup notes")); + assert!( + staged.contains("\"origin\":\"conformance\""), + "metadata rides in the entry's front-matter" + ); + assert!( + on_disk + .file_name() + .and_then(|name| name.to_str()) + .is_some_and(|name| name.starts_with(&at.timestamp_millis().to_string())), + "the entry is filed under the caller's timestamp, not the driver's now" + ); + + // Seeded through the engine's own writer, as every tree test here is, so + // the reads cannot pass against a layout the store does not use. + for node_id in ["root", "2024"] { + let summary = format!("summary of {node_id}"); + tinymemory_core::tree::tree_runtime::store::write_node( + &config, + &TreeNode { + node_id: node_id.into(), + namespace: "team".into(), + level: level_from_node_id(node_id), + parent_id: derive_parent_id(node_id), + summary: summary.clone(), + token_count: estimate_tokens(&summary), + child_count: 0, + created_at: at, + updated_at: at, + metadata: None, + }, + ) + .expect("write a node"); + } + + let year = tree + .runtime_read_node("team", "2024") + .await + .expect("read the year node") + .expect("the year node exists"); + assert_eq!(year.node_id, "2024"); + assert_eq!(year.level, NodeLevel::Year); + assert_eq!(year.parent_id.as_deref(), Some("root")); + assert_eq!(year.summary, "summary of 2024"); + + let children = tree + .runtime_read_children("team", "root") + .await + .expect("read the root's children"); + assert_eq!(children.len(), 1); + assert_eq!(children[0].node_id, "2024"); + + // An hour leaf has nothing under it by construction: empty, not an error. + assert!(tree + .runtime_read_children("team", "2024/03/15/09") + .await + .expect("a leaf's children") + .is_empty()); + + let status = tree + .runtime_tree_status("team") + .await + .expect("status over a seeded tree"); + assert_eq!(status.namespace, "team"); + assert_eq!(status.total_nodes, 2); +} + +/// The two provider-backed runtime doors: validation first, then the provider, +/// and only then the engine. +/// +/// This suite configures no chat host, which is what makes both halves +/// assertable: a bad namespace answers `Invalid` — proof the check runs before +/// any provider is reached for — and a good one answers a backend failure even +/// though the buffer and the tree are empty, because these are a person's +/// explicit "run now" and a runner that could not have run must say so rather +/// than report a pass that ran nothing. (`seal`/`cascade` keep their empty +/// short-circuits; they are the scheduler's, called unconditionally.) +#[tokio::test(flavor = "multi_thread")] +async fn the_provider_backed_runtime_doors_refuse_before_they_reach_the_engine() { + use tinymemory_api::error::MemoryError; + use tinymemory_api::provider::MemoryProvider; + + let workspace = tempfile::tempdir().expect("workspace"); + let provider = provider_over(workspace.path()); + let tree = provider.as_tree().expect("Tree"); + let at = chrono::DateTime::from_timestamp(1_700_000_000, 0).expect("timestamp"); + + assert!(matches!( + tree.runtime_summarize("../escape", at).await, + Err(MemoryError::Invalid(_)) + )); + assert!(matches!( + tree.runtime_rebuild("../escape").await, + Err(MemoryError::Invalid(_)) + )); + + assert!( + matches!( + tree.runtime_summarize("team", at).await, + Err(MemoryError::Other(_)) + ), + "an unresolvable summariser is a failure, not an empty pass" + ); + assert!( + matches!( + tree.runtime_rebuild("team").await, + Err(MemoryError::Other(_)) + ), + "an unresolvable summariser fails a rebuild before any status is read" + ); +} + +/// The flavour door: `None` until a body exists, then the whole artifact. +#[tokio::test(flavor = "multi_thread")] +async fn the_flavour_door_answers_none_until_a_body_exists_then_serves_the_whole_artifact() { + use tinymemory_api::error::MemoryError; + use tinymemory_api::provider::MemoryProvider; + use tinymemory_core::store::trees::{ + store::insert_tree, Tree, TreeKind, TreeStatus as StoreTreeStatus, + }; + + const SCOPE: &str = "persona/communication"; + + let workspace = tempfile::tempdir().expect("workspace"); + let provider = provider_over(workspace.path()); + let config = provider_config(workspace.path(), serde_json::Value::Null); + let tree_door = provider.as_tree().expect("Tree"); + let at = chrono::DateTime::from_timestamp(1_700_000_000, 0).expect("timestamp"); + + assert!(matches!( + tree_door.flavour_profile(" ").await, + Err(MemoryError::Invalid(_)) + )); + + // A scope no tree was ever written under is not built. It is also + // indistinguishable from a scope the caller misspelled — deliberately, per + // the contract: the vocabulary is the caller's. + assert_eq!( + tree_door + .flavour_profile(SCOPE) + .await + .expect("an unknown scope is not an error"), + None + ); + + // A flavoured tree that exists but has never sealed compiles to + // front-matter over an empty body: still `None` — and the compile really + // ran, which the freshly staged artifact proves. + insert_tree( + &config, + &Tree { + id: "tree-flavour-1".to_string(), + kind: TreeKind::Flavoured, + scope: SCOPE.to_string(), + root_id: None, + max_level: 0, + status: StoreTreeStatus::Active, + created_at: at, + last_sealed_at: None, + ask: Some("distil how this person communicates".to_string()), + }, + ) + .expect("insert the flavoured tree row"); + assert_eq!( + tree_door + .flavour_profile(SCOPE) + .await + .expect("an unsealed tree is not an error"), + None, + "front-matter over an empty body is not a profile" + ); + let artifact = tinycortex::memory::tree::flavoured_root_abs_path( + &tinymemory_core::engine::engine_config(&config), + SCOPE, + ); + assert!( + artifact.is_file(), + "the unsealed lookup still staged the fixed-path artifact" + ); + + // Once a body exists at the fixed path, the door serves the artifact + // whole — front-matter included, stripping left to the caller. Written at + // the path the engine itself derives, so the fast path is read exactly + // where the compiler stages. + let compiled = format!("---\nscope: {SCOPE}\n---\nTalks in short declaratives.\n"); + std::fs::write(&artifact, &compiled).expect("stage a compiled artifact"); + let served = tree_door + .flavour_profile(SCOPE) + .await + .expect("a built profile is served") + .expect("a body-bearing artifact is Some"); + assert_eq!( + served, compiled, + "the artifact crosses whole: front-matter intact, byte for byte" + ); + assert!(served.starts_with("---\n")); +}