diff --git a/package.json b/package.json index 6802feb..26e82f5 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@thinkfleet/memory-sdk", - "version": "0.1.0", + "version": "0.2.0", "description": "TypeScript SDK for memory.thinkfleet.ai — admin + project memory CRUD, semantic search, feedback, and Lattice behavioral patterns", "type": "module", "main": "./dist/index.cjs", @@ -27,6 +27,7 @@ "scripts": { "build": "tsup", "typecheck": "tsc --noEmit", + "check:proto": "node scripts/check-proto-drift.mjs", "test:integration": "tsx test-app.ts", "seed": "tsx scripts/seed-memory.ts", "seed:lattice": "tsx scripts/seed-lattice.ts" diff --git a/proto/coverage.json b/proto/coverage.json new file mode 100644 index 0000000..a7b2a3a --- /dev/null +++ b/proto/coverage.json @@ -0,0 +1,38 @@ +{ + "_comment": "Tracks how each engine RPC (proto/memory.proto) is surfaced in the SDK. check-proto-drift.mjs fails CI if the proto gains an RPC that is in neither list, so the SDK can't silently fall behind the engine. Vendored from thinkfleet-memory-engine main @ d96330867a99.", + "covered": { + "Save": "memory.admin.create / memory.observe", + "Observe": "memory.observe", + "Search": "memory.admin.search", + "GetStats": "memory.admin.stats", + "MineMemories": "lattice.mineMemories", + "Predict": "lattice.predict", + "Estimate": "lattice.estimate", + "Consolidate": "memory.admin.dedup", + "Backfill": "memory.admin.backfillEmbeddings", + "GetCalibration": "lattice.getCalibration", + "GetProfile": "lattice.getProfile", + "BuildContext": "context.build", + "EmitEvent": "events.emit", + "PollEvents": "events.poll", + "CreateAlertRule": "alerts.create", + "ListAlertRules": "alerts.list", + "GetAlertRule": "alerts.get", + "UpdateAlertRule": "alerts.update", + "DeleteAlertRule": "alerts.delete", + "ListAlertFires": "alerts.listFires", + "ExportSubject": "compliance.exportSubject", + "HardDeleteSubject": "compliance.hardDeleteSubject", + "ListAuditEvents": "compliance.listAuditEvents", + "ListPacks": "compliance.listPacks", + "GetCohort": "lattice.getCohort", + "PredictByCohort": "lattice.predictByCohort" + }, + "internal": { + "ResolveOrCreateEntity": "graph primitive; entity resolution runs server-side during extraction", + "ExtractAndWire": "graph primitive; extraction is automatic/gated server-side", + "BudgetGate": "platform-internal spend gate, not a consumer operation", + "RecordCost": "platform-internal cost accounting, not a consumer operation", + "Health": "liveness probe; not part of the consumer surface" + } +} diff --git a/proto/memory.proto b/proto/memory.proto new file mode 100644 index 0000000..dc1b3a9 --- /dev/null +++ b/proto/memory.proto @@ -0,0 +1,1051 @@ +// memory.thinkfleet.ai — gRPC contract. +// +// Defines the boundary between the TypeScript Fastify shell (cold +// path: Cognito JWT verification, Lago entitlement lookup, REST +// routing, admin UI, billing event emission) and the Rust hot path +// (crates/memory-retrieve, sub-30ms p99 retrieval that wraps the +// thinkfleet-memory-engine library). +// +// Auth model: the TS shell validates Cognito JWTs at the edge and +// passes the resolved platform_id + user_id as gRPC metadata. The +// Rust service trusts this metadata — it never sees a raw JWT. This +// matches the GrowthOS super-memory-retrieve pattern and keeps the +// gRPC service internal-only. +// +// Versioning: any breaking change goes to memory.v2. Rust generates +// via tonic-build, TS via ts-proto. + +syntax = "proto3"; + +package thinkfleet.memory.v1; + +option go_package = "github.com/rrader26/memory-thinkfleet/proto/memory/v1;memoryv1"; + +// ─── Service ──────────────────────────────────────────────────────── + +service Memory { + // Direct save of a fully-formed memory item. Used by the TS + // shell when the caller already knows what it wants written + // (no extraction). Engine generates id + timestamps if missing. + rpc Save(SaveRequest) returns (MemoryItem); + + // Save a raw text observation. The engine runs heuristic + // extraction and writes resulting MemoryItem rows. Returns the + // count of saved items plus the items themselves so the caller + // can display them or push a websocket update. + rpc Observe(ObserveRequest) returns (ObserveResponse); + + // Search memory items by query. Implements hybrid scoring + // (semantic + recency + importance) inside the engine; gRPC + // surface is just text + filters in, ranked rows out. + rpc Search(SearchRequest) returns (SearchResponse); + + // Find an entity in the project/platform scope by name or alias. + // Creates one with default_type if no match exists. Used by the + // graph-extraction pipeline. + rpc ResolveOrCreateEntity(ResolveEntityRequest) returns (Entity); + + // Run the heuristic graph extractor on a text payload and wire + // up the resulting entities + edges. Returns extraction stats so + // the caller can surface "N entities, M edges" feedback. + rpc ExtractAndWire(ExtractAndWireRequest) returns (ExtractAndWireResponse); + + // Check whether a projected cost would exceed the platform's + // monthly cap. Returns the current budget state; the caller + // emits the actual spend via RecordCost after a successful op. + rpc BudgetGate(BudgetGateRequest) returns (BudgetState); + + // Record a cost against the platform's monthly spend. Idempotent + // only if the caller makes it so — usually called once per + // successful LLM-backed operation. + rpc RecordCost(RecordCostRequest) returns (BudgetState); + + // Liveness probe. TS shell hits this on startup + periodically; + // if unreachable the shell can fail-fast or fall back per + // operator policy. + rpc Health(HealthRequest) returns (HealthResponse); + + // Aggregate observability stats for an admin dashboard: memory counts by + // scope/status/type, vector-embedding coverage, and behavior-pattern + // count. Scoped by platform_id (required) + optional project_id. + rpc GetStats(GetStatsRequest) returns (StatsReport); + + // Mine behavioral patterns from the memory corpus. Subject- + // agnostic: works on any subject kind (contact, user, team, + // workspace, service). Patterns are written back as MemoryItem + // rows with kind="behavior_pattern" and provenance pointing at + // the source memory ids — patterns ARE memories on the storage + // side, recallable by the same search. + rpc MineMemories(MineMemoriesRequest) returns (MineMemoriesResult); + + // Project future events for a subject by reading their active + // behavior_pattern memories and extending each pattern forward + // by its cadence. Returns ranked predictions with confidence + // and source provenance. + rpc Predict(PredictRequest) returns (PredictResult); + + // Compute a derived estimate (e.g. biological age via PhenoAge) for a + // subject from their lab-signal memories. Deterministic; returns a + // wellness-framed Score with contributors + provenance, or the list of + // missing required signals. Generative narration is the caller's job. + rpc Estimate(EstimateRequest) returns (EstimateResult); + + // Collapse near-duplicate memories (cosine >= threshold): keep the + // strongest of each group, supersede the rest. Deterministic; no deletion. + // Intended to be run on a schedule by the SaaS. + rpc Consolidate(ConsolidateRequest) returns (ConsolidateResult); + + // Generate embeddings for items that don't have one yet (the backfill + // work-list). generate-on-save only covers new writes; this catches up the + // existing corpus. No-op if the engine's embedding provider is disabled. + rpc Backfill(BackfillRequest) returns (BackfillResult); + + // Report prediction calibration for a project: active patterns + // bucketed by stated confidence, with the realized hit-rate of + // their past predictions per bucket. Answers "of the patterns we + // rate ~0.8, do ~80% of their predictions actually fire?" — the + // signal for whether the engine's confidence is honest. Read-only; + // the same data the mine run logs, exposed for querying. + rpc GetCalibration(GetCalibrationRequest) returns (CalibrationReport); + + // Build a behavioral profile snapshot for a subject. Aggregates + // their active patterns into a single payload: RFM segment, top + // entity, weekly cadence, risk indicators (churn, declining + // engagement). The non-temporal counterpart to Predict — answers + // "who is this subject?" rather than "what will they do?". + rpc GetProfile(GetProfileRequest) returns (Profile); + + // ─── Context (the LLM-ready bundle) ───────────────────────────── + + // Build a unified context bundle for a subject. Aggregates + // profile + active patterns + predictions + memories + + // observations into one token-budgeted payload. Every claim + // carries source ids so any inference can be traced. + rpc BuildContext(BuildContextRequest) returns (ContextBundle); + + // ─── Events (the proactive event log) ─────────────────────────── + + // Append an event to the durable event log. Idempotent — + // re-emitting the same (projectId, dedupeKey) silently drops. + // The alert evaluator runs synchronously inside this call so + // that "event emitted" implies "rules matched and dispatched". + rpc EmitEvent(EmitEventRequest) returns (EmitEventResult); + + // Pull events newer than `since`. Subscribers walk the log by + // setting `since` to the last returned event's occurredAt. + rpc PollEvents(PollEventsRequest) returns (PollEventsResult); + + // ─── Alert rules (user-defined reactions) ─────────────────────── + + rpc CreateAlertRule(CreateAlertRuleRequest) returns (AlertRule); + rpc ListAlertRules(ListAlertRulesRequest) returns (ListAlertRulesResult); + rpc GetAlertRule(GetAlertRuleRequest) returns (AlertRule); + rpc UpdateAlertRule(UpdateAlertRuleRequest) returns (AlertRule); + rpc DeleteAlertRule(DeleteAlertRuleRequest) returns (DeleteAlertRuleResult); + rpc ListAlertFires(ListAlertFiresRequest) returns (ListAlertFiresResult); + + // ─── Phase 3f Compliance (Rust) ───────────────────────────────── + + // Subject-access export — GDPR Art. 15 "right of access." + // Returns every memory, pattern, observation, event, alert fire, + // and audit row that references the named subject, in a single + // bundle a controller can hand to the data subject. + rpc ExportSubject(ExportSubjectRequest) returns (SubjectExport); + + // Hard-delete cascade — GDPR Art. 17 "right to erasure." + // Removes every artifact tied to a subject: memories, derived + // patterns + observations, the subject's event rows, and the + // alert fires referencing those events. Writes a single + // tombstone memory_audit_event row so the deletion itself is + // discoverable in the audit log. + rpc HardDeleteSubject(HardDeleteSubjectRequest) returns (HardDeleteSubjectResult); + + // Read the memory_audit_event log. Powers the "who accessed my + // data and when" surface (GDPR Art. 15 + EU AI Act audit-trail + // requirements). Cursor-paginated. + rpc ListAuditEvents(ListAuditEventsRequest) returns (ListAuditEventsResult); + + // Introspection — return manifests for all compliance packs + // loaded into the engine. The admin UI uses this to surface + // which regulatory frameworks the deployment satisfies. + rpc ListPacks(ListPacksRequest) returns (ListPacksResult); + + // Phase 5 cohort similarity — find subjects whose behavior + // looks like the target's. Returns top-K nearest neighbors + // ranked by a blended similarity score (RFM + entity Jaccard + // + pattern-kind Jaccard). + rpc GetCohort(GetCohortRequest) returns (GetCohortResult); + + // Phase 5 cohort-aware predictions — aggregate predictions + // across the target's cohort so "people like Sarah also + // ordered pizza on Fridays" surfaces even when Sarah's own + // signal is weak. Returns ranked CohortPrediction with full + // provenance (which cohort members + which patterns + // contributed). + rpc PredictByCohort(PredictByCohortRequest) returns (PredictByCohortResult); +} + +// ─── Common ───────────────────────────────────────────────────────── + +enum MemoryScope { + SCOPE_UNSPECIFIED = 0; + SCOPE_PLATFORM = 1; + SCOPE_PROJECT = 2; + SCOPE_LOCATION = 3; + SCOPE_AGENT = 4; + SCOPE_USER = 5; + SCOPE_SESSION = 6; +} + +enum MemoryStatus { + STATUS_UNSPECIFIED = 0; + STATUS_PENDING = 1; + STATUS_CONFIRMED = 2; + STATUS_SUPERSEDED = 3; + STATUS_REJECTED = 4; +} + +message MemoryItem { + string id = 1; + string created = 2; // RFC3339 + string updated = 3; // RFC3339 + MemoryScope scope = 4; + MemoryStatus status = 5; + string platform_id = 6; + optional string project_id = 7; + optional string user_id = 8; + optional string agent_id = 9; + optional string session_id = 10; + string kind = 11; // e.g. "fact", "preference" + string content = 12; + double importance = 13; + string valid_from = 14; // RFC3339 + optional string valid_to = 15; + optional string last_accessed_at = 16; + string metadata_json = 17; // serialized JSON +} + +// ─── Save (direct write) ──────────────────────────────────────────── + +message SaveRequest { + // Optional. Engine assigns a short_id if blank. + string id = 1; + // Required: where to attribute the item. May be overridden from + // metadata if the platform/project isn't in the body. + optional string platform_id = 2; + optional string project_id = 3; + optional string location_id = 4; + optional string agent_id = 5; + optional string user_id = 6; + optional string session_id = 7; + // Required content. + string kind = 10; // "fact" | "preference" | "rule" | ... + string content = 11; + optional string category = 12; + optional double importance = 13; // default 5.0 + optional string source = 14; + // JSON-encoded metadata (the schema is opaque to the engine). + optional string metadata_json = 15; + MemoryScope scope = 20; + MemoryStatus status = 21; // default STATUS_CONFIRMED + optional double confidence = 22; // default 1.0 + // RFC3339 timestamps. Engine fills in `now()` defaults when blank. + optional string valid_from = 30; + optional string valid_to = 31; +} + +// ─── Observe ──────────────────────────────────────────────────────── + +message ObserveRequest { + string text = 1; + optional string role = 2; // "user" | "assistant" | "system" + // Tenancy is normally injected from gRPC metadata; these fields + // let callers override when running in test or as service-to- + // service ingestion. + optional string platform_id = 10; + optional string project_id = 11; + optional string user_id = 12; + optional string agent_id = 13; + optional string session_id = 14; +} + +message ObserveResponse { + repeated MemoryItem saved = 1; + // How many extraction candidates were found before the budget + // cap / dedupe pass. saved.len() <= candidate_count. + uint32 candidate_count = 2; +} + +// ─── Search ───────────────────────────────────────────────────────── + +message SearchRequest { + optional string text_match = 1; + optional MemoryScope scope = 2; + optional string platform_id = 3; + optional string project_id = 4; + optional string user_id = 5; + optional string agent_id = 6; + optional string session_id = 7; + optional string kind = 8; + optional MemoryStatus status = 9; + optional uint32 limit = 10; + optional uint32 offset = 11; + // Phase 3f min-necessary access — categories to drop from the + // response. Same primitive as BuildContextRequest.exclude_categories. + repeated string exclude_categories = 12; +} + +message SearchResponse { + repeated MemoryItem items = 1; +} + +// ─── Entities + graph ────────────────────────────────────────────── + +message Entity { + string id = 1; + string canonical_name = 2; + repeated string aliases = 3; + string type = 4; // "person" | "org" | "concept" | ... + string platform_id = 5; + optional string project_id = 6; + MemoryScope scope = 7; + string created = 8; + string updated = 9; +} + +message ResolveEntityRequest { + string name = 1; + string default_type = 2; + optional MemoryScope scope = 3; + // platform_id + project_id normally come from gRPC metadata; + // these are overrides for service-to-service calls. + optional string platform_id = 10; + optional string project_id = 11; +} + +message ExtractAndWireRequest { + string text = 1; + optional string source_memory_id = 2; + optional string platform_id = 10; + optional string project_id = 11; +} + +message ExtractAndWireResponse { + uint32 entities_resolved = 1; + uint32 edges_created = 2; +} + +// ─── Budget ───────────────────────────────────────────────────────── + +message BudgetGateRequest { + string platform_id = 1; + int64 cents = 2; // Projected spend +} + +message RecordCostRequest { + string platform_id = 1; + int64 cents = 2; +} + +message BudgetState { + string platform_id = 1; + string period_start = 2; // RFC3339, calendar-month boundary + int64 spent_cents = 3; + optional int64 cap_cents = 4; +} + +// ─── Health ───────────────────────────────────────────────────────── + +message HealthRequest {} + +message HealthResponse { + bool ok = 1; + string version = 2; +} + +message GetStatsRequest { + // Tenancy normally injected from gRPC metadata; these override for + // service-to-service callers. + optional string platform_id = 10; + optional string project_id = 11; +} + +// Admin-dashboard observability snapshot. All counts are scoped to the +// request's platform_id (+ optional project_id). +message StatsReport { + int64 total_items = 1; + map by_scope = 2; + map by_status = 3; + map by_type = 4; + // Vector index coverage — how much of the corpus is semantically indexed. + int64 items_with_embedding = 5; + int64 items_without_embedding = 6; + // Active embedding provider on this engine ("none" if semantic search is + // disabled) and its output dimension. + string embedding_model = 7; + uint32 embedding_dim = 8; + // Count of persisted behavior_pattern items (the lattice output). + int64 pattern_count = 9; +} + +// ─── Mining ───────────────────────────────────────────────────────── +// +// Subject-agnostic behavioral mining. A subject is just a `(kind, +// external_id)` pair so the same engine can mine contacts, users, +// teams, workspaces, services, or anything else. The mining +// algorithms read MemoryItem rows that carry a matching subject in +// metadata and treat them as the activity stream. + +message Subject { + // Subject kind: "contact", "user", "team", "workspace", "service", + // "session", etc. Free-form so new kinds don't need a proto change. + string kind = 1; + // Stable external id within `kind`. Opaque to the engine. + string external_id = 2; +} + +message MineMemoriesRequest { + // Restrict mining to a single subject. Omit for project-wide bulk. + optional Subject subject = 1; + // Look-back window in days. Defaults to 90; clamped to [7, 730]. + optional uint32 window_days = 2; + // Force re-extraction even when a recent pattern already exists. + // Otherwise the engine skips subjects whose patterns were + // refreshed inside the last refresh-cooldown window. + bool force = 3; + // Tenancy normally injected from gRPC metadata; these fields let + // service-to-service callers override. + optional string platform_id = 10; + optional string project_id = 11; +} + +// A persisted behavior pattern. Stored as a MemoryItem of +// kind="behavior_pattern" with the structured fields below denormalized +// from metadata_json for convenience. `source_memory_ids` is the +// right-to-explanation primitive: every prediction can name the raw +// memories that produced it. +message BehaviorPatternRecord { + string id = 1; + Subject subject = 2; + // Pattern kind: "recurring_event", "day_of_week", "time_of_day", + // "entity_preference", "entity_bundle", "declining_engagement", + // "offer_responsiveness", "rfm_segment", "price_tier", + // "seasonal", "cart_abandonment", etc. Free-form string so new + // detectors don't need a proto bump. + string pattern_kind = 3; + // Human-readable summary — also the `content` of the underlying + // memory item (e.g. "Sarah orders every Friday at 3pm"). + string summary = 4; + // Full metadata JSON. The shape per pattern_kind is documented in + // the engine but kept opaque at the proto layer. + string metadata_json = 5; + double confidence = 6; + uint32 observation_count = 7; + string last_observed_at = 8; // RFC3339 + optional string next_expected_at = 9; // RFC3339 + bool active = 10; + // Provenance: ids of the MemoryItem rows that produced this + // pattern. Powers GDPR Art. 22 explanation requests and the + // deletion-cascade hook (deleting a source memory deactivates + // any patterns derived from it). + repeated string source_memory_ids = 11; + string created = 12; // RFC3339 + string updated = 13; // RFC3339 +} + +message MineMemoriesResult { + uint32 subjects_processed = 1; + uint32 patterns_created = 2; + uint32 patterns_refreshed = 3; + uint32 patterns_deactivated = 4; + uint64 duration_ms = 5; + // The patterns produced or refreshed during this run. May be + // truncated for very large bulk runs — see `subjects_processed` + // for the true totals. + repeated BehaviorPatternRecord patterns = 6; +} + +// ─── Predict ──────────────────────────────────────────────────────── + +message PredictRequest { + // Subject to project predictions for. + Subject subject = 1; + // How far forward to project (in days). Default 30, clamped [1, 365]. + optional uint32 horizon_days = 2; + // Max predictions to return. Default 20, max 200. + optional uint32 limit = 3; + // Minimum confidence (0..1). Predictions below this are dropped. + // Default 0.5. + optional double min_confidence = 4; + // How many future occurrences to project per pattern within the + // horizon. 1 (default) = just the next firing. Higher values yield a + // forward view ("the next N firings"), each occurrence carrying its + // own horizon-decayed confidence — later firings are less certain. + // Only patterns with a fixed cadence can project past the first. + optional uint32 occurrences_per_pattern = 5; + // Tenancy override; normally injected from gRPC metadata. + optional string platform_id = 10; + optional string project_id = 11; + // When true, emit a `prediction.imminent` event for each prediction due + // within `imminent_within_hours` — the predictions→events bridge that lets + // the commerce side act on behavior (campaigns, nudges). Per-day dedupe + // means a daily run won't double-fire. Default false. + bool emit_events = 12; + // Imminence window for event emission, in hours. Default 48, clamped + // [1, 720]. Only used when emit_events is true. + optional uint32 imminent_within_hours = 13; +} + +// One projected event the engine expects based on the subject's +// active patterns. `pattern_id` is the behavior_pattern memory id +// that produced it — the caller can fetch its full provenance via +// the right-to-explanation API. +message PredictedEvent { + string pattern_id = 1; + // Kind of pattern that drove this prediction (e.g. "recurring_event"). + string pattern_kind = 2; + // Human-readable description. Mirrors the pattern's summary by + // default — engines may produce richer text in follow-ups. + string description = 3; + // ISO timestamp the event is expected at. + string expected_at = 4; + // 0..1 confidence inherited from the pattern's dominance score. + double confidence = 5; + // Tolerance window in minutes (cadence-dependent — daily patterns + // get hours of slack, monthly patterns get days). + uint32 window_minutes = 6; + // Provenance: ids of the raw memories that produced the pattern + // (and therefore this prediction). Powers the right-to- + // explanation API. + repeated string source_memory_ids = 7; +} + +message PredictResult { + Subject subject = 1; + repeated PredictedEvent predictions = 2; + // Total active patterns considered (whether or not each produced + // a prediction inside the horizon). + uint32 active_pattern_count = 3; + // ISO timestamp the prediction was generated at. + string generated_at = 4; + uint64 duration_ms = 5; + // Number of `prediction.imminent` events emitted this call (0 unless + // emit_events was set; dedupe may make it less than the imminent count). + uint32 events_emitted = 6; +} + +message EstimateRequest { + Subject subject = 1; + // Which estimator to run. v1: "phenoage". + string estimator_id = 2; + // Persist the score as a memory (kind="estimate") so it builds a trajectory + // and feeds the calibration loop. Default false (compute-only). + bool persist = 3; + // Tenancy normally injected from gRPC metadata; override for s2s callers. + optional string platform_id = 10; + optional string project_id = 11; +} + +message ScoreContributor { + string signal = 1; + // Signed contribution to the score (positive pushed it up). + double contribution = 2; +} + +message EstimateResult { + Subject subject = 1; + string estimator_id = 2; + // True when every required signal was present and a score was produced. + bool ok = 3; + double value = 4; + string unit = 5; + repeated ScoreContributor contributors = 6; + double confidence = 7; + repeated string provenance = 8; + // Always set: "estimate" + a not-a-diagnosis disclaimer. + string framing = 9; + string disclaimer = 10; + // Required signals with no reading, when ok=false. + repeated string missing_signals = 11; +} + +message ConsolidateRequest { + // Cosine threshold for "same memory". Default 0.92. + optional double threshold = 1; + // Max items to scan this pass. Default 1000, clamped [1, 10000]. + optional uint32 scan_limit = 2; + // Tenancy override; normally injected from gRPC metadata. + optional string platform_id = 10; + optional string project_id = 11; +} + +message ConsolidateResult { + uint32 scanned = 1; + uint32 groups = 2; + uint32 superseded = 3; +} + +message BackfillRequest { + // Max items to embed this call. Default 500, clamped [1, 10000]. Call + // repeatedly until embedded == 0 to drain a large corpus. + optional uint32 batch = 1; + // Tenancy override; normally injected from gRPC metadata. + optional string platform_id = 10; + optional string project_id = 11; +} + +message BackfillResult { + uint32 embedded = 1; +} + +// ─── Calibration ──────────────────────────────────────────────────── + +message GetCalibrationRequest { + optional string project_id = 1; + optional string platform_id = 2; + // Number of equal-width confidence buckets over [0,1]. Defaults to + // 5; clamped to [1, 20]. + optional uint32 bucket_count = 3; +} + +message CalibrationBucket { + double lower = 1; + double upper = 2; + // Active patterns whose stated confidence falls in this band. + uint32 patterns = 3; + // Predictions from those patterns that have been scored (hits+misses). + uint32 predictions = 4; + uint32 hits = 5; + uint32 misses = 6; + // hits / predictions; only meaningful when has_data is true. + double realized_hit_rate = 7; + // False when no prediction in this band has been scored yet. + bool has_data = 8; +} + +message CalibrationReport { + repeated CalibrationBucket buckets = 1; + // Totals across all buckets. + uint32 total_patterns = 2; + uint32 total_predictions = 3; +} + +// ─── Profile ──────────────────────────────────────────────────────── + +message GetProfileRequest { + Subject subject = 1; + optional string platform_id = 10; + optional string project_id = 11; +} + +// One risk signal active for the subject (declining_engagement, +// cant_lose_them RFM segment, long silence after a recurring cadence...). +message RiskIndicator { + // Stable identifier: "declining_engagement", "rfm_cant_lose_them", + // "cadence_missed", etc. + string kind = 1; + // Free-text explanation for the human dashboard. + string description = 2; + // 0..1 — how strongly the signal is firing. + double severity = 3; + // Pattern memory id that produced the signal (for provenance). + string source_pattern_id = 4; +} + +// Behavioral profile snapshot. The non-temporal "who is this +// subject" view, aggregated from their active behavior patterns. +message Profile { + Subject subject = 1; + // RFM segment label when an rfm_segment pattern is active. + optional string rfm_segment = 2; + optional uint32 recency_score = 3; + optional uint32 frequency_score = 4; + optional uint32 monetary_score = 5; + // Subject's dominant entity (from entity_preference). Empty when + // no preference pattern is active. + optional string top_entity = 6; + // Cadence summary when a temporal pattern is active. + // "weekly", "every 14 days", "Fridays at 19:00", etc. + optional string cadence_summary = 7; + // Active risk signals — agents should surface these before + // proceeding with non-trivial actions. + repeated RiskIndicator risks = 8; + // Ids of every active pattern that fed this profile (provenance). + repeated string contributing_pattern_ids = 9; + string generated_at = 10; + uint64 duration_ms = 11; +} + +// ─── Context bundle ───────────────────────────────────────────────── + +message BuildContextRequest { + Subject subject = 1; + // Which sections to include. Empty = all five. + repeated string include = 2; + // Hard cap on bundle size. Sections truncate to fit. Default 2000. + optional uint32 max_tokens = 3; + optional uint32 memory_limit = 4; + optional uint32 prediction_limit = 5; + // Phase 3f min-necessary access — categories to drop from the + // bundle. Used to keep medical / HR / PII memories out of + // sales-context queries against the same project. + repeated string exclude_categories = 6; + optional string platform_id = 10; + optional string project_id = 11; +} + +message ContextProfileSummary { + optional string rfm_segment = 1; + optional uint32 recency_score = 2; + optional uint32 frequency_score = 3; + optional uint32 monetary_score = 4; + optional string top_entity = 5; + optional string cadence_summary = 6; + repeated RiskIndicator risks = 7; +} + +message ContextPattern { + string id = 1; + string pattern_kind = 2; + string summary = 3; + double confidence = 4; + optional string next_expected_at = 5; +} + +message ContextMemoryRow { + string id = 1; + string type = 2; + string content = 3; + double importance = 4; + string created = 5; +} + +message ContextObservationRow { + string id = 1; + string content = 2; + uint32 proof_count = 3; + repeated string source_memory_ids = 4; + string created = 5; +} + +message ContextProvenance { + repeated string memory_ids = 1; + repeated string pattern_ids = 2; + repeated string observation_ids = 3; +} + +message ContextBundle { + Subject subject = 1; + optional ContextProfileSummary profile = 2; + repeated ContextPattern patterns = 3; + repeated PredictedEvent predictions = 4; + repeated ContextMemoryRow memories = 5; + repeated ContextObservationRow observations = 6; + ContextProvenance provenance = 7; + uint32 tokens_estimate = 8; + // Sections that were dropped or truncated to fit the budget. + repeated string truncated = 9; + string generated_at = 10; + uint64 duration_ms = 11; + // Ground-truth authority directive: place at the top of the prompt so the + // model treats this memory as already-known and doesn't re-derive it. + string guidance = 12; +} + +// ─── Events ───────────────────────────────────────────────────────── + +message MemoryEventRecord { + string id = 1; + string event_type = 2; + optional Subject subject = 3; + string severity = 4; // "info" | "warn" | "critical" + string payload_json = 5; // free-form JSON + repeated string source_memory_ids = 6; + optional string source_pattern_id = 7; + optional string emitted_by_pack = 8; + string occurred_at = 9; +} + +message EmitEventRequest { + string event_type = 1; + optional Subject subject = 2; + // info | warn | critical. Defaults to "info" when blank. + string severity = 3; + string payload_json = 4; + repeated string source_memory_ids = 5; + optional string source_pattern_id = 6; + optional string emitted_by_pack = 7; + optional string platform_id = 10; + optional string project_id = 11; +} + +message EmitEventResult { + // Empty when the dedupe collision suppressed the insert. + optional MemoryEventRecord event = 1; + // Number of alert rules that matched + dispatched. The TS shell + // can surface this in a 'X-Alert-Dispatches' header. + uint32 alert_dispatches = 2; +} + +message PollEventsRequest { + optional string since_iso = 1; + optional uint32 limit = 2; + repeated string event_types = 3; + optional string platform_id = 10; + optional string project_id = 11; +} + +message PollEventsResult { + repeated MemoryEventRecord events = 1; +} + +// ─── Alert rules ──────────────────────────────────────────────────── + +message AlertRule { + string id = 1; + string project_id = 2; + string name = 3; + optional string description = 4; + bool enabled = 5; + // Free-form JSON to keep proto stable as triggers evolve. + string trigger_json = 6; + optional string filter_json = 7; + string notify_json = 8; + optional string throttle_json = 9; + string created = 10; + string updated = 11; +} + +message CreateAlertRuleRequest { + string name = 1; + optional string description = 2; + optional bool enabled = 3; + string trigger_json = 4; + optional string filter_json = 5; + string notify_json = 6; + optional string throttle_json = 7; + optional string created_by_user_id = 8; + optional string platform_id = 10; + optional string project_id = 11; +} + +message ListAlertRulesRequest { + optional string platform_id = 10; + optional string project_id = 11; +} + +message ListAlertRulesResult { + repeated AlertRule rules = 1; +} + +message GetAlertRuleRequest { + string alert_rule_id = 1; + optional string platform_id = 10; + optional string project_id = 11; +} + +message UpdateAlertRuleRequest { + string alert_rule_id = 1; + optional string name = 2; + optional string description = 3; + optional bool enabled = 4; + optional string trigger_json = 5; + // Pass empty string to clear; absent = no-op. + optional string filter_json = 6; + optional string notify_json = 7; + optional string throttle_json = 8; + optional string platform_id = 10; + optional string project_id = 11; +} + +message DeleteAlertRuleRequest { + string alert_rule_id = 1; + optional string platform_id = 10; + optional string project_id = 11; +} + +message DeleteAlertRuleResult { + bool deleted = 1; +} + +message AlertFire { + string id = 1; + string alert_rule_id = 2; + optional string event_id = 3; + string dedupe_key = 4; + // JSON array of { channel, ok, error? }. + string delivery_results_json = 5; + string fired_at = 6; +} + +message ListAlertFiresRequest { + string alert_rule_id = 1; + optional uint32 limit = 2; + optional string platform_id = 10; + optional string project_id = 11; +} + +message ListAlertFiresResult { + repeated AlertFire fires = 1; +} + +// ─── Compliance ───────────────────────────────────────────────────── + +message ExportSubjectRequest { + Subject subject = 1; + optional string platform_id = 10; + optional string project_id = 11; +} + +// JSON-serialized bundles so downstream tooling can serialize without +// needing to know every memory type. Stable shape: top-level keys are +// memories[], patterns[], observations[], events[], alert_fires[]. +message SubjectExport { + Subject subject = 1; + string export_json = 2; // full bundle as JSON + uint32 memory_count = 3; + uint32 pattern_count = 4; + uint32 observation_count = 5; + uint32 event_count = 6; + uint32 alert_fire_count = 7; + string generated_at = 8; + uint64 duration_ms = 9; +} + +message HardDeleteSubjectRequest { + Subject subject = 1; + // Free-text reason kept on the tombstone audit row. Required — + // GDPR Art. 17 requests should carry a reference (case id, etc.). + string reason = 2; + // When true, the engine returns a dry-run preview (counts only, + // no rows touched). Default false. + bool dry_run = 3; + optional string platform_id = 10; + optional string project_id = 11; +} + +message HardDeleteSubjectResult { + Subject subject = 1; + uint32 memories_deleted = 2; + uint32 patterns_deleted = 3; + uint32 observations_deleted = 4; + uint32 events_deleted = 5; + uint32 alert_fires_deleted = 6; + bool dry_run = 7; + optional string audit_event_id = 8; + string generated_at = 9; + uint64 duration_ms = 10; +} + +// ─── Audit log read ──────────────────────────────────────────────── + +message ListAuditEventsRequest { + // Restrict to events touching this subject. Matches against + // both the structured subject metadata and the `query` column. + optional Subject subject = 1; + optional string actor = 2; + // Event types to include. Empty = all. + repeated string event_types = 3; + optional string since_iso = 4; + optional uint32 limit = 5; + optional string platform_id = 10; + optional string project_id = 11; +} + +message AuditEventRecord { + string id = 1; + string created = 2; // RFC3339 + string actor = 3; + string event_type = 4; + optional string query = 5; + optional string memory_ids = 6; + uint32 result_count = 7; + string metadata_json = 8; +} + +message ListAuditEventsResult { + repeated AuditEventRecord events = 1; +} + +// ─── Pack introspection ──────────────────────────────────────────── + +message ListPacksRequest { + optional string platform_id = 10; + optional string project_id = 11; +} + +message PackManifest { + string id = 1; + string version = 2; + string description = 3; + repeated string owns_classes = 4; + repeated string regulatory_tags = 5; +} + +message ListPacksResult { + repeated PackManifest packs = 1; +} + +// ─── Phase 5 cohort similarity ───────────────────────────────────── + +message GetCohortRequest { + Subject subject = 1; + // How many nearest neighbors to return. Server clamps to [1, 50]. + uint32 k = 2; + // Minimum similarity score (0.0..1.0) to include a member. + // 0 = no cutoff (returns top-K regardless). + double min_similarity = 3; + optional string platform_id = 10; + optional string project_id = 11; +} + +message CohortMember { + Subject subject = 1; + double similarity = 2; + // RFM segment label of the cohort member (e.g. "loyal_customer"). + // Lets the admin UI annotate the cohort list without a second + // GetProfile round-trip per member. + string rfm_segment = 3; + // Pattern kinds active on this member — surfaces "Mike has the + // same recurring_event + entity_preference as Sarah" annotation. + repeated string pattern_kinds = 4; +} + +message GetCohortResult { + Subject target = 1; + repeated CohortMember members = 2; + // How many candidate subjects we scored before truncating to k. + uint32 candidate_count = 3; + string generated_at = 4; + uint64 duration_ms = 5; +} + +message PredictByCohortRequest { + Subject subject = 1; + // Cohort size. Same clamping as GetCohort. + uint32 cohort_k = 2; + // Max predictions to return. + uint32 prediction_limit = 3; + // Pass-through to GetCohort. + double min_similarity = 4; + optional string platform_id = 10; + optional string project_id = 11; +} + +message CohortPrediction { + string pattern_kind = 1; + string description = 2; + string expected_at = 3; + double confidence = 4; + uint32 window_minutes = 5; + repeated Subject supporting_subjects = 6; + repeated string source_memory_ids = 7; +} + +message PredictByCohortResult { + Subject target = 1; + repeated CohortMember cohort = 2; + repeated CohortPrediction predictions = 3; + string generated_at = 4; + uint64 duration_ms = 5; +} diff --git a/scripts/check-proto-drift.mjs b/scripts/check-proto-drift.mjs new file mode 100644 index 0000000..a571171 --- /dev/null +++ b/scripts/check-proto-drift.mjs @@ -0,0 +1,57 @@ +#!/usr/bin/env node +// Guards the SDK against silently falling behind the engine's RPC surface. +// +// Parses the RPC names out of the vendored proto/memory.proto and checks that +// every one is classified in proto/coverage.json (either `covered` — has an SDK +// method — or `internal` — intentionally not exposed). Fails if: +// - a proto RPC is in neither list (new engine capability → classify it), or +// - coverage.json lists an RPC that no longer exists in the proto (stale). +// +// Run: `npm run check:proto`. Wire into CI so bumping the vendored proto forces +// a conscious decision about each new RPC. + +import { readFileSync } from 'node:fs' +import { fileURLToPath } from 'node:url' +import { dirname, join } from 'node:path' + +const root = join(dirname(fileURLToPath(import.meta.url)), '..') +const proto = readFileSync(join(root, 'proto', 'memory.proto'), 'utf8') +const coverage = JSON.parse(readFileSync(join(root, 'proto', 'coverage.json'), 'utf8')) + +const protoRpcs = new Set( + [...proto.matchAll(/^\s*rpc\s+([A-Za-z0-9_]+)\s*\(/gm)].map((m) => m[1]), +) +const classified = new Set([ + ...Object.keys(coverage.covered ?? {}), + ...Object.keys(coverage.internal ?? {}), +]) + +const unclassified = [...protoRpcs].filter((r) => !classified.has(r)) +const stale = [...classified].filter((r) => !protoRpcs.has(r)) + +let failed = false +if (unclassified.length > 0) { + failed = true + console.error( + `\n✗ ${unclassified.length} engine RPC(s) not classified in proto/coverage.json:\n` + + unclassified.map((r) => ` - ${r}`).join('\n') + + `\n Add each to "covered" (with its SDK method) or "internal" (with a reason).`, + ) +} +if (stale.length > 0) { + failed = true + console.error( + `\n✗ ${stale.length} RPC(s) in coverage.json no longer exist in the proto:\n` + + stale.map((r) => ` - ${r}`).join('\n'), + ) +} + +if (failed) { + process.exit(1) +} + +console.log( + `✓ proto in sync: ${protoRpcs.size} RPCs — ` + + `${Object.keys(coverage.covered ?? {}).length} covered, ` + + `${Object.keys(coverage.internal ?? {}).length} internal.`, +) diff --git a/src/resources/events.ts b/src/resources/events.ts index 647a2e3..a5d61d0 100644 --- a/src/resources/events.ts +++ b/src/resources/events.ts @@ -1,5 +1,6 @@ import type { HttpClient } from '../core/http-client.js' import type { RequestOptions } from '../core/types.js' +import type { EmitEventRequest, EmitEventResult } from '../types/lattice.js' /** Severity tier emitted on every event. */ export type EventSeverity = 'info' | 'warn' | 'critical' @@ -75,6 +76,27 @@ export class EventsResource { return this.http.get('/memory-events', query, options) } + /** + * Append an event to the durable log. Matching alert rules fire + * synchronously. The write-side counterpart to poll(). + * + * @example + * ```ts + * await tf.events.emit({ + * eventType: 'cart.abandoned', + * subject: { kind: 'contact', externalId: 'sarah' }, + * severity: 'warn', + * payloadJson: JSON.stringify({ cartValue: 84 }), + * }) + * ``` + */ + async emit( + body: EmitEventRequest, + options?: RequestOptions, + ): Promise { + return this.http.post('/lattice/events/emit', body, options) + } + /** * Convenience: poll in a loop and invoke `handler` for each event. * Returns an unsubscribe function that cancels the loop. diff --git a/src/resources/lattice.ts b/src/resources/lattice.ts index b4d117f..3a3903f 100644 --- a/src/resources/lattice.ts +++ b/src/resources/lattice.ts @@ -13,6 +13,10 @@ import type { MonitorTickResult, PredictRequest, PredictResult, + EstimateRequest, + EstimateResult, + CalibrationReport, + GetCalibrationParams, RiskIndicator, Subject, SubjectProfile, @@ -256,6 +260,43 @@ export class LatticeResource { ): Promise { return this.http.post('/lattice/cohort/predict', body, options) } + + /** + * Run a deterministic estimator (e.g. PhenoAge biological age) over a + * subject's biomarker signals. Returns a wellness estimate with per-signal + * contributors and a not-a-diagnosis disclaimer — never a medical verdict. + * + * @example + * ```ts + * const est = await tf.lattice.estimate({ + * subject: { kind: 'patient', externalId: 'p-123' }, + * estimatorId: 'phenoage', + * }) + * if (est.ok) console.log(`${est.value} ${est.unit} — ${est.disclaimer}`) + * else console.log('missing:', est.missingSignals) + * ``` + */ + async estimate( + body: EstimateRequest, + options?: RequestOptions, + ): Promise { + return this.http.post('/lattice/estimate', body, options) + } + + /** + * Prediction calibration report: confidence buckets mapped to realized + * hit-rates. Tells you whether "80% confident" predictions actually fire + * ~80% of the time. + */ + async getCalibration( + params: GetCalibrationParams = {}, + options?: RequestOptions, + ): Promise { + const query: Record = { + bucketCount: params.bucketCount, + } + return this.http.get('/lattice/calibration', query, options) + } } export interface GetCohortRequest { diff --git a/src/resources/memory.ts b/src/resources/memory.ts index 17c2ed5..a90d4ed 100644 --- a/src/resources/memory.ts +++ b/src/resources/memory.ts @@ -14,6 +14,10 @@ import { type MemoryStats, type ConsolidateRequest, type ConsolidateResult, + type BackfillEmbeddingsRequest, + type BackfillEmbeddingsResult, + type DedupRequest, + type DedupResult, type ObserveAttachmentRequest, type ObserveDocumentRequest, type ObserveRequest, @@ -496,6 +500,36 @@ export class AdminMemoryResource { return this.http.post('/admin/memory/llm-consolidate', body, options) } + /** + * Vectorize memory items that have no embedding yet (catch-up + * embedding work-list). Call repeatedly until `embedded` is 0 to drain a + * large corpus. Idempotent — already-embedded items are skipped. + * + * @example + * ```ts + * let n: number + * do { ({ embedded: n } = await tf.memory.admin.backfillEmbeddings({ batch: 500 })) } while (n > 0) + * ``` + */ + async backfillEmbeddings( + body: BackfillEmbeddingsRequest = {}, + options?: RequestOptions, + ): Promise { + return this.http.post('/admin/memory/embeddings/backfill', body, options) + } + + /** + * Semantic dedup: collapse near-duplicate memories (cosine ≥ threshold), + * keep the strongest, supersede the rest. Distinct from consolidate(), which + * is the LLM-driven deductive-observations pass. + */ + async dedup( + body: DedupRequest = {}, + options?: RequestOptions, + ): Promise { + return this.http.post('/admin/memory/dedup', body, options) + } + /** * List the feedback records attached to a memory item — useful when * inspecting auto-flagged items to decide whether to confirm or reject. diff --git a/src/types/lattice.ts b/src/types/lattice.ts index bedbfa2..2c75775 100644 --- a/src/types/lattice.ts +++ b/src/types/lattice.ts @@ -222,6 +222,20 @@ export interface PredictRequest { limit?: number /** Minimum confidence (0..1) to surface. Default 0.5. */ minConfidence?: number + /** + * Forward view: how many future occurrences to project per fixed-cadence + * pattern. 1 (default) = just the next firing; higher yields "the next N + * firings", each with its own horizon-decayed confidence. + */ + occurrencesPerPattern?: number + /** + * Predictions→events bridge: when true, emit a `prediction.imminent` event + * for each prediction due within `imminentWithinHours`. Per-day dedupe means + * a daily run won't double-fire. Default false. + */ + emitEvents?: boolean + /** Imminence window for event emission, in hours. Default 48, clamped [1, 720]. */ + imminentWithinHours?: number } /** One projected event derived from one active behavior pattern. */ @@ -245,6 +259,8 @@ export interface PredictResult { predictions: PredictedEvent[] /** Total active patterns considered, whether or not each produced a prediction. */ activePatternCount: number + /** Number of `prediction.imminent` events emitted (when emitEvents=true). */ + eventsEmitted?: number /** ISO timestamp the prediction was generated. */ generatedAt: string durationMs: number @@ -295,3 +311,94 @@ export interface MonitorStatus { /** Total active patterns the monitor is watching. */ activePatternCount: number } + +// ── Estimate (deterministic estimators, e.g. PhenoAge bio-age) ─────── + +export interface EstimateRequest { + /** Subject whose signals to score. */ + subject: Subject + /** Which estimator to run. v1: "phenoage". */ + estimatorId: string + /** + * Persist the score as a memory (kind="estimate") so it builds a trajectory + * and feeds the calibration loop. Default false (compute-only). + */ + persist?: boolean +} + +export interface ScoreContributor { + signal: string + /** Signed contribution to the score (positive pushed it up). */ + contribution: number +} + +export interface EstimateResult { + subject: Subject + estimatorId: string + /** True when every required signal was present and a score was produced. */ + ok: boolean + value: number + unit: string + contributors: ScoreContributor[] + confidence: number + provenance: string[] + /** Always set: an "estimate" framing, never a diagnosis. */ + framing: string + disclaimer: string + /** Required signals with no reading, when ok=false. */ + missingSignals: string[] +} + +// ── Calibration (prediction reliability) ───────────────────────────── + +export interface CalibrationBucket { + lower: number + upper: number + /** Active patterns whose stated confidence falls in this band. */ + patterns: number + /** Predictions from those patterns that have been scored (hits+misses). */ + predictions: number + hits: number + misses: number + /** hits / predictions; only meaningful when hasData is true. */ + realizedHitRate: number + /** False when no prediction in this band has been scored yet. */ + hasData: boolean +} + +export interface CalibrationReport { + buckets: CalibrationBucket[] + totalPatterns: number + totalPredictions: number +} + +export interface GetCalibrationParams { + /** Number of equal-width confidence buckets over [0,1]. Default 5, clamped [1,20]. */ + bucketCount?: number +} + +// ── Emit event (write-side of the event log) ───────────────────────── + +export interface EmitEventRequest { + eventType: string + subject?: Subject + /** info | warn | critical. Defaults to "info". */ + severity?: string + /** Free-form JSON payload (can include "value", "channel", etc.). */ + payloadJson?: string + sourceMemoryIds?: string[] + sourcePatternId?: string +} + +export interface EmitEventResult { + /** False when a dedupe collision suppressed the insert. */ + emitted: boolean + event: { + id: string + eventType: string + severity: string + occurredAt: string + } | null + /** Number of alert rules that matched + dispatched. */ + alertDispatches: number +} diff --git a/src/types/memory.ts b/src/types/memory.ts index bda7245..4b96667 100644 --- a/src/types/memory.ts +++ b/src/types/memory.ts @@ -230,3 +230,28 @@ export interface MemoryStats { byScope: Record byStatus: Record } + +// ── Maintenance ops (embedding backfill + semantic dedup) ──────────── + +export interface BackfillEmbeddingsRequest { + /** Max items to embed this call. Default 500, clamped [1, 10000]. Call repeatedly until embedded=0. */ + batch?: number +} + +export interface BackfillEmbeddingsResult { + /** Items vectorized on this pass. */ + embedded: number +} + +export interface DedupRequest { + /** Cosine threshold for "same memory". Default 0.92. */ + threshold?: number + /** Max items to scan this pass. Default 1000, clamped [1, 10000]. */ + scanLimit?: number +} + +export interface DedupResult { + scanned: number + groups: number + superseded: number +}