diff --git a/mc-ctl b/mc-ctl index b46a4a4..0431255 100755 --- a/mc-ctl +++ b/mc-ctl @@ -1665,6 +1665,106 @@ cmd_stats() { echo "" } +# --------------------------------------------------------------------------- +# JME utility dashboard (Phase 3) +# --------------------------------------------------------------------------- + +cmd_jme_stats() { + need_db + echo -e "${BOLD}=== JME Utility Dashboard ===${NC}" + echo "" + + local now_ms + now_ms=$(date +%s%3N) + local today + today=$(date '+%Y-%m-%d') + + # ── Fact store ────────────────────────────────────────────────────────────── + echo -e "${BOLD}Fact store:${NC}" + local facts_total facts_emb facts_expiring + facts_total=$(sql_raw "SELECT COUNT(*) FROM jme_facts;" 2>/dev/null || echo "0") + facts_emb=$(sql_raw "SELECT COUNT(*) FROM jme_facts WHERE embedding IS NOT NULL;" 2>/dev/null || echo "0") + facts_expiring=$(sql_raw "SELECT COUNT(*) FROM jme_facts WHERE expires_at IS NOT NULL AND expires_at BETWEEN ${now_ms} AND ${now_ms} + 604800000;" 2>/dev/null || echo "0") + + echo -e " Facts total: ${facts_total}" + echo -e " With embeddings: ${facts_emb} (vector query LIMIT=500, ceiling warn=400)" + echo -e " Expiring in 7d: ${facts_expiring}" + + if [ "$facts_emb" -ge 400 ] 2>/dev/null; then + echo -e " ${YELLOW}⚠️ Vector ceiling approaching — consider pruning low-confidence facts${NC}" + fi + + echo "" + + # ── Turn buffer ───────────────────────────────────────────────────────────── + echo -e "${BOLD}Turn buffer:${NC}" + local turns_total + turns_total=$(sql_raw "SELECT COUNT(*) FROM jme_turns;" 2>/dev/null || echo "0") + echo -e " Turns (pending consolidation): ${turns_total}" + echo "" + + # ── Recall utility ────────────────────────────────────────────────────────── + echo -e "${BOLD}Recall utility (source='jme'):${NC}" + + local total_jme used_jme unused_jme null_jme + total_jme=$(sql_raw "SELECT COUNT(*) FROM recall_audit WHERE source='jme';" 2>/dev/null || echo "0") + used_jme=$(sql_raw "SELECT COUNT(*) FROM recall_audit WHERE source='jme' AND was_used=1;" 2>/dev/null || echo "0") + unused_jme=$(sql_raw "SELECT COUNT(*) FROM recall_audit WHERE source='jme' AND was_used=0;" 2>/dev/null || echo "0") + null_jme=$(sql_raw "SELECT COUNT(*) FROM recall_audit WHERE source='jme' AND was_used IS NULL;" 2>/dev/null || echo "0") + + local util_pct="n/a" + if [ "$total_jme" -gt 0 ] 2>/dev/null && [ "$((used_jme + unused_jme))" -gt 0 ] 2>/dev/null; then + util_pct=$(awk "BEGIN { printf \"%.1f%%\", 100 * ${used_jme} / (${used_jme} + ${unused_jme}) }") + fi + + echo -e " Total recalls: ${total_jme}" + echo -e " was_used=1 (used): ${used_jme}" + echo -e " was_used=0 (miss): ${unused_jme}" + echo -e " was_used=NULL: ${null_jme} (unmatched — normal for recent turns)" + echo -e " Utility rate: ${util_pct}" + echo "" + + # ── Daily breakdown (last 7 days) ────────────────────────────────────────── + echo -e "${BOLD}Daily breakdown (last 7 days, source='jme'):${NC}" + sql "SELECT + date(created_at) AS day, + COUNT(*) AS recalls, + SUM(CASE WHEN was_used=1 THEN 1 ELSE 0 END) AS used, + SUM(CASE WHEN was_used=0 THEN 1 ELSE 0 END) AS unused, + ROUND(100.0 * SUM(CASE WHEN was_used=1 THEN 1 ELSE 0 END) / NULLIF(SUM(CASE WHEN was_used IS NOT NULL THEN 1 ELSE 0 END), 0), 1) AS util_pct + FROM recall_audit + WHERE source='jme' + AND created_at >= date('now', '-7 days') + GROUP BY day + ORDER BY day DESC;" 2>/dev/null || echo -e " ${DIM}No JME recall data yet.${NC}" + + echo "" + + # ── Baseline comparison ───────────────────────────────────────────────────── + echo -e "${BOLD}Baseline comparison (last 7 days, all sources):${NC}" + sql "SELECT + source, + COUNT(*) AS recalls, + SUM(CASE WHEN was_used=1 THEN 1 ELSE 0 END) AS used, + ROUND(100.0 * SUM(CASE WHEN was_used=1 THEN 1 ELSE 0 END) / NULLIF(SUM(CASE WHEN was_used IS NOT NULL THEN 1 ELSE 0 END), 0), 1) AS util_pct + FROM recall_audit + WHERE created_at >= date('now', '-7 days') + GROUP BY source + ORDER BY util_pct DESC NULLS LAST;" 2>/dev/null || echo -e " ${DIM}No recall audit data.${NC}" + + echo "" + + # ── Verdict runway ────────────────────────────────────────────────────────── + echo -e "${BOLD}Verdict runway:${NC}" + local verdict_date="2026-07-27" + local days_left + days_left=$(( ( $(date -d "$verdict_date" +%s) - $(date +%s) ) / 86400 )) + echo -e " Verdict date: ${verdict_date}" + echo -e " Days remaining: ${days_left}" + echo -e " Success criteria: util_pct ≥ 39% (baseline hybrid), p50 latency < 350ms" + echo "" +} + # --------------------------------------------------------------------------- # Task trace (V8.5 Phase 6) # --------------------------------------------------------------------------- @@ -1796,6 +1896,7 @@ usage() { echo -e " ${BOLD}Memory${NC}" echo -e " ${GREEN}conversations${NC} [N] Last N conversations (default 10)" echo -e " ${GREEN}memory${NC} Memory backend status" + echo -e " ${GREEN}jme-stats${NC} JME utility dashboard — facts, recall rate, verdict runway" echo "" echo -e " ${BOLD}Hindsight${NC}" echo -e " ${GREEN}hindsight-status${NC} Container + API health" @@ -2873,6 +2974,7 @@ case "$cmd" in db) cmd_db "$@" ;; stats) cmd_stats ;; + jme-stats) cmd_jme_stats ;; validation) cmd_validation "$@" ;; diff --git a/src/memory/jme.test.ts b/src/memory/jme.test.ts index 84c7dc0..1a27fa2 100644 --- a/src/memory/jme.test.ts +++ b/src/memory/jme.test.ts @@ -641,3 +641,147 @@ describe("JME — pruneStaleTurns", () => { expect(deleted).toBe(0); }); }); + +// --------------------------------------------------------------------------- +// deduplicateFacts — Phase 3 temporal filter +// --------------------------------------------------------------------------- +describe("JME — deduplicateFacts (Phase 3 temporal filter)", () => { + beforeEach(() => { + vi.resetModules(); + vi.clearAllMocks(); + }); + + /** + * Build a minimal JmeRecallResult for use in deduplicateFacts tests. + * Embeddings are plain Float32Arrays — cosineSimilarity is mocked globally. + */ + function makeResult( + id: number, + ts: number, + score = 0.8, + ): import("./jme.js").JmeRecallResult { + return { + id, + factText: `fact-${id}`, + category: "project" as const, + sourceTask: "test", + score, + ts, + }; + } + + function makeEmbedding(id: number, value: number): [number, Float32Array] { + const vec = new Float32Array(4); + vec.fill(value); + return [id, vec]; + } + + it("passes through single-element list unchanged", async () => { + const { deduplicateFacts, TEMPORAL_DEDUP_THRESHOLD } = await getJme(); + const r = makeResult(1, 1000); + const embeddings = new Map([makeEmbedding(1, 0.5)]); + + // cosineSimilarity mock returns 0 — no clustering possible + const { cosineSimilarity: cosineSim } = await import("./embeddings.js"); + vi.mocked(cosineSim).mockReturnValue(0); + + const out = deduplicateFacts([r], embeddings); + expect(out).toHaveLength(1); + expect(out[0].id).toBe(1); + void TEMPORAL_DEDUP_THRESHOLD; // import check + }); + + it("passes through two dissimilar facts unchanged", async () => { + const { deduplicateFacts } = await getJme(); + const r1 = makeResult(1, 1000, 0.9); + const r2 = makeResult(2, 2000, 0.7); + const embeddings = new Map([makeEmbedding(1, 0.1), makeEmbedding(2, 0.9)]); + + const { cosineSimilarity: cosineSim } = await import("./embeddings.js"); + // Low similarity — below threshold + vi.mocked(cosineSim).mockReturnValue(0.3); + + const out = deduplicateFacts([r1, r2], embeddings); + expect(out).toHaveLength(2); + }); + + it("removes older fact when two facts are semantically similar", async () => { + const { deduplicateFacts, TEMPORAL_DEDUP_THRESHOLD } = await getJme(); + + const older = makeResult(1, 1000, 0.9); // higher score but older + const newer = makeResult(2, 9000, 0.7); // lower score but newer + const embeddings = new Map([makeEmbedding(1, 0.5), makeEmbedding(2, 0.5)]); + + const { cosineSimilarity: cosineSim } = await import("./embeddings.js"); + // High similarity — above threshold + vi.mocked(cosineSim).mockReturnValue(TEMPORAL_DEDUP_THRESHOLD + 0.01); + + const out = deduplicateFacts([older, newer], embeddings); + // Should keep the newer fact, drop the older one + expect(out).toHaveLength(1); + expect(out[0].id).toBe(2); + }); + + it("collapses a 3+ member cluster to ONLY the newest fact (v1/v2/v3 regression)", async () => { + const { deduplicateFacts, TEMPORAL_DEDUP_THRESHOLD } = await getJme(); + + // Same topic, three versions: ranked v1 > v2 > v3 by score, aged + // v1 < v2 < v3. Spec: only v3 reaches the context. The pre-fix shape + // returned [v2, v3] — the anchor broke on the FIRST newer candidate and + // never absorbed the rest of the cluster. 2-member tests cannot catch + // this; keep this one 3-wide. + const v1 = makeResult(1, 100, 0.9); + const v2 = makeResult(2, 200, 0.8); + const v3 = makeResult(3, 300, 0.7); + const embeddings = new Map([ + makeEmbedding(1, 0.5), + makeEmbedding(2, 0.5), + makeEmbedding(3, 0.5), + ]); + + const { cosineSimilarity: cosineSim } = await import("./embeddings.js"); + vi.mocked(cosineSim).mockReturnValue(TEMPORAL_DEDUP_THRESHOLD + 0.01); + + const out = deduplicateFacts([v1, v2, v3], embeddings); + expect(out).toHaveLength(1); + expect(out[0].id).toBe(3); + }); + + it("keeps older fact when anchor is more recent than candidate", async () => { + const { deduplicateFacts, TEMPORAL_DEDUP_THRESHOLD } = await getJme(); + + const anchor = makeResult(1, 9000, 0.9); // higher score AND newer + const candidate = makeResult(2, 1000, 0.7); // lower score AND older + const embeddings = new Map([makeEmbedding(1, 0.5), makeEmbedding(2, 0.5)]); + + const { cosineSimilarity: cosineSim } = await import("./embeddings.js"); + vi.mocked(cosineSim).mockReturnValue(TEMPORAL_DEDUP_THRESHOLD + 0.01); + + const out = deduplicateFacts([anchor, candidate], embeddings); + // Anchor is newer — keep anchor, drop candidate + expect(out).toHaveLength(1); + expect(out[0].id).toBe(1); + }); + + it("skips facts without embeddings (keeps them unconditionally)", async () => { + const { deduplicateFacts, TEMPORAL_DEDUP_THRESHOLD } = await getJme(); + + const withEmb = makeResult(1, 9000, 0.9); + const noEmb = makeResult(2, 1000, 0.8); + // Only id=1 has an embedding; id=2 has none + const embeddings = new Map([makeEmbedding(1, 0.5)]); + + const { cosineSimilarity: cosineSim } = await import("./embeddings.js"); + vi.mocked(cosineSim).mockReturnValue(TEMPORAL_DEDUP_THRESHOLD + 0.01); + + const out = deduplicateFacts([withEmb, noEmb], embeddings); + // Both kept: id=2 has no embedding so cannot be clustered + expect(out).toHaveLength(2); + }); + + it("returns empty list unchanged", async () => { + const { deduplicateFacts } = await getJme(); + const out = deduplicateFacts([], new Map()); + expect(out).toHaveLength(0); + }); +}); diff --git a/src/memory/jme.ts b/src/memory/jme.ts index 130ff43..d863993 100644 --- a/src/memory/jme.ts +++ b/src/memory/jme.ts @@ -27,6 +27,23 @@ import { logRecall } from "./recall-utility.js"; import { infer } from "../inference/adapter.js"; import { HAIKU_MODEL_ID } from "../inference/claude-sdk.js"; +// ── Phase 3 constants ──────────────────────────────────────────────────────── + +/** + * Cosine similarity threshold above which two facts are considered to be + * about the same topic. When grouping recall candidates, only the most-recent + * fact per cluster survives. Keeps contradictory v1/v3 facts out of the same + * context window. (Phase 3, Pieza 1) + */ +export const TEMPORAL_DEDUP_THRESHOLD = 0.85; + +/** + * When jme_facts with embeddings exceeds this count, the nightly consolidator + * emits a warn log. LIMIT 500 in queryMemory() starts being a bottleneck at + * ~400 rows. (Phase 3, Pieza 2) + */ +export const VECTOR_CEILING_WARN = 400; + // ── Types ──────────────────────────────────────────────────────────────────── export type JmeTurnRole = "user" | "jarvis"; @@ -257,6 +274,71 @@ function extractKeywords(query: string): string[] { * * Falls back to FTS5-only if embedding fails. */ +/** + * Temporal deduplication of recall candidates. + * + * Groups results by semantic similarity (cosine > TEMPORAL_DEDUP_THRESHOLD). + * Within each cluster, keeps only the most-recent fact (highest `ts`). + * Prevents contradictory v1/v3 facts from appearing in the same context window. + * + * Algorithm is O(n²) over the recall set — acceptable because k ≤ 20 and + * this runs entirely in-process with pre-loaded Float32Arrays. + * + * @param results Already-ranked recall results (best first) + * @param queryVec Query embedding used during the original vector search; + * used to re-compute inter-fact similarities. If null, no + * deduplication is performed (FTS5-only path). + * @returns Filtered results with at most one representative per cluster, + * preserving the original ranking order among survivors. + */ +export function deduplicateFacts( + results: JmeRecallResult[], + factEmbeddings: Map, +): JmeRecallResult[] { + if (results.length <= 1) return results; + + const kept: JmeRecallResult[] = []; + // Indices already claimed by an earlier anchor's cluster. + const absorbed = new Set(); + + for (let i = 0; i < results.length; i++) { + if (absorbed.has(i)) continue; + + const anchor = results[i]; + const anchorVec = factEmbeddings.get(anchor.id); + + // If we have no embedding for the anchor, keep it unconditionally — + // we can't meaningfully cluster it. + if (!anchorVec) { + kept.push(anchor); + continue; + } + + // Absorb the FULL cluster around this anchor first, then keep only its + // most-recent member (at the anchor's rank position). Breaking out on + // the first newer candidate — the original shape — left the rest of the + // cluster un-absorbed, so a v1/v2/v3 trio surfaced BOTH v2 and v3 + // (repro: identical embeddings, ts 100/200/300 → [v2, v3]). + let newest = anchor; + for (let j = i + 1; j < results.length; j++) { + if (absorbed.has(j)) continue; + const candidate = results[j]; + const candidateVec = factEmbeddings.get(candidate.id); + if (!candidateVec) continue; + + const sim = cosineSimilarity(anchorVec, candidateVec); + if (sim >= TEMPORAL_DEDUP_THRESHOLD) { + absorbed.add(j); + if (candidate.ts > newest.ts) newest = candidate; + } + } + + kept.push(newest); + } + + return kept; +} + export async function queryMemory( query: string, options: { k?: number; minScore?: number } = {}, @@ -278,6 +360,8 @@ export async function queryMemory( // ── 1. Vector search ─────────────────────────────────────────────────────── const vectorScores = new Map(); + // Phase 3 Pieza 1: stash deserialized embeddings for temporal dedup below + const factEmbeddings = new Map(); let queryVec: Float32Array | null = null; try { @@ -304,6 +388,7 @@ export async function queryMemory( const factVec = deserializeEmbedding(row.embedding); const sim = cosineSimilarity(queryVec, factVec); vectorScores.set(row.id, sim * row.confidence); + factEmbeddings.set(row.id, factVec); } catch { // skip malformed embedding } @@ -389,18 +474,18 @@ export async function queryMemory( }>; // ── 5. Fuse scores ───────────────────────────────────────────────────────── - const results: JmeRecallResult[] = candidateRows + const fused: JmeRecallResult[] = candidateRows .map((row) => { const vScore = vectorScores.get(row.id) ?? 0; const kScore = keywordScores.get(row.id) ?? 0; - const fused = queryVec ? vScore * 0.7 + kScore * 0.3 : kScore; + const fusedScore = queryVec ? vScore * 0.7 + kScore * 0.3 : kScore; return { id: row.id, factText: row.fact_text, category: row.category as JmeFactCategory, sourceTask: row.source_task, - score: fused, + score: fusedScore, ts: row.ts, }; }) @@ -408,6 +493,19 @@ export async function queryMemory( .sort((a, b) => b.score - a.score) .slice(0, k); + // ── 5.5 Temporal dedup (Phase 3 Pieza 1) ────────────────────────────────── + // Group semantically similar facts and keep only the most-recent per cluster. + // Prevents contradictory v1/v3 versions of the same fact from appearing in + // the same context window. Only runs when we have embeddings to compare. + const results = + factEmbeddings.size > 0 ? deduplicateFacts(fused, factEmbeddings) : fused; + + if (results.length < fused.length) { + console.log( + `[jme] temporal dedup: ${fused.length} → ${results.length} facts (removed ${fused.length - results.length} stale duplicate(s))`, + ); + } + // Telemetry: record this recall in the shared audit table with source:'jme' // so JME retrievals are observable alongside the hindsight/sqlite paths. return logAndReturn(results); @@ -669,6 +767,23 @@ export async function consolidateAll(): Promise { role: JmeTurnRole; content: string; }>; + // Phase 3 Pieza 2: vector ceiling surveillance. + // When jme_facts with embeddings approaches the LIMIT 500 in queryMemory(), + // recall quality degrades (oldest facts get cut off). Warn early at 400 so + // there's runway before it becomes a problem. Phase 4 will add auto-pruning. + const embeddingCount = ( + db + .prepare( + "SELECT COUNT(*) as n FROM jme_facts WHERE embedding IS NOT NULL", + ) + .get() as { n: number } + ).n; + if (embeddingCount >= VECTOR_CEILING_WARN) { + console.warn( + `[jme] consolidateAll: vector ceiling warning — ${embeddingCount} facts with embeddings (warn threshold=${VECTOR_CEILING_WARN}, query LIMIT=500). Consider pruning low-confidence facts.`, + ); + } + if (turns.length === 0) return result; result.turnsProcessed = turns.length; if (turns.length === CONSOLIDATOR_MAX_TURNS) {