fix(memory): make evidence and corrections durable across replay

.github/workflows/ci.yml

@@ -152,6 +152,9 @@ jobs:
             bridge_intelligence.py \
             tests/contract/test_phase5_invariants.py \
             supervisor/dispatch.py supervisor/collector.py supervisor/router.py \
+            engines/synthesis.py intelligence/memory/hybrid_retriever.py \
+            intelligence/memory/maintenance.py scripts/memory_maintenance.py \
+            tests/test_correction_acceptance.py \
             store_registry.py tests/test_dispatch_integrity.py \
             tests/test_orchestration_pipeline.py tests/test_model_router.py \
             tests/test_doctor_json_reset.py tests/test_mcp_research_tools.py tests/network_guard.py
@@ -247,6 +250,31 @@ jobs:
           /tmp/wheelvenv/bin/python -c "import cli, mcp_server, store_registry, recall_confidence; print('wheel imports OK')"
           /tmp/wheelvenv/bin/cortex --help
 
+      - name: Installed correction acceptance
+        run: |
+          cd /tmp
+          /tmp/wheelvenv/bin/python - <<'PYTHON'
+          import json, os, subprocess, sys, tempfile
+          with tempfile.TemporaryDirectory() as state:
+              os.environ["CORTEX_STATE_DIR"] = state
+              os.environ["CORTEX_HOME"] = state + "/wrong"
+              from mcp_handlers import record_learning_decision
+              from engines.synthesis import ContextGraph
+              from intelligence.memory.hybrid_retriever import HybridRetriever
+              old = record_learning_decision("obsolete canary advice")
+              graph = ContextGraph()
+              graph.import_decisions()
+              recall = HybridRetriever([], include_conversation_digests=False)
+              new = record_learning_decision("corrected canary advice", supersedes=old["decision_id"])
+              expected = ["decision:" + new["decision_id"]]
+              assert [p.id for p, _ in recall.search("canary", alpha=0)] == expected
+              assert [n.id for n in graph.query("canary")] == expected
+              output = subprocess.check_output([sys.executable, "-m", "intelligence.memory.maintenance", "--graph-only"], text=True)
+              assert json.loads(output)["ok"] is True
+              assert [n.id for n in ContextGraph().query("canary")] == expected
+          print("Installed correction acceptance passed")
+          PYTHON
+
   integration:
     # Separate job so integration tests don't bitrot. They're skipped by
     # default in `pytest` (see addopts in pyproject.toml); this job runs

api/routes/decisions.py

@@ -17,17 +17,15 @@
 
 from __future__ import annotations
 
-import json
-import time
+import uuid
 from datetime import datetime
-from pathlib import Path
 from typing import Any, Dict, Optional
 
 from fastapi import APIRouter, HTTPException
 from pydantic import BaseModel, Field
 
 
-DECISIONS_FILE = Path.home() / ".cortex" / "decisions.jsonl"
+DECISIONS_FILE = None  # Optional override; resolve configured state at call time.
 
 
 class DecisionRecordRequest(BaseModel):
@@ -57,8 +55,10 @@ async def record_decision(req: DecisionRecordRequest) -> Dict[str, Any]:
     persisted record.
     """
     try:
-        DECISIONS_FILE.parent.mkdir(parents=True, exist_ok=True)
-        decision_id = f"dec_{int(time.time())}_{req.prediction_id[:8]}"
+        from intelligence.durable_jsonl import append_decision
+        from state_paths import get_cortex_dir
+
+        decision_id = f"dec_{uuid.uuid4().hex[:12]}"
         entry = {
             "decision_id": decision_id,
             "prediction_id": req.prediction_id,
@@ -68,8 +68,7 @@ async def record_decision(req: DecisionRecordRequest) -> Dict[str, Any]:
             "override_reason": req.override_reason,
             "timestamp": datetime.now().isoformat(),
         }
-        with open(DECISIONS_FILE, "a", encoding="utf-8") as f:
-            f.write(json.dumps(entry) + "\n")
+        append_decision(DECISIONS_FILE or get_cortex_dir() / "decisions.jsonl", entry)
         return {
             "recorded": True,
             "decision_id": decision_id,
@@ -93,7 +92,10 @@ class LearningDecisionRequest(BaseModel):
     alternatives: str = Field(default="", description="Other options considered")
     rationale: str = Field(default="", description="Why this option was chosen over alternatives")
     project: str = Field(default="", description="Project this decision belongs to (optional)")
-    supersedes: str = Field(default="", description="decision_id of a prior decision this one replaces (tombstoned out of recall)")
+    supersedes: str = Field(
+        default="",
+        description="decision_id of a prior decision this one replaces (tombstoned out of recall)",
+    )
 
 
 @router.post("/decisions/learning")

cli/commands/demo.py

@@ -7,8 +7,8 @@
 No API key required. No network. Read-only against the system Cortex install
 (uses a tempdir for its data).
 
-This is the falsifiable demonstration of the "compounding intelligence" claim:
-if it prints links, the FK contract is live in this build.
+This demonstrates synthetic advisory associations, not causal learning or
+verified user outcomes.
 """
 
 from __future__ import annotations
@@ -87,7 +87,7 @@ def cmd_demo(args) -> None:
         print("⚠ No links produced — FK contract is BROKEN in this build.")
         sys.exit(1)
 
-    print("FK trail:")
+    print("Advisory FK trail (unverified; not learning eligible):")
     print("─" * 56)
     for entry in linked:
         score = entry["outcome_score"]
@@ -99,7 +99,7 @@ def cmd_demo(args) -> None:
         print(f"  {flag} [score {score:>4.2f}] {prompt[:48]:<48}{commit_str}")
     print("─" * 56)
     print()
-    print(f"Score components (per intelligence/outcome_linker.py):")
+    print("Score components (per intelligence/outcome_linker.py):")
     print("  0.4 * test_pass_ratio  +  0.4 * commit_landed  +  0.2 * activity")
     print()
     print("This output was generated with NO API key and NO network call.")

docs/design/durable-evidence-memory.md

@@ -0,0 +1,123 @@
+# Durable evidence and memory corrections
+
+This tranche builds on truthful execution (PR #21). It repairs demonstrated
+linkage and memory failures without certifying task outcomes or enabling adaptive
+learning. No state migration or runtime cutover is performed by the code change.
+
+## Evidence contract
+
+- Only git_commit and test_result events are associated. Session must be known;
+  session and project must match exactly (including a missing project).
+- Producer prompt_id/event_id identities are scoped by session and project.
+  Legacy prompt identities use a content hash, not queue position. Indistinguishable
+  legacy duplicates cannot be separated; producer IDs are the migration target.
+- An explicit prompt_id on evidence may link after 90 seconds and after a later
+  prompt. Unreferenced events remain temporal hints within 90 seconds of a unique
+  latest prompt. Explicit reference does not prove causality or verification.
+- The persisted outcome_score is retained for demo/reader compatibility as an
+  activity heuristic. All records remain verification_status=unverified and
+  learning_eligible=false. This does not switch off every other Cortex heuristic.
+- Writers lock a separate file, read the current snapshot, merge evidence by
+  stable identity, and atomically replace the JSONL file. Concurrent/stale workers
+  cannot erase newer evidence. Conflicting IDs fail the transaction. Revisions
+  retain evidence digests, counts and heuristic scores; evidence itself is retained.
+- Old position-based rows remain audit records and are not silently matched to
+  new identities. Historical duplicate IDs require explicit repair; malformed
+  lines are preserved. Old and new identities may coexist and inflate raw counts
+  until a reviewed migration; historical metrics are not proof of learning.
+- New/revised evidence must have a new event ID. Reusing an ID with changed content
+  is rejected. Retraction and evaluator revision schemas remain future work.
+
+## Memory contract
+
+The primary journal update commits the replacement decision and its compatibility
+tombstone in one snapshot. Readers also honor the replacement's own supersedes
+field, repairing recall for historical partial writes and spooled corrections.
+Original journal lines are retained verbatim; the physical file is replaced.
+
+MCP decision writes, the HTTP decision route, spool replay and importance backfill
+use the same lock. A temporary snapshot is fsynced before replacement and the
+parent directory is fsynced before acknowledgement. A failed primary write falls
+back to an atomically written spool; if both fail, the call raises. A spooled
+record is pending replay, not immediately recallable. Conflicting replay content
+is retained for inspection rather than silently discarded.
+
+Decision indexing and hybrid retrieval honor CORTEX_STATE_DIR before CORTEX_HOME.
+The hybrid retriever replaces cached decision patterns with the live journal at
+construction, so a stale pattern index cannot resurrect corrected advice. Cache
+identity includes the state path, inode, nanosecond timestamp and size. Long-lived
+retrievers refresh journal decisions before search; a changed decision set
+invalidates that instance's vectors and uses current BM25 until reconstruction.
+Graph query/traversal APIs reconcile decision status against the journal on reads.
+Retired graph nodes and edges remain audit records, excluded from active context.
+An absent/unreadable journal hides imported decision nodes. This is not
+system-wide memory erasure or a guarantee for callers reading raw graph storage.
+
+## Deployment and repository alignment
+
+These are POSIX local-file guarantees for cooperating writers on Linux/macOS.
+They are not distributed transactions, network-filesystem guarantees or machine
+memory synchronization. Updates rewrite the file and cost O(file size). Before
+cutover, stop old writers, back up the state, validate the journal, install one
+reviewed version, and verify its actual imported modules and state directory.
+For growing multi-writer workloads, prefer a measured SQLite migration with one
+local owner, then a service protocol if multiple machines need concurrent access.
+
+The standalone main remains 90ca32d8 at review. PR #21 is open; this tranche is
+stacked on its tested head 8d75a115. Repository reconciliation remains a separate
+reviewed process: preserve unique integrations, port reusable behavior through
+PRs, and avoid publishing an entire divergent tree over the standalone source.
+No observed repository change establishes what either live machine imports.
+
+## Verification and remaining acceptance gates
+
+Failure tests cover cross-session/project attribution, stable IDs, late evidence,
+stale replay, concurrent writers, conflict rollback, interrupted replacement,
+spool failure and correction replay. A fresh-process test records a correction
+through the public handler, reindexes in another process, and searches through
+HybridRetriever using real local BM25 with no state-path monkeypatches.
+
+Still required: producer IDs wired end to end; evaluator evidence tied to a task,
+attempt and acceptance criterion; live
+machine path checks; one bounded pilot task that resumes after interruption and
+produces independently verified user-visible output. Green synthetic tests alone
+do not establish trusted compounding or autonomous project progress.
+
+## Reassessment: finish a user outcome before expanding architecture
+
+The first acceptance outcome is: record advice, correct it, then retrieve only
+the correction through both recall and graph context in a new process. It must
+also work in a long-lived reader, and after graph saving is interrupted. Old
+nodes remain auditable; no graph path may use them as active context. Maintenance
+must report failure when persistence fails. The new correction acceptance suite
+exercises these conditions, including real local embedding-cache invalidation.
+
+The decision journal is authoritative. The graph is a derived view; its two
+legacy snapshot files are not a multi-file transaction. Reconciliation on reads
+protects the correction outcome even if graph persistence is interrupted. This
+does not repair every graph concurrency or historical corruption problem.
+
+Do not equate this completed acceptance path with useful autonomous learning.
+Next, install one reviewed artifact on one development machine and exercise one
+real project task with explicit user acceptance. Resolve the runtime dependency
+closure needed for that pilot before cutover; keep the broader repository
+inventory as a staged reconciliation queue. A blanket review of every historical
+file is not the release gate for one isolated pilot.
+
+Defer general-purpose self-improvement, cross-machine memory synchronization,
+and broad unattended orchestration until the pilot produces accepted work,
+recovers after interruption, and uses corrected context. Model-generated
+success scores and a larger test count cannot substitute for that evidence.
+
+The installed pilot command is:
+
+```sh
+python -m intelligence.memory.maintenance --graph-only
+```
+
+Run it with CORTEX_STATE_DIR pointing at the pilot state. A persistence failure
+returns a nonzero exit status. The existing scripts/memory_maintenance.py remains
+a source-checkout compatibility entrypoint. The package command was added because
+the installed-wheel acceptance test exposed that scripts/ is excluded from the
+wheel. The default full maintenance job also emits operational failure signals;
+that separate learning path is not part of this correction acceptance proof.