From b76fe3fba4e77cfc545b84ec16973dff23ebf682 Mon Sep 17 00:00:00 2001 From: ElxMaj <66059051+ElxMaj@users.noreply.github.com> Date: Sun, 12 Jul 2026 03:29:24 +0200 Subject: [PATCH] core+cli: marrow redact part two, cascade and the completeness check (FOUNDER-GATED) Stacked on part one; the same gate applies: neither PR merges without the founder's explicit yes. - --cascade: every citing node is retracted (through the same store path as marrow retract), its text columns tombstoned with a per-node marker (unique, so tombstones never trip the duplicate guard), and its embedding rows deleted; ids, citations, and history survive. - Decided citing nodes refuse without --force, the refusal destroys nothing, and the audit row (written first, append-only, never the secret) names each decided node the human forced over. - marrow redact --check : the completeness audit (tombstone payload, retracted + tombstoned + embedding-free citing nodes); marrow doctor gains a bounded Redactions sweep over every recorded redaction, warning cleanly on schemas from before 0018. - Still CLI-only; the MCP no-redact pin from part one holds. Roadmap: R26 part two of two. Both PRs now await sign-off. Co-Authored-By: Claude Fable 5 --- .changeset/redact-cascade.md | 17 ++++++++ packages/cli/src/cli.ts | 16 +++++-- packages/core/src/doctor.ts | 47 ++++++++++++++++++++ packages/core/src/loop.test.ts | 63 +++++++++++++++++++++++++++ packages/core/src/marrow.ts | 78 ++++++++++++++++++++++++++++++---- packages/core/src/store.ts | 65 ++++++++++++++++++++++++++++ 6 files changed, 274 insertions(+), 12 deletions(-) create mode 100644 .changeset/redact-cascade.md diff --git a/.changeset/redact-cascade.md b/.changeset/redact-cascade.md new file mode 100644 index 0000000..97440d2 --- /dev/null +++ b/.changeset/redact-cascade.md @@ -0,0 +1,17 @@ +--- +"@marrowhq/core": minor +"@marrowhq/cli": minor +--- + +marrow redact, part two: cascade and the completeness check. + +--cascade extends the redaction over the nodes that quote the leaked row: +each is retracted, its text columns tombstoned, and its embedding rows +deleted, while the row ids, citations, and history survive. A decided +citing node refuses without --force (settled truth needs the same explicit +override as a direct retract), the refusal destroys nothing, and the +append-only audit row names every decided node the human forced over, +never the secret. marrow redact --check verifies a redaction end to end +(tombstone, retractions, tombstoned text, zero embeddings), and marrow +doctor sweeps every recorded redaction with a bounded completeness check. +Still CLI-only: no MCP path exists. diff --git a/packages/cli/src/cli.ts b/packages/cli/src/cli.ts index 31a16e7..6a870d8 100644 --- a/packages/cli/src/cli.ts +++ b/packages/cli/src/cli.ts @@ -243,7 +243,8 @@ Add to the room (transcripts in many formats: vtt, srt, json, txt, md): answer --text "..." [--decide ] The human promote-to-decided step retract --reason "..." [--force] Human-only: a false memory stops surfacing (kept, never erased) history The replacement lineage: what replaced what, when, and why - redact --reason "..." Human-only, audited: destroy ONE row's payload (a leaked secret) + redact --reason "..." [--cascade] [--force] Human-only, audited: destroy ONE row's payload (a leaked secret) + redact --check Verify a redaction is complete (also runs in marrow doctor) goal author "" [--type product|user] [--description "..."] [--entity <id>] Author a decided goal (the human commitment path) goal propose "<title>" --type product|user --evidence <id> [--start N --end N] @@ -573,11 +574,20 @@ export async function runCommand(core: Marrow, argv: string[]): Promise<unknown> case "redact": { const evidenceId = positional(rest); + if (rest.includes("--check")) { + if (!evidenceId) throw new Error("Usage: marrow redact --check <evidenceId>"); + return { evidenceId, ...(await core.redactCheck(evidenceId)) }; + } const reason = flagValue(rest, "--reason"); if (!evidenceId || reason === undefined) { - throw new Error('Usage: marrow redact <evidenceId> --reason "what leaked and why"'); + throw new Error( + 'Usage: marrow redact <evidenceId> --reason "what leaked and why" [--cascade] [--force] | marrow redact --check <evidenceId>', + ); } - return core.redact(evidenceId, reason); + return core.redact(evidenceId, reason, { + cascade: rest.includes("--cascade"), + force: rest.includes("--force"), + }); } case "history": { diff --git a/packages/core/src/doctor.ts b/packages/core/src/doctor.ts index aa9256d..c82eeca 100644 --- a/packages/core/src/doctor.ts +++ b/packages/core/src/doctor.ts @@ -75,6 +75,53 @@ export async function doctor( remedy: "Run `marrow migrate`.", }); } + + // Redaction completeness: every redacted evidence row must be a full + // tombstone with its citing nodes retracted, tombstoned, and stripped + // of embeddings. Bounded; skipped cleanly on schemas from before 0018. + try { + const redacted = await pool.query<{ id: string }>( + "select id from evidence where redacted_at is not null order by redacted_at asc limit 100", + ); + if (redacted.rows.length === 0) { + checks.push({ name: "Redactions", status: "ok", detail: "none recorded" }); + } else { + const incomplete: string[] = []; + for (const row of redacted.rows) { + const bad = await pool.query<{ node_id: string }>( + `select p.node_id from provenance p + join evidence e on e.id = p.evidence_id + where p.evidence_id = $1 + and exists ( + select 1 from embedding em + where em.node_id = p.node_id and em.node_kind = p.node_kind) + limit 1`, + [row.id], + ); + if ((bad.rowCount ?? 0) > 0) incomplete.push(row.id); + } + if (incomplete.length === 0) { + checks.push({ + name: "Redactions", + status: "ok", + detail: `${redacted.rows.length} recorded, all complete`, + }); + } else { + checks.push({ + name: "Redactions", + status: "error", + detail: `${incomplete.length} incomplete (${incomplete.slice(0, 3).join(", ")})`, + remedy: "Run `marrow redact --check <evidenceId>` for the exact gaps.", + }); + } + } + } catch { + checks.push({ + name: "Redactions", + status: "warn", + detail: "skipped (schema predates redaction)", + }); + } } catch (err) { const code = (err as { code?: string }).code; checks.push({ diff --git a/packages/core/src/loop.test.ts b/packages/core/src/loop.test.ts index 55e82c3..a53c4a3 100644 --- a/packages/core/src/loop.test.ts +++ b/packages/core/src/loop.test.ts @@ -736,6 +736,69 @@ describe("agent decision gate and truth maintenance", () => { await expect(core.redact(cited.id, " ")).rejects.toThrow(/reason is required/); }); + it("redact --cascade retracts, tombstones, and strips citing nodes, then --check passes", async () => { + const leak = await store.insertEvidence({ + text: "the deploy key is horse-battery-staple-42, keep it quiet", + source: "session/cascade-leak.md", + }); + const openNode = (await core.proposeNode({ + kind: "decision", + title: "Deploy key rotation is manual", + provenance: [{ evidenceId: leak.id, start: 0, end: 14 }], + })) as { id: string }; + const decidedNode = await store.insertDecision({ + title: "Deploys use the shared key", + rationale: "", + constraint: false, + status: "decided", + confidence: human, + provenance: [{ evidenceId: leak.id, start: 0, end: 14 }], + }); + + // cascade without force refuses, naming the decided node: settled truth + // needs the same explicit override as a direct retract. + await expect(core.redact(leak.id, "leaked key", { cascade: true })).rejects.toThrow( + decidedNode.id, + ); + // and nothing was destroyed by the refusal. + expect((await store.getEvidence(leak.id))?.text).toContain("horse-battery"); + + const receipt = await core.redact(leak.id, "leaked key", { cascade: true, force: true }); + expect(receipt.retractedNodeIds.sort()).toEqual([openNode.id, decidedNode.id].sort()); + + // the payload is gone; the audit names the forced decided node, never the secret. + expect((await store.getEvidence(leak.id))?.text).toBe("[redacted: leaked key]"); + const audit = await store.getEvidence(receipt.auditEvidenceId); + expect(audit?.text).toContain(`forced over decided node ${decidedNode.id}`); + expect(audit?.text).not.toContain("horse-battery"); + + // citing nodes: retracted, tombstoned, embedding-free, still inspectable. + for (const id of receipt.retractedNodeIds) { + const node = await core.getNode(id); + expect(node?.status).toBe("retracted"); + const title = node && "title" in node ? node.title : ""; + expect(title.startsWith("[redacted")).toBe(true); + expect(await store.hasEmbedding(id, node?.kind ?? "decision")).toBe(false); + } + + // the completeness audit agrees, and doctor's sweep has nothing to flag. + const check = await core.redactCheck(leak.id); + expect(check.problems).toEqual([]); + expect(check.ok).toBe(true); + + // break it deliberately: a stray embedding row must fail the check. + await store.insertEmbedding({ + nodeId: openNode.id, + nodeKind: "decision", + model: "fake-emb", + dim: 4, + vector: [0, 0, 0, 0], + }); + const broken = await core.redactCheck(leak.id); + expect(broken.ok).toBe(false); + expect(broken.problems.join(" ")).toContain("embedding"); + }); + it("maintainTruth surfaces the undistilled evidence backlog with a drain action", async () => { // a session-end hook write: evidence appended, never distilled. await store.insertEvidence({ diff --git a/packages/core/src/marrow.ts b/packages/core/src/marrow.ts index dd65851..7fd62ca 100644 --- a/packages/core/src/marrow.ts +++ b/packages/core/src/marrow.ts @@ -2307,7 +2307,8 @@ export class Marrow { async redact( evidenceId: string, reason: string, - ): Promise<{ evidenceId: string; auditEvidenceId: string }> { + opts: { cascade?: boolean; force?: boolean } = {}, + ): Promise<{ evidenceId: string; auditEvidenceId: string; retractedNodeIds: string[] }> { if (!reason || reason.trim().length === 0) { throw new Error("redact: a reason is required"); } @@ -2317,21 +2318,80 @@ export class Marrow { throw new Error(`redact: evidence ${evidenceId} is already redacted`); } const citing = await this.store.getNodesForEvidence(evidenceId); - if (citing.length > 0) { - const list = citing - .map((node) => `${node.id} [${node.status}] ${nodeTitle(node)}`) - .join("; "); + const describe = (node: Distilled): string => `${node.id} [${node.status}] ${nodeTitle(node)}`; + if (citing.length > 0 && !opts.cascade) { throw new Error( - `redact: ${citing.length} distilled node${citing.length === 1 ? "" : "s"} cite this evidence and would keep quoting it: ${list}. Cascade support arrives with the second redaction PR; until then retract the citing nodes first.`, + `redact: ${citing.length} distilled node${citing.length === 1 ? "" : "s"} cite this evidence and would keep quoting it: ${citing.map(describe).join("; ")}. Re-run with --cascade to retract and tombstone them too.`, ); } - await this.store.redactEvidence(evidenceId, reason); + // the blast radius rule for settled truth: cascading over a DECIDED node + // needs the same explicit force as retracting one directly. + const decided = citing.filter((node) => node.status === "decided"); + if (decided.length > 0 && !opts.force) { + throw new Error( + `redact: --cascade would retract ${decided.length} DECIDED node${decided.length === 1 ? "" : "s"}: ${decided.map(describe).join("; ")}. Settled truth is normally replaced through the answer loop; pass --force to cascade over it anyway.`, + ); + } + // the audit trail is ordinary append-only evidence, and never the secret. + // It is written FIRST so every cascaded retraction can cite it, and it + // names each decided node the human forced over. + const auditLines = [ + `redacted ${evidenceId}: ${reason}`, + ...(citing.length > 0 + ? [`cascaded over ${citing.length} citing node${citing.length === 1 ? "" : "s"}.`] + : []), + ...decided.map((node) => `forced over decided node ${node.id} (${nodeTitle(node)}).`), + ]; const audit = await this.store.insertEvidence({ - text: `redacted ${evidenceId}: ${reason}`, + text: auditLines.join("\n"), source: `redactions/${evidenceId}`, }); - return { evidenceId, auditEvidenceId: audit.id }; + const auditSpan = { evidenceId: audit.id, start: 0, end: auditLines[0]?.length ?? 1 }; + + const retractedNodeIds: string[] = []; + for (const node of citing) { + if (node.status !== "retracted") { + await this.store.retract(node.id, node.kind, auditSpan); + } + await this.store.tombstoneNodeText( + node.id, + node.kind, + `[redacted ${node.id}: cited ${evidenceId}]`, + ); + await this.store.deleteEmbeddingsFor(node.id, node.kind); + retractedNodeIds.push(node.id); + } + + await this.store.redactEvidence(evidenceId, reason); + return { evidenceId, auditEvidenceId: audit.id, retractedNodeIds }; + } + + /** + * The completeness audit for one redaction: the payload is the tombstone, + * every citing node is retracted with tombstoned text, and no embedding + * rows remain for them. Read-only; doctor runs it over every redacted row. + */ + async redactCheck(evidenceId: string): Promise<{ ok: boolean; problems: string[] }> { + const problems: string[] = []; + const evidence = await this.store.getEvidence(evidenceId); + if (!evidence) return { ok: false, problems: [`evidence ${evidenceId} not found`] }; + if (evidence.redactedAt === undefined) problems.push("redacted_at is not set"); + if (!evidence.text.startsWith("[redacted:")) problems.push("payload is not the tombstone"); + for (const node of await this.store.getNodesForEvidence(evidenceId)) { + // every node still citing the row once quoted the secret: each must be + // retracted, tombstoned, and stripped of its embedding. + if (node.status !== "retracted") { + problems.push(`${node.id} is ${node.status}, not retracted`); + } + if (!nodeTitle(node).startsWith("[redacted")) { + problems.push(`${node.id} text is not tombstoned`); + } + if (await this.store.hasEmbedding(node.id, node.kind)) { + problems.push(`${node.id} still has an embedding row`); + } + } + return { ok: problems.length === 0, problems }; } /** diff --git a/packages/core/src/store.ts b/packages/core/src/store.ts index 55fa203..9ab3596 100644 --- a/packages/core/src/store.ts +++ b/packages/core/src/store.ts @@ -418,6 +418,71 @@ export class Store { } } + /** Cascade support for redaction: overwrite a citing node's text columns + * with a per-node tombstone. Only the redaction path calls this. */ + async tombstoneNodeText(nodeId: string, kind: DistilledKind, tombstone: string): Promise<void> { + if (kind === "entity") { + await this.pool.query("update entity set name = $2, description = null where id = $1", [ + nodeId, + tombstone, + ]); + } else if (kind === "decision") { + await this.pool.query("update decision set title = $2, rationale = '' where id = $1", [ + nodeId, + tombstone, + ]); + } else if (kind === "goal") { + await this.pool.query("update goal set title = $2, description = null where id = $1", [ + nodeId, + tombstone, + ]); + } else { + await this.pool.query("update question set prompt = $2 where id = $1", [nodeId, tombstone]); + } + } + + /** Remove a node's embedding rows: the secret's vector is derived data, and + * deleting it matches the dedupe-delete precedent. */ + async deleteEmbeddingsFor(nodeId: string, kind: DistilledKind): Promise<void> { + await this.pool.query("delete from embedding where node_id = $1 and node_kind = $2", [ + nodeId, + kind, + ]); + } + + /** Whether a node still has an embedding row (the completeness audit). */ + async hasEmbedding(nodeId: string, kind: DistilledKind): Promise<boolean> { + const res = await this.pool.query( + "select 1 from embedding where node_id = $1 and node_kind = $2 limit 1", + [nodeId, kind], + ); + return (res.rowCount ?? 0) > 0; + } + + /** Every redacted evidence row, oldest first, bounded: the completeness + * audit's worklist. */ + async listRedactedEvidence(limit = 100): Promise<Evidence[]> { + const res = await this.pool.query<{ + id: string; + text: string; + source: string; + created_at: Date; + redacted_at: Date; + }>( + `select id, text, source, created_at, redacted_at from evidence + where redacted_at is not null order by redacted_at asc limit $1`, + [limit], + ); + return res.rows.map((row) => ({ + id: row.id, + kind: "evidence" as const, + text: row.text, + source: row.source, + createdAt: iso(row.created_at), + redactedAt: iso(row.redacted_at), + })); + } + async getEvidence(id: string): Promise<Evidence | undefined> { const res = await this.pool.query<{ id: string;