diff --git a/HANDOFF.md b/HANDOFF.md index 3d390e4..d4b5e94 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -4,10 +4,10 @@ - GitHub: `https://github.com/Jdelg718/BotFucker` - Default branch: `main` -- Latest merged milestone: Phase 12 real n8n import/dry-run validation (`docs: validate n8n import dry run`, PR #13) -- Current working branch: `phase-13-reviewed-action-bridge-promotion-plan` -- Current PR target: Phase 13 Reviewed Action Bridge Promotion Plan — docs/tests only, no OAuth/live provider mutation -- Current local demo target: demonstrate deterministic local review, optional mocked LLM classifier fallback/validation, approved-action export, dry-run n8n bridge contract, fail-closed YOLO policy checks, real n8n import validation results, and reviewed bridge-promotion gate +- Latest merged milestone: Phase 13 Reviewed Action Bridge Promotion Plan (`docs: plan reviewed action bridge promotion`, main at 4619bf5) +- Current working branch: `phase-14-durable-bridge-ledger` +- Current PR target: Phase 14 Durable Bridge Ledger Scaffold — code/docs/tests only, no OAuth/live provider mutation +- Current local demo target: demonstrate deterministic local review, optional mocked LLM classifier fallback/validation, approved-action export, dry-run n8n bridge contract, fail-closed YOLO policy checks, real n8n import validation results, reviewed bridge-promotion gate, and durable bridge ledger dedupe scaffold - Current promo artifact: `promo/botfucker-animated-explainer/renders/botfucker-animated-explainer_narrated-final.mp4` ## What BotFucker Is @@ -35,6 +35,7 @@ docs/n8n-approved-action-bridge.json # importable n8n approved-action dry-run br docs/n8n-approved-action-bridge.md # approved-action bridge operator guide docs/provider-auth-plan.md # provider auth/action boundary plan docs/reviewed-action-bridge-promotion-plan.md # Phase 13 reviewed live-bridge gate; no OAuth/live mutation +docs/bridge-ledger-scaffold.md # Phase 14 durable processed-audit ledger scaffold; no OAuth/live mutation README.md # user-facing setup and project overview outreach_filter.py # compatibility CLI wrapper botfucker/models.py # normalized email/classification/review models @@ -42,6 +43,7 @@ botfucker/classifier.py # deterministic classifier botfucker/history.py # SQLite sender history + strike state botfucker/review_queue.py # review item/audit models and sample data helpers botfucker/review_store.py # durable SQLite review queue and audit store +botfucker/bridge_ledger.py # durable bridge processed-audit ledger scaffold botfucker/review_cli.py # durable local review CLI botfucker/webhook_contract.py # n8n/webhook payload sanitizer/import adapter botfucker/local_ui.py # local browser review UI server @@ -64,6 +66,7 @@ These are non-negotiable: - n8n/provider credentials stay in n8n or the provider layer, not BotFucker core. - Local UI and review CLI actions affect SQLite review state only. - Provider-side actions are future bridge work, not local UI behavior. +- The Phase 14 durable bridge ledger records `audit_id` state only (`bridge_ledger_state_only`) before provider mutation; it is not a provider action engine. ## Current Test Commands @@ -267,15 +270,15 @@ python3 -m unittest discover -s tests -v ## Next PR Recommendation -After Phase 13, keep OAuth on hold. The next safe step is either documentation review cleanup for the promotion gate or a mocked/sandbox-only processed-audit state prototype that still performs no live provider mutation. +After Phase 14, keep OAuth on hold. The next safe step is emergency-stop proof or a sandbox-only bridge rehearsal that uses the durable bridge ledger and still performs no production provider mutation. Do **not** add real OAuth, provider credentials, or live n8n provider mutation nodes until the Phase 13 gate has Rex/Gus review and provider-specific sandbox evidence. Suggested follow-up scope: -1. Review Phase 13 plan with Rex/Gus. -2. Decide the first provider/action pair for sandbox review, likely `approve_warning` only. -3. Prototype processed-`audit_id` state with fake/sample data only. +1. Review Phase 14 ledger scaffold with Rex/Gus. +2. Keep the first provider/action pair to `approve_warning` only. +3. Use processed-`audit_id` state before any provider mutation attempt. 4. Keep credentials in n8n only. 5. Require rollback and emergency-stop proof before any live provider action node is connected. @@ -284,18 +287,18 @@ Suggested follow-up scope: ```text You are working on BotFucker, an AI-era inbox defense app. -Read DESIGN.md, ROADMAP.md, HANDOFF.md, README.md, docs/webhook-contract.md, docs/n8n-workflow.md, docs/n8n-approved-action-bridge.md, docs/n8n-import-validation.md, and docs/provider-auth-plan.md. +Read DESIGN.md, ROADMAP.md, HANDOFF.md, README.md, docs/webhook-contract.md, docs/n8n-workflow.md, docs/n8n-approved-action-bridge.md, docs/n8n-import-validation.md, docs/provider-auth-plan.md, docs/reviewed-action-bridge-promotion-plan.md, and docs/bridge-ledger-scaffold.md. -First, verify the current Phase 12 branch without changing behavior: +First, verify the current Phase 14 branch without changing behavior: - run python3 scripts/validate_n8n_workflow_exports.py - run python3 -m py_compile outreach_filter.py botfucker/*.py scripts/validate_n8n_workflow_exports.py - run python3 -m unittest discover -s tests -v -- inspect docs/n8n-import-validation.md and samples/approved-actions.sample.json -- confirm n8n workflows include explicit ids, are inactive, and use /home/node/.n8n-files for Read/Write Files paths +- inspect docs/bridge-ledger-scaffold.md and botfucker/bridge_ledger.py +- confirm the ledger stores durable audit_id/status data only and has no provider calls or credential fields -Then review Phase 12 only: Real n8n Import/Dry-Run Validation. +Then review Phase 14 only: Durable Bridge Ledger Scaffold. -Check that both workflows imported into n8n-vps as inactive/manual, approved-action bridge executed sample-only dry-run, final output was provider_execution:not_performed, and cleanup removed validation rows/temp files. +Check that claim_action records pending state before provider mutation, duplicate audit_id claims fail closed, unsafe approved-action exports are rejected, and no OAuth/provider credentials/live provider mutation nodes were added. Do not add real OAuth. Do not add provider credentials. Do not attach Gmail/Microsoft/IMAP/SMTP mutation credentials. Do not enable live n8n provider actions. Preserve the provider boundary: live provider execution remains separately reviewed and guarded. ``` @@ -303,9 +306,9 @@ Do not add real OAuth. Do not add provider credentials. Do not attach Gmail/Micr ## Team Plan - **Amy**: orchestration and scope control. She keeps the product from wandering into OAuth swamp country before bridge promotion is reviewed. -- **Chip**: owns reviewed action bridge promotion docs/tests if Phase 12 review passes. +- **Chip**: owns durable bridge ledger scaffold and bridge promotion safety docs/tests. - **Rex**: security veto on processed-audit dedupe, credential absence, live-action safety gates, provider-boundary isolation, and XSS regressions. -- **Gus**: n8n operator verification, dry-run bridge observability, cleanup steps, CI, and operator docs. +- **Gus**: n8n operator verification, dry-run bridge observability, cleanup steps, CI, ledger operability, and operator docs. - **Fred**: provider sandbox/action-limit research only; no direct OAuth implementation yet. ## Known Follow-Up Issues @@ -315,13 +318,14 @@ Do not add real OAuth. Do not add provider credentials. Do not attach Gmail/Micr - No production OAuth yet. - Real n8n import validation passed on n8n-vps with sample-only dry-run and cleanup; do not activate those workflows without a separate reviewed bridge-promotion plan. - n8n approved action bridge is dry-run only; live provider actions still need a separate explicit reviewed workflow. +- Phase 14 durable bridge ledger scaffold exists for processed-`audit_id` dedupe, but it is not connected to live provider mutation. - YOLO guardrails exist but live provider actions still require explicit operator configuration and must not be casually enabled. ## Tomorrow Restart -- PR #13 is open and CI green: `https://github.com/Jdelg718/BotFucker/pull/13`. -- First move tomorrow: re-check PR #13, merge if green, pull `main`, branch Phase 13. -- Phase 13 target: **Reviewed Action Bridge Promotion Plan**. +- Phase 14 branch is local: `phase-14-durable-bridge-ledger`. +- First move next: re-check tests, review diff, and open/squash Phase 14 if green. +- Phase 15 target should be emergency-stop proof or sandbox-only bridge rehearsal using the durable ledger. - Do **not** add OAuth, provider credentials, or live n8n provider mutation nodes. - Keep live provider execution separate, reviewed, audited, deduped by processed `audit_id`, rollback-ready, and security/ops-reviewed. diff --git a/README.md b/README.md index f29618d..df23141 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,7 @@ The current core is split into reusable modules under `botfucker/`: - `classifier.py` returns structured deterministic classifications with reasons. - `history.py` tracks sender history, warning counts, and strike levels in SQLite. - `review_store.py` persists local review queue items and audit events in SQLite. +- `bridge_ledger.py` provides a durable bridge ledger scaffold keyed by approved-action `audit_id` values. - `webhook_contract.py` normalizes bounded n8n/webhook email JSON into local review items. - `review_cli.py` provides a provider-safe local review workflow around seeded/imported items. - `responses.py` contains human-reviewable warning templates. @@ -37,6 +38,7 @@ See [DESIGN.md](DESIGN.md) for the proposed architecture and roadmap. - Imports bounded n8n/webhook JSON after the provider layer has already fetched mail. - Exports approved local audit events as an idempotent JSON bundle for an n8n/provider bridge. - Provides an inactive n8n approved-action bridge starter that validates/dedupes actions in dry-run mode. +- Provides a Phase 14 durable bridge ledger scaffold for recording processed `audit_id` state before provider mutation. - Keeps provider credentials and live mailbox side effects outside the local UI and review queue. ## Safety First @@ -487,6 +489,23 @@ Artifact: The plan requires persistent processed-`audit_id` state, rollback and emergency-stop procedures, provider-specific sandbox/manual tests, and Rex/Gus review before any live mutation node is connected. +## Phase 14 Durable Bridge Ledger Scaffold + +Phase 14 adds a durable bridge ledger scaffold for future reviewed provider bridges. It is not OAuth, not provider auth, and not live mailbox automation. The scaffold records approved-action `audit_id` state before provider mutation so a future bridge can fail closed on duplicates. + +Artifacts: + +- [`botfucker/bridge_ledger.py`](botfucker/bridge_ledger.py) — standard-library SQLite ledger keyed by `audit_id`, with `pending`, `processed`, `failed`, and `rolled_back` states. +- [`docs/bridge-ledger-scaffold.md`](docs/bridge-ledger-scaffold.md) — operator/security notes for using the ledger before any provider mutation. + +Safety constraints: + +- effect scope is `bridge_ledger_state_only` +- validates `botfucker.approved_actions.v1`, `provider_action_export_only`, and `provider_execution: not_performed` +- stores IDs/status only, not subject, snippet, body, headers, OAuth tokens, API keys, passwords, cookies, or private provider headers +- no OAuth, no provider credentials, and no live provider mutation nodes are added +- checked-in n8n workflows remain inactive/dry-run starters + ## Test Before Going Live Compile-check the script and package: diff --git a/ROADMAP.md b/ROADMAP.md index 34632c6..a8dc5ca 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -28,7 +28,9 @@ Merged through PR #8: - provider auth boundary plan - FF2K-branded local browser UI and hero art - HyperFrames animated/narrated product explainer under `promo/botfucker-animated-explainer/` -- tests for classifier/history/safety/review/webhook/docs/branding behavior +- reviewed live-bridge promotion gate +- durable bridge ledger scaffold for processed `audit_id` state before provider mutation +- tests for classifier/history/safety/review/webhook/docs/branding/bridge-ledger behavior The repo is ready to pull locally into Kodex/Codex and demonstrate the local cockpit without connecting to any live mail provider. @@ -317,6 +319,28 @@ Acceptance criteria: - No live provider credentials are committed or exported. - No OAuth, live provider mutation nodes, or provider behavior changes are added by this phase. +### Phase 14 — Durable Bridge Ledger Scaffold ✅ + +Status: implemented on `phase-14-durable-bridge-ledger`. + +Goal: provide a durable processed-`audit_id` ledger scaffold that a future reviewed bridge can use before any provider mutation, without adding OAuth, credentials, or live mutation nodes. + +Delivered: + +- `botfucker.bridge_ledger.DurableBridgeLedger` SQLite scaffold keyed by `audit_id` +- `pending`, `processed`, `failed`, and `rolled_back` statuses +- `claim_action()` flow that inserts durable `pending` state before provider mutation +- validation for `botfucker.approved_actions.v1`, `provider_action_export_only`, and `provider_execution: not_performed` +- docs in `docs/bridge-ledger-scaffold.md` +- tests proving dedupe, status transitions, unsafe export rejection, and no message-content/secret columns + +Acceptance criteria: + +- Durable state is keyed by `audit_id`. +- Repeated claims of an `audit_id` do not acquire a second mutation slot. +- Ledger stores IDs/status only, not message body/header/credential material. +- No OAuth, no provider credentials, no live provider mutation nodes, and no checked-in n8n activation changes are added. + ## Local Kodex/Codex Demo Plan Kent is pulling this locally onto Kodex/Codex next. The demo should show what exists now, not pretend Phase 8 is already done. Revolutionary concept, apparently. @@ -363,27 +387,27 @@ Use fake or sanitized JSON only. Real mailbox payloads stay out of the repo. ## Near-Term Recommendation -After Phase 13 is reviewed and merged, the next PR should be **Phase 14: durable bridge ledger design or sandbox-only live bridge scaffold**, not broad OAuth implementation. +After Phase 14 is reviewed and merged, the next PR should be **Phase 15: emergency-stop proof or sandbox-only bridge rehearsal**, not broad OAuth implementation. Recommended scope: -- choose one provider/action pair only -- implement or document the durable processed-`audit_id` ledger before any provider mutation +- keep one provider/action pair only (`approve_warning`) +- use the durable bridge ledger before any provider mutation - keep credentials in n8n only - keep dry-run as the default path - prove emergency stop exits before provider mutation - require provider-specific sandbox/manual tests - require Rex/Gus security/ops review before any live mutation node is connected -OAuth can still wait. We proved the n8n package imports and dry-runs in the real beast, and Phase 13 defines the promotion gate. Next is one tiny, reviewed step toward a provider bridge — not a live-mail fireworks show. +OAuth can still wait. Phase 14 gives the bridge a seatbelt: durable dedupe before action. Next is proving the brakes, not flooring it into a live inbox. -### Restart checklist after Phase 13 +### Restart checklist after Phase 14 -1. Re-check Phase 13 PR CI and mergeability. -2. Squash-merge Phase 13 into `main` if still green. -3. Pull updated `main` and create a Phase 14 branch. -4. Pick exactly one sandbox provider/action pair, likely `approve_warning` only if Kent explicitly wants reply-send tested. -5. Build durable dedupe/ledger scaffolding and emergency-stop proof before wiring any provider mutation. +1. Re-check Phase 14 branch CI and mergeability. +2. Squash-merge Phase 14 into `main` if still green. +3. Pull updated `main` and create a Phase 15 branch. +4. Keep exactly one sandbox provider/action pair, likely `approve_warning` only if Kent explicitly wants reply-send tested. +5. Prove emergency-stop and dry-run behavior against the durable ledger before wiring any provider mutation. ## Team Utilization diff --git a/botfucker/bridge_ledger.py b/botfucker/bridge_ledger.py new file mode 100644 index 0000000..d79816c --- /dev/null +++ b/botfucker/bridge_ledger.py @@ -0,0 +1,339 @@ +"""Durable provider-bridge ledger scaffold. + +This module records bridge idempotency state only. It does not call Gmail, +Microsoft, IMAP, SMTP, n8n, HTTP APIs, or any provider mutation surface. A +future operator-owned bridge can use this before provider mutation to claim an +approved ``audit_id`` and avoid duplicate execution. +""" + +from __future__ import annotations + +import sqlite3 +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +BRIDGE_LEDGER_EFFECT_SCOPE = "bridge_ledger_state_only" +APPROVED_ACTIONS_SCHEMA = "botfucker.approved_actions.v1" +REQUIRED_SAFETY_SCOPE = "provider_action_export_only" +REQUIRED_PROVIDER_EXECUTION = "not_performed" +ALLOWED_APPROVED_ACTIONS = {"approve_warning"} + +PENDING = "pending" +PROCESSED = "processed" +FAILED = "failed" +ROLLED_BACK = "rolled_back" +VALID_STATUSES = {PENDING, PROCESSED, FAILED, ROLLED_BACK} + + +class BridgeLedgerError(ValueError): + """Raised when bridge ledger state cannot be read or mutated safely.""" + + +@dataclass(frozen=True) +class BridgeLedgerRecord: + """One durable idempotency record keyed by approved-action ``audit_id``.""" + + audit_id: str + action_id: str + provider: str + approved_action: str + message_id: str + thread_id: str + status: str + dry_run: bool + provider_result_id: str + processed_at: str + processed_by_workflow: str + effect_scope: str = BRIDGE_LEDGER_EFFECT_SCOPE + + +@dataclass(frozen=True) +class BridgeLedgerClaim: + """Result of trying to claim an ``audit_id`` before provider mutation.""" + + acquired: bool + record: BridgeLedgerRecord + + +class DurableBridgeLedger: + """SQLite-backed durable bridge idempotency ledger. + + The ledger is intentionally narrow: it stores only IDs and bridge status. It + stores no OAuth tokens, provider credentials, message subjects, snippets, raw + headers, or message bodies, and it performs no provider-side work. + """ + + def __init__(self, path: str | Path = "botfucker_bridge_ledger.sqlite3") -> None: + self.path = Path(path) + if self.path.parent and str(self.path.parent) != ".": + self.path.parent.mkdir(parents=True, exist_ok=True) + self.connection = sqlite3.connect(self.path) + self.connection.row_factory = sqlite3.Row + self._ensure_schema() + + def __enter__(self) -> "DurableBridgeLedger": + return self + + def __exit__(self, *_exc: object) -> None: + self.close() + + def close(self) -> None: + self.connection.close() + + def _ensure_schema(self) -> None: + self.connection.execute( + """ + CREATE TABLE IF NOT EXISTS bridge_processed_audits ( + audit_id TEXT PRIMARY KEY, + action_id TEXT NOT NULL DEFAULT '', + provider TEXT NOT NULL DEFAULT '', + approved_action TEXT NOT NULL DEFAULT '', + message_id TEXT NOT NULL DEFAULT '', + thread_id TEXT NOT NULL DEFAULT '', + status TEXT NOT NULL DEFAULT 'pending', + dry_run INTEGER NOT NULL DEFAULT 1, + provider_result_id TEXT NOT NULL DEFAULT '', + processed_at TEXT NOT NULL, + processed_by_workflow TEXT NOT NULL DEFAULT '', + effect_scope TEXT NOT NULL DEFAULT 'bridge_ledger_state_only' + ) + """ + ) + self.connection.execute( + "CREATE INDEX IF NOT EXISTS idx_bridge_processed_status ON bridge_processed_audits(status)" + ) + self.connection.execute( + "CREATE INDEX IF NOT EXISTS idx_bridge_processed_provider ON bridge_processed_audits(provider)" + ) + self.connection.commit() + + def claim_action( + self, + bundle: dict[str, Any], + action: dict[str, Any], + *, + processed_by_workflow: str, + dry_run: bool = True, + ) -> BridgeLedgerClaim: + """Claim an approved action's ``audit_id`` before provider mutation. + + Returns ``acquired=True`` only for the first claim of an ``audit_id``. + Repeated claims return the existing record with ``acquired=False``. + This method validates the approved-action export safety markers before + inserting ledger state. + """ + + normalized = _normalize_action(bundle, action) + workflow = _require_nonempty(processed_by_workflow, "processed_by_workflow") + now = _now() + + with self.connection: + cursor = self.connection.execute( + """ + INSERT OR IGNORE INTO bridge_processed_audits ( + audit_id, action_id, provider, approved_action, message_id, + thread_id, status, dry_run, provider_result_id, processed_at, + processed_by_workflow, effect_scope + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + ( + normalized["audit_id"], + normalized["action_id"], + normalized["provider"], + normalized["approved_action"], + normalized["message_id"], + normalized["thread_id"], + PENDING, + 1 if dry_run else 0, + "", + now, + workflow, + BRIDGE_LEDGER_EFFECT_SCOPE, + ), + ) + acquired = cursor.rowcount == 1 + row = self.connection.execute( + "SELECT * FROM bridge_processed_audits WHERE audit_id = ?", + (normalized["audit_id"],), + ).fetchone() + + if row is None: # pragma: no cover - defensive guard for corrupted DB handles. + raise BridgeLedgerError(f"Could not read claimed audit_id: {normalized['audit_id']}") + return BridgeLedgerClaim(acquired=acquired, record=_record_from_row(row)) + + def mark_processed( + self, + audit_id: str, + *, + provider_result_id: str = "", + processed_by_workflow: str | None = None, + ) -> BridgeLedgerRecord: + """Mark a claimed audit as processed after provider-side work completes.""" + + return self._update_status( + audit_id, + PROCESSED, + provider_result_id=provider_result_id, + processed_by_workflow=processed_by_workflow, + ) + + def mark_failed( + self, + audit_id: str, + *, + provider_result_id: str = "", + processed_by_workflow: str | None = None, + ) -> BridgeLedgerRecord: + """Record an attempted action that failed and requires manual review.""" + + return self._update_status( + audit_id, + FAILED, + provider_result_id=provider_result_id, + processed_by_workflow=processed_by_workflow, + ) + + def mark_rolled_back( + self, + audit_id: str, + *, + provider_result_id: str = "", + processed_by_workflow: str | None = None, + ) -> BridgeLedgerRecord: + """Record rollback/remediation status without retrying the action.""" + + return self._update_status( + audit_id, + ROLLED_BACK, + provider_result_id=provider_result_id, + processed_by_workflow=processed_by_workflow, + ) + + def _update_status( + self, + audit_id: str, + status: str, + *, + provider_result_id: str = "", + processed_by_workflow: str | None = None, + ) -> BridgeLedgerRecord: + audit_id = _require_nonempty(audit_id, "audit_id") + if status not in VALID_STATUSES: + raise BridgeLedgerError(f"Unsupported ledger status: {status}") + + existing = self.get(audit_id) + _validate_status_transition(existing.status, status) + workflow = processed_by_workflow if processed_by_workflow is not None else existing.processed_by_workflow + workflow = _require_nonempty(workflow, "processed_by_workflow") + + with self.connection: + self.connection.execute( + """ + UPDATE bridge_processed_audits + SET status = ?, provider_result_id = ?, processed_at = ?, processed_by_workflow = ? + WHERE audit_id = ? + """, + (status, provider_result_id or "", _now(), workflow, audit_id), + ) + return self.get(audit_id) + + def get(self, audit_id: str) -> BridgeLedgerRecord: + audit_id = _require_nonempty(audit_id, "audit_id") + row = self.connection.execute( + "SELECT * FROM bridge_processed_audits WHERE audit_id = ?", + (audit_id,), + ).fetchone() + if row is None: + raise BridgeLedgerError(f"Unknown bridge ledger audit_id: {audit_id}") + return _record_from_row(row) + + def has_processed(self, audit_id: str) -> bool: + """Return true only when an audit is durably marked processed.""" + + try: + return self.get(audit_id).status == PROCESSED + except BridgeLedgerError: + return False + + def list_records(self) -> list[BridgeLedgerRecord]: + rows = self.connection.execute( + "SELECT * FROM bridge_processed_audits ORDER BY processed_at ASC, audit_id ASC" + ).fetchall() + return [_record_from_row(row) for row in rows] + + +def _normalize_action(bundle: dict[str, Any], action: dict[str, Any]) -> dict[str, str]: + actions = bundle.get("actions") + if not isinstance(actions, list): + raise BridgeLedgerError("Approved-actions bundle must include an actions list") + if action not in actions: + raise BridgeLedgerError("Approved action must be present in the approved-actions bundle") + + if bundle.get("schema") != APPROVED_ACTIONS_SCHEMA: + raise BridgeLedgerError(f"Unsupported approved-actions schema: {bundle.get('schema')!r}") + if bundle.get("safety_scope") != REQUIRED_SAFETY_SCOPE: + raise BridgeLedgerError("Approved-actions bundle has unsafe safety_scope") + if bundle.get("provider_execution") != REQUIRED_PROVIDER_EXECUTION: + raise BridgeLedgerError("Approved-actions bundle must not have provider execution already performed") + if action.get("safety_scope") != REQUIRED_SAFETY_SCOPE: + raise BridgeLedgerError("Approved action has unsafe safety_scope") + if action.get("provider_execution") != REQUIRED_PROVIDER_EXECUTION: + raise BridgeLedgerError("Approved action must have provider_execution: not_performed") + + audit_id = _require_nonempty(action.get("audit_id", ""), "audit_id") + approved_action = _require_nonempty(action.get("approved_action", ""), "approved_action") + if approved_action not in ALLOWED_APPROVED_ACTIONS: + raise BridgeLedgerError(f"Unsupported approved action for bridge ledger scaffold: {approved_action}") + return { + "audit_id": audit_id, + "action_id": _require_nonempty(action.get("action_id") or f"bf-action-{audit_id}", "action_id"), + "provider": _require_nonempty(action.get("provider", ""), "provider"), + "approved_action": approved_action, + "message_id": _require_nonempty(action.get("message_id", ""), "message_id"), + "thread_id": _require_nonempty(action.get("thread_id", ""), "thread_id"), + } + + +def _validate_status_transition(current: str, target: str) -> None: + if current not in VALID_STATUSES: + raise BridgeLedgerError(f"Unknown current bridge ledger status: {current}") + if current == target: + return + allowed = { + PENDING: {PROCESSED, FAILED, ROLLED_BACK}, + FAILED: {ROLLED_BACK}, + PROCESSED: {ROLLED_BACK}, + ROLLED_BACK: set(), + } + if target not in allowed[current]: + raise BridgeLedgerError(f"Unsafe bridge ledger status transition: {current} -> {target}") + + +def _record_from_row(row: sqlite3.Row) -> BridgeLedgerRecord: + return BridgeLedgerRecord( + audit_id=row["audit_id"], + action_id=row["action_id"], + provider=row["provider"], + approved_action=row["approved_action"], + message_id=row["message_id"], + thread_id=row["thread_id"], + status=row["status"], + dry_run=bool(row["dry_run"]), + provider_result_id=row["provider_result_id"], + processed_at=row["processed_at"], + processed_by_workflow=row["processed_by_workflow"], + effect_scope=row["effect_scope"], + ) + + +def _require_nonempty(value: Any, name: str) -> str: + text = str(value or "").strip() + if not text: + raise BridgeLedgerError(f"Bridge ledger requires {name}") + return text + + +def _now() -> str: + return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z") diff --git a/docs/bridge-ledger-scaffold.md b/docs/bridge-ledger-scaffold.md new file mode 100644 index 0000000..1f2841b --- /dev/null +++ b/docs/bridge-ledger-scaffold.md @@ -0,0 +1,147 @@ +# Phase 14 Durable Bridge Ledger Scaffold + +Phase 14 adds a durable bridge ledger scaffold for future reviewed provider bridges. It is **not** a provider implementation: no OAuth, no provider credentials, no live provider mutation nodes, no Gmail/Microsoft/IMAP/SMTP calls, and no n8n activation changes. + +The purpose is narrow and boring in the useful way: give a future n8n/operator-owned bridge a tested way to record approved `audit_id` state durably **before provider mutation** so duplicate sends/moves/actions are blocked by construction. + +## Artifact + +- `botfucker.bridge_ledger.DurableBridgeLedger`: a standard-library SQLite ledger keyed by `audit_id`. + +Default local file name if used directly: + +```text +botfucker_bridge_ledger.sqlite3 +``` + +A future n8n deployment may instead map this shape to n8n Data Store, Postgres, or another operator-owned durable store. The required behavior is the same: claim `audit_id` before provider mutation, then mark the record processed/failed/rolled back after the bridge attempt is known. + +## Safety boundary + +This scaffold only records bridge ledger state: + +```text +bridge_ledger_state_only +``` + +It does not: + +- implement Gmail OAuth +- implement Microsoft OAuth +- store IMAP/SMTP passwords +- store OAuth access or refresh tokens +- store provider API keys, cookies, or private headers +- call Gmail, Microsoft, IMAP, SMTP, n8n, HTTP APIs, or live mailbox providers +- send replies +- move, delete, archive, label, whitelist, or blacklist provider-side messages +- add live provider mutation nodes to checked-in n8n workflows + +The ledger stores only provider/action identifiers and execution state. It intentionally omits message subject, snippet, body, raw headers, and credential material. + +## Required input + +`claim_action()` requires the existing approved-action export contract: + +```json +{ + "schema": "botfucker.approved_actions.v1", + "safety_scope": "provider_action_export_only", + "provider_execution": "not_performed", + "actions": [ + { + "audit_id": "audit-0001", + "action_id": "bf-action-audit-0001", + "message_id": "gmail-msg-123", + "thread_id": "gmail-thread-7", + "provider": "gmail", + "approved_action": "approve_warning", + "safety_scope": "provider_action_export_only", + "provider_execution": "not_performed" + } + ] +} +``` + +The scaffold rejects bundles/actions that are missing the expected schema, safety scope, or `provider_execution: not_performed` marker. + +## Ledger lifecycle + +A future reviewed bridge should use this sequence: + +1. Validate the approved-actions bundle. +2. Check emergency stop and provider/action enablement. +3. Call `claim_action()` for the action. +4. If `acquired` is false, stop before provider mutation because the `audit_id` was already claimed. +5. If `acquired` is true, the durable row is already `pending`. +6. Only after that pending row exists may the reviewed bridge approach a provider mutation node. +7. Mark the row `processed`, `failed`, or `rolled_back` after the attempt is known. +8. Never blindly retry a processed or ambiguous `audit_id`; require manual review. + +Minimal Python example using fake/local IDs only: + +```python +from botfucker.bridge_ledger import DurableBridgeLedger + +bundle = {...} # botfucker.approved_actions.v1 export +action = bundle["actions"][0] + +with DurableBridgeLedger("botfucker_bridge_ledger.sqlite3") as ledger: + claim = ledger.claim_action( + bundle, + action, + processed_by_workflow="botfucker-reviewed-live-bridge-v1", + dry_run=True, + ) + if not claim.acquired: + raise SystemExit("audit_id already claimed; stop before provider mutation") + + # Future reviewed provider bridge would run only after this point. + # Phase 14 deliberately does not include that provider node/action. + + ledger.mark_processed(action["audit_id"], provider_result_id="dry-run-result-id") +``` + +## Stored fields + +The SQLite table is `bridge_processed_audits`. + +Stored fields: + +- `audit_id` — primary idempotency key +- `action_id` +- `provider` +- `approved_action` +- `message_id` +- `thread_id` +- `status` — `pending`, `processed`, `failed`, or `rolled_back` +- `dry_run` +- `provider_result_id` +- `processed_at` +- `processed_by_workflow` +- `effect_scope` — always `bridge_ledger_state_only` + +Not stored: + +- message subject/snippet/body +- raw headers +- OAuth tokens +- API keys +- passwords +- cookies +- private provider headers + +## Promotion gate remains closed + +Phase 14 does not make the bridge live. Before any live provider mutation is connected, the Phase 13 gate still applies: + +- one provider/action pair only +- dry-run evidence exists +- provider-specific sandbox/manual evidence exists +- emergency stop exits before provider mutation +- rollback/remediation is documented +- provider credentials remain in n8n/operator infrastructure only +- Rex/security approval is recorded +- Gus/ops approval is recorded +- checked-in starter workflows remain inactive and free of live mutation nodes + +This phase is the seatbelt mount, not the rocket engine. diff --git a/docs/n8n-approved-action-bridge.md b/docs/n8n-approved-action-bridge.md index 0e11b14..36b2232 100644 --- a/docs/n8n-approved-action-bridge.md +++ b/docs/n8n-approved-action-bridge.md @@ -66,7 +66,7 @@ export BOTFUCKER_APPROVED_ACTIONS="/path/to/approved-actions.json" export BOTFUCKER_PROCESSED_AUDIT_IDS="audit-0001,audit-0002" ``` -`BOTFUCKER_PROCESSED_AUDIT_IDS` is the starter dry-run dedupe source. A later live bridge should replace that with durable storage, but it must still dedupe by `audit_id` before touching a provider. +`BOTFUCKER_PROCESSED_AUDIT_IDS` is the starter dry-run dedupe source. Phase 14 adds a durable bridge ledger scaffold for the reviewed live-bridge path; live bridges must replace environment-variable dedupe with durable processed-`audit_id` state before touching a provider. ## Workflow behavior diff --git a/tests/test_bridge_ledger.py b/tests/test_bridge_ledger.py new file mode 100644 index 0000000..f69b9b9 --- /dev/null +++ b/tests/test_bridge_ledger.py @@ -0,0 +1,224 @@ +import sqlite3 +import tempfile +import unittest +from pathlib import Path + +from botfucker.bridge_ledger import ( + BRIDGE_LEDGER_EFFECT_SCOPE, + BridgeLedgerError, + DurableBridgeLedger, + FAILED, + PENDING, + PROCESSED, + ROLLED_BACK, +) + + +REPO_ROOT = Path(__file__).resolve().parents[1] +DOC_PATH = REPO_ROOT / "docs" / "bridge-ledger-scaffold.md" +README_PATH = REPO_ROOT / "README.md" +ROADMAP_PATH = REPO_ROOT / "ROADMAP.md" +HANDOFF_PATH = REPO_ROOT / "HANDOFF.md" + + +def sample_bundle(action=None): + action = action or sample_action() + return { + "schema": "botfucker.approved_actions.v1", + "safety_scope": "provider_action_export_only", + "provider_execution": "not_performed", + "cursor": {"since_audit_id": None, "last_audit_id": action["audit_id"]}, + "actions": [action], + } + + +def sample_action(**overrides): + action = { + "audit_id": "audit-0001", + "action_id": "bf-action-audit-0001", + "item_id": "webhook:gmail:gmail-msg-123", + "message_id": "gmail-msg-123", + "thread_id": "gmail-thread-7", + "provider": "gmail", + "approved_action": "approve_warning", + "approved_by": "human", + "approved_at": "2026-05-14T21:45:00Z", + "draft_reply": "Human-reviewed warning text", + "safety_scope": "provider_action_export_only", + "provider_execution": "not_performed", + } + action.update(overrides) + return action + + +class DurableBridgeLedgerTests(unittest.TestCase): + def test_claim_action_inserts_pending_record_before_provider_mutation(self): + with tempfile.TemporaryDirectory() as tmpdir: + with DurableBridgeLedger(Path(tmpdir) / "ledger.sqlite3") as ledger: + action = sample_action() + claim = ledger.claim_action( + sample_bundle(action), + action, + processed_by_workflow="botfucker-reviewed-live-bridge-v1", + dry_run=False, + ) + + self.assertTrue(claim.acquired) + self.assertEqual(PENDING, claim.record.status) + self.assertFalse(claim.record.dry_run) + self.assertEqual("audit-0001", claim.record.audit_id) + self.assertEqual("bf-action-audit-0001", claim.record.action_id) + self.assertEqual("gmail", claim.record.provider) + self.assertEqual("approve_warning", claim.record.approved_action) + self.assertEqual("gmail-msg-123", claim.record.message_id) + self.assertEqual("gmail-thread-7", claim.record.thread_id) + self.assertEqual(BRIDGE_LEDGER_EFFECT_SCOPE, claim.record.effect_scope) + + def test_claim_action_dedupes_by_audit_id(self): + with tempfile.TemporaryDirectory() as tmpdir: + with DurableBridgeLedger(Path(tmpdir) / "ledger.sqlite3") as ledger: + action = sample_action() + first = ledger.claim_action( + sample_bundle(action), + action, + processed_by_workflow="bridge-v1", + ) + second = ledger.claim_action( + sample_bundle(action), + action, + processed_by_workflow="bridge-v1", + ) + + self.assertTrue(first.acquired) + self.assertFalse(second.acquired) + self.assertEqual(first.record.audit_id, second.record.audit_id) + self.assertEqual(1, len(ledger.list_records())) + + def test_mark_processed_and_failed_update_existing_claim_only(self): + with tempfile.TemporaryDirectory() as tmpdir: + with DurableBridgeLedger(Path(tmpdir) / "ledger.sqlite3") as ledger: + action = sample_action() + ledger.claim_action(sample_bundle(action), action, processed_by_workflow="bridge-v1") + + processed = ledger.mark_processed("audit-0001", provider_result_id="provider-reply-123") + self.assertEqual(PROCESSED, processed.status) + self.assertEqual("provider-reply-123", processed.provider_result_id) + self.assertTrue(ledger.has_processed("audit-0001")) + + failed_action = sample_action(audit_id="audit-0002", action_id="bf-action-audit-0002") + ledger.claim_action(sample_bundle(failed_action), failed_action, processed_by_workflow="bridge-v1") + failed = ledger.mark_failed("audit-0002", provider_result_id="provider-error-429") + self.assertEqual(FAILED, failed.status) + self.assertFalse(ledger.has_processed("audit-0002")) + + with self.assertRaises(BridgeLedgerError): + ledger.mark_processed("audit-missing") + + def test_rejects_unsafe_or_already_executed_exports(self): + with tempfile.TemporaryDirectory() as tmpdir: + with DurableBridgeLedger(Path(tmpdir) / "ledger.sqlite3") as ledger: + action = sample_action() + unsafe_bundle = sample_bundle(action) + unsafe_bundle["provider_execution"] = "performed" + + with self.assertRaises(BridgeLedgerError): + ledger.claim_action(unsafe_bundle, action, processed_by_workflow="bridge-v1") + + unsafe_action = sample_action(provider_execution="performed") + with self.assertRaises(BridgeLedgerError): + ledger.claim_action(sample_bundle(unsafe_action), unsafe_action, processed_by_workflow="bridge-v1") + + missing_id_action = sample_action(audit_id="") + with self.assertRaises(BridgeLedgerError): + ledger.claim_action(sample_bundle(missing_id_action), missing_id_action, processed_by_workflow="bridge-v1") + + missing_thread_action = sample_action(thread_id="") + with self.assertRaises(BridgeLedgerError): + ledger.claim_action(sample_bundle(missing_thread_action), missing_thread_action, processed_by_workflow="bridge-v1") + + unsupported_action = sample_action(approved_action="delete_message") + with self.assertRaises(BridgeLedgerError): + ledger.claim_action(sample_bundle(unsupported_action), unsupported_action, processed_by_workflow="bridge-v1") + + action_outside_bundle = sample_action(audit_id="audit-outside", action_id="bf-action-audit-outside") + with self.assertRaises(BridgeLedgerError): + ledger.claim_action(sample_bundle(action), action_outside_bundle, processed_by_workflow="bridge-v1") + + def test_status_transitions_fail_closed_after_terminal_states(self): + with tempfile.TemporaryDirectory() as tmpdir: + with DurableBridgeLedger(Path(tmpdir) / "ledger.sqlite3") as ledger: + processed_action = sample_action() + ledger.claim_action(sample_bundle(processed_action), processed_action, processed_by_workflow="bridge-v1") + ledger.mark_processed("audit-0001", provider_result_id="provider-reply-123") + + with self.assertRaises(BridgeLedgerError): + ledger.mark_failed("audit-0001", provider_result_id="late-error") + + rolled_back = ledger.mark_rolled_back("audit-0001", provider_result_id="manual-remediation-1") + self.assertEqual(ROLLED_BACK, rolled_back.status) + + with self.assertRaises(BridgeLedgerError): + ledger.mark_processed("audit-0001", provider_result_id="retry-after-rollback") + + failed_action = sample_action(audit_id="audit-0002", action_id="bf-action-audit-0002") + ledger.claim_action(sample_bundle(failed_action), failed_action, processed_by_workflow="bridge-v1") + ledger.mark_failed("audit-0002", provider_result_id="provider-error-429") + + with self.assertRaises(BridgeLedgerError): + ledger.mark_processed("audit-0002", provider_result_id="retry-after-failure") + + def test_schema_excludes_message_content_and_secret_storage(self): + with tempfile.TemporaryDirectory() as tmpdir: + db_path = Path(tmpdir) / "ledger.sqlite3" + with DurableBridgeLedger(db_path): + pass + connection = sqlite3.connect(db_path) + try: + columns = { + row[1] + for row in connection.execute("PRAGMA table_info(bridge_processed_audits)").fetchall() + } + finally: + connection.close() + + forbidden_columns = { + "subject", + "snippet", + "body", + "raw_headers", + "headers", + "oauth_token", + "refresh_token", + "api_key", + "password", + "cookie", + } + self.assertFalse(columns & forbidden_columns) + self.assertIn("audit_id", columns) + self.assertIn("status", columns) + self.assertIn("provider_result_id", columns) + + def test_docs_describe_phase_14_as_scaffold_without_provider_side_effects(self): + doc = DOC_PATH.read_text(encoding="utf-8") + readme = README_PATH.read_text(encoding="utf-8") + roadmap = ROADMAP_PATH.read_text(encoding="utf-8") + handoff = HANDOFF_PATH.read_text(encoding="utf-8") + + for text in (doc, readme, roadmap, handoff): + self.assertIn("Phase 14", text) + self.assertIn("durable bridge ledger", text.lower()) + + for phrase in ( + "no OAuth", + "no provider credentials", + "no live provider mutation nodes", + "before provider mutation", + "bridge_ledger_state_only", + ): + self.assertIn(phrase, doc) + + self.assertIn("botfucker.bridge_ledger.DurableBridgeLedger", doc) + + +if __name__ == "__main__": + unittest.main()