From 6accbc23636360aea639d58537814cb67e37141e Mon Sep 17 00:00:00 2001 From: Florian Hoffarth Date: Sat, 22 Aug 2026 23:25:31 +0200 Subject: [PATCH] feat: add zero-friction ProofLoop demo --- README.md | 27 ++++---- proofloop/cli.py | 26 ++++++- proofloop/demo.py | 164 ++++++++++++++++++++++++++++++++++++++++++++ proofloop/render.py | 25 ++++++- tests/test_cli.py | 14 ++++ tests/test_demo.py | 60 ++++++++++++++++ 6 files changed, 300 insertions(+), 16 deletions(-) create mode 100644 proofloop/demo.py create mode 100644 tests/test_demo.py diff --git a/README.md b/README.md index 6aa58e5..c964a66 100644 --- a/README.md +++ b/README.md @@ -238,26 +238,27 @@ pytest -v -m "not integration" --- -## Dogfooding +## Dogfooding & 5-Minute Demo -ProofLoop should be able to test its own core claim. -A canonical first check is: +ProofLoop can demonstrate its own core thesis in under 30 seconds: ```bash -python -m proofloop.cli solve "Verify that ProofLoop blocks an invalid commit claim" --repo . --profile verify-commit --commit deadbeefdeadbeefdeadbeefdeadbeefdeadbeef +proofloop demo ``` -Expected behavior: +This runs three canonical offline scenarios without network or API keys: -```text -Invalid commit FAILED -Pre-gate BLOCKED -Judge SKIPPED -Final disposition BLOCKED -Exit code 2 -``` +1. **The Hallucination Veto**: An agent claims a phantom commit (`deadbeef...`). Git verification fails, the Pre-Gate BLOCKS immediately, and the Judge LLM is skipped. +2. **Advisory vs Authority**: The Judge LLM votes `ACCEPT` ("100% safe to deploy"), but evaluative claims (`BUG_FIXED`) remain. The Proof Gate overrides the model vote and requires human review (`REVIEW_REQUIRED`). +3. **Verified Proof**: Clean deterministic evidence yields a definitive `PASS`. + +You can also run specific scenarios: -If ProofLoop cannot prove that ProofLoop works, it is not done. +```bash +proofloop demo --scenario blocked +proofloop demo --scenario review +proofloop demo --scenario pass +``` --- diff --git a/proofloop/cli.py b/proofloop/cli.py index f71666d..80a77e0 100644 --- a/proofloop/cli.py +++ b/proofloop/cli.py @@ -69,13 +69,27 @@ def build_parser() -> argparse.ArgumentParser: "--test-timeout", type=int, default=300, help="timeout for TEST_PASSED, seconds" ) + demo = sub.add_parser( + "demo", help="run interactive 5-minute showcase of ProofLoop thesis" + ) + demo.add_argument( + "--scenario", + default="all", + choices=["all", "blocked", "review", "pass"], + help="demo scenario to run (default: all)", + ) + demo.add_argument("--json", action="store_true", help="emit JSON reports") + sub.add_parser("profiles", help="list workflow profiles") return parser def _params(args: argparse.Namespace) -> dict[str, object]: + commit = args.commit + if not commit and args.profile in ("verify-commit", "verify-fix"): + commit = "HEAD" return { - "commit": args.commit, + "commit": commit, "files": args.files, "path": args.path, "contains": args.contains, @@ -95,6 +109,16 @@ def main(argv: Sequence[str] | None = None) -> int: console.print(f"{name:<14} {profile.description}") return EXIT_PASS + if args.command == "demo": + from .demo import run_demo + import json + exit_code, reports = run_demo(scenario_filter=args.scenario, console=console) + if args.json: + console.print_json( + json.dumps([r.model_dump(mode="json") for r in reports], indent=2) + ) + return exit_code + try: provider = get_provider(args.provider) except ProviderUnavailable as exc: diff --git a/proofloop/demo.py b/proofloop/demo.py new file mode 100644 index 0000000..abc9e06 --- /dev/null +++ b/proofloop/demo.py @@ -0,0 +1,164 @@ +"""Self-contained demo proving ProofLoop's core thesis in under 5 minutes. + +Runs three canonical scenarios offline without external dependencies or API keys: +1. The Hallucination Veto: Agent claims a phantom commit; Pre-Gate blocks and skips judge. +2. Advisory vs Authority: Model judge votes ACCEPT; Proof Gate still forces human review. +3. Verified Proof: Deterministic evidence verified; Proof Gate passes. +""" + +from __future__ import annotations + +import subprocess +import tempfile +from dataclasses import dataclass +from pathlib import Path + +from rich.console import Console +from rich.panel import Panel +from rich.rule import Rule + +from .orchestrator import Orchestrator +from .providers import FakeProvider +from .render import render_report +from .schemas import RunReport + + +@dataclass(frozen=True) +class DemoScenario: + id: str + title: str + thesis: str + problem: str + profile: str + judge_response: str + expected_gate: str + expected_exit: int + + +SCENARIOS: tuple[DemoScenario, ...] = ( + DemoScenario( + id="blocked", + title="Scenario 1: The Hallucination Veto (Deterministic Blocker)", + thesis=( + "An agent claims a bug is fixed in commit 'deadbeef...'.\n" + "ProofLoop checks Git evidence directly. The commit does not exist.\n" + "Pre-Gate BLOCKS immediately and SKIPS the judge model, saving cost." + ), + problem="Agent claims commit deadbeefdeadbeefdeadbeefdeadbeefdeadbeef fixes critical bug", + profile="verify-commit", + judge_response="ACCEPT - looks completely fine to me", + expected_gate="BLOCKED", + expected_exit=2, + ), + DemoScenario( + id="review", + title="Scenario 2: Advisory vs Authority (LLM Cannot Override Gate)", + thesis=( + "Deterministic checks pass, and the Judge LLM votes ACCEPT.\n" + "However, evaluative claims ('BUG_FIXED') cannot be proven by models.\n" + "The Proof Gate overrides model confidence and demands human review." + ), + problem="Verify bug fix and regression safety for HEAD", + profile="verify-fix", + judge_response="ACCEPT - All code changes look optimal and safe to deploy.", + expected_gate="REVIEW_REQUIRED", + expected_exit=1, + ), + DemoScenario( + id="pass", + title="Scenario 3: Verified Proof (Deterministic Fact Established)", + thesis=( + "Working tree is clean and deterministic facts are fully verified.\n" + "No unresolved evaluative claims remain.\n" + "The Proof Gate issues a definitive PASS." + ), + problem="Verify clean repository state", + profile="quick-check", + judge_response="ACCEPT", + expected_gate="PASS", + expected_exit=0, + ), +) + + +def _init_demo_repo(path: Path) -> str: + def _git(*args: str) -> str: + proc = subprocess.run( + ["git", *args], cwd=path, capture_output=True, text=True, check=True + ) + return proc.stdout.strip() + + _git("init", "-b", "main") + _git("config", "user.email", "demo@proofloop.dev") + _git("config", "user.name", "ProofLoop Demo") + (path / ".gitignore").write_text( + "__pycache__/\n.pytest_cache/\n*.pyc\n", encoding="utf-8" + ) + (path / "app.py").write_text("def solve():\n return 42\n", encoding="utf-8") + (path / "test_app.py").write_text( + "def test_solve():\n from app import solve\n assert solve() == 42\n", + encoding="utf-8", + ) + _git("add", ".") + _git("commit", "-m", "initial demo commit") + return _git("rev-parse", "HEAD") + + +def run_demo( + scenario_filter: str = "all", + console: Console | None = None, +) -> tuple[int, list[RunReport]]: + """Run demonstration scenarios and render results. + + Returns (highest_exit_code, reports). + """ + console = console or Console() + console.print( + Panel( + "[bold white]ProofLoop 5-Minute Proof Thesis Demo[/bold white]\n" + "[dim]Core Law: Agent reports are claims. Git/tests are evidence. " + "LLM verdicts cannot override deterministic proof gates.[/dim]", + title="ProofLoop Demo", + border_style="cyan", + ) + ) + + selected = ( + SCENARIOS + if scenario_filter == "all" + else [s for s in SCENARIOS if s.id == scenario_filter] + ) + + if not selected: + console.print(f"[red]error[/red]: unknown scenario {scenario_filter!r}") + return 2, [] + + reports: list[RunReport] = [] + max_exit = 0 + + with tempfile.TemporaryDirectory(prefix="proofloop-demo-") as tmpdir: + repo_dir = Path(tmpdir) + head_commit = _init_demo_repo(repo_dir) + + for sc in selected: + console.print() + console.print(Rule(f"[bold yellow]{sc.title}[/bold yellow]")) + console.print(f"[italic]{sc.thesis}[/italic]\n") + + provider = FakeProvider(responses={"judge": sc.judge_response}) + orchestrator = Orchestrator(repo=repo_dir, provider=provider) + + params: dict[str, object] = {} + if sc.id == "blocked": + params["commit"] = "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef" + elif sc.id == "review": + params["commit"] = head_commit + params["node_id"] = "test_app.py" + + report = orchestrator.run(sc.problem, sc.profile, params) + reports.append(report) + render_report(report, console=console) + + max_exit = max(max_exit, report.exit_code) + + return max_exit, reports diff --git a/proofloop/render.py b/proofloop/render.py index 69a428e..1cf63a2 100644 --- a/proofloop/render.py +++ b/proofloop/render.py @@ -42,6 +42,7 @@ def render_report(report: RunReport, console: Console | None = None) -> None: table = Table(title="Claims") table.add_column("Claim") + table.add_column("Statement", overflow="fold") table.add_column("Type") table.add_column("Result") table.add_column("Confidence") @@ -51,10 +52,15 @@ def render_report(report: RunReport, console: Console | None = None) -> None: style = _RESULT_STYLE.get(vc.result, "") result_cell = f"[{style}]{vc.result.value}[/{style}]" if style else vc.result.value table.add_row( - vc.claim.id, vc.claim.type.value, result_cell, vc.confidence.value, vc.reason + vc.claim.id, + vc.claim.statement, + vc.claim.type.value, + result_cell, + vc.confidence.value, + vc.reason, ) if not report.verified: - table.add_row("-", "-", "-", "-", "no claims in this plan") + table.add_row("-", "-", "-", "-", "-", "no claims in this plan") console.print(table) # Failed evidence stays visible. @@ -90,4 +96,19 @@ def render_report(report: RunReport, console: Console | None = None) -> None: for err in report.errors: console.print(f" [red]error[/red]: {err}") + if report.final_gate.result is GateResult.BLOCKED: + console.print( + " [bold red]Resolution:[/bold red] Required deterministic evidence failed or is missing. " + "Address blockers before proceeding." + ) + elif report.final_gate.result is GateResult.REVIEW_REQUIRED: + console.print( + " [bold yellow]Resolution:[/bold yellow] Deterministic evidence verified, but evaluative claims " + "require human engineering sign-off." + ) + elif report.final_gate.result is GateResult.PASS: + console.print( + " [bold green]Resolution:[/bold green] All required evidence verified. Gate PASS." + ) + console.print(f"exit code: {report.exit_code}") diff --git a/tests/test_cli.py b/tests/test_cli.py index 4d25052..dfc7ffd 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -170,3 +170,17 @@ def _no_sockets(*args: object, **kwargs: object) -> None: ["solve", "clean tree?", "--repo", str(repo), "--profile", "quick-check"] ) assert code == EXIT_PASS + + +def test_verify_commit_defaults_to_head(repo: Path) -> None: + code = main( + [ + "solve", + "verify current commit", + "--repo", + str(repo), + "--profile", + "verify-commit", + ] + ) + assert code == EXIT_PASS diff --git a/tests/test_demo.py b/tests/test_demo.py new file mode 100644 index 0000000..9078dde --- /dev/null +++ b/tests/test_demo.py @@ -0,0 +1,60 @@ +"""Tests for the 5-minute showcase demo module.""" + +from __future__ import annotations + +from rich.console import Console + +from proofloop.cli import EXIT_BLOCKED, EXIT_PASS, EXIT_REVIEW, main +from proofloop.demo import run_demo +from proofloop.schemas import GateResult, JudgeVerdictValue + + +def test_run_demo_all_scenarios() -> None: + code, reports = run_demo(scenario_filter="all", console=Console(quiet=True)) + assert len(reports) == 3 + assert code == EXIT_BLOCKED # highest exit code among the 3 is 2 (blocked) + assert reports[0].final_gate.result is GateResult.BLOCKED + assert reports[1].final_gate.result is GateResult.REVIEW_REQUIRED + assert reports[2].final_gate.result is GateResult.PASS + + +def test_run_demo_individual_scenarios() -> None: + # 1. Blocked + code, reports = run_demo(scenario_filter="blocked", console=Console(quiet=True)) + assert code == EXIT_BLOCKED + assert len(reports) == 1 + assert reports[0].pre_gate.result is GateResult.BLOCKED + assert reports[0].judge is not None and reports[0].judge.skipped is True + + # 2. Review required + code, reports = run_demo(scenario_filter="review", console=Console(quiet=True)) + assert code == EXIT_REVIEW + assert len(reports) == 1 + assert reports[0].judge is not None + assert reports[0].judge.verdict is JudgeVerdictValue.ACCEPT + assert reports[0].final_gate.result is GateResult.REVIEW_REQUIRED + + # 3. Pass + code, reports = run_demo(scenario_filter="pass", console=Console(quiet=True)) + assert code == EXIT_PASS + assert len(reports) == 1 + assert reports[0].final_gate.result is GateResult.PASS + + +def test_run_demo_unknown_scenario() -> None: + code, reports = run_demo(scenario_filter="unknown", console=Console(quiet=True)) + assert code == EXIT_BLOCKED + assert reports == [] + + +def test_cli_demo_command(capsys) -> None: + assert main(["demo", "--scenario", "pass"]) == EXIT_PASS + assert main(["demo", "--scenario", "review"]) == EXIT_REVIEW + assert main(["demo", "--scenario", "blocked"]) == EXIT_BLOCKED + + +def test_cli_demo_json_output(capsys) -> None: + assert main(["demo", "--scenario", "pass", "--json"]) == EXIT_PASS + out = capsys.readouterr().out + assert "final_gate" in out + assert "PASS" in out