Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 14 additions & 13 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -238,26 +238,27 @@ pytest -v -m "not integration"

---

## Dogfooding
## Dogfooding & 5-Minute Demo

ProofLoop should be able to test its own core claim.
A canonical first check is:
ProofLoop can demonstrate its own core thesis in under 30 seconds:

```bash
python -m proofloop.cli solve "Verify that ProofLoop blocks an invalid commit claim" --repo . --profile verify-commit --commit deadbeefdeadbeefdeadbeefdeadbeefdeadbeef
proofloop demo
```

Expected behavior:
This runs three canonical offline scenarios without network or API keys:

```text
Invalid commit FAILED
Pre-gate BLOCKED
Judge SKIPPED
Final disposition BLOCKED
Exit code 2
```
1. **The Hallucination Veto**: An agent claims a phantom commit (`deadbeef...`). Git verification fails, the Pre-Gate BLOCKS immediately, and the Judge LLM is skipped.
2. **Advisory vs Authority**: The Judge LLM votes `ACCEPT` ("100% safe to deploy"), but evaluative claims (`BUG_FIXED`) remain. The Proof Gate overrides the model vote and requires human review (`REVIEW_REQUIRED`).
3. **Verified Proof**: Clean deterministic evidence yields a definitive `PASS`.

You can also run specific scenarios:

If ProofLoop cannot prove that ProofLoop works, it is not done.
```bash
proofloop demo --scenario blocked
proofloop demo --scenario review
proofloop demo --scenario pass
```

---

Expand Down
26 changes: 25 additions & 1 deletion proofloop/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -69,13 +69,27 @@ def build_parser() -> argparse.ArgumentParser:
"--test-timeout", type=int, default=300, help="timeout for TEST_PASSED, seconds"
)

demo = sub.add_parser(
"demo", help="run interactive 5-minute showcase of ProofLoop thesis"
)
demo.add_argument(
"--scenario",
default="all",
choices=["all", "blocked", "review", "pass"],
help="demo scenario to run (default: all)",
)
demo.add_argument("--json", action="store_true", help="emit JSON reports")

sub.add_parser("profiles", help="list workflow profiles")
return parser


def _params(args: argparse.Namespace) -> dict[str, object]:
commit = args.commit
if not commit and args.profile in ("verify-commit", "verify-fix"):
commit = "HEAD"
return {
"commit": args.commit,
"commit": commit,
"files": args.files,
"path": args.path,
"contains": args.contains,
Expand All @@ -95,6 +109,16 @@ def main(argv: Sequence[str] | None = None) -> int:
console.print(f"{name:<14} {profile.description}")
return EXIT_PASS

if args.command == "demo":
from .demo import run_demo
import json
exit_code, reports = run_demo(scenario_filter=args.scenario, console=console)
if args.json:
console.print_json(
json.dumps([r.model_dump(mode="json") for r in reports], indent=2)
)
return exit_code

try:
provider = get_provider(args.provider)
except ProviderUnavailable as exc:
Expand Down
164 changes: 164 additions & 0 deletions proofloop/demo.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,164 @@
"""Self-contained demo proving ProofLoop's core thesis in under 5 minutes.

Runs three canonical scenarios offline without external dependencies or API keys:
1. The Hallucination Veto: Agent claims a phantom commit; Pre-Gate blocks and skips judge.
2. Advisory vs Authority: Model judge votes ACCEPT; Proof Gate still forces human review.
3. Verified Proof: Deterministic evidence verified; Proof Gate passes.
"""

from __future__ import annotations

import subprocess
import tempfile
from dataclasses import dataclass
from pathlib import Path

from rich.console import Console
from rich.panel import Panel
from rich.rule import Rule

from .orchestrator import Orchestrator
from .providers import FakeProvider
from .render import render_report
from .schemas import RunReport


@dataclass(frozen=True)
class DemoScenario:
id: str
title: str
thesis: str
problem: str
profile: str
judge_response: str
expected_gate: str
expected_exit: int


SCENARIOS: tuple[DemoScenario, ...] = (
DemoScenario(
id="blocked",
title="Scenario 1: The Hallucination Veto (Deterministic Blocker)",
thesis=(
"An agent claims a bug is fixed in commit 'deadbeef...'.\n"
"ProofLoop checks Git evidence directly. The commit does not exist.\n"
"Pre-Gate BLOCKS immediately and SKIPS the judge model, saving cost."
),
problem="Agent claims commit deadbeefdeadbeefdeadbeefdeadbeefdeadbeef fixes critical bug",
profile="verify-commit",
judge_response="ACCEPT - looks completely fine to me",
expected_gate="BLOCKED",
expected_exit=2,
),
DemoScenario(
id="review",
title="Scenario 2: Advisory vs Authority (LLM Cannot Override Gate)",
thesis=(
"Deterministic checks pass, and the Judge LLM votes ACCEPT.\n"
"However, evaluative claims ('BUG_FIXED') cannot be proven by models.\n"
"The Proof Gate overrides model confidence and demands human review."
),
problem="Verify bug fix and regression safety for HEAD",
profile="verify-fix",
judge_response="ACCEPT - All code changes look optimal and safe to deploy.",
expected_gate="REVIEW_REQUIRED",
expected_exit=1,
),
DemoScenario(
id="pass",
title="Scenario 3: Verified Proof (Deterministic Fact Established)",
thesis=(
"Working tree is clean and deterministic facts are fully verified.\n"
"No unresolved evaluative claims remain.\n"
"The Proof Gate issues a definitive PASS."
),
problem="Verify clean repository state",
profile="quick-check",
judge_response="ACCEPT",
expected_gate="PASS",
expected_exit=0,
),
)


def _init_demo_repo(path: Path) -> str:
def _git(*args: str) -> str:
proc = subprocess.run(
["git", *args], cwd=path, capture_output=True, text=True, check=True
)
return proc.stdout.strip()

_git("init", "-b", "main")
_git("config", "user.email", "demo@proofloop.dev")
_git("config", "user.name", "ProofLoop Demo")
(path / ".gitignore").write_text(
"__pycache__/\n.pytest_cache/\n*.pyc\n", encoding="utf-8"
)
(path / "app.py").write_text("def solve():\n return 42\n", encoding="utf-8")
(path / "test_app.py").write_text(
"def test_solve():\n from app import solve\n assert solve() == 42\n",
encoding="utf-8",
)
_git("add", ".")
_git("commit", "-m", "initial demo commit")
return _git("rev-parse", "HEAD")


def run_demo(
scenario_filter: str = "all",
console: Console | None = None,
) -> tuple[int, list[RunReport]]:
"""Run demonstration scenarios and render results.

Returns (highest_exit_code, reports).
"""
console = console or Console()
console.print(
Panel(
"[bold white]ProofLoop 5-Minute Proof Thesis Demo[/bold white]\n"
"[dim]Core Law: Agent reports are claims. Git/tests are evidence. "
"LLM verdicts cannot override deterministic proof gates.[/dim]",
title="ProofLoop Demo",
border_style="cyan",
)
)

selected = (
SCENARIOS
if scenario_filter == "all"
else [s for s in SCENARIOS if s.id == scenario_filter]
)

if not selected:
console.print(f"[red]error[/red]: unknown scenario {scenario_filter!r}")
return 2, []

reports: list[RunReport] = []
max_exit = 0

with tempfile.TemporaryDirectory(prefix="proofloop-demo-") as tmpdir:
repo_dir = Path(tmpdir)
head_commit = _init_demo_repo(repo_dir)

for sc in selected:
console.print()
console.print(Rule(f"[bold yellow]{sc.title}[/bold yellow]"))
console.print(f"[italic]{sc.thesis}[/italic]\n")

provider = FakeProvider(responses={"judge": sc.judge_response})
orchestrator = Orchestrator(repo=repo_dir, provider=provider)

params: dict[str, object] = {}
if sc.id == "blocked":
params["commit"] = "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef"
elif sc.id == "review":
params["commit"] = head_commit
params["node_id"] = "test_app.py"

report = orchestrator.run(sc.problem, sc.profile, params)
reports.append(report)
render_report(report, console=console)

max_exit = max(max_exit, report.exit_code)

return max_exit, reports
25 changes: 23 additions & 2 deletions proofloop/render.py
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ def render_report(report: RunReport, console: Console | None = None) -> None:

table = Table(title="Claims")
table.add_column("Claim")
table.add_column("Statement", overflow="fold")
table.add_column("Type")
table.add_column("Result")
table.add_column("Confidence")
Expand All @@ -51,10 +52,15 @@ def render_report(report: RunReport, console: Console | None = None) -> None:
style = _RESULT_STYLE.get(vc.result, "")
result_cell = f"[{style}]{vc.result.value}[/{style}]" if style else vc.result.value
table.add_row(
vc.claim.id, vc.claim.type.value, result_cell, vc.confidence.value, vc.reason
vc.claim.id,
vc.claim.statement,
vc.claim.type.value,
result_cell,
vc.confidence.value,
vc.reason,
)
if not report.verified:
table.add_row("-", "-", "-", "-", "no claims in this plan")
table.add_row("-", "-", "-", "-", "-", "no claims in this plan")
console.print(table)

# Failed evidence stays visible.
Expand Down Expand Up @@ -90,4 +96,19 @@ def render_report(report: RunReport, console: Console | None = None) -> None:
for err in report.errors:
console.print(f" [red]error[/red]: {err}")

if report.final_gate.result is GateResult.BLOCKED:
console.print(
" [bold red]Resolution:[/bold red] Required deterministic evidence failed or is missing. "
"Address blockers before proceeding."
)
elif report.final_gate.result is GateResult.REVIEW_REQUIRED:
console.print(
" [bold yellow]Resolution:[/bold yellow] Deterministic evidence verified, but evaluative claims "
"require human engineering sign-off."
)
elif report.final_gate.result is GateResult.PASS:
console.print(
" [bold green]Resolution:[/bold green] All required evidence verified. Gate PASS."
)

console.print(f"exit code: {report.exit_code}")
14 changes: 14 additions & 0 deletions tests/test_cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -170,3 +170,17 @@ def _no_sockets(*args: object, **kwargs: object) -> None:
["solve", "clean tree?", "--repo", str(repo), "--profile", "quick-check"]
)
assert code == EXIT_PASS


def test_verify_commit_defaults_to_head(repo: Path) -> None:
code = main(
[
"solve",
"verify current commit",
"--repo",
str(repo),
"--profile",
"verify-commit",
]
)
assert code == EXIT_PASS
60 changes: 60 additions & 0 deletions tests/test_demo.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
"""Tests for the 5-minute showcase demo module."""

from __future__ import annotations

from rich.console import Console

from proofloop.cli import EXIT_BLOCKED, EXIT_PASS, EXIT_REVIEW, main
from proofloop.demo import run_demo
from proofloop.schemas import GateResult, JudgeVerdictValue


def test_run_demo_all_scenarios() -> None:
code, reports = run_demo(scenario_filter="all", console=Console(quiet=True))
assert len(reports) == 3
assert code == EXIT_BLOCKED # highest exit code among the 3 is 2 (blocked)
assert reports[0].final_gate.result is GateResult.BLOCKED
assert reports[1].final_gate.result is GateResult.REVIEW_REQUIRED
assert reports[2].final_gate.result is GateResult.PASS


def test_run_demo_individual_scenarios() -> None:
# 1. Blocked
code, reports = run_demo(scenario_filter="blocked", console=Console(quiet=True))
assert code == EXIT_BLOCKED
assert len(reports) == 1
assert reports[0].pre_gate.result is GateResult.BLOCKED
assert reports[0].judge is not None and reports[0].judge.skipped is True

# 2. Review required
code, reports = run_demo(scenario_filter="review", console=Console(quiet=True))
assert code == EXIT_REVIEW
assert len(reports) == 1
assert reports[0].judge is not None
assert reports[0].judge.verdict is JudgeVerdictValue.ACCEPT
assert reports[0].final_gate.result is GateResult.REVIEW_REQUIRED

# 3. Pass
code, reports = run_demo(scenario_filter="pass", console=Console(quiet=True))
assert code == EXIT_PASS
assert len(reports) == 1
assert reports[0].final_gate.result is GateResult.PASS


def test_run_demo_unknown_scenario() -> None:
code, reports = run_demo(scenario_filter="unknown", console=Console(quiet=True))
assert code == EXIT_BLOCKED
assert reports == []


def test_cli_demo_command(capsys) -> None:
assert main(["demo", "--scenario", "pass"]) == EXIT_PASS
assert main(["demo", "--scenario", "review"]) == EXIT_REVIEW
assert main(["demo", "--scenario", "blocked"]) == EXIT_BLOCKED


def test_cli_demo_json_output(capsys) -> None:
assert main(["demo", "--scenario", "pass", "--json"]) == EXIT_PASS
out = capsys.readouterr().out
assert "final_gate" in out
assert "PASS" in out
Loading