diff --git a/README.md b/README.md index 78aeaad..e8eb1bc 100644 --- a/README.md +++ b/README.md @@ -71,7 +71,7 @@ The repository includes sanitized fixture scans so reviewers can inspect real ou | `risky-ci-agent` | `blocked` | Shows a privileged pull request workflow with write access being stopped. | | `risky-mcp-agent` | `needs_review` | Shows an MCP server receiving a credential reference and requiring human review. | -Start with [the public fixture manifest](docs/demo-artifacts/public-fixture-scans/manifest.json), then inspect the blocked CI [summary](docs/demo-artifacts/public-fixture-scans/risky-ci-agent/summary.md), [permit](docs/demo-artifacts/public-fixture-scans/risky-ci-agent/permit.yaml), and [raw findings](docs/demo-artifacts/public-fixture-scans/risky-ci-agent/raw-findings.json). +Start with [the public fixture manifest](docs/demo-artifacts/public-fixture-scans/manifest.json), then inspect the blocked CI [summary](docs/demo-artifacts/public-fixture-scans/risky-ci-agent/summary.md), [permit](docs/demo-artifacts/public-fixture-scans/risky-ci-agent/permit.yaml), [raw findings](docs/demo-artifacts/public-fixture-scans/risky-ci-agent/raw-findings.json), and [finding baseline](docs/demo-artifacts/public-fixture-scans/risky-ci-agent/finding-baseline.json). Regenerate them with: diff --git a/docs-site/content/docs/public-demo-artifacts.mdx b/docs-site/content/docs/public-demo-artifacts.mdx index 89c1a59..dee9b9d 100644 --- a/docs-site/content/docs/public-demo-artifacts.mdx +++ b/docs-site/content/docs/public-demo-artifacts.mdx @@ -50,6 +50,7 @@ docs/demo-artifacts/public-fixture-scans//permit.yaml docs/demo-artifacts/public-fixture-scans//raw-findings.json docs/demo-artifacts/public-fixture-scans//graph-paths.json docs/demo-artifacts/public-fixture-scans//controls.json +docs/demo-artifacts/public-fixture-scans//finding-baseline.json docs/demo-artifacts/public-fixture-scans//run-metrics.json ``` @@ -64,6 +65,7 @@ Then compare the blocked CI example: - [Permit](https://github.com/IntelIP/agent-permit-office/blob/main/docs/demo-artifacts/public-fixture-scans/risky-ci-agent/permit.yaml) - [Raw findings](https://github.com/IntelIP/agent-permit-office/blob/main/docs/demo-artifacts/public-fixture-scans/risky-ci-agent/raw-findings.json) - [Graph paths](https://github.com/IntelIP/agent-permit-office/blob/main/docs/demo-artifacts/public-fixture-scans/risky-ci-agent/graph-paths.json) +- [Finding baseline](https://github.com/IntelIP/agent-permit-office/blob/main/docs/demo-artifacts/public-fixture-scans/risky-ci-agent/finding-baseline.json) ## How to interpret it @@ -75,7 +77,8 @@ Read the artifacts in this order: 4. `raw-findings.json`: exact scanner findings with evidence locations. 5. `graph-paths.json`: source-to-sink paths, such as credential to MCP server. 6. `controls.json`: review controls that must pass before approval. -7. `run-metrics.json`: counts used by dashboards and CI. +7. `finding-baseline.json`: frozen finding keys for future diff scans. +8. `run-metrics.json`: counts used by dashboards and CI. The important pattern: scanner output owns the decision. Deep Agent review may explain these artifacts later, but it should not invent a different permit status. diff --git a/docs/demo-artifacts/public-fixture-scans/manifest.json b/docs/demo-artifacts/public-fixture-scans/manifest.json index 82adc10..4278696 100644 --- a/docs/demo-artifacts/public-fixture-scans/manifest.json +++ b/docs/demo-artifacts/public-fixture-scans/manifest.json @@ -3,6 +3,7 @@ "fixtures": [ { "artifact_dir": "docs/demo-artifacts/public-fixture-scans/safe-agent", + "baseline": "docs/demo-artifacts/public-fixture-scans/safe-agent/finding-baseline.json", "controls": 0, "findings": 0, "graph_paths": 0, @@ -13,6 +14,7 @@ }, { "artifact_dir": "docs/demo-artifacts/public-fixture-scans/risky-ci-agent", + "baseline": "docs/demo-artifacts/public-fixture-scans/risky-ci-agent/finding-baseline.json", "controls": 5, "findings": 4, "graph_paths": 1, @@ -23,6 +25,7 @@ }, { "artifact_dir": "docs/demo-artifacts/public-fixture-scans/risky-mcp-agent", + "baseline": "docs/demo-artifacts/public-fixture-scans/risky-mcp-agent/finding-baseline.json", "controls": 3, "findings": 2, "graph_paths": 1, diff --git a/docs/demo-artifacts/public-fixture-scans/risky-ci-agent/finding-baseline.json b/docs/demo-artifacts/public-fixture-scans/risky-ci-agent/finding-baseline.json new file mode 100644 index 0000000..a82b829 --- /dev/null +++ b/docs/demo-artifacts/public-fixture-scans/risky-ci-agent/finding-baseline.json @@ -0,0 +1,51 @@ +{ + "findings": [ + { + "category": "runtime_policy", + "finding_id": "finding:ci-pr-target-write-token:.github/workflows/agent.yml:7", + "key": "283fcff6a21dee45c569dc9cbe7953a5852415f937f36b072daddd94e12ba425", + "line_end": 4, + "line_start": 4, + "path": ".github/workflows/agent.yml", + "rule_id": "ci-pr-target-write-token", + "severity": "critical", + "title": "PR-target workflow has write token permissions" + }, + { + "category": "credential_scope", + "finding_id": "finding:ci-secret-reference:.github/workflows/agent.yml:16", + "key": "373477bfd7de2192d376abaa70a197bb31a5d7459e32c2c05eb6206982e31e3d", + "line_end": 16, + "line_start": 16, + "path": ".github/workflows/agent.yml", + "rule_id": "ci-secret-reference", + "severity": "medium", + "title": "Workflow references repository secrets" + }, + { + "category": "runtime_policy", + "finding_id": "finding:ci-write-all-permissions:.github/workflows/agent.yml:7", + "key": "5aa6c21ac8aad6819887091cb2d69d82a80246ba3a0aff8957c52cbc6cf8bb00", + "line_end": 7, + "line_start": 7, + "path": ".github/workflows/agent.yml", + "rule_id": "ci-write-all-permissions", + "severity": "high", + "title": "Workflow grants write-all permissions" + }, + { + "category": "runtime_policy", + "finding_id": "finding:ci-pull-request-target:.github/workflows/agent.yml:4", + "key": "b87dfcfe645ab40389e4fc1d31ad8bb9ea643a2c967af8746f4e6911f2635d93", + "line_end": 4, + "line_start": 4, + "path": ".github/workflows/agent.yml", + "rule_id": "ci-pull-request-target", + "severity": "high", + "title": "Workflow uses pull_request_target" + } + ], + "generated_at": "2026-01-01T00:00:00Z", + "scan_run_id": "public-demo-risky-ci-agent", + "version": 1 +} diff --git a/docs/demo-artifacts/public-fixture-scans/risky-mcp-agent/finding-baseline.json b/docs/demo-artifacts/public-fixture-scans/risky-mcp-agent/finding-baseline.json new file mode 100644 index 0000000..53cd9ed --- /dev/null +++ b/docs/demo-artifacts/public-fixture-scans/risky-mcp-agent/finding-baseline.json @@ -0,0 +1,29 @@ +{ + "findings": [ + { + "category": "credential_scope", + "finding_id": "finding:mcp-stdio-credential-ref:.mcp.json:github-tools", + "key": "e5c680f51bfa51d075d6d913c5a337001ae788dcc4d605ac9316b9dc984949fa", + "line_end": null, + "line_start": 3, + "path": ".mcp.json", + "rule_id": "mcp-stdio-credential-ref", + "severity": "high", + "title": "Stdio MCP server receives credential references" + }, + { + "category": "supply_chain", + "finding_id": "finding:mcp-unpinned-package-command:.mcp.json:github-tools", + "key": "f9e305d3e60f3a64ef84f3a6dcd1791916772995c8e99da24b2f163cccbabbb2", + "line_end": null, + "line_start": 3, + "path": ".mcp.json", + "rule_id": "mcp-unpinned-package-command", + "severity": "medium", + "title": "MCP server package is not version pinned" + } + ], + "generated_at": "2026-01-01T00:00:00Z", + "scan_run_id": "public-demo-risky-mcp-agent", + "version": 1 +} diff --git a/docs/demo-artifacts/public-fixture-scans/safe-agent/finding-baseline.json b/docs/demo-artifacts/public-fixture-scans/safe-agent/finding-baseline.json new file mode 100644 index 0000000..27f736e --- /dev/null +++ b/docs/demo-artifacts/public-fixture-scans/safe-agent/finding-baseline.json @@ -0,0 +1,6 @@ +{ + "findings": [], + "generated_at": "2026-01-01T00:00:00Z", + "scan_run_id": "public-demo-safe-agent", + "version": 1 +} diff --git a/tools/build_public_demo_artifacts.py b/tools/build_public_demo_artifacts.py index df4106e..6ba2ca7 100644 --- a/tools/build_public_demo_artifacts.py +++ b/tools/build_public_demo_artifacts.py @@ -1,6 +1,7 @@ from __future__ import annotations from contextlib import redirect_stderr +from datetime import datetime, timezone from io import StringIO import json from pathlib import Path @@ -8,7 +9,9 @@ import tempfile from typing import Any +from agent_permit.baseline import build_finding_baseline from agent_permit.cli import run_scan +from agent_permit.models import Finding REPO_ROOT = Path(__file__).resolve().parents[1] @@ -23,6 +26,7 @@ "controls.json", "run-metrics.json", ) +PUBLIC_BASELINE_GENERATED_AT = datetime(2026, 1, 1, tzinfo=timezone.utc) FIXTURES = ( { "id": "safe-agent", @@ -116,6 +120,7 @@ def main() -> int: encoding="utf-8", ) + _write_public_baseline(public_dir) metrics = json.loads((public_dir / "run-metrics.json").read_text()) manifest["fixtures"].append( { @@ -127,6 +132,9 @@ def main() -> int: "graph_paths": metrics["graph_paths"], "controls": metrics["controls"], "artifact_dir": str(public_dir.relative_to(REPO_ROOT)), + "baseline": str( + (public_dir / "finding-baseline.json").relative_to(REPO_ROOT) + ), } ) @@ -155,6 +163,23 @@ def _write_sanitized_json( ) +def _write_public_baseline(public_dir: Path) -> None: + raw_findings = json.loads((public_dir / "raw-findings.json").read_text()) + findings = [ + Finding.model_validate(finding) + for finding in raw_findings.get("findings", []) + ] + baseline = build_finding_baseline( + findings, + scan_run_id=str(raw_findings.get("scan_run_id")), + generated_at=PUBLIC_BASELINE_GENERATED_AT, + ) + (public_dir / "finding-baseline.json").write_text( + json.dumps(baseline.model_dump(mode="json"), indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + + def _sanitize_json(value: Any, replacements: dict[str, str]) -> Any: if isinstance(value, str): return _sanitize_text(value, replacements)