From 57f377f6567f9b058acb69f1281e3c6fca4d6a13 Mon Sep 17 00:00:00 2001 From: Tom Ballard Date: Fri, 31 Jul 2026 08:09:47 +0100 Subject: [PATCH 1/2] feat(benchmarks): add deterministic Sentry evaluation --- .github/scripts/install-asdecided.sh | 19 +- .github/workflows/ci.yml | 6 +- README.md | 1 + sentry/README.md | 68 ++ sentry/baseline.json | 46 ++ sentry/cases.json | 585 ++++++++++++++++++ sentry/config.json | 15 + sentry/corpus/enforcement.md | 66 ++ sentry/corpus/human-review.md | 35 ++ sentry/corpus/unclassified.md | 27 + .../SEN-ADR-0001-sentrybench-contract.md | 61 ++ sentry/repository/docs/readme.md | 3 + sentry/repository/src/audit.rs | 3 + sentry/repository/src/domain.rs | 5 + sentry/repository/src/index.ts | 3 + sentry/repository/src/service.py | 4 + sentry/repository/src/sql/users.sql | 1 + ...ch-deterministic-enforcement-evaluation.md | 59 ++ sentry/run.py | 18 + sentry/sentry_benchmark.py | 446 +++++++++++++ tests/test_repo_invariants.py | 16 +- tests/test_sentry_benchmark.py | 95 +++ 22 files changed, 1570 insertions(+), 12 deletions(-) create mode 100644 sentry/README.md create mode 100644 sentry/baseline.json create mode 100644 sentry/cases.json create mode 100644 sentry/config.json create mode 100644 sentry/corpus/enforcement.md create mode 100644 sentry/corpus/human-review.md create mode 100644 sentry/corpus/unclassified.md create mode 100644 sentry/decisions/SEN-ADR-0001-sentrybench-contract.md create mode 100644 sentry/repository/docs/readme.md create mode 100644 sentry/repository/src/audit.rs create mode 100644 sentry/repository/src/domain.rs create mode 100644 sentry/repository/src/index.ts create mode 100644 sentry/repository/src/service.py create mode 100644 sentry/repository/src/sql/users.sql create mode 100644 sentry/requirements/sentrybench-deterministic-enforcement-evaluation.md create mode 100644 sentry/run.py create mode 100644 sentry/sentry_benchmark.py create mode 100644 tests/test_sentry_benchmark.py diff --git a/.github/scripts/install-asdecided.sh b/.github/scripts/install-asdecided.sh index 3826e37..e26cffc 100755 --- a/.github/scripts/install-asdecided.sh +++ b/.github/scripts/install-asdecided.sh @@ -1,15 +1,22 @@ #!/usr/bin/env bash set -euo pipefail -version="${1:-0.23.1}" +version="${1:-0.26.0}" version="${version#v}" -if [[ "$version" != "0.23.1" ]]; then - echo "::error::No verified checksum is recorded for asdecided-core $version" - exit 1 -fi archive="asdecided-x86_64-unknown-linux-gnu.tar.gz" -digest="51cce8025a7cb2f8b2caea93a8ea71be0ad8c5c316fd0ecced688267bf97b8ac" +case "$version" in + 0.23.1) + digest="51cce8025a7cb2f8b2caea93a8ea71be0ad8c5c316fd0ecced688267bf97b8ac" + ;; + 0.26.0) + digest="4d7f2fa85686af8d1006aa530928f60e2cd3d13d8560b303495f2784d1b8bbed" + ;; + *) + echo "::error::No verified checksum is recorded for asdecided-core $version" + exit 1 + ;; +esac install_dir="${RUNNER_TEMP}/asdecided-${version}" download="${RUNNER_TEMP}/${archive}" url="https://github.com/asdecided/core/releases/download/v${version}/${archive}" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d321d18..7a33279 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -32,14 +32,14 @@ jobs: cache: pip - name: Install AsDecided (external CLI on PATH, never an import) - run: bash .github/scripts/install-asdecided.sh 0.23.1 + run: bash .github/scripts/install-asdecided.sh 0.26.0 - name: Install the harness run: pip install -e ".[dev]" - name: Validate every fixture corpus run: | - for bench in search-artifacts find-decisions get-artifact get-related get-summary; do + for bench in search-artifacts find-decisions get-artifact get-related get-summary sentry; do decided validate "$bench/corpus" done @@ -48,6 +48,6 @@ jobs: - name: Gate every benchmark against its committed baseline run: | - for bench in search-artifacts find-decisions get-artifact get-related get-summary; do + for bench in search-artifacts find-decisions get-artifact get-related get-summary sentry; do python "$bench/run.py" --check done diff --git a/README.md b/README.md index ca8f12b..7439c9c 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,7 @@ corpus. | [`get-artifact/`](get-artifact/) | Exact-id resolution contract: alias and case-insensitive hits, duplicate and not-found error shapes. Conformance gated at 1.0. | | [`get-related/`](get-related/) | Relationship-edge retrieval: exact incoming AND outgoing edge sets per artifact. Conformance gated at 1.0. | | [`get-summary/`](get-summary/) | Portfolio summary contract: counts by type, empty-corpus shape, byte stability. Conformance gated at 1.0. | +| [`sentry/`](sentry/) | Deterministic decision-to-code enforcement: violation recall, clean-pass behaviour, attribution, diff isolation, SARIF, gate parity, and byte determinism. | | [`gitchameleon/`](gitchameleon/) | External evidence run (scaffold): does grounding in recorded version-pin decisions improve version-correct codegen on GitChameleon 2.0? Upstream executable-test scoring; never a merge gate. | ## Shared harness diff --git a/sentry/README.md b/sentry/README.md new file mode 100644 index 0000000..5a8d5d1 --- /dev/null +++ b/sentry/README.md @@ -0,0 +1,68 @@ +# SentryBench + +SentryBench evaluates whether AsDecided's deterministic code enforcement +blocks known decision violations without blocking compliant or unrelated code. +It consumes the published `decided` CLI as an external process and never +imports Core. + +This is not a retrieval benchmark. It scores enforcement correctness: + +- `forbid_pattern`, `require_pattern`, and `forbid_import` +- SQL, Python, Rust, JavaScript/TypeScript, and unsupported-language failure +- full-tree certification and Git diff isolation +- invalid constraint fail-closed behaviour +- decision, rule, path, and line attribution in Sentry JSON +- SARIF source locations +- `decided sentry` / `decided gate --code` parity over their shared projection +- byte-identical JSON on repeated unchanged runs + +The initial frozen set contains 63 contract cases, including 30 seeded +violations and close-neighbour allow cases. + +The committed fixture includes an eligible constrained decision, an explicitly +ineligible decision, and an intentionally unclassified decision. Coverage is +asserted but remains distinct from correctness. + +## Run + +```sh +python3 sentry/run.py +python3 sentry/run.py --json +python3 sentry/run.py --check +``` + +Set `RAC_BIN` to test a particular native executable: + +```sh +RAC_BIN=/path/to/decided python3 sentry/run.py --check +``` + +## Performance evidence + +Timing is deliberately outside the scored metrics block: + +```sh +python3 sentry/run.py --performance --iterations 30 +``` + +This reports median, p95, minimum, and maximum engine time for a clean +full-tree profile and a violating diff profile. It does not gate CI until a +controlled runner profile and stable scale matrix are committed. + +## Known boundary + +`decided gate --code` does not yet expose Sentry's decision and rule +provenance. Parity therefore compares the public fields shared by both +surfaces: outcome, coverage, code, path, and line. The dedicated Sentry JSON +remains authoritative for decision and rule attribution. + +## Gate + +Every correctness metric is gated at `1.0` with zero tolerance. Baselines may +only be updated through the explicit, human-reviewed command: + +```sh +python3 sentry/run.py --update-baseline +``` + +See `decisions/` and `requirements/` for the benchmark's governing contract. diff --git a/sentry/baseline.json b/sentry/baseline.json new file mode 100644 index 0000000..0ca28c7 --- /dev/null +++ b/sentry/baseline.json @@ -0,0 +1,46 @@ +{ + "overall": { + "conformance": 1.0, + "cases_passed": 63, + "cases_total": 63, + "negative_violations": 0, + "violation_recall": 1.0, + "clean_pass_rate": 1.0, + "attribution_accuracy": 1.0, + "sarif_accuracy": 1.0, + "gate_parity": 1.0, + "byte_determinism": 1.0 + }, + "by_category": { + "clean": { + "conformance": 1.0 + }, + "diff_isolation": { + "conformance": 1.0 + }, + "forbid_import_javascript": { + "conformance": 1.0 + }, + "forbid_import_python": { + "conformance": 1.0 + }, + "forbid_import_rust": { + "conformance": 1.0 + }, + "forbid_pattern": { + "conformance": 1.0 + }, + "invalid_constraint": { + "conformance": 1.0 + }, + "reporting": { + "conformance": 1.0 + }, + "require_pattern": { + "conformance": 1.0 + }, + "unsupported_language": { + "conformance": 1.0 + } + } +} diff --git a/sentry/cases.json b/sentry/cases.json new file mode 100644 index 0000000..11c5ca8 --- /dev/null +++ b/sentry/cases.json @@ -0,0 +1,585 @@ +{ + "cases": [ + { + "id": "C01", + "category": "clean", + "mode": "full", + "expected_findings": [], + "expected_coverage": { + "live_decisions": 3, + "classified_decisions": 2, + "unclassified_decisions": 1, + "eligible_decisions": 1, + "constrained_decisions": 1, + "active_rules": 6, + "corpus_adoption_percent": 33.333333333333336, + "eligible_coverage_percent": 100.0 + } + }, + { + "id": "C02", + "category": "clean", + "mode": "diff", + "changes": {"docs/readme.md": "# Fixture service\n\nUnrelated documentation changed.\n"}, + "expected_findings": [] + }, + { + "id": "C03", + "category": "clean", + "mode": "diff", + "changes": {"src/sql/users.sql": "UPDATE users SET deleted_at = CURRENT_TIMESTAMP WHERE id = 1;\n"}, + "expected_findings": [] + }, + { + "id": "C04", + "category": "clean", + "mode": "diff", + "changes": {"src/service.py": "from domain.users import User\n\nuser = User(\"1\")\n"}, + "expected_findings": [] + }, + { + "id": "C05", + "category": "clean", + "mode": "diff", + "changes": {"src/domain.rs": "use crate::domain::Account;\n\npub fn account() -> Account { Account::default() }\n"}, + "expected_findings": [] + }, + { + "id": "C06", + "category": "clean", + "mode": "diff", + "changes": {"src/index.ts": "import { Account } from \"./domain\";\n\nexport const account = new Account();\n"}, + "expected_findings": [] + }, + { + "id": "C07", + "category": "clean", + "mode": "full", + "expected_findings": [], + "byte_stable": true + }, + { + "id": "C08", + "category": "clean", + "mode": "full", + "changes": {"src/sql/users.sql": "DELETE FROM audit_users WHERE user_id = 1;\n"}, + "expected_findings": [] + }, + { + "id": "C09", + "category": "clean", + "mode": "full", + "changes": {"src/sql/users.sql": "SELECT delete_user(1);\n"}, + "expected_findings": [] + }, + { + "id": "C10", + "category": "clean", + "mode": "full", + "changes": {"src/service.py": "import domain.db\n"}, + "expected_findings": [] + }, + { + "id": "C11", + "category": "clean", + "mode": "full", + "changes": {"src/domain.rs": "use crate::domain::Repository;\n"}, + "expected_findings": [] + }, + { + "id": "C12", + "category": "clean", + "mode": "full", + "changes": {"src/index.ts": "import { adapter } from \"./pg\";\n"}, + "expected_findings": [] + }, + + { + "id": "V01", + "category": "forbid_pattern", + "mode": "full", + "changes": {"src/sql/users.sql": "SELECT 1;\nDELETE FROM users WHERE id = 1;\n"}, + "violation": true, + "expected_findings": [ + {"code": "code-constraint-violation", "decision_path": "corpus/enforcement.md", "rule_id": "no-hard-delete", "path": "src/sql/users.sql", "line": 2} + ] + }, + { + "id": "V02", + "category": "forbid_pattern", + "mode": "diff", + "changes": {"src/sql/users.sql": "SELECT 1;\nDELETE FROM users WHERE id = 1;\n"}, + "violation": true, + "expected_findings": [ + {"code": "code-constraint-violation", "decision_path": "corpus/enforcement.md", "rule_id": "no-hard-delete", "path": "src/sql/users.sql", "line": 2} + ] + }, + { + "id": "V03", + "category": "require_pattern", + "mode": "full", + "changes": {"src/audit.rs": "fn audit_internal(event: &str) -> bool { !event.is_empty() }\n"}, + "violation": true, + "expected_findings": [ + {"code": "code-constraint-violation", "decision_path": "corpus/enforcement.md", "rule_id": "require-audit-entrypoint", "path": "src/audit.rs", "line": null} + ] + }, + { + "id": "V04", + "category": "require_pattern", + "mode": "diff", + "changes": {"src/audit.rs": "fn audit_internal(event: &str) -> bool { !event.is_empty() }\n"}, + "violation": true, + "expected_findings": [ + {"code": "code-constraint-violation", "decision_path": "corpus/enforcement.md", "rule_id": "require-audit-entrypoint", "path": "src/audit.rs", "line": null} + ] + }, + { + "id": "V05", + "category": "forbid_import_python", + "mode": "full", + "changes": {"src/service.py": "from sqlalchemy.orm import Session\n"}, + "violation": true, + "expected_findings": [ + {"code": "code-constraint-violation", "decision_path": "corpus/enforcement.md", "rule_id": "no-python-db-import", "path": "src/service.py", "line": 1} + ] + }, + { + "id": "V06", + "category": "forbid_import_python", + "mode": "diff", + "changes": {"src/service.py": "import psycopg\n"}, + "violation": true, + "expected_findings": [ + {"code": "code-constraint-violation", "decision_path": "corpus/enforcement.md", "rule_id": "no-python-db-import", "path": "src/service.py", "line": 1} + ] + }, + { + "id": "V07", + "category": "forbid_import_rust", + "mode": "full", + "changes": {"src/domain.rs": "use diesel::prelude::*;\n"}, + "violation": true, + "expected_findings": [ + {"code": "code-constraint-violation", "decision_path": "corpus/enforcement.md", "rule_id": "no-rust-db-import", "path": "src/domain.rs", "line": 1} + ] + }, + { + "id": "V08", + "category": "forbid_import_rust", + "mode": "diff", + "changes": {"src/domain.rs": "use sqlx::Pool;\n"}, + "violation": true, + "expected_findings": [ + {"code": "code-constraint-violation", "decision_path": "corpus/enforcement.md", "rule_id": "no-rust-db-import", "path": "src/domain.rs", "line": 1} + ] + }, + { + "id": "V09", + "category": "forbid_import_javascript", + "mode": "full", + "changes": {"src/index.ts": "import { Client } from \"pg\";\n"}, + "violation": true, + "expected_findings": [ + {"code": "code-constraint-violation", "decision_path": "corpus/enforcement.md", "rule_id": "no-js-db-import", "path": "src/index.ts", "line": 1} + ] + }, + { + "id": "V10", + "category": "forbid_import_javascript", + "mode": "diff", + "changes": {"src/legacy.cjs": "const db = require(\"typeorm\");\n"}, + "violation": true, + "expected_findings": [ + {"code": "code-constraint-violation", "decision_path": "corpus/enforcement.md", "rule_id": "no-js-db-import", "path": "src/legacy.cjs", "line": 1} + ] + }, + { + "id": "V11", + "category": "unsupported_language", + "mode": "full", + "changes": {"src/storage.go": "package storage\n\nimport \"database/sql\"\n"}, + "violation": true, + "expected_findings": [ + {"code": "code-constraint-unsupported-language", "decision_path": "corpus/enforcement.md", "rule_id": "reject-unsupported-import-language", "path": "src/storage.go", "line": null} + ] + }, + { + "id": "V12", + "category": "unsupported_language", + "mode": "diff", + "changes": {"src/storage.go": "package storage\n\nimport \"database/sql\"\n"}, + "violation": true, + "expected_findings": [ + {"code": "code-constraint-unsupported-language", "decision_path": "corpus/enforcement.md", "rule_id": "reject-unsupported-import-language", "path": "src/storage.go", "line": null} + ] + }, + { + "id": "V13", + "category": "forbid_pattern", + "mode": "full", + "changes": {"src/sql/users.sql": "delete from users where id = 1;\n"}, + "violation": true, + "expected_findings": [ + {"code": "code-constraint-violation", "decision_path": "corpus/enforcement.md", "rule_id": "no-hard-delete", "path": "src/sql/users.sql", "line": 1} + ] + }, + { + "id": "V14", + "category": "forbid_pattern", + "mode": "full", + "changes": {"src/sql/archive/users.sql": "DELETE FROM users;\n"}, + "violation": true, + "expected_findings": [ + {"code": "code-constraint-violation", "decision_path": "corpus/enforcement.md", "rule_id": "no-hard-delete", "path": "src/sql/archive/users.sql", "line": 1} + ] + }, + { + "id": "V15", + "category": "forbid_pattern", + "mode": "diff", + "changes": {"src/sql/archive/users.sql": "DELETE FROM users;\n"}, + "violation": true, + "expected_findings": [ + {"code": "code-constraint-violation", "decision_path": "corpus/enforcement.md", "rule_id": "no-hard-delete", "path": "src/sql/archive/users.sql", "line": 1} + ] + }, + { + "id": "V16", + "category": "forbid_import_python", + "mode": "full", + "changes": {"src/service.py": "import sqlalchemy as sa\n"}, + "violation": true, + "expected_findings": [ + {"code": "code-constraint-violation", "decision_path": "corpus/enforcement.md", "rule_id": "no-python-db-import", "path": "src/service.py", "line": 1} + ] + }, + { + "id": "V17", + "category": "forbid_import_python", + "mode": "diff", + "changes": {"src/service.py": "import psycopg, os\n"}, + "violation": true, + "expected_findings": [ + {"code": "code-constraint-violation", "decision_path": "corpus/enforcement.md", "rule_id": "no-python-db-import", "path": "src/service.py", "line": 1} + ] + }, + { + "id": "V18", + "category": "forbid_import_rust", + "mode": "full", + "changes": {"src/domain.rs": "use diesel::prelude::*;\n"}, + "violation": true, + "expected_findings": [ + {"code": "code-constraint-violation", "decision_path": "corpus/enforcement.md", "rule_id": "no-rust-db-import", "path": "src/domain.rs", "line": 1} + ] + }, + { + "id": "V19", + "category": "forbid_import_rust", + "mode": "diff", + "changes": {"src/domain.rs": "use sqlx::{Pool, Row};\n"}, + "violation": true, + "expected_findings": [ + {"code": "code-constraint-violation", "decision_path": "corpus/enforcement.md", "rule_id": "no-rust-db-import", "path": "src/domain.rs", "line": 1} + ] + }, + { + "id": "V20", + "category": "forbid_import_javascript", + "mode": "full", + "changes": {"src/bootstrap.js": "import \"pg\";\n"}, + "violation": true, + "expected_findings": [ + {"code": "code-constraint-violation", "decision_path": "corpus/enforcement.md", "rule_id": "no-js-db-import", "path": "src/bootstrap.js", "line": 1} + ] + }, + { + "id": "V21", + "category": "forbid_import_javascript", + "mode": "full", + "changes": {"src/view.jsx": "import { DataSource } from \"typeorm/driver\";\n"}, + "violation": true, + "expected_findings": [ + {"code": "code-constraint-violation", "decision_path": "corpus/enforcement.md", "rule_id": "no-js-db-import", "path": "src/view.jsx", "line": 1} + ] + }, + { + "id": "V22", + "category": "forbid_import_javascript", + "mode": "diff", + "changes": {"src/view.tsx": "const client = require(\"pg\");\n"}, + "violation": true, + "expected_findings": [ + {"code": "code-constraint-violation", "decision_path": "corpus/enforcement.md", "rule_id": "no-js-db-import", "path": "src/view.tsx", "line": 1} + ] + }, + { + "id": "V23", + "category": "forbid_import_javascript", + "mode": "full", + "changes": {"src/worker.mjs": "import client from \"pg\";\n"}, + "violation": true, + "expected_findings": [ + {"code": "code-constraint-violation", "decision_path": "corpus/enforcement.md", "rule_id": "no-js-db-import", "path": "src/worker.mjs", "line": 1} + ] + }, + { + "id": "V24", + "category": "forbid_import_javascript", + "mode": "full", + "changes": {"src/worker.cjs": "const orm = require(\"typeorm\");\n"}, + "violation": true, + "expected_findings": [ + {"code": "code-constraint-violation", "decision_path": "corpus/enforcement.md", "rule_id": "no-js-db-import", "path": "src/worker.cjs", "line": 1} + ] + }, + { + "id": "V25", + "category": "forbid_import_javascript", + "mode": "diff", + "changes": {"src/nested/store.ts": "import { Pool } from \"pg\";\n"}, + "violation": true, + "expected_findings": [ + {"code": "code-constraint-violation", "decision_path": "corpus/enforcement.md", "rule_id": "no-js-db-import", "path": "src/nested/store.ts", "line": 1} + ] + }, + { + "id": "V26", + "category": "forbid_import_python", + "mode": "full", + "changes": {"src/rows.py": "from psycopg.rows import dict_row\n"}, + "violation": true, + "expected_findings": [ + {"code": "code-constraint-violation", "decision_path": "corpus/enforcement.md", "rule_id": "no-python-db-import", "path": "src/rows.py", "line": 1} + ] + }, + + { + "id": "D01", + "category": "diff_isolation", + "mode": "diff", + "base_changes": {"src/sql/users.sql": "DELETE FROM users WHERE id = 1;\n"}, + "changes": {"docs/readme.md": "# Fixture service\n\nOnly docs changed.\n"}, + "expected_findings": [] + }, + { + "id": "D02", + "category": "diff_isolation", + "mode": "diff", + "base_changes": {"src/service.py": "import sqlalchemy\n"}, + "changes": {"docs/readme.md": "# Fixture service\n\nOnly docs changed.\n"}, + "expected_findings": [] + }, + { + "id": "D03", + "category": "diff_isolation", + "mode": "diff", + "base_changes": {"src/sql/users.sql": "DELETE FROM users WHERE id = 1;\n"}, + "changes": {"src/sql/users.sql": "UPDATE users SET deleted_at = CURRENT_TIMESTAMP WHERE id = 1;\n"}, + "expected_findings": [] + }, + { + "id": "D04", + "category": "diff_isolation", + "mode": "diff", + "base_changes": {"src/sql/users.sql": "DELETE FROM users WHERE id = 1;\nSELECT 1;\n"}, + "changes": {"src/sql/users.sql": "DELETE FROM users WHERE id = 1;\nSELECT 2;\n"}, + "expected_findings": [] + }, + { + "id": "D05", + "category": "diff_isolation", + "mode": "diff", + "base_changes": {"src/index.ts": "import { Client } from \"pg\";\n"}, + "changes": {"docs/readme.md": "# Fixture service\n\nUnrelated to TypeScript.\n"}, + "expected_findings": [] + }, + { + "id": "D06", + "category": "diff_isolation", + "mode": "diff", + "base_changes": {"src/domain.rs": "use sqlx::Pool;\n\npub fn old() {}\n"}, + "changes": {"src/domain.rs": "use sqlx::Pool;\n\npub fn renamed() {}\n"}, + "expected_findings": [] + }, + { + "id": "D07", + "category": "diff_isolation", + "mode": "diff", + "changes": {"src/nested/service.py": "from domain.users import User\n"}, + "expected_findings": [] + }, + { + "id": "D08", + "category": "diff_isolation", + "mode": "diff", + "changes": {"src/nested/service.py": "from sqlalchemy.orm import Session\n"}, + "violation": true, + "expected_findings": [ + {"code": "code-constraint-violation", "decision_path": "corpus/enforcement.md", "rule_id": "no-python-db-import", "path": "src/nested/service.py", "line": 1} + ] + }, + + { + "id": "I01", + "category": "invalid_constraint", + "mode": "full", + "constraint_yaml": "version: 2\neligibility: eligible\nrules: []", + "expected_findings": [ + {"code": "unsupported-code-constraints-version", "decision_path": "corpus/enforcement.md", "rule_id": null, "path": "corpus/enforcement.md", "line": null} + ] + }, + { + "id": "I02", + "category": "invalid_constraint", + "mode": "full", + "constraint_yaml": "version: 1\neligibility: eligible\nrules:\n - id: duplicate-rule\n kind: forbid_pattern\n path_glob: \"src/**/*.sql\"\n pattern: \"DELETE\"\n - id: duplicate-rule\n kind: forbid_pattern\n path_glob: \"src/**/*.sql\"\n pattern: \"DROP\"", + "expected_findings": [ + {"code": "duplicate-code-constraint-id", "decision_path": "corpus/enforcement.md", "rule_id": "duplicate-rule", "path": "corpus/enforcement.md", "line": null} + ] + }, + { + "id": "I03", + "category": "invalid_constraint", + "mode": "full", + "constraint_yaml": "version: 1\neligibility: eligible\nrules:\n - id: unsafe-glob\n kind: forbid_pattern\n path_glob: \"../src/**\"\n pattern: \"DELETE\"", + "expected_findings": [ + {"code": "invalid-code-constraint", "decision_path": "corpus/enforcement.md", "rule_id": "unsafe-glob", "path": "corpus/enforcement.md", "line": null} + ] + }, + { + "id": "I04", + "category": "invalid_constraint", + "mode": "full", + "constraint_yaml": "version: 1\neligibility: eligible\nrules:\n - id: broken-glob\n kind: forbid_pattern\n path_glob: \"[\"\n pattern: \"DELETE\"", + "expected_findings": [ + {"code": "invalid-code-constraint", "decision_path": "corpus/enforcement.md", "rule_id": "broken-glob", "path": "corpus/enforcement.md", "line": null} + ] + }, + { + "id": "I05", + "category": "invalid_constraint", + "mode": "full", + "constraint_yaml": "version: 1\neligibility: eligible\nrules:\n - id: broken-regex\n kind: forbid_pattern\n path_glob: \"src/**\"\n pattern: \"(\"", + "expected_findings": [ + {"code": "invalid-code-constraint", "decision_path": "corpus/enforcement.md", "rule_id": "broken-regex", "path": "corpus/enforcement.md", "line": null} + ] + }, + { + "id": "I06", + "category": "invalid_constraint", + "mode": "full", + "constraint_yaml": "version: 1\neligibility: ineligible\nreason: \"Human review.\"\nrules:\n - id: forbidden-rule\n kind: forbid_pattern\n path_glob: \"src/**\"\n pattern: \"DELETE\"", + "expected_findings": [ + {"code": "invalid-code-constraint", "decision_path": "corpus/enforcement.md", "rule_id": null, "path": "corpus/enforcement.md", "line": null} + ] + }, + { + "id": "I07", + "category": "invalid_constraint", + "mode": "full", + "constraint_yaml": "version: 1\neligibility: ineligible\nrules: []", + "expected_findings": [ + {"code": "invalid-code-constraint", "decision_path": "corpus/enforcement.md", "rule_id": null, "path": "corpus/enforcement.md", "line": null} + ] + }, + { + "id": "I08", + "category": "invalid_constraint", + "mode": "full", + "constraint_section": "```yaml\nversion: [\n```", + "expected_findings": [ + {"code": "malformed-code-constraints", "decision_path": "corpus/enforcement.md", "rule_id": null, "path": "corpus/enforcement.md", "line": null} + ] + }, + { + "id": "I09", + "category": "invalid_constraint", + "mode": "full", + "constraint_yaml": "version: 1\neligibility: eligible\nrules:\n - id: BadRule\n kind: forbid_pattern\n path_glob: \"src/**\"\n pattern: \"DELETE\"", + "expected_findings": [ + {"code": "invalid-code-constraint", "decision_path": "corpus/enforcement.md", "rule_id": "BadRule", "path": "corpus/enforcement.md", "line": null} + ] + }, + { + "id": "I10", + "category": "invalid_constraint", + "mode": "full", + "constraint_yaml": "version: 1\neligibility: eligible\nrules:\n - id: empty-message\n kind: forbid_pattern\n path_glob: \"src/**\"\n pattern: \"DELETE\"\n message: \"\"", + "expected_findings": [ + {"code": "invalid-code-constraint", "decision_path": "corpus/enforcement.md", "rule_id": "empty-message", "path": "corpus/enforcement.md", "line": null} + ] + }, + { + "id": "I11", + "category": "invalid_constraint", + "mode": "full", + "constraint_yaml": "version: 1\neligibility: eligible\ninvented_field: true\nrules: []", + "expected_findings": [ + {"code": "malformed-code-constraints", "decision_path": "corpus/enforcement.md", "rule_id": null, "path": "corpus/enforcement.md", "line": null} + ] + }, + { + "id": "I12", + "category": "invalid_constraint", + "mode": "full", + "constraint_section": "```yaml\nversion: 1\neligibility: ineligible\nreason: \"First block.\"\nrules: []\n```\n\n## Code Constraints\n\n```yaml\nversion: 1\neligibility: ineligible\nreason: \"Second block.\"\nrules: []\n```", + "expected_findings": [ + {"code": "malformed-code-constraints", "decision_path": "corpus/enforcement.md", "rule_id": null, "path": "corpus/enforcement.md", "line": null} + ] + }, + + { + "id": "R01", + "category": "reporting", + "mode": "full", + "changes": {"src/sql/users.sql": "SELECT 1;\nDELETE FROM users WHERE id = 1;\n"}, + "violation": true, + "sarif": true, + "expected_findings": [ + {"code": "code-constraint-violation", "decision_path": "corpus/enforcement.md", "rule_id": "no-hard-delete", "path": "src/sql/users.sql", "line": 2} + ] + }, + { + "id": "R02", + "category": "reporting", + "mode": "full", + "changes": {"src/sql/users.sql": "SELECT 1;\nDELETE FROM users WHERE id = 1;\n"}, + "violation": true, + "gate_parity": true, + "expected_findings": [ + {"code": "code-constraint-violation", "decision_path": "corpus/enforcement.md", "rule_id": "no-hard-delete", "path": "src/sql/users.sql", "line": 2} + ] + }, + { + "id": "R03", + "category": "reporting", + "mode": "diff", + "changes": {"docs/readme.md": "# Fixture service\n\nDeterministic docs change.\n"}, + "byte_stable": true, + "gate_parity": true, + "expected_findings": [] + }, + { + "id": "R04", + "category": "reporting", + "mode": "diff", + "changes": {"src/service.py": "from sqlalchemy.orm import Session\n"}, + "violation": true, + "byte_stable": true, + "sarif": true, + "gate_parity": true, + "expected_findings": [ + {"code": "code-constraint-violation", "decision_path": "corpus/enforcement.md", "rule_id": "no-python-db-import", "path": "src/service.py", "line": 1} + ] + }, + { + "id": "R05", + "category": "reporting", + "mode": "full", + "sarif": true, + "expected_findings": [] + } + ] +} diff --git a/sentry/config.json b/sentry/config.json new file mode 100644 index 0000000..c1eff59 --- /dev/null +++ b/sentry/config.json @@ -0,0 +1,15 @@ +{ + "tolerance": 0.0, + "floors": { + "negative_violations": 0, + "overall": { + "conformance": 1.0, + "violation_recall": 1.0, + "clean_pass_rate": 1.0, + "attribution_accuracy": 1.0, + "sarif_accuracy": 1.0, + "gate_parity": 1.0, + "byte_determinism": 1.0 + } + } +} diff --git a/sentry/corpus/enforcement.md b/sentry/corpus/enforcement.md new file mode 100644 index 0000000..a55c064 --- /dev/null +++ b/sentry/corpus/enforcement.md @@ -0,0 +1,66 @@ +--- +schema_version: 1 +id: SEN-C3D4E5F6G7H8 +type: decision +tags: [fixture, enforcement] +--- +# Decision: Enforce Repository Boundaries + +## Status + +Accepted + +## Context + +The fixture repository needs representative source boundaries. + +## Decision + +Hard deletion and direct database dependencies are forbidden, while the audit +entry point must remain present. + +## Consequences + +Sentry can exercise every version-one rule family. + +## Code Constraints + +```yaml +version: 1 +eligibility: eligible +rules: + - id: no-hard-delete + kind: forbid_pattern + path_glob: "src/**/*.sql" + pattern: "(?i)DELETE\\s+FROM\\s+users" + message: "User records must not be hard-deleted." + - id: require-audit-entrypoint + kind: require_pattern + path_glob: "src/audit.rs" + pattern: "pub fn audit" + message: "The public audit entry point must remain present." + - id: no-python-db-import + kind: forbid_import + path_glob: "src/**/*.py" + pattern: "^(sqlalchemy|psycopg)(\\.|$)" + message: "Python services must not import database clients directly." + - id: no-rust-db-import + kind: forbid_import + path_glob: "src/**/*.rs" + pattern: "^(diesel|sqlx)(::|$)" + message: "Rust services must not import database clients directly." + - id: no-js-db-import + kind: forbid_import + path_glob: "src/**/*.{js,jsx,ts,tsx,mjs,cjs}" + pattern: "^(pg|typeorm)(/|$)" + message: "JavaScript services must not import database clients directly." + - id: reject-unsupported-import-language + kind: forbid_import + path_glob: "src/**/*.go" + pattern: "^database/sql$" + message: "Selected languages without a deterministic adapter must fail closed." +``` + +## Category + +Architecture diff --git a/sentry/corpus/human-review.md b/sentry/corpus/human-review.md new file mode 100644 index 0000000..dadcc4b --- /dev/null +++ b/sentry/corpus/human-review.md @@ -0,0 +1,35 @@ +--- +schema_version: 1 +id: SEN-D4E5F6G7H8J9 +type: decision +tags: [fixture, coverage] +--- +# Decision: Product Voice Requires Human Review + +## Status + +Accepted + +## Context + +Product voice cannot be reduced to a deterministic source check. + +## Decision + +Keep product-voice review human. + +## Consequences + +Sentry reports the decision as classified but ineligible. + +## Code Constraints + +```yaml +version: 1 +eligibility: ineligible +reason: "Product voice requires human judgement." +``` + +## Category + +Product diff --git a/sentry/corpus/unclassified.md b/sentry/corpus/unclassified.md new file mode 100644 index 0000000..246bb86 --- /dev/null +++ b/sentry/corpus/unclassified.md @@ -0,0 +1,27 @@ +--- +schema_version: 1 +id: SEN-E5F6G7H8J9K1 +type: decision +tags: [fixture, coverage] +--- +# Decision: Keep a Visible Unclassified Record + +## Status + +Accepted + +## Context + +Coverage must not imply that every live decision has been assessed. + +## Decision + +Leave this fixture intentionally unclassified. + +## Consequences + +The report exposes one unclassified decision. + +## Category + +Process diff --git a/sentry/decisions/SEN-ADR-0001-sentrybench-contract.md b/sentry/decisions/SEN-ADR-0001-sentrybench-contract.md new file mode 100644 index 0000000..2a3928d --- /dev/null +++ b/sentry/decisions/SEN-ADR-0001-sentrybench-contract.md @@ -0,0 +1,61 @@ +--- +schema_version: 1 +id: SEN-A1B2C3D4E5F6 +type: decision +tags: [benchmark, sentry, enforcement] +--- +# SEN-ADR-0001: SentryBench Correctness Contract + +## Status + +Accepted + +## Context + +Sentry is a deterministic enforcement surface, not a retrieval system. A useful +evaluation must prove that known violations are blocked, compliant and +unrelated changes remain green, findings cite the correct decision and rule, +and the dedicated and composed gate surfaces do not drift. + +A single blended coverage percentage would obscure two different questions: +whether enforcement is correct where rules exist, and how much of a corpus has +been classified for enforcement. + +## Decision + +SentryBench is a contract-shaped benchmark consumed through the published +`decided` executable only. + +1. Correctness cases are deterministic, offline, and gated at 1.0 with zero + tolerance. +2. Every enforceable rule family has both `must_block` and `must_allow` + examples. +3. Diff cases distinguish newly introduced violations from pre-existing, + removed, adjacent, and unrelated code. +4. Findings are scored by code, governing decision, rule ID, path, and line + when the engine provides one. +5. SARIF accuracy, `decided sentry` / `decided gate --code` parity over their + shared public finding projection, and byte-identical repeated JSON are + first-class cases. Sentry-specific decision/rule provenance is scored on + the dedicated JSON surface until the composed gate exposes equivalent + fields. +6. Corpus adoption and eligible enforcement coverage remain diagnostic + metadata and are never collapsed into correctness. +7. Wall-clock performance is a separate, non-scored mode. Correctness metrics + contain no clock, network, randomness, embedding, or model output. + +## Consequences + +The benchmark can block a semantic enforcement regression without importing +Core internals or using an LLM judge. Fixture expansion is additive. Timing +results can guide performance work but cannot make an unchanged correctness run +non-deterministic. Gate-level decision and rule provenance remains an explicit +follow-up rather than an inferred capability. + +## Related Requirements + +- sentrybench-deterministic-enforcement-evaluation + +## Category + +Process diff --git a/sentry/repository/docs/readme.md b/sentry/repository/docs/readme.md new file mode 100644 index 0000000..422c9ed --- /dev/null +++ b/sentry/repository/docs/readme.md @@ -0,0 +1,3 @@ +# Fixture service + +This file represents an unrelated documentation change. diff --git a/sentry/repository/src/audit.rs b/sentry/repository/src/audit.rs new file mode 100644 index 0000000..7f1e95f --- /dev/null +++ b/sentry/repository/src/audit.rs @@ -0,0 +1,3 @@ +pub fn audit(event: &str) -> bool { + !event.is_empty() +} diff --git a/sentry/repository/src/domain.rs b/sentry/repository/src/domain.rs new file mode 100644 index 0000000..6a4bc86 --- /dev/null +++ b/sentry/repository/src/domain.rs @@ -0,0 +1,5 @@ +use crate::domain::User; + +pub fn load_user() -> User { + User::default() +} diff --git a/sentry/repository/src/index.ts b/sentry/repository/src/index.ts new file mode 100644 index 0000000..188598b --- /dev/null +++ b/sentry/repository/src/index.ts @@ -0,0 +1,3 @@ +import { User } from "./domain"; + +export const loadUser = (): User => new User(); diff --git a/sentry/repository/src/service.py b/sentry/repository/src/service.py new file mode 100644 index 0000000..21a348d --- /dev/null +++ b/sentry/repository/src/service.py @@ -0,0 +1,4 @@ +from domain.users import User + +def load_user(user_id: str) -> User: + return User(user_id) diff --git a/sentry/repository/src/sql/users.sql b/sentry/repository/src/sql/users.sql new file mode 100644 index 0000000..bd2a8d2 --- /dev/null +++ b/sentry/repository/src/sql/users.sql @@ -0,0 +1 @@ +SELECT id, email FROM users WHERE deleted_at IS NULL; diff --git a/sentry/requirements/sentrybench-deterministic-enforcement-evaluation.md b/sentry/requirements/sentrybench-deterministic-enforcement-evaluation.md new file mode 100644 index 0000000..2bf962c --- /dev/null +++ b/sentry/requirements/sentrybench-deterministic-enforcement-evaluation.md @@ -0,0 +1,59 @@ +--- +schema_version: 1 +id: SEN-B2C3D4E5F6G7 +type: requirement +tags: [benchmark, sentry, conformance] +--- +# Requirement: SentryBench Deterministic Enforcement Evaluation + +## Status + +Accepted + +## Problem + +AsDecided needs reproducible evidence that Sentry catches machine-checkable +decision violations without blocking compliant changes or overstating corpus +coverage. + +## Requirements + +- [REQ-001] The benchmark MUST invoke AsDecided only through an external `decided` CLI. +- [REQ-002] Scored correctness MUST be deterministic, offline, and free of model judgement. +- [REQ-003] Every supported rule kind MUST have blocking and allowing cases. +- [REQ-004] Diff mode MUST prove new-line isolation for introduced, pre-existing, removed, adjacent, and unrelated changes. +- [REQ-005] Findings MUST be checked for code, decision, rule, path, and available line attribution. +- [REQ-006] JSON output MUST be byte-identical across repeated unchanged runs. +- [REQ-007] SARIF MUST identify the same violation and source location as JSON. +- [REQ-008] `decided sentry` and `decided gate --code` MUST agree on their shared finding projection and coverage; dedicated Sentry JSON MUST retain decision and rule attribution. +- [REQ-009] Invalid constraints and unsupported selected import languages MUST fail closed. +- [REQ-010] Performance measurements MUST remain outside the scored metrics block. +- [REQ-011] The gate MUST require perfect conformance, violation recall, clean-pass rate, attribution, report accuracy, parity, and determinism. + +## Success Metrics + +- All committed correctness cases pass. +- Violation recall and clean-patch pass rate are both 1.0. +- Attribution, SARIF, gate parity, and determinism are all 1.0. +- A deliberately contradicted case fails the benchmark gate. + +## Risks + +- Synthetic fixtures may be easier than real repositories; mutation and external-repository tranches must follow. +- Regex rules can be correct for a fixture but too broad for production; every rule therefore needs a near-neighbour allow case. +- Runtime measurements vary by host; they are diagnostic until a controlled runner profile is established. +- The composed gate does not yet expose Sentry decision and rule fields; parity is limited to code, path, line, outcome, and coverage until that payload grows additively. + +## Assumptions + +- Git is available locally for diff fixtures. +- The tested AsDecided release supports `decided sentry` and `decided gate --code`. + +## Related Decisions + +- SEN-ADR-0001 + +## Verified By + +- sentry/run.py +- tests/test_sentry_benchmark.py diff --git a/sentry/run.py b/sentry/run.py new file mode 100644 index 0000000..63d2d1b --- /dev/null +++ b/sentry/run.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +"""Run SentryBench through the published AsDecided CLI.""" + +from __future__ import annotations + +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[1] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from sentry_benchmark import main + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/sentry/sentry_benchmark.py b/sentry/sentry_benchmark.py new file mode 100644 index 0000000..a505cf7 --- /dev/null +++ b/sentry/sentry_benchmark.py @@ -0,0 +1,446 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Deterministic contract and mutation evaluation for AsDecided Sentry.""" + +from __future__ import annotations + +import argparse +import json +import os +import shutil +import statistics +import subprocess +import tempfile +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from harness.errors import UsageError +from harness.gate import evaluate_gate +from harness.scorecard import ( + Scorecard, + build_metadata, + render_metrics_json, + render_scorecard_human, + render_scorecard_json, +) + +BENCHMARK_DIR = Path(__file__).resolve().parent +FINDING_KEYS = ("code", "decision_path", "rule_id", "path", "line") + + +@dataclass(frozen=True) +class Invocation: + argv: tuple[str, ...] + exit_code: int + stdout: str + stderr: str + + def payload(self) -> dict[str, Any]: + try: + value = json.loads(self.stdout) + except json.JSONDecodeError as exc: + raise UsageError(f"non-JSON output from {' '.join(self.argv)}: {exc}") from None + if not isinstance(value, dict): + raise UsageError(f"non-object output from {' '.join(self.argv)}") + return value + + +def _load_json(path: Path, label: str) -> dict[str, Any]: + if not path.is_file(): + raise UsageError(f"{label} not found: {path}") + try: + value = json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc: + raise UsageError(f"cannot read {label}: {path}: {exc}") from None + if not isinstance(value, dict): + raise UsageError(f"malformed {label}: expected an object") + return value + + +def _load_cases(path: Path) -> list[dict[str, Any]]: + value = _load_json(path, "case set").get("cases") + if not isinstance(value, list) or not value: + raise UsageError(f"malformed case set: {path}: expected a non-empty cases list") + ids: set[str] = set() + for index, case in enumerate(value): + if not isinstance(case, dict): + raise UsageError(f"malformed case set: case {index} is not an object") + for required in ("id", "category", "mode", "expected_findings"): + if required not in case: + raise UsageError(f"malformed case set: case {index} missing {required}") + if case["id"] in ids: + raise UsageError(f"malformed case set: duplicate id {case['id']}") + if case["mode"] not in ("full", "diff"): + raise UsageError(f"malformed case set: case {case['id']} has invalid mode") + if not isinstance(case["expected_findings"], list): + raise UsageError(f"malformed case set: case {case['id']} findings must be a list") + ids.add(str(case["id"])) + return value + + +def _write_files(root: Path, values: dict[str, str]) -> None: + for relative, content in values.items(): + target = root / relative + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(content, encoding="utf-8") + + +def _replace_constraint(corpus: Path, case: dict[str, Any]) -> None: + if "constraint_yaml" not in case and "constraint_section" not in case: + return + target = corpus / "enforcement.md" + text = target.read_text(encoding="utf-8") + before, remainder = text.split("## Code Constraints\n\n", 1) + _, after = remainder.split("\n## Category", 1) + section = case.get("constraint_section") + if section is None: + section = f"```yaml\n{case['constraint_yaml']}\n```" + target.write_text( + before + "## Code Constraints\n\n" + str(section) + "\n\n## Category" + after, + encoding="utf-8", + ) + + +def _git(repo: Path, *args: str) -> str: + completed = subprocess.run( + ["git", "-C", str(repo), *args], + capture_output=True, + text=True, + check=False, + ) + if completed.returncode != 0: + raise UsageError(f"git {' '.join(args)} failed: {completed.stderr.strip()}") + return completed.stdout.strip() + + +def _prepare(case: dict[str, Any], parent: Path) -> tuple[Path, Path, str]: + corpus = parent / "corpus" + repository = parent / "repo" + shutil.copytree(BENCHMARK_DIR / "corpus", corpus) + shutil.copytree(BENCHMARK_DIR / "repository", repository) + _replace_constraint(corpus, case) + _write_files(repository, case.get("base_changes", {})) + _git(repository, "init", "-q") + _git(repository, "config", "user.email", "sentrybench@example.invalid") + _git(repository, "config", "user.name", "SentryBench") + _git(repository, "add", ".") + _git(repository, "commit", "-qm", "fixture base") + base = _git(repository, "rev-parse", "HEAD") + _write_files(repository, case.get("changes", {})) + for relative in case.get("deletes", []): + target = repository / relative + if target.exists(): + target.unlink() + # Git diff does not include untracked additions. Intent-to-add gives new + # fixture files the same diff visibility they have in a committed PR. + _git(repository, "add", "-N", ".") + return corpus, repository, base + + +def _invoke( + executable: str, + cwd: Path, + mode: str, + base: str, + *, + surface: str = "sentry", + output: str = "json", +) -> Invocation: + args = [executable, surface, "corpus"] + if surface == "gate": + args.append("--code") + args.extend(["--repository", "repo"]) + if mode == "full": + args.append("--full") + else: + args.extend(["--base", base]) + args.append("--sarif" if output == "sarif" else "--json") + completed = subprocess.run(args, cwd=cwd, capture_output=True, text=True, check=False) + return Invocation(tuple(args), completed.returncode, completed.stdout, completed.stderr) + + +def _normalise_findings(payload: dict[str, Any]) -> list[dict[str, Any]]: + findings = payload.get("findings") + if not isinstance(findings, list): + raise UsageError("Sentry JSON has no findings list") + return [ + {key: finding.get(key) for key in FINDING_KEYS} + for finding in findings + if isinstance(finding, dict) + ] + + +def _sentry_gate_projection(payload: dict[str, Any]) -> list[dict[str, Any]]: + findings = payload.get("findings") + if not isinstance(findings, list): + raise UsageError("gate JSON has no findings list") + projected = [] + for finding in findings: + if not isinstance(finding, dict) or finding.get("source") != "sentry": + continue + projected.append( + { + "code": finding.get("code"), + "path": finding.get("path"), + "line": finding.get("line"), + } + ) + return projected + + +def _sarif_projection(payload: dict[str, Any]) -> list[dict[str, Any]]: + try: + results = payload["runs"][0]["results"] + except (KeyError, IndexError, TypeError): + raise UsageError("Sentry SARIF has no runs[0].results") from None + projected = [] + for result in results: + location = result["locations"][0]["physicalLocation"] + projected.append( + { + "code": result.get("ruleId"), + "path": location["artifactLocation"]["uri"], + "line": location.get("region", {}).get("startLine"), + } + ) + return projected + + +def _run_case(case: dict[str, Any], executable: str) -> dict[str, Any]: + with tempfile.TemporaryDirectory(prefix=f"sentrybench-{case['id']}-") as raw: + parent = Path(raw) + _, _, base = _prepare(case, parent) + result = _invoke(executable, parent, case["mode"], base) + payload = result.payload() + actual = _normalise_findings(payload) + expected = case["expected_findings"] + checks = { + "exit_code": result.exit_code == (1 if expected else 0), + "ok": payload.get("ok") is (not expected), + "findings": actual == expected, + } + if "expected_coverage" in case: + coverage = payload.get("coverage", {}) + checks["coverage"] = all( + coverage.get(key) == value for key, value in case["expected_coverage"].items() + ) + + sarif_ok: bool | None = None + if case.get("sarif"): + sarif = _invoke(executable, parent, case["mode"], base, output="sarif") + sarif_payload = sarif.payload() + sarif_actual = _sarif_projection(sarif_payload) + sarif_expected = [ + {"code": item["code"], "path": item["path"], "line": item["line"]} + for item in expected + ] + sarif_ok = sarif.exit_code == (1 if expected else 0) and sarif_actual == sarif_expected + checks["sarif"] = sarif_ok + + parity_ok: bool | None = None + if case.get("gate_parity"): + gate = _invoke(executable, parent, case["mode"], base, surface="gate") + gate_payload = gate.payload() + expected_projection = [ + { + "code": item["code"], + "path": item["path"], + "line": item["line"], + } + for item in expected + ] + coverage = gate_payload.get("code_coverage") + parity_ok = ( + gate.exit_code == (1 if expected else 0) + and _sentry_gate_projection(gate_payload) == expected_projection + and coverage == payload.get("coverage") + ) + checks["gate_parity"] = parity_ok + + deterministic_ok: bool | None = None + if case.get("byte_stable"): + repeated = _invoke(executable, parent, case["mode"], base) + deterministic_ok = ( + repeated.exit_code == result.exit_code + and repeated.stdout == result.stdout + and repeated.stderr == result.stderr + ) + checks["byte_stable"] = deterministic_ok + + passed = all(checks.values()) + return { + "id": case["id"], + "category": case["category"], + "mode": case["mode"], + "violation": bool(case.get("violation")), + "passed": passed, + "checks": checks, + "expected_findings": expected, + "actual_findings": actual, + "sarif_ok": sarif_ok, + "gate_parity_ok": parity_ok, + "byte_stable": deterministic_ok, + } + + +def _ratio(numerator: int, denominator: int) -> float: + return 1.0 if denominator == 0 else numerator / denominator + + +def _aggregate(results: list[dict[str, Any]]) -> dict[str, Any]: + violating = [result for result in results if result["violation"]] + clean = [ + result + for result in results + if not result["expected_findings"] and not result["violation"] + ] + attributed = [result for result in results if result["expected_findings"]] + sarif = [result for result in results if result["sarif_ok"] is not None] + parity = [result for result in results if result["gate_parity_ok"] is not None] + deterministic = [result for result in results if result["byte_stable"] is not None] + by_category: dict[str, Any] = {} + for category in sorted({result["category"] for result in results}): + selected = [result for result in results if result["category"] == category] + by_category[category] = { + "conformance": _ratio(sum(result["passed"] for result in selected), len(selected)) + } + return { + "overall": { + "conformance": _ratio(sum(result["passed"] for result in results), len(results)), + "cases_passed": sum(result["passed"] for result in results), + "cases_total": len(results), + "negative_violations": sum( + 1 for result in clean if result["actual_findings"] + ), + "violation_recall": _ratio( + sum(result["passed"] for result in violating), len(violating) + ), + "clean_pass_rate": _ratio(sum(result["passed"] for result in clean), len(clean)), + "attribution_accuracy": _ratio( + sum(result["checks"]["findings"] for result in attributed), len(attributed) + ), + "sarif_accuracy": _ratio( + sum(result["sarif_ok"] is True for result in sarif), len(sarif) + ), + "gate_parity": _ratio( + sum(result["gate_parity_ok"] is True for result in parity), len(parity) + ), + "byte_determinism": _ratio( + sum(result["byte_stable"] is True for result in deterministic), + len(deterministic), + ), + }, + "by_category": by_category, + } + + +def _version(executable: str) -> str: + completed = subprocess.run( + [executable, "--version"], capture_output=True, text=True, check=False + ) + if completed.returncode != 0: + raise UsageError(f"{executable} --version failed: {completed.stderr.strip()}") + return completed.stdout.strip() + + +def run_scorecard(cases_path: Path, executable: str) -> Scorecard: + if shutil.which(executable) is None: + raise UsageError(f"'{executable}' not found on PATH") + cases = _load_cases(cases_path) + results = [_run_case(case, executable) for case in cases] + results.sort(key=lambda result: result["id"]) + metadata = build_metadata( + rac_version=_version(executable), + root=str(BENCHMARK_DIR / "corpus"), + queries_path=str(cases_path), + n_queries=len(cases), + ) + metadata["benchmark"] = "sentry" + return Scorecard(metrics=_aggregate(results), metadata=metadata, per_query=results) + + +def _percentile(values: list[float], percentile: float) -> float: + ordered = sorted(values) + index = min(len(ordered) - 1, max(0, int(round((len(ordered) - 1) * percentile)))) + return ordered[index] + + +def run_performance(executable: str, iterations: int) -> dict[str, Any]: + if iterations < 1: + raise UsageError("--iterations must be at least 1") + cases = {case["id"]: case for case in _load_cases(BENCHMARK_DIR / "cases.json")} + output: dict[str, Any] = { + "schema_version": "1", + "benchmark": "sentry-performance", + "engine": _version(executable), + "iterations": iterations, + "profiles": {}, + } + for name, case_id in (("full_clean", "C01"), ("diff_violation", "V02")): + with tempfile.TemporaryDirectory(prefix=f"sentrybench-perf-{name}-") as raw: + parent = Path(raw) + _, _, base = _prepare(cases[case_id], parent) + samples: list[float] = [] + for _ in range(iterations): + started = time.perf_counter_ns() + result = _invoke(executable, parent, cases[case_id]["mode"], base) + samples.append((time.perf_counter_ns() - started) / 1_000_000) + if result.exit_code not in (0, 1): + raise UsageError(f"performance profile {name} failed: {result.stderr}") + output["profiles"][name] = { + "median_ms": round(statistics.median(samples), 3), + "p95_ms": round(_percentile(samples, 0.95), 3), + "min_ms": round(min(samples), 3), + "max_ms": round(max(samples), 3), + } + return output + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description="Run deterministic SentryBench evaluation.") + mode = parser.add_mutually_exclusive_group() + mode.add_argument("--check", action="store_true") + mode.add_argument("--update-baseline", action="store_true") + mode.add_argument("--performance", action="store_true") + parser.add_argument("--json", action="store_true") + parser.add_argument("--cases", type=Path, default=BENCHMARK_DIR / "cases.json") + parser.add_argument("--baseline", type=Path, default=BENCHMARK_DIR / "baseline.json") + parser.add_argument("--config", type=Path, default=BENCHMARK_DIR / "config.json") + parser.add_argument("--iterations", type=int, default=10) + return parser + + +def main(argv: list[str] | None = None) -> int: + args = _parser().parse_args(argv) + executable = os.environ.get("RAC_BIN", "decided") + try: + if args.performance: + print(json.dumps(run_performance(executable, args.iterations), indent=2)) + return 0 + scorecard = run_scorecard(args.cases, executable) + if args.update_baseline: + args.baseline.write_text( + render_metrics_json(scorecard.metrics) + "\n", encoding="utf-8" + ) + print(f"sentry: baseline updated -> {args.baseline}") + return 0 + if args.check: + baseline = _load_json(args.baseline, "baseline") + config = _load_json(args.config, "config") + failures = evaluate_gate(scorecard.metrics, baseline, config) + if failures: + for failure in failures: + print(failure.render()) + return 1 + print("sentry: gate PASS") + return 0 + print( + render_scorecard_json(scorecard) + if args.json + else render_scorecard_human(scorecard, "conformance") + ) + return 0 + except UsageError as exc: + print(f"sentry: {exc}", file=os.sys.stderr) + return 2 diff --git a/tests/test_repo_invariants.py b/tests/test_repo_invariants.py index 4b61381..54c9e6c 100644 --- a/tests/test_repo_invariants.py +++ b/tests/test_repo_invariants.py @@ -12,14 +12,14 @@ _ENGINE_IMPORT = re.compile(r"^\s*(?:import\s+rac\b|from\s+rac\b)", re.MULTILINE) -# Non-gated evidence-run subdirs are inside the engine boundary too. -EVIDENCE_DIRS = ("gitchameleon",) +# Suites with custom input shapes are inside the engine boundary too. +CUSTOM_INPUT_DIRS = ("gitchameleon", "sentry") def _suite_python_files(): yield from (REPO_ROOT / "harness").rglob("*.py") yield from (REPO_ROOT / "tests").rglob("*.py") - for bench in BENCHMARKS + EVIDENCE_DIRS: + for bench in BENCHMARKS + CUSTOM_INPUT_DIRS: yield from (REPO_ROOT / bench).rglob("*.py") @@ -57,3 +57,13 @@ def test_every_benchmark_ships_its_committed_inputs(): assert (root / required).is_file(), f"{bench}/{required} missing" assert (root / "corpus").is_dir(), f"{bench}/corpus missing" assert (root / "decisions").is_dir(), f"{bench}/decisions missing" + + +def test_sentry_benchmark_ships_its_committed_inputs(): + root = REPO_ROOT / "sentry" + for required in ("run.py", "cases.json", "baseline.json", "config.json", "README.md"): + assert (root / required).is_file(), f"sentry/{required} missing" + assert (root / "corpus").is_dir() + assert (root / "repository").is_dir() + assert (root / "decisions").is_dir() + assert (root / "requirements").is_dir() diff --git a/tests/test_sentry_benchmark.py b/tests/test_sentry_benchmark.py new file mode 100644 index 0000000..fe0d47a --- /dev/null +++ b/tests/test_sentry_benchmark.py @@ -0,0 +1,95 @@ +# SPDX-License-Identifier: Apache-2.0 +"""SentryBench proof: perfect baseline, determinism, regression and usage gates.""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +from pathlib import Path + +from conftest import REPO_ROOT + +SENTRY = REPO_ROOT / "sentry" + + +def run_sentry(*args: str) -> subprocess.CompletedProcess: + return subprocess.run( + [sys.executable, str(SENTRY / "run.py"), *args], + capture_output=True, + text=True, + check=False, + env=os.environ.copy(), + ) + + +def scorecard() -> dict: + completed = run_sentry("--json") + assert completed.returncode == 0, completed.stderr + return json.loads(completed.stdout) + + +def test_sentry_scorecard_is_perfect(): + card = scorecard() + overall = card["metrics"]["overall"] + assert overall["cases_total"] == 63 + assert overall["cases_passed"] == 63 + for metric in ( + "conformance", + "violation_recall", + "clean_pass_rate", + "attribution_accuracy", + "sarif_accuracy", + "gate_parity", + "byte_determinism", + ): + assert overall[metric] == 1.0 + assert overall["negative_violations"] == 0 + + +def test_sentry_metrics_are_byte_identical(): + first = json.dumps(scorecard()["metrics"]) + second = json.dumps(scorecard()["metrics"]) + assert first == second + + +def test_sentry_baseline_matches_fresh_run(): + baseline = json.loads((SENTRY / "baseline.json").read_text(encoding="utf-8")) + assert baseline == scorecard()["metrics"] + + +def test_sentry_gate_passes(): + completed = run_sentry("--check") + assert completed.returncode == 0, completed.stdout + completed.stderr + assert "sentry: gate PASS" in completed.stdout + + +def test_sentry_contradiction_fails_named_floor(tmp_path): + cases = json.loads((SENTRY / "cases.json").read_text(encoding="utf-8")) + case = next(item for item in cases["cases"] if item["id"] == "V01") + case["expected_findings"][0]["line"] = 999 + path = tmp_path / "contradicted.json" + path.write_text(json.dumps(cases), encoding="utf-8") + completed = run_sentry("--check", "--cases", str(path)) + assert completed.returncode == 1 + assert "[floor] overall.conformance" in completed.stdout + + +def test_sentry_malformed_case_set_is_usage_error(tmp_path): + path = tmp_path / "bad.json" + path.write_text('{"cases": [{"id": "bad"}]}', encoding="utf-8") + completed = run_sentry("--cases", str(path)) + assert completed.returncode == 2 + assert "malformed case set" in completed.stderr + + +def test_sentry_performance_is_diagnostic_only(): + completed = run_sentry("--performance", "--iterations", "2") + assert completed.returncode == 0, completed.stderr + report = json.loads(completed.stdout) + assert "metrics" not in report + assert set(report["profiles"]) == {"full_clean", "diff_violation"} + for profile in report["profiles"].values(): + assert profile["median_ms"] > 0 + assert profile["p95_ms"] > 0 From c043a98f10190b68f31319a7888c34fa1a9b4f18 Mon Sep 17 00:00:00 2001 From: Tom Ballard Date: Fri, 31 Jul 2026 08:33:17 +0100 Subject: [PATCH 2/2] test(benchmarks): exercise Sentry at 5k decisions --- .github/workflows/ci.yml | 3 + README.md | 2 +- sentry/README.md | 20 +- sentry/baseline.json | 13 +- sentry/cases.json | 179 ++++++++++++++++++ .../SEN-ADR-0001-sentrybench-contract.md | 10 +- ...ch-deterministic-enforcement-evaluation.md | 3 + sentry/sentry_benchmark.py | 159 ++++++++++++++++ tests/test_sentry_benchmark.py | 26 ++- 9 files changed, 406 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7a33279..6d9f128 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -51,3 +51,6 @@ jobs: for bench in search-artifacts find-decisions get-artifact get-related get-summary sentry; do python "$bench/run.py" --check done + + - name: Verify Sentry at the supported 5,000-decision scale + run: python sentry/run.py --scale --corpus-size 5000 diff --git a/README.md b/README.md index 7439c9c..aa90058 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ corpus. | [`get-artifact/`](get-artifact/) | Exact-id resolution contract: alias and case-insensitive hits, duplicate and not-found error shapes. Conformance gated at 1.0. | | [`get-related/`](get-related/) | Relationship-edge retrieval: exact incoming AND outgoing edge sets per artifact. Conformance gated at 1.0. | | [`get-summary/`](get-summary/) | Portfolio summary contract: counts by type, empty-corpus shape, byte stability. Conformance gated at 1.0. | -| [`sentry/`](sentry/) | Deterministic decision-to-code enforcement: violation recall, clean-pass behaviour, attribution, diff isolation, SARIF, gate parity, and byte determinism. | +| [`sentry/`](sentry/) | Deterministic decision-to-code enforcement: 80 contract cases plus generated 5,000-decision scale evidence for recall, clean-pass behaviour, attribution, diff isolation, SARIF, gate parity, and byte determinism. | | [`gitchameleon/`](gitchameleon/) | External evidence run (scaffold): does grounding in recorded version-pin decisions improve version-correct codegen on GitChameleon 2.0? Upstream executable-test scoring; never a merge gate. | ## Shared harness diff --git a/sentry/README.md b/sentry/README.md index 5a8d5d1..541f35e 100644 --- a/sentry/README.md +++ b/sentry/README.md @@ -16,8 +16,9 @@ This is not a retrieval benchmark. It scores enforcement correctness: - `decided sentry` / `decided gate --code` parity over their shared projection - byte-identical JSON on repeated unchanged runs -The initial frozen set contains 63 contract cases, including 30 seeded -violations and close-neighbour allow cases. +The frozen set contains 80 contract cases, including 38 seeded violations, +multi-finding and ordering cases, diff-isolation edges, fail-closed behaviour, +and close-neighbour allow cases. The committed fixture includes an eligible constrained decision, an explicitly ineligible decision, and an intentionally unclassified decision. Coverage is @@ -37,6 +38,21 @@ Set `RAC_BIN` to test a particular native executable: RAC_BIN=/path/to/decided python3 sentry/run.py --check ``` +## Supported-scale evidence + +The scale profile deterministically generates a 5,000-decision corpus, then +checks clean and violating full-tree runs, a violating diff, composed-gate +parity, attribution, coverage accounting, and byte stability: + +```sh +python3 sentry/run.py --scale +python3 sentry/run.py --scale --corpus-size 5000 +``` + +The generated corpus is temporary: the repository does not carry 5,000 +low-information fixture files. The profile reports elapsed time for each +surface, but correctness does not depend on a wall-clock threshold. + ## Performance evidence Timing is deliberately outside the scored metrics block: diff --git a/sentry/baseline.json b/sentry/baseline.json index 0ca28c7..b78c684 100644 --- a/sentry/baseline.json +++ b/sentry/baseline.json @@ -1,8 +1,8 @@ { "overall": { "conformance": 1.0, - "cases_passed": 63, - "cases_total": 63, + "cases_passed": 80, + "cases_total": 80, "negative_violations": 0, "violation_recall": 1.0, "clean_pass_rate": 1.0, @@ -15,9 +15,15 @@ "clean": { "conformance": 1.0 }, + "coverage": { + "conformance": 1.0 + }, "diff_isolation": { "conformance": 1.0 }, + "fail_closed": { + "conformance": 1.0 + }, "forbid_import_javascript": { "conformance": 1.0 }, @@ -33,6 +39,9 @@ "invalid_constraint": { "conformance": 1.0 }, + "multi_finding": { + "conformance": 1.0 + }, "reporting": { "conformance": 1.0 }, diff --git a/sentry/cases.json b/sentry/cases.json index 11c5ca8..4618c84 100644 --- a/sentry/cases.json +++ b/sentry/cases.json @@ -580,6 +580,185 @@ "mode": "full", "sarif": true, "expected_findings": [] + }, + + { + "id": "C13", + "category": "clean", + "mode": "full", + "changes": {"src/service.py": "# import sqlalchemy\nvalue = \"from psycopg import connect\"\n"}, + "expected_findings": [] + }, + { + "id": "C14", + "category": "clean", + "mode": "full", + "changes": {"src/domain.rs": "// use sqlx::Pool;\nconst IMPORT: &str = \"diesel::Connection\";\n"}, + "expected_findings": [] + }, + { + "id": "C15", + "category": "clean", + "mode": "full", + "changes": {"src/index.ts": "const packageName = \"pg\";\nconst packageExample = \"typeorm\";\n"}, + "expected_findings": [] + }, + { + "id": "C16", + "category": "clean", + "mode": "full", + "changes": {"scripts/users.sql": "DELETE FROM users WHERE id = 1;\n"}, + "expected_findings": [] + }, + { + "id": "C17", + "category": "clean", + "mode": "full", + "changes": {"tools/service.py": "import sqlalchemy\n"}, + "expected_findings": [] + }, + { + "id": "C18", + "category": "clean", + "mode": "full", + "changes": {"src/index.ts": "import client from \"pg-native\";\n"}, + "expected_findings": [] + }, + { + "id": "C19", + "category": "coverage", + "mode": "full", + "constraint_yaml": "version: 1\neligibility: eligible\nrules: []", + "expected_coverage": { + "live_decisions": 3, + "classified_decisions": 2, + "unclassified_decisions": 1, + "eligible_decisions": 1, + "constrained_decisions": 0, + "active_rules": 0, + "percent": 0.0, + "eligible_coverage_percent": 0.0 + }, + "expected_findings": [] + }, + { + "id": "M01", + "category": "multi_finding", + "mode": "full", + "changes": { + "src/domain.rs": "use sqlx::Pool;\n", + "src/index.ts": "import { Client } from \"pg\";\n", + "src/service.py": "from sqlalchemy.orm import Session\n", + "src/sql/users.sql": "SELECT 1;\nDELETE FROM users WHERE id = 1;\nDELETE FROM users WHERE id = 2;\n" + }, + "violation": true, + "expected_findings": [ + {"code": "code-constraint-violation", "decision_path": "corpus/enforcement.md", "rule_id": "no-rust-db-import", "path": "src/domain.rs", "line": 1}, + {"code": "code-constraint-violation", "decision_path": "corpus/enforcement.md", "rule_id": "no-js-db-import", "path": "src/index.ts", "line": 1}, + {"code": "code-constraint-violation", "decision_path": "corpus/enforcement.md", "rule_id": "no-python-db-import", "path": "src/service.py", "line": 1}, + {"code": "code-constraint-violation", "decision_path": "corpus/enforcement.md", "rule_id": "no-hard-delete", "path": "src/sql/users.sql", "line": 2}, + {"code": "code-constraint-violation", "decision_path": "corpus/enforcement.md", "rule_id": "no-hard-delete", "path": "src/sql/users.sql", "line": 3} + ] + }, + { + "id": "M02", + "category": "multi_finding", + "mode": "diff", + "changes": { + "src/nested/one.py": "import sqlalchemy\n", + "src/nested/two.py": "import psycopg\n" + }, + "violation": true, + "expected_findings": [ + {"code": "code-constraint-violation", "decision_path": "corpus/enforcement.md", "rule_id": "no-python-db-import", "path": "src/nested/one.py", "line": 1}, + {"code": "code-constraint-violation", "decision_path": "corpus/enforcement.md", "rule_id": "no-python-db-import", "path": "src/nested/two.py", "line": 1} + ] + }, + { + "id": "M03", + "category": "multi_finding", + "mode": "full", + "changes": {"src/service.py": "import sqlalchemy, psycopg\n"}, + "violation": true, + "expected_findings": [ + {"code": "code-constraint-violation", "decision_path": "corpus/enforcement.md", "rule_id": "no-python-db-import", "path": "src/service.py", "line": 1}, + {"code": "code-constraint-violation", "decision_path": "corpus/enforcement.md", "rule_id": "no-python-db-import", "path": "src/service.py", "line": 1} + ] + }, + { + "id": "M04", + "category": "multi_finding", + "mode": "full", + "changes": {"src/index.ts": "import { Client } from \"pg\";\nconst orm = require(\"typeorm\");\n"}, + "violation": true, + "expected_findings": [ + {"code": "code-constraint-violation", "decision_path": "corpus/enforcement.md", "rule_id": "no-js-db-import", "path": "src/index.ts", "line": 1}, + {"code": "code-constraint-violation", "decision_path": "corpus/enforcement.md", "rule_id": "no-js-db-import", "path": "src/index.ts", "line": 2} + ] + }, + { + "id": "M05", + "category": "multi_finding", + "mode": "diff", + "changes": {"src/sql/users.sql": "DELETE FROM users WHERE id = 1;\nDELETE FROM users WHERE id = 2;\n"}, + "violation": true, + "sarif": true, + "gate_parity": true, + "byte_stable": true, + "expected_findings": [ + {"code": "code-constraint-violation", "decision_path": "corpus/enforcement.md", "rule_id": "no-hard-delete", "path": "src/sql/users.sql", "line": 1}, + {"code": "code-constraint-violation", "decision_path": "corpus/enforcement.md", "rule_id": "no-hard-delete", "path": "src/sql/users.sql", "line": 2} + ] + }, + { + "id": "D09", + "category": "diff_isolation", + "mode": "diff", + "base_changes": {"src/sql/users.sql": "DELETE FROM users WHERE id = 1;\n"}, + "changes": {"src/sql/users.sql": "DELETE FROM users WHERE id = 1;\nDELETE FROM users WHERE id = 2;\n"}, + "violation": true, + "expected_findings": [ + {"code": "code-constraint-violation", "decision_path": "corpus/enforcement.md", "rule_id": "no-hard-delete", "path": "src/sql/users.sql", "line": 2} + ] + }, + { + "id": "D10", + "category": "diff_isolation", + "mode": "diff", + "base_changes": {"src/service.py": "import sqlalchemy\n"}, + "changes": {"src/service.py": "import sqlalchemy\nimport psycopg\n"}, + "violation": true, + "expected_findings": [ + {"code": "code-constraint-violation", "decision_path": "corpus/enforcement.md", "rule_id": "no-python-db-import", "path": "src/service.py", "line": 2} + ] + }, + { + "id": "D11", + "category": "diff_isolation", + "mode": "diff", + "base_changes": {"src/index.ts": "import { Client } from \"pg\";\n"}, + "changes": {"src/index.ts": "import { Client } from \"pg\";\nimport { DataSource } from \"typeorm\";\n"}, + "violation": true, + "expected_findings": [ + {"code": "code-constraint-violation", "decision_path": "corpus/enforcement.md", "rule_id": "no-js-db-import", "path": "src/index.ts", "line": 2} + ] + }, + { + "id": "F01", + "category": "fail_closed", + "mode": "full", + "constraint_yaml": "version: 1\neligibility: eligible\nrules:\n - id: missing-required-file\n kind: require_pattern\n path_glob: \"src/missing.rs\"\n pattern: \"pub fn required\"\n message: \"Required source file must exist.\"", + "expected_findings": [ + {"code": "code-constraint-empty-match", "decision_path": "corpus/enforcement.md", "rule_id": "missing-required-file", "path": "corpus/enforcement.md", "line": null} + ] + }, + { + "id": "F02", + "category": "fail_closed", + "mode": "diff", + "constraint_yaml": "version: 1\neligibility: eligible\nrules:\n - id: missing-required-file\n kind: require_pattern\n path_glob: \"src/missing.rs\"\n pattern: \"pub fn required\"\n message: \"Required source file must exist.\"", + "changes": {"docs/readme.md": "# Fixture service\n\nUnrelated diff.\n"}, + "expected_findings": [] } ] } diff --git a/sentry/decisions/SEN-ADR-0001-sentrybench-contract.md b/sentry/decisions/SEN-ADR-0001-sentrybench-contract.md index 2a3928d..ccd513a 100644 --- a/sentry/decisions/SEN-ADR-0001-sentrybench-contract.md +++ b/sentry/decisions/SEN-ADR-0001-sentrybench-contract.md @@ -43,14 +43,20 @@ SentryBench is a contract-shaped benchmark consumed through the published metadata and are never collapsed into correctness. 7. Wall-clock performance is a separate, non-scored mode. Correctness metrics contain no clock, network, randomness, embedding, or model output. +8. Supported-scale evidence runs the same enforcement contract against a + deterministically generated 5,000-decision corpus. Generated scale fixtures + are temporary; corpus size, findings, attribution, coverage, parity, and + repeated output are checked, while elapsed time remains diagnostic. ## Consequences The benchmark can block a semantic enforcement regression without importing Core internals or using an LLM judge. Fixture expansion is additive. Timing results can guide performance work but cannot make an unchanged correctness run -non-deterministic. Gate-level decision and rule provenance remains an explicit -follow-up rather than an inferred capability. +non-deterministic. The 5,000-decision profile proves the supported corpus scale +without committing thousands of repetitive artifacts. Gate-level decision and +rule provenance remains an explicit follow-up rather than an inferred +capability. ## Related Requirements diff --git a/sentry/requirements/sentrybench-deterministic-enforcement-evaluation.md b/sentry/requirements/sentrybench-deterministic-enforcement-evaluation.md index 2bf962c..2b3e99e 100644 --- a/sentry/requirements/sentrybench-deterministic-enforcement-evaluation.md +++ b/sentry/requirements/sentrybench-deterministic-enforcement-evaluation.md @@ -29,6 +29,7 @@ coverage. - [REQ-009] Invalid constraints and unsupported selected import languages MUST fail closed. - [REQ-010] Performance measurements MUST remain outside the scored metrics block. - [REQ-011] The gate MUST require perfect conformance, violation recall, clean-pass rate, attribution, report accuracy, parity, and determinism. +- [REQ-012] A generated 5,000-decision corpus profile MUST preserve clean and violating enforcement outcomes, attribution, coverage accounting, composed-gate parity, and byte determinism. ## Success Metrics @@ -36,12 +37,14 @@ coverage. - Violation recall and clean-patch pass rate are both 1.0. - Attribution, SARIF, gate parity, and determinism are all 1.0. - A deliberately contradicted case fails the benchmark gate. +- The supported-scale profile passes with exactly 5,000 live decisions. ## Risks - Synthetic fixtures may be easier than real repositories; mutation and external-repository tranches must follow. - Regex rules can be correct for a fixture but too broad for production; every rule therefore needs a near-neighbour allow case. - Runtime measurements vary by host; they are diagnostic until a controlled runner profile is established. +- Synthetic scale decisions exercise corpus traversal and classification but do not approximate the semantic diversity of 5,000 independently authored decisions. - The composed gate does not yet expose Sentry decision and rule fields; parity is limited to code, path, line, outcome, and coverage until that payload grows additively. ## Assumptions diff --git a/sentry/sentry_benchmark.py b/sentry/sentry_benchmark.py index a505cf7..e710920 100644 --- a/sentry/sentry_benchmark.py +++ b/sentry/sentry_benchmark.py @@ -366,6 +366,159 @@ def _percentile(values: list[float], percentile: float) -> float: return ordered[index] +def _scale_decision(index: int) -> str: + artifact_id = f"SCL-{index:012d}" + return f"""--- +schema_version: 1 +id: {artifact_id} +type: decision +tags: [fixture, scale] +--- +# Decision: Scale Fixture {index:05d} + +## Status + +Accepted + +## Context + +This deterministic artifact expands the SentryBench corpus. + +## Decision + +Keep this synthetic decision outside machine-checkable source enforcement. + +## Consequences + +Sentry must classify it without producing a code finding. + +## Code Constraints + +```yaml +version: 1 +eligibility: ineligible +reason: "Synthetic scale fixture has no source-code constraint." +``` + +## Category + +Process +""" + + +def _expand_corpus(corpus: Path, corpus_size: int) -> None: + existing = len(list(corpus.rglob("*.md"))) + if corpus_size < existing: + raise UsageError( + f"--corpus-size must be at least {existing} for the committed fixture" + ) + scale = corpus / "scale" + scale.mkdir(exist_ok=True) + for index in range(1, corpus_size - existing + 1): + (scale / f"decision-{index:05d}.md").write_text( + _scale_decision(index), encoding="utf-8" + ) + + +def _timed_invoke( + executable: str, + cwd: Path, + mode: str, + base: str, + *, + surface: str = "sentry", +) -> tuple[Invocation, float]: + started = time.perf_counter_ns() + result = _invoke(executable, cwd, mode, base, surface=surface) + elapsed_ms = (time.perf_counter_ns() - started) / 1_000_000 + return result, round(elapsed_ms, 3) + + +def run_scale(executable: str, corpus_size: int) -> dict[str, Any]: + if shutil.which(executable) is None: + raise UsageError(f"'{executable}' not found on PATH") + cases = {case["id"]: case for case in _load_cases(BENCHMARK_DIR / "cases.json")} + with tempfile.TemporaryDirectory(prefix="sentrybench-scale-") as raw: + parent = Path(raw) + corpus, repository, base = _prepare(cases["C01"], parent) + _expand_corpus(corpus, corpus_size) + + clean, clean_ms = _timed_invoke(executable, parent, "full", base) + clean_payload = clean.payload() + + _write_files( + repository, + {"src/sql/users.sql": "SELECT 1;\nDELETE FROM users WHERE id = 1;\n"}, + ) + _git(repository, "add", "-N", ".") + full, full_ms = _timed_invoke(executable, parent, "full", base) + diff, diff_ms = _timed_invoke(executable, parent, "diff", base) + gate, gate_ms = _timed_invoke( + executable, parent, "diff", base, surface="gate" + ) + repeated = _invoke(executable, parent, "diff", base) + + full_payload = full.payload() + diff_payload = diff.payload() + gate_payload = gate.payload() + expected_finding = { + "code": "code-constraint-violation", + "decision_path": "corpus/enforcement.md", + "rule_id": "no-hard-delete", + "path": "src/sql/users.sql", + "line": 2, + } + expected_projection = { + "code": expected_finding["code"], + "path": expected_finding["path"], + "line": expected_finding["line"], + } + expected_coverage = { + "live_decisions": corpus_size, + "classified_decisions": corpus_size - 1, + "unclassified_decisions": 1, + "eligible_decisions": 1, + "constrained_decisions": 1, + "active_rules": 6, + "percent": 100.0 / corpus_size, + "metric": "corpus_adoption", + "corpus_adoption_percent": 100.0 / corpus_size, + "eligible_coverage_percent": 100.0, + } + checks = { + "corpus_size": clean_payload.get("coverage", {}).get("live_decisions") + == corpus_size, + "coverage_accounting": clean_payload.get("coverage") == expected_coverage, + "clean_full": clean.exit_code == 0 + and clean_payload.get("ok") is True + and _normalise_findings(clean_payload) == [], + "violation_full": full.exit_code == 1 + and _normalise_findings(full_payload) == [expected_finding], + "violation_diff": diff.exit_code == 1 + and _normalise_findings(diff_payload) == [expected_finding], + "gate_parity": gate.exit_code == 1 + and _sentry_gate_projection(gate_payload) == [expected_projection] + and gate_payload.get("code_coverage") == expected_coverage, + "byte_determinism": repeated.exit_code == diff.exit_code + and repeated.stdout == diff.stdout + and repeated.stderr == diff.stderr, + } + return { + "schema_version": "1", + "benchmark": "sentry-scale", + "engine": _version(executable), + "corpus_size": corpus_size, + "passed": all(checks.values()), + "checks": checks, + "profiles": { + "clean_full": {"elapsed_ms": clean_ms}, + "violation_full": {"elapsed_ms": full_ms}, + "violation_diff": {"elapsed_ms": diff_ms}, + "gate_diff": {"elapsed_ms": gate_ms}, + }, + } + + def run_performance(executable: str, iterations: int) -> dict[str, Any]: if iterations < 1: raise UsageError("--iterations must be at least 1") @@ -403,11 +556,13 @@ def _parser() -> argparse.ArgumentParser: mode.add_argument("--check", action="store_true") mode.add_argument("--update-baseline", action="store_true") mode.add_argument("--performance", action="store_true") + mode.add_argument("--scale", action="store_true") parser.add_argument("--json", action="store_true") parser.add_argument("--cases", type=Path, default=BENCHMARK_DIR / "cases.json") parser.add_argument("--baseline", type=Path, default=BENCHMARK_DIR / "baseline.json") parser.add_argument("--config", type=Path, default=BENCHMARK_DIR / "config.json") parser.add_argument("--iterations", type=int, default=10) + parser.add_argument("--corpus-size", type=int, default=5000) return parser @@ -415,6 +570,10 @@ def main(argv: list[str] | None = None) -> int: args = _parser().parse_args(argv) executable = os.environ.get("RAC_BIN", "decided") try: + if args.scale: + report = run_scale(executable, args.corpus_size) + print(json.dumps(report, indent=2)) + return 0 if report["passed"] else 1 if args.performance: print(json.dumps(run_performance(executable, args.iterations), indent=2)) return 0 diff --git a/tests/test_sentry_benchmark.py b/tests/test_sentry_benchmark.py index fe0d47a..d4d3bb5 100644 --- a/tests/test_sentry_benchmark.py +++ b/tests/test_sentry_benchmark.py @@ -33,8 +33,8 @@ def scorecard() -> dict: def test_sentry_scorecard_is_perfect(): card = scorecard() overall = card["metrics"]["overall"] - assert overall["cases_total"] == 63 - assert overall["cases_passed"] == 63 + assert overall["cases_total"] == 80 + assert overall["cases_passed"] == 80 for metric in ( "conformance", "violation_recall", @@ -93,3 +93,25 @@ def test_sentry_performance_is_diagnostic_only(): for profile in report["profiles"].values(): assert profile["median_ms"] > 0 assert profile["p95_ms"] > 0 + + +def test_sentry_scale_profile_preserves_contract(): + completed = run_sentry("--scale", "--corpus-size", "25") + assert completed.returncode == 0, completed.stderr + report = json.loads(completed.stdout) + assert report["benchmark"] == "sentry-scale" + assert report["corpus_size"] == 25 + assert report["passed"] is True + assert all(report["checks"].values()) + assert set(report["profiles"]) == { + "clean_full", + "violation_full", + "violation_diff", + "gate_diff", + } + + +def test_sentry_scale_rejects_too_small_corpus(): + completed = run_sentry("--scale", "--corpus-size", "2") + assert completed.returncode == 2 + assert "--corpus-size must be at least 3" in completed.stderr