diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..67402e1 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,31 @@ +# Automated dependency-update monitoring (AC-039). +# +# Dependabot opens pull requests when a newer version of a pinned dependency or +# a pinned action is published, so constraints.txt and the action SHAs cannot +# quietly rot. Each PR runs the full CI matrix, the falsification harness, and +# the dependency-advisory audit before it can be considered. +# +# NOTE ON PROVIDER STATE: version-update PRs come from this file alone. +# Dependabot SECURITY alerts (and security-update PRs) are a separate repository +# setting that cannot be enabled from a file in the repository — the owner must +# turn on Dependabot alerts under Settings -> Advanced Security. That is why +# .github/workflows/security.yml exists: the advisory gate it enforces is +# repository-controlled and does not depend on that provider setting. + +version: 2 +updates: + - package-ecosystem: "pip" + directory: "/" + schedule: + interval: "weekly" + open-pull-requests-limit: 5 + commit-message: + prefix: "deps" + + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + open-pull-requests-limit: 5 + commit-message: + prefix: "ci" diff --git a/.github/workflows/authcontract-gate.yml b/.github/workflows/authcontract-gate.yml index 78730f1..0f53c6f 100644 --- a/.github/workflows/authcontract-gate.yml +++ b/.github/workflows/authcontract-gate.yml @@ -13,13 +13,19 @@ name: AuthContract Gate on: pull_request: +# Least privilege (AC-039). The gate reads the repository and adjudicates; it +# writes nothing back to GitHub. The default token grant is deliberately not +# relied on. +permissions: + contents: read + jobs: gate: name: AuthContract Gate runs-on: ubuntu-latest steps: - name: Checkout PR test-merge composition - uses: actions/checkout@v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: # Default ref for a pull_request-triggered workflow is GitHub's # own ephemeral merge commit (refs/pull//merge, i.e. @@ -32,12 +38,12 @@ jobs: - name: Fetch current base run: git fetch origin "${{ github.event.pull_request.base.ref }}" - - uses: actions/setup-python@v5 + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.12" - name: Install - run: pip install -e ".[test]" + run: pip install -e ".[test]" -c constraints.txt - name: Run AuthContract test suite against the merge composition id: tests diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 08c5342..acfd72d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,6 +4,13 @@ on: push: pull_request: +# Least privilege (AC-039). This workflow only reads the repository and runs +# tests: it publishes nothing, comments nothing, and touches no GitHub state. +# The default token grant is not relied on — prior runs succeeding under the +# default does not establish that the default is appropriate. +permissions: + contents: read + jobs: test: runs-on: ubuntu-latest @@ -11,9 +18,19 @@ jobs: matrix: python-version: ["3.10", "3.12"] steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + # Actions are pinned to immutable commit SHAs, not to mutable major tags: + # a tag can be repointed by its publisher, so `@v4` does not identify the + # code that will actually run. Each SHA below is the exact commit its + # named tag resolved to when it was pinned. + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: ${{ matrix.python-version }} - - run: pip install -e ".[test]" + # -c constraints.txt fixes the resolved dependency versions, so CI + # measures the same dependency set the benchmarks were bound to. + - run: pip install -e ".[test]" -c constraints.txt + - name: Record the exact installed dependency set + run: pip freeze --exclude-editable - run: pytest -q + - name: Public falsification harness + run: python3 falsify.py diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml new file mode 100644 index 0000000..d860808 --- /dev/null +++ b/.github/workflows/security.yml @@ -0,0 +1,100 @@ +name: Security + +# Automated dependency-advisory monitoring (AC-039). +# +# Audits the CONTROLLED dependency set in constraints.txt — the exact versions +# CI installs and the benchmarks are bound to — against the PyPI/OSV advisory +# databases via pip-audit, the PyPA-maintained tool for this ecosystem. +# +# The schedule matters: an advisory can be published against an unchanged +# dependency set, so a check that only ran on push would go stale silently. +# +# SCOPE, stated exactly: this audits THIRD-PARTY DEPENDENCIES only. It does not +# analyse AuthContract's own source for security defects, and a green run is +# not an audit. See SECURITY.md §5. + +on: + push: + pull_request: + schedule: + # Weekly, so a newly published advisory against an unchanged dependency set + # surfaces without waiting for the next commit. + - cron: "17 6 * * 1" + +permissions: + contents: read + +jobs: + dependency-advisories: + name: Dependency advisory audit + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - name: Install the auditor at a pinned version + run: pip install "pip-audit==2.10.1" + + - name: Audit the controlled dependency set + run: | + # --no-deps audits exactly the pinned closure in constraints.txt + # rather than re-resolving it, so the audited set is the installed + # set. --strict fails if any listed dependency could not be audited, + # so "no findings" cannot mean "nothing was checked". + # + # There is deliberately no severity threshold and no ignore list: + # ANY known advisory against a pinned dependency fails this job. That + # is stricter than the HIGH/CRITICAL floor it has to satisfy. A + # finding is never suppressed to make this workflow green — it is + # fixed by upgrading, or adjudicated as an explicit exception by a + # human, in the open. + set +e + pip-audit -r constraints.txt --no-deps --strict \ + --format json --output pip-audit.json + rc=$? + set -e + echo "--- pip-audit report (inspectable evidence) ---" + cat pip-audit.json + echo + echo "--- summary ---" + python3 - <<'PY' + import json, sys + + def norm(name): + return name.lower().replace("_", "-") + + pins = {} + for line in open("constraints.txt"): + line = line.split("#", 1)[0].strip() + if line: + name, version = line.split("==") + pins[norm(name)] = version + + report = json.load(open("pip-audit.json")) + deps = report.get("dependencies", []) + audited = {norm(d["name"]) for d in deps} + + findings = [d for d in deps if d.get("vulns")] + print(f"pinned: {len(pins)} audited: {len(audited)}") + for d in findings: + for v in d["vulns"]: + fixed = ", ".join(v.get("fix_versions") or []) or "no fix listed" + print(f"VULNERABLE {d['name']}=={d['version']} {v['id']} (fix: {fixed})") + print(f"{len(findings)} dependency/dependencies with known advisories") + + # Coverage assertion. The auditor drops a pin it cannot resolve in the + # advisory service and still exits 0, so "no known vulnerabilities" + # can otherwise mean "this package was never checked". Treat an + # unaudited pin as a failure, not as a pass. There is deliberately no + # allowlist here: the fix is to pin a version with advisory coverage. + unaudited = sorted(set(pins) - audited) + if unaudited: + print("UNAUDITED PINS (advisory coverage missing):") + for name in unaudited: + print(f" {name}=={pins[name]}") + sys.exit(1) + print("coverage OK: every pinned dependency was audited") + PY + exit $rc diff --git a/AGENTS.md b/AGENTS.md index 9dc7687..feb539f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -56,8 +56,15 @@ python3 -m venv .venv && source .venv/bin/activate pip install -e ".[test]" ``` +For the exact dependency versions CI and the benchmarks use, add the controlled +set: `pip install -e ".[test]" -c constraints.txt`. See +[`docs/VERSIONING.md`](docs/VERSIONING.md). + Verify: `pytest -q` → `342 passed`. +Falsify: `python3 falsify.py` → 5 cases, all matching their declared expected +disposition, exit `0`. Run this rather than assuming the refusal paths work. + **Not on PyPI.** `pip install authcontract` will not work. Install from source only. ## 5. Supported CLI commands @@ -241,11 +248,38 @@ Do not soften, omit, or paraphrase this ceiling when summarizing the project. ## 15. Licensing **No license is declared.** No `LICENSE` file exists and `pyproject.toml` -declares no license field, so default copyright applies and no usage rights are -granted. Treat this as source-available for evaluation and reading. **Do not +declares no SPDX license field, so default copyright applies and no usage rights +are granted. `pyproject.toml` carries the Trove classifier +`License :: Other/Proprietary License` — a machine-readable statement that this +is **not** open source. That classifier is a description, not a grant. Treat this as source-available for evaluation and reading. **Do not describe it as open source**, and do not assume redistribution or derivative rights. Direct licensing questions to the repository owner. +## 15a. Version and interface stability + +Package version is `0.0.1` and has never been published or incremented per +change. **The commit SHA is the only reliable identity** — two checkouts both +reporting `0.0.1` may differ. Semantic Versioning is **not** implemented or +claimed. Pre-1.0 interfaces may change without a major bump. The one commitment +made is that reason codes will not change meaning *silently* under the same +version. Full policy and the declared public surface: +[`docs/VERSIONING.md`](docs/VERSIONING.md). + +## 15b. Falsification harness + +`python3 falsify.py` runs five public cases — valid ALLOW, undeclared action, +stale fact, untampered receipt, tampered receipt binding — each against a +declared expected disposition, and exits non-zero on any mismatch in either +direction. Use it to verify refusal behaviour rather than asserting it. + +## 15c. Security reporting + +Do not open a public issue for a suspected vulnerability. See +[`SECURITY.md`](SECURITY.md). A private reporting route is **not yet +established** — that is recorded there as requiring owner action, not glossed +over. Automated dependency-advisory checking runs in CI; a passing scan is +**not** an audit and establishes nothing about this project's own code. + ## 16. Prohibition on invention If asked to do something this repository does not support: @@ -258,6 +292,9 @@ If asked to do something this repository does not support: 5. Do **not** weaken or bypass a refusal to produce a desired outcome. Further reading: [`README.md`](README.md) · measured evidence -[`docs/BENCHMARKS.md`](docs/BENCHMARKS.md) · maturity +[`docs/BENCHMARKS-AC-039.md`](docs/BENCHMARKS-AC-039.md) (current; `docs/BENCHMARKS.md` is +the superseded earlier baseline) · versioning +[`docs/VERSIONING.md`](docs/VERSIONING.md) · security +[`SECURITY.md`](SECURITY.md) · maturity [`docs/TRL-ASSESSMENT.md`](docs/TRL-ASSESSMENT.md) · planned work [`docs/ROADMAP.md`](docs/ROADMAP.md). diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..9fe8d2b --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,56 @@ +# Contributing + +## Current status: no unsolicited contribution workflow is established + +This is stated plainly rather than left ambiguous, because an unstated process +wastes a contributor's time. + +There is **no** established process for unsolicited pull requests. Specifically, +none of the following exists: a contributor licence agreement, a review-time +commitment, a maintainer rota, a triage service level, a governance model, a +code of conduct process, or a merge policy for outside contributions. + +Nothing here promises any of those will exist. This is an experimental +reference implementation maintained as research and engineering evidence, not a +community-maintained project. + +**Contribution is also constrained by licensing.** No license is declared (see +[`README.md`](README.md) § License), so the terms under which a contribution +could be accepted and redistributed are themselves unsettled. That is an owner +decision, not a process gap someone can work around. + +## What is genuinely useful right now + +**Open an issue.** Issues are read, and they are the reliable path. + +The most valuable thing you can send is a **falsification**: a case where +AuthContract's documented behaviour and its actual behaviour disagree. + +- Run `python3 falsify.py` — the public falsification harness — and include its + output if a case failed. +- Include your OS, Python version, and the exact commit SHA. +- Reproduce against the committed fixtures in `fixtures/` where possible. No + credentials or network are needed. +- If a claim in `README.md`, `AGENTS.md`, or any document under `docs/` does not + hold, say which sentence and what you observed instead. A documented claim + that turns out to be false is a defect, and it is recorded rather than quietly + edited away. + +For a **suspected security vulnerability**, do not open a public issue — follow +[`SECURITY.md`](SECURITY.md) instead. + +## Before writing code + +**Open an issue first.** A pull request that arrives without prior discussion +may sit unreviewed, and given the licensing situation above it may not be +mergeable at all. That is a genuine risk of wasted effort, so it is said up +front rather than discovered afterwards. + +If a change is agreed, the practical expectations are the same ones this +repository applies to itself: + +- The full suite passes: `pytest -q` → 342 passed. +- The falsification harness passes: `python3 falsify.py`. +- A fix comes with a regression test that **fails without the fix**. +- Tests are not weakened, skipped, or deleted to make CI green. +- No claim is added that the repository's own measurements do not support. diff --git a/README.md b/README.md index f9d7b47..26c0497 100644 --- a/README.md +++ b/README.md @@ -150,6 +150,35 @@ Note that **no receipt is issued on refusal** — a refused decision never produ --- +## Falsify it yourself + +Reproducing a happy path proves very little. The useful question is whether this +system refuses when it should — and whether you can watch it do so. + +```bash +python3 falsify.py +``` + +One command, no credentials, no network. It runs five cases against the +committed fixtures and checks each against a **declared expected disposition**: + +| Case | Expected | +|---|---| +| Valid specimen | `PASS` / `OK` / exit 0, receipt issued | +| Undeclared action | `REFUSED` / `RUN_UNCLASSIFIED_ACTION` / exit 1, no receipt | +| Stale runtime fact | `REFUSED` / `RUN_FACT_STALE` / exit 1, no receipt | +| Untampered receipt | `PASS` / `OK` / exit 0 | +| Tampered receipt binding | `REFUSED` / `VEIP_RECEIPT_MISMATCH` / exit 1 | + +A case fails if what happens differs from what was expected **in either +direction**. An unexpected refusal fails; so does an unexpected pass. The +harness exits non-zero on any mismatch, so it works as a check in your own CI. + +If you find a mismatch, that is a real result. Please +[open an issue](https://github.com/veraxis-protocol/AuthContract/issues). + +--- + ## What the result means | Field | Meaning | @@ -242,7 +271,7 @@ Read this before forming expectations. The worked conceptual examples later in t - No concurrency, distribution, or measured multi-core scaling. - Evidence scope is one synthetic banking specimen family — not a general solution. -Measured evidence and its limits: [`docs/BENCHMARKS.md`](docs/BENCHMARKS.md) · maturity assessment: [`docs/TRL-ASSESSMENT.md`](docs/TRL-ASSESSMENT.md). +Measured evidence and its limits: [`docs/BENCHMARKS-AC-039.md`](docs/BENCHMARKS-AC-039.md) (current; [`docs/BENCHMARKS.md`](docs/BENCHMARKS.md) is the superseded earlier baseline) · maturity assessment: [`docs/TRL-ASSESSMENT.md`](docs/TRL-ASSESSMENT.md). --- @@ -250,19 +279,26 @@ Measured evidence and its limits: [`docs/BENCHMARKS.md`](docs/BENCHMARKS.md) · **No license is currently declared.** This repository contains no `LICENSE` file and `pyproject.toml` declares no license field. Absent an explicit grant, default copyright applies and no usage rights are conferred — so treat this as source-available for evaluation and reading, not as open source. If you need licensed use, ask the repository owner. +`pyproject.toml` declares the Trove classifier `License :: Other/Proprietary License`. That is the machine-readable statement of the situation described above — it marks the project as **not** open source without inventing a grant. It is a description of the current state, not a license, and it confers nothing. No SPDX identifier is declared, because declaring one would be false. Selecting an actual license is an owner decision that has not been made. + --- ## Where to go next | If you want to… | Go to | |---|---| -| See measured performance and correctness evidence | [`docs/BENCHMARKS.md`](docs/BENCHMARKS.md) | +| See measured performance and correctness evidence | [`docs/BENCHMARKS-AC-039.md`](docs/BENCHMARKS-AC-039.md) | +| See the earlier, superseded baseline | [`docs/BENCHMARKS.md`](docs/BENCHMARKS.md) | | Reproduce the benchmarks yourself | [`benchmarks/README.md`](benchmarks/README.md) | | Understand current maturity honestly | [`docs/TRL-ASSESSMENT.md`](docs/TRL-ASSESSMENT.md) | | See what is planned and why | [`docs/ROADMAP.md`](docs/ROADMAP.md) | | Validate a clean clone yourself | [`docs/CLEANROOM-VALIDATION-RUNBOOK.md`](docs/CLEANROOM-VALIDATION-RUNBOOK.md) | | See how this repository was usability-tested | [`docs/REPOSITORY-USABILITY.md`](docs/REPOSITORY-USABILITY.md) | | See the release-readiness verification record | [`docs/RELEASE-READINESS.md`](docs/RELEASE-READINESS.md) | +| Try to falsify it | run `python3 falsify.py` | +| Know what you can depend on across versions | [`docs/VERSIONING.md`](docs/VERSIONING.md) | +| Report a suspected vulnerability | [`SECURITY.md`](SECURITY.md) | +| Understand the contribution situation | [`CONTRIBUTING.md`](CONTRIBUTING.md) | | Understand the terminology | [`docs/DEVELOPER-LANGUAGE.md`](docs/DEVELOPER-LANGUAGE.md) | | See how this compares to other systems | [`docs/SOTA.md`](docs/SOTA.md) | | Understand the full conceptual model | keep reading below | @@ -281,7 +317,12 @@ Measured evidence and its limits: [`docs/BENCHMARKS.md`](docs/BENCHMARKS.md) · **Contributions.** There is no contribution process established yet, and no contributor licence or review policy exists. Issues are the reliable path today. If you are considering a substantive contribution, open an issue first so it -isn't wasted effort. +isn't wasted effort. [`CONTRIBUTING.md`](CONTRIBUTING.md) states exactly what +does and does not exist. + +**Security.** Do not report a suspected vulnerability through a public issue. +[`SECURITY.md`](SECURITY.md) sets out the triage policy and the current state of +the private reporting route. **A note on scope of support.** This is an experimental reference implementation maintained as research and engineering evidence. There is no support commitment, diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..f812e22 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,102 @@ +# Security policy + +## 1. What this repository is + +AuthContract is an **experimental reference implementation at TRL 4**. It is +not audited, not security-certified, and not production-ready. Maturity is +assessed in [`docs/TRL-ASSESSMENT.md`](docs/TRL-ASSESSMENT.md). + +Treat it as research and engineering evidence. Do not place it on a path where a +security failure would have real consequences. + +## 2. Supported versions + +**Only the current `main` is supported.** + +There is no released version, no tag, no published package, and no backport +branch. A fix, if one is made, lands on `main` and nowhere else. Any older +commit you have checked out is unsupported, and nothing in this repository +promises that a fix will be applied to it. + +## 3. Reporting a vulnerability + +**Do not open a public GitHub issue for a suspected vulnerability**, and do not +attach exploit details to a pull request. Public issues are the right channel +for ordinary bugs and for claims that do not hold — not for security reports. + +**Current status of the private reporting route: NOT ESTABLISHED.** + +At the time of writing, this repository publishes no verified private +vulnerability-reporting channel: + +- GitHub Private Vulnerability Reporting could not be confirmed as enabled. The + repository metadata reachable from this project's tooling does not expose + that setting, so its state is *unknown*, not *enabled*. +- No security contact address is published here. None is invented for this + document — a reporting address that does not demonstrably reach someone is + worse than an honest absence, because it silently swallows reports. + +**Owner action required.** The smallest sufficient fix is to enable GitHub +Private Vulnerability Reporting on this repository (Settings → Advanced +Security → Private vulnerability reporting), which gives reporters a +first-class private channel at +`https://github.com/veraxis-protocol/AuthContract/security/advisories/new` +without publishing any address. Once enabled, this section should be replaced +with that link. + +Until then, a reporter who needs a private channel should contact the owner +through [veraxis.io](https://veraxis.io) and **withhold technical detail until a +private channel is agreed**. + +## 4. Triage policy + +Bounded to what an experimental TRL 4 reference implementation can honestly +commit to. These are handling intentions, not a service-level agreement, and no +response time is promised. + +| Severity | Meaning here | Handling | +|---|---|---| +| **Critical** | Enables an unauthorized ALLOW, forges or defeats receipt verification, or breaks the fail-closed property | Work stops on other changes. Fixed on `main`, or the affected capability is documented as unsafe and its limitation stated in `README.md` and `AGENTS.md`. | +| **High** | Compromises canonical identity, digest binding, projection closure, fact admissibility, or the merge-result gate without directly producing a false ALLOW | Fixed on `main` before further feature work, with a regression test that fails without the fix. | +| **Medium** | Correctness or robustness defect with no path to a false ALLOW — crashes, unhandled input, misleading output | Recorded as a finding and scheduled. May be fixed alongside other work. | +| **Low** | Hardening, defence in depth, documentation that could mislead a reader into an unsafe assumption | Recorded. Fixed opportunistically. | + +Every accepted finding gets a regression test. A fix without a test that +demonstrates the original failure is not considered closed. + +## 5. What automated checks do and do not establish + +This repository runs an automated dependency-advisory check +(`.github/workflows/security.yml`) against its declared, version-controlled +dependency set. + +**A passing scanner is not an audit.** It establishes only that the specific +advisory database consulted contained no matching published advisory for those +specific pinned versions at that moment. It does **not** establish: + +- that no vulnerability exists — absence of a published advisory is not absence of a defect; +- that this project's own code is free of vulnerabilities — no scanner in this repository analyses it for security defects; +- that any third party has reviewed the design, the cryptographic binding, or the refusal logic; +- any certification, accreditation, or compliance status whatsoever. + +No independent security review of AuthContract has ever been performed. That +gap is recorded in [`docs/TRL-ASSESSMENT.md`](docs/TRL-ASSESSMENT.md) as a +condition for TRL 6 and is not closed by any check in this repository. + +## 6. Responsible disclosure + +If you find something: + +- **Report privately first** if the finding could enable a false ALLOW, forge a + receipt, or defeat a refusal — even though the private route above is not yet + established, initiate contact before publishing detail. +- **Give the owner a reasonable chance to respond** before public disclosure. + Because no response-time commitment exists, a reporter is entitled to set + their own reasonable deadline and say what it is. +- **Do not test against systems you do not own.** Everything needed to + reproduce a finding is in this repository: the fixtures, the specimens, and + the falsification harness (`falsify.py`) all run locally with no network and + no credentials. +- **Publishing a finding is welcome once disclosed responsibly.** This project's + stated purpose is to be falsifiable. A demonstrated failure is a contribution, + not an attack, and it will be recorded rather than quietly repaired. diff --git a/benchmarks/harness.py b/benchmarks/harness.py index eda4746..462770e 100644 --- a/benchmarks/harness.py +++ b/benchmarks/harness.py @@ -216,6 +216,57 @@ def sustained_throughput( # Paths that constitute the device under test. The benchmark measures these and # must not modify them; the harness itself lives outside this set. +def capture_dependency_identity() -> dict[str, Any]: + """Record the exact third-party distributions present while measuring. + + Performance and correctness are properties of the code *plus* the + dependencies it runs against. Recording the declared constraints alongside + the versions actually installed makes the pair checkable: if they disagree, + the run was not measuring the controlled set it claims to measure. + """ + constraints_path = REPO_ROOT / "constraints.txt" + declared: dict[str, str] = {} + if constraints_path.exists(): + for line in constraints_path.read_text(encoding="utf-8").splitlines(): + line = line.split("#", 1)[0].strip() + if line and "==" in line: + name, version = line.split("==", 1) + declared[name.strip().lower().replace("_", "-")] = version.strip() + + installed: dict[str, str] = {} + try: + from importlib import metadata + + for dist in metadata.distributions(): + name = (dist.metadata["Name"] or "").lower().replace("_", "-") + if name: + installed[name] = dist.version + except Exception as exc: # pragma: no cover - defensive + return {"declared": declared, "error": f"could not enumerate installed set: {exc}"} + + relevant = {name: installed.get(name) for name in sorted(declared)} + mismatched = { + name: {"declared": declared[name], "installed": relevant[name]} + for name in declared + if relevant[name] is not None and relevant[name] != declared[name] + } + absent = sorted(name for name in declared if relevant[name] is None) + return { + "constraints_file": "constraints.txt", + "declared": declared, + "installed_for_declared": relevant, + "mismatched": mismatched, + "declared_but_not_installed": absent, + "matches_declared_set": not mismatched, + "note": ( + "declared_but_not_installed is expected for distributions pinned only " + "for older Python versions (pytest pulls exceptiongroup and tomli on " + "Python < 3.11 only). A non-empty 'mismatched' means this run did not " + "measure the controlled dependency set." + ), + } + + DUT_PATHS = ( "authcontract", "tests", diff --git a/benchmarks/results/AC-039-CORRECTNESS-MATRIX.json b/benchmarks/results/AC-039-CORRECTNESS-MATRIX.json new file mode 100644 index 0000000..169792f --- /dev/null +++ b/benchmarks/results/AC-039-CORRECTNESS-MATRIX.json @@ -0,0 +1,761 @@ +{ + "adversarial_matrix": { + "by_category": { + "action_outside_scope": { + "FAIL": 0, + "PASS": 1 + }, + "admission_evidence_binding": { + "FAIL": 0, + "PASS": 2 + }, + "control": { + "FAIL": 0, + "PASS": 1 + }, + "corroboration_missing": { + "FAIL": 0, + "PASS": 1 + }, + "digest_mutation": { + "FAIL": 0, + "PASS": 2 + }, + "digest_scope_violation": { + "FAIL": 0, + "PASS": 1 + }, + "evidence_mismatch": { + "FAIL": 0, + "PASS": 4 + }, + "inactive_contract": { + "FAIL": 0, + "PASS": 1 + }, + "malformed_structured_input": { + "FAIL": 0, + "PASS": 4 + }, + "malformed_type": { + "FAIL": 0, + "PASS": 5 + }, + "missing_required_field": { + "FAIL": 0, + "PASS": 2 + }, + "out_of_domain_value": { + "FAIL": 0, + "PASS": 1 + }, + "post_binding_mutation": { + "FAIL": 0, + "PASS": 2 + }, + "receipt_context_substitution": { + "FAIL": 0, + "PASS": 1 + }, + "reordered_structured_input": { + "FAIL": 0, + "PASS": 1 + }, + "replay": { + "FAIL": 0, + "PASS": 1 + }, + "stale_fact": { + "FAIL": 0, + "PASS": 1 + }, + "trust_basis_violation": { + "FAIL": 0, + "PASS": 1 + }, + "unknown_field_forbidden": { + "FAIL": 0, + "PASS": 5 + }, + "validly_bound_variant": { + "FAIL": 0, + "PASS": 1 + } + }, + "results": [ + { + "category": "control", + "description": "Control: the valid specimen must still be ALLOWed", + "expected_result": "ALLOW", + "failure_stage": null, + "notes": "", + "observed_result": "ALLOW", + "pass_fail": "PASS", + "reason_code": "OK", + "receipt_generated": true, + "receipt_verified": true, + "specimen": "ADV-00-control" + }, + { + "category": "missing_required_field", + "description": "Missing required action parameter", + "expected_result": "REFUSE", + "failure_stage": "authorization_decision", + "notes": "", + "observed_result": "REFUSE", + "pass_fail": "PASS", + "reason_code": "RUN_DOMAIN_ESCAPE", + "receipt_generated": false, + "receipt_verified": null, + "specimen": "ADV-01" + }, + { + "category": "unknown_field_forbidden", + "description": "Unknown action parameter where unknown fields are forbidden", + "expected_result": "REFUSE", + "failure_stage": "authorization_decision", + "notes": "", + "observed_result": "REFUSE", + "pass_fail": "PASS", + "reason_code": "RUN_DOMAIN_ESCAPE", + "receipt_generated": false, + "receipt_verified": null, + "specimen": "ADV-02" + }, + { + "category": "out_of_domain_value", + "description": "Out-of-domain enum value", + "expected_result": "REFUSE", + "failure_stage": "authorization_decision", + "notes": "", + "observed_result": "REFUSE", + "pass_fail": "PASS", + "reason_code": "RUN_DOMAIN_ESCAPE", + "receipt_generated": false, + "receipt_verified": null, + "specimen": "ADV-03" + }, + { + "category": "malformed_type", + "description": "Lossy decimal representation in action parameter", + "expected_result": "REFUSE", + "failure_stage": "authorization_decision", + "notes": "", + "observed_result": "REFUSE", + "pass_fail": "PASS", + "reason_code": "RUN_DOMAIN_ESCAPE", + "receipt_generated": false, + "receipt_verified": null, + "specimen": "ADV-04" + }, + { + "category": "action_outside_scope", + "description": "Action type outside declared mediated scope", + "expected_result": "REFUSE", + "failure_stage": "authorization_decision", + "notes": "", + "observed_result": "REFUSE", + "pass_fail": "PASS", + "reason_code": "RUN_UNCLASSIFIED_ACTION", + "receipt_generated": false, + "receipt_verified": null, + "specimen": "ADV-05" + }, + { + "category": "missing_required_field", + "description": "Required fact absent from bundle", + "expected_result": "REFUSE", + "failure_stage": "authorization_decision", + "notes": "", + "observed_result": "REFUSE", + "pass_fail": "PASS", + "reason_code": "VEIP_FACT_BUNDLE_INCOMPLETE", + "receipt_generated": false, + "receipt_verified": null, + "specimen": "ADV-06" + }, + { + "category": "stale_fact", + "description": "Stale fact beyond freshness window", + "expected_result": "REFUSE", + "failure_stage": "authorization_decision", + "notes": "", + "observed_result": "REFUSE", + "pass_fail": "PASS", + "reason_code": "RUN_FACT_STALE", + "receipt_generated": false, + "receipt_verified": null, + "specimen": "ADV-07" + }, + { + "category": "malformed_type", + "description": "Future-dated fact timestamp", + "expected_result": "REFUSE", + "failure_stage": "authorization_decision", + "notes": "", + "observed_result": "REFUSE", + "pass_fail": "PASS", + "reason_code": "RUN_FACT_FUTURE_TIMESTAMP", + "receipt_generated": false, + "receipt_verified": null, + "specimen": "ADV-08" + }, + { + "category": "malformed_type", + "description": "Timezone-naive fact timestamp (unverifiable ordering)", + "expected_result": "REFUSE", + "failure_stage": "authorization_decision", + "notes": "", + "observed_result": "REFUSE", + "pass_fail": "PASS", + "reason_code": "RUN_FACT_TIME_UNVERIFIABLE", + "receipt_generated": false, + "receipt_verified": null, + "specimen": "ADV-09" + }, + { + "category": "trust_basis_violation", + "description": "Self-asserted fact where policy prohibits self-assertion", + "expected_result": "REFUSE", + "failure_stage": "authorization_decision", + "notes": "", + "observed_result": "REFUSE", + "pass_fail": "PASS", + "reason_code": "RUN_FACT_SELF_ASSERTED", + "receipt_generated": false, + "receipt_verified": null, + "specimen": "ADV-10" + }, + { + "category": "malformed_type", + "description": "Lossy wire representation of fact value", + "expected_result": "REFUSE", + "failure_stage": "authorization_decision", + "notes": "", + "observed_result": "REFUSE", + "pass_fail": "PASS", + "reason_code": "RUN_FACT_REPRESENTATION", + "receipt_generated": false, + "receipt_verified": null, + "specimen": "ADV-11" + }, + { + "category": "malformed_structured_input", + "description": "Duplicate fact_id in bundle", + "expected_result": "REFUSE", + "failure_stage": "authorization_decision", + "notes": "", + "observed_result": "REFUSE", + "pass_fail": "PASS", + "reason_code": "VEIP_MALFORMED_INPUT", + "receipt_generated": false, + "receipt_verified": null, + "specimen": "ADV-12" + }, + { + "category": "unknown_field_forbidden", + "description": "Unknown field on fact object", + "expected_result": "REFUSE", + "failure_stage": "authorization_decision", + "notes": "", + "observed_result": "REFUSE", + "pass_fail": "PASS", + "reason_code": "VEIP_MALFORMED_INPUT", + "receipt_generated": false, + "receipt_verified": null, + "specimen": "ADV-13" + }, + { + "category": "unknown_field_forbidden", + "description": "Unknown field on fact bundle", + "expected_result": "REFUSE", + "failure_stage": "authorization_decision", + "notes": "", + "observed_result": "REFUSE", + "pass_fail": "PASS", + "reason_code": "VEIP_MALFORMED_INPUT", + "receipt_generated": false, + "receipt_verified": null, + "specimen": "ADV-14" + }, + { + "category": "unknown_field_forbidden", + "description": "Unknown field on verified-evidence object", + "expected_result": "REFUSE", + "failure_stage": "authorization_decision", + "notes": "", + "observed_result": "REFUSE", + "pass_fail": "PASS", + "reason_code": "VEIP_MALFORMED_INPUT", + "receipt_generated": false, + "receipt_verified": null, + "specimen": "ADV-15" + }, + { + "category": "evidence_mismatch", + "description": "Claimed value diverges from verifier-established evidence value", + "expected_result": "REFUSE", + "failure_stage": "authorization_decision", + "notes": "", + "observed_result": "REFUSE", + "pass_fail": "PASS", + "reason_code": "RUN_FACT_EVIDENCE_MISMATCH", + "receipt_generated": false, + "receipt_verified": null, + "specimen": "ADV-16" + }, + { + "category": "evidence_mismatch", + "description": "Claimed asserter diverges from verifier-established asserter", + "expected_result": "REFUSE", + "failure_stage": "authorization_decision", + "notes": "", + "observed_result": "REFUSE", + "pass_fail": "PASS", + "reason_code": "RUN_FACT_EVIDENCE_MISMATCH", + "receipt_generated": false, + "receipt_verified": null, + "specimen": "ADV-17" + }, + { + "category": "evidence_mismatch", + "description": "Claimed fact_id diverges from verifier-established fact_id", + "expected_result": "REFUSE", + "failure_stage": "authorization_decision", + "notes": "", + "observed_result": "REFUSE", + "pass_fail": "PASS", + "reason_code": "RUN_FACT_IDENTITY_MISMATCH", + "receipt_generated": false, + "receipt_verified": null, + "specimen": "ADV-18" + }, + { + "category": "evidence_mismatch", + "description": "Stale verified evidence presented with a fresh caller claim", + "expected_result": "REFUSE", + "failure_stage": "authorization_decision", + "notes": "", + "observed_result": "REFUSE", + "pass_fail": "PASS", + "reason_code": "RUN_FACT_EVIDENCE_MISMATCH", + "receipt_generated": false, + "receipt_verified": null, + "specimen": "ADV-19" + }, + { + "category": "validly_bound_variant", + "description": "Variant contract, mutated AND correctly re-sealed (binding intact)", + "expected_result": "ALLOW", + "failure_stage": null, + "notes": "", + "observed_result": "ALLOW", + "pass_fail": "PASS", + "reason_code": "OK", + "receipt_generated": true, + "receipt_verified": true, + "specimen": "ADV-20" + }, + { + "category": "digest_mutation", + "description": "Sibling digest disagreement across bound locations", + "expected_result": "REFUSE", + "failure_stage": "authorization_decision", + "notes": "", + "observed_result": "REFUSE", + "pass_fail": "PASS", + "reason_code": "AC_DIGEST", + "receipt_generated": false, + "receipt_verified": null, + "specimen": "ADV-21" + }, + { + "category": "digest_scope_violation", + "description": "Self-referential contract digest", + "expected_result": "REFUSE", + "failure_stage": "authorization_decision", + "notes": "", + "observed_result": "REFUSE", + "pass_fail": "PASS", + "reason_code": "AC_DIGEST_SCOPE", + "receipt_generated": false, + "receipt_verified": null, + "specimen": "ADV-22" + }, + { + "category": "digest_mutation", + "description": "Cross-object digest substitution", + "expected_result": "REFUSE", + "failure_stage": "authorization_decision", + "notes": "", + "observed_result": "REFUSE", + "pass_fail": "PASS", + "reason_code": "AC_DIGEST", + "receipt_generated": false, + "receipt_verified": null, + "specimen": "ADV-23" + }, + { + "category": "malformed_type", + "description": "Malformed artifact", + "expected_result": "REFUSE", + "failure_stage": "authorization_decision", + "notes": "", + "observed_result": "REFUSE", + "pass_fail": "PASS", + "reason_code": "RUN_DOMAIN_ESCAPE", + "receipt_generated": false, + "receipt_verified": null, + "specimen": "ADV-24" + }, + { + "category": "inactive_contract", + "description": "Suspended contract (activation state not ACTIVE)", + "expected_result": "REFUSE", + "failure_stage": "authorization_decision", + "notes": "", + "observed_result": "REFUSE", + "pass_fail": "PASS", + "reason_code": "RUN_INACTIVE_CONTRACT", + "receipt_generated": false, + "receipt_verified": null, + "specimen": "ADV-25" + }, + { + "category": "admission_evidence_binding", + "description": "Admission carrying approvals, contract binding intact", + "expected_result": "ALLOW", + "failure_stage": null, + "notes": "", + "observed_result": "ALLOW", + "pass_fail": "PASS", + "reason_code": "OK", + "receipt_generated": true, + "receipt_verified": true, + "specimen": "ADV-26" + }, + { + "category": "malformed_structured_input", + "description": "Admission present as JSON null", + "expected_result": "REFUSE", + "failure_stage": "authorization_decision", + "notes": "", + "observed_result": "REFUSE", + "pass_fail": "PASS", + "reason_code": "VEIP_MALFORMED_INPUT", + "receipt_generated": false, + "receipt_verified": null, + "specimen": "ADV-27" + }, + { + "category": "malformed_structured_input", + "description": "Admission present as JSON list", + "expected_result": "REFUSE", + "failure_stage": "authorization_decision", + "notes": "", + "observed_result": "REFUSE", + "pass_fail": "PASS", + "reason_code": "VEIP_MALFORMED_INPUT", + "receipt_generated": false, + "receipt_verified": null, + "specimen": "ADV-28" + }, + { + "category": "malformed_structured_input", + "description": "Duplicate required-fact declaration", + "expected_result": "REFUSE", + "failure_stage": "authorization_decision", + "notes": "", + "observed_result": "REFUSE", + "pass_fail": "PASS", + "reason_code": "VEIP_MALFORMED_INPUT", + "receipt_generated": false, + "receipt_verified": null, + "specimen": "ADV-29" + }, + { + "category": "unknown_field_forbidden", + "description": "Unknown field on required-fact declaration", + "expected_result": "REFUSE", + "failure_stage": "authorization_decision", + "notes": "", + "observed_result": "REFUSE", + "pass_fail": "PASS", + "reason_code": "VEIP_MALFORMED_INPUT", + "receipt_generated": false, + "receipt_verified": null, + "specimen": "ADV-30" + }, + { + "category": "corroboration_missing", + "description": "Corroboration required but not satisfiable as declared", + "expected_result": "REFUSE", + "failure_stage": "authorization_decision", + "notes": "", + "observed_result": "REFUSE", + "pass_fail": "PASS", + "reason_code": "VEIP_MALFORMED_INPUT", + "receipt_generated": false, + "receipt_verified": null, + "specimen": "ADV-31" + }, + { + "category": "reordered_structured_input", + "description": "Reordered JSON object keys must not alter canonical identity or decision", + "expected_result": "ALLOW", + "failure_stage": null, + "notes": "digest invariant under key reordering: True", + "observed_result": "ALLOW", + "pass_fail": "PASS", + "reason_code": "OK", + "receipt_generated": true, + "receipt_verified": null, + "specimen": "ADV-32" + }, + { + "category": "replay", + "description": "Replayed identical request", + "expected_result": "ALLOW", + "failure_stage": null, + "notes": "Replay yields an identical receipt. NOTE: this documents determinism, not replay *protection* \u2014 there is no nonce, sequence number, or single-use semantics at this commit, so an intercepted receipt is indistinguishable from a legitimately re-derived one. Recorded as an architectural gap.", + "observed_result": "ALLOW", + "pass_fail": "PASS", + "reason_code": "OK", + "receipt_generated": true, + "receipt_verified": null, + "specimen": "ADV-33" + }, + { + "category": "receipt_context_substitution", + "description": "Receipt presented against a different action than it was issued for", + "expected_result": "REFUSE", + "failure_stage": "receipt_verification", + "notes": "", + "observed_result": "REFUSE", + "pass_fail": "PASS", + "reason_code": "RUN_DOMAIN_ESCAPE", + "receipt_generated": true, + "receipt_verified": false, + "specimen": "ADV-34" + }, + { + "category": "post_binding_mutation", + "description": "Contract version altered, digest binding left stale", + "expected_result": "REFUSE", + "failure_stage": "digest_binding", + "notes": "", + "observed_result": "REFUSE", + "pass_fail": "PASS", + "reason_code": "AC_DIGEST", + "receipt_generated": false, + "receipt_verified": null, + "specimen": "ADV-35" + }, + { + "category": "post_binding_mutation", + "description": "Projection domain widened (amount retyped), digest binding left stale", + "expected_result": "REFUSE", + "failure_stage": "digest_binding", + "notes": "", + "observed_result": "REFUSE", + "pass_fail": "PASS", + "reason_code": "AC_DIGEST", + "receipt_generated": false, + "receipt_verified": null, + "specimen": "ADV-36" + }, + { + "category": "admission_evidence_binding", + "description": "Forged admission approvals must alter the bound evidence", + "expected_result": "ALLOW", + "failure_stage": null, + "notes": "Decision is ALLOW because approvals are not an authorization gate at this commit (finding AC-035-F1). The evidence binding nonetheless holds: admission_digest and receipt_digest both change (True), and a receipt issued for the unforged admission does not verify against the forged one (VEIP_RECEIPT_MISMATCH).", + "observed_result": "ALLOW", + "pass_fail": "PASS", + "reason_code": "OK", + "receipt_generated": true, + "receipt_verified": false, + "specimen": "ADV-37" + } + ], + "totals": { + "failed": 0, + "not_evaluated": 0, + "passed": 38, + "total": 38 + } + }, + "claim_ceiling": [ + "This benchmark measures a bounded MVP-alpha implementation on one synthetic banking specimen family.", + "It does NOT establish production readiness.", + "It does NOT establish regulatory or legal correctness.", + "It does NOT establish universal source-to-rule derivation.", + "It does NOT establish arbitrary-domain compatibility.", + "It does NOT establish security certification.", + "It does NOT establish distributed or concurrent scalability.", + "It does NOT constitute a formal proof.", + "It does NOT establish comparative superiority over any other system.", + "Latency and throughput figures are single-process, single-machine, and environment-specific." + ], + "end_to_end_matrix": [ + { + "category": "happy_path", + "description": "Happy path: valid contract, valid facts, permitted action", + "expected_result": "ALLOW", + "failure_stage": null, + "notes": "", + "observed_result": "ALLOW", + "pass_fail": "PASS", + "reason_code": "OK", + "receipt_generated": true, + "receipt_verified": true, + "specimen": "E2E-01" + }, + { + "category": "stale_fact", + "description": "Stale runtime fact: freshness window exceeded", + "expected_result": "REFUSE", + "failure_stage": "authorization_decision", + "notes": "", + "observed_result": "REFUSE", + "pass_fail": "PASS", + "reason_code": "RUN_FACT_STALE", + "receipt_generated": false, + "receipt_verified": null, + "specimen": "E2E-02" + }, + { + "category": "malformed_contract", + "description": "Malformed contract artifact", + "expected_result": "REFUSE", + "failure_stage": "authorization_decision", + "notes": "", + "observed_result": "REFUSE", + "pass_fail": "PASS", + "reason_code": "RUN_DOMAIN_ESCAPE", + "receipt_generated": false, + "receipt_verified": null, + "specimen": "E2E-03" + }, + { + "category": "unsupported_domain", + "description": "Action outside the declared projection domain", + "expected_result": "REFUSE", + "failure_stage": "authorization_decision", + "notes": "", + "observed_result": "REFUSE", + "pass_fail": "PASS", + "reason_code": "RUN_UNCLASSIFIED_ACTION", + "receipt_generated": false, + "receipt_verified": null, + "specimen": "E2E-04" + }, + { + "category": "source_version_mutation", + "description": "Source/version material changed with stale (non-recomputed) digest binding", + "expected_result": "REFUSE", + "failure_stage": "digest_binding", + "notes": "contract.identity.version altered; activation/admission/proof digests left stale", + "observed_result": "REFUSE", + "pass_fail": "PASS", + "reason_code": "AC_DIGEST", + "receipt_generated": false, + "receipt_verified": null, + "specimen": "E2E-06" + }, + { + "category": "receipt_mutation", + "description": "Receipt mutation and truncation across every protected field", + "expected_result": "REFUSE", + "failure_stage": "receipt_verification", + "notes": "20 mutation/truncation variants tested; 20 detected", + "observed_result": "REFUSE", + "pass_fail": "PASS", + "reason_code": "VEIP_RECEIPT_MISMATCH/MALFORMED", + "receipt_generated": true, + "receipt_verified": false, + "specimen": "E2E-05" + }, + { + "category": "deterministic_replay", + "description": "Deterministic replay, 100 executions of the identical specimen", + "expected_result": "ALLOW", + "failure_stage": null, + "notes": "all protected receipt fields byte-identical across replays", + "observed_result": "ALLOW", + "pass_fail": "PASS", + "reason_code": "OK", + "receipt_generated": true, + "receipt_verified": true, + "specimen": "E2E-07" + } + ], + "environment": { + "commit_sha": "389e9ff557f0c1f12996b7ebbc478689f38abdda", + "dependency_versions": { + "pytest": "9.1.1", + "rfc8785": "0.1.4" + }, + "git_status_clean": false, + "machine": "x86_64", + "platform": "Linux-6.18.44-fc-v21-x86_64-with-glibc2.39", + "processor": "x86_64", + "python_implementation": "CPython", + "python_version": "3.11.15", + "tree_sha": "edd3b56bc794720672ce9853e8173e1a2b446e17" + }, + "generated_at_utc": "2026-08-24T21:23:53Z", + "provenance": { + "benchmark_harness_sha": "389e9ff557f0c1f12996b7ebbc478689f38abdda", + "dependency_identity": { + "constraints_file": "constraints.txt", + "declared": { + "exceptiongroup": "1.3.1", + "iniconfig": "2.3.0", + "packaging": "25.0", + "pluggy": "1.6.0", + "pygments": "2.21.0", + "pytest": "9.1.1", + "rfc8785": "0.1.4", + "tomli": "2.4.1" + }, + "declared_but_not_installed": [ + "exceptiongroup" + ], + "installed_for_declared": { + "exceptiongroup": null, + "iniconfig": "2.3.0", + "packaging": "25.0", + "pluggy": "1.6.0", + "pygments": "2.21.0", + "pytest": "9.1.1", + "rfc8785": "0.1.4", + "tomli": "2.4.1" + }, + "matches_declared_set": true, + "mismatched": {}, + "note": "declared_but_not_installed is expected for distributions pinned only for older Python versions (pytest pulls exceptiongroup and tomli on Python < 3.11 only). A non-empty 'mismatched' means this run did not measure the controlled dependency set." + }, + "dut_base_sha": "389e9ff557f0c1f12996b7ebbc478689f38abdda", + "dut_verification": { + "dut_base_sha": "389e9ff557f0c1f12996b7ebbc478689f38abdda", + "dut_paths": [ + "authcontract", + "tests", + "fixtures", + ".github", + "pyproject.toml", + "README.md", + "docs/SOTA.md", + "docs/SOTA-EVIDENCE.md", + "docs/DEVELOPER-LANGUAGE.md", + "docs/CLEANROOM-VALIDATION-RUNBOOK.md" + ], + "modified_dut_files": [], + "statement": "All device-under-test paths are byte-identical to 389e9ff557f0c1f12996b7ebbc478689f38abdda; the benchmark harness changed nothing under measurement.", + "verified": true + }, + "note": "DUT_BASE_SHA is the AuthContract implementation being measured. BENCHMARK_HARNESS_SHA is the commit containing the harness that measured it \u2014 necessarily a later commit, since the harness did not exist at DUT_BASE_SHA. Reproduce from BENCHMARK_HARNESS_SHA, not from DUT_BASE_SHA." + }, + "work_order": "AC-035 / AC-035A / AC-039" +} diff --git a/benchmarks/results/AC-039-DETERMINISM.json b/benchmarks/results/AC-039-DETERMINISM.json new file mode 100644 index 0000000..53c23d5 --- /dev/null +++ b/benchmarks/results/AC-039-DETERMINISM.json @@ -0,0 +1,131 @@ +{ + "claim_ceiling": [ + "This benchmark measures a bounded MVP-alpha implementation on one synthetic banking specimen family.", + "It does NOT establish production readiness.", + "It does NOT establish regulatory or legal correctness.", + "It does NOT establish universal source-to-rule derivation.", + "It does NOT establish arbitrary-domain compatibility.", + "It does NOT establish security certification.", + "It does NOT establish distributed or concurrent scalability.", + "It does NOT constitute a formal proof.", + "It does NOT establish comparative superiority over any other system.", + "Latency and throughput figures are single-process, single-machine, and environment-specific." + ], + "determinism": { + "determinism_statement": "Every observed output is stable across repeated execution of a fixed specimen: decision, reason code, contract digest, projection, and all protected receipt fields including decision_time. decision_time is stable because it is bound to the fact bundle's own declared `now`, not to wall-clock time at invocation \u2014 so at this commit there is no intentionally-varying receipt field. This is determinism over fixed inputs in a single process; it is not a claim about cross-version, cross-platform, or cross-implementation reproducibility, none of which was tested.", + "fully_deterministic_over_fixed_inputs": true, + "primitives": { + "contract_digest_distinct_values": 1, + "contract_digest_stable": true, + "projection_distinct_values": 1, + "projection_stable": true + }, + "refused_specimen": { + "decision_stable": true, + "decision_values_observed": [ + "REFUSED" + ], + "executions": 100, + "reason_code_stable": true, + "reason_codes_observed": [ + "RUN_FACT_STALE" + ], + "receipt_emitted": false, + "stable_receipt_fields": [], + "varying_receipt_fields": [] + }, + "valid_specimen": { + "decision_stable": true, + "decision_values_observed": [ + "ALLOW" + ], + "executions": 100, + "reason_code_stable": true, + "reason_codes_observed": [ + "OK" + ], + "receipt_emitted": true, + "stable_receipt_fields": [ + "activation_id", + "admission_digest", + "contract_digest", + "decision", + "decision_time", + "exact_action_digest", + "execution_result", + "projection_digest", + "receipt_digest", + "runtime_fact_set_digest" + ], + "varying_receipt_fields": [] + } + }, + "environment": { + "commit_sha": "389e9ff557f0c1f12996b7ebbc478689f38abdda", + "dependency_versions": { + "pytest": "9.1.1", + "rfc8785": "0.1.4" + }, + "git_status_clean": false, + "machine": "x86_64", + "platform": "Linux-6.18.44-fc-v21-x86_64-with-glibc2.39", + "processor": "x86_64", + "python_implementation": "CPython", + "python_version": "3.11.15", + "tree_sha": "edd3b56bc794720672ce9853e8173e1a2b446e17" + }, + "generated_at_utc": "2026-08-24T21:23:53Z", + "provenance": { + "benchmark_harness_sha": "389e9ff557f0c1f12996b7ebbc478689f38abdda", + "dependency_identity": { + "constraints_file": "constraints.txt", + "declared": { + "exceptiongroup": "1.3.1", + "iniconfig": "2.3.0", + "packaging": "25.0", + "pluggy": "1.6.0", + "pygments": "2.21.0", + "pytest": "9.1.1", + "rfc8785": "0.1.4", + "tomli": "2.4.1" + }, + "declared_but_not_installed": [ + "exceptiongroup" + ], + "installed_for_declared": { + "exceptiongroup": null, + "iniconfig": "2.3.0", + "packaging": "25.0", + "pluggy": "1.6.0", + "pygments": "2.21.0", + "pytest": "9.1.1", + "rfc8785": "0.1.4", + "tomli": "2.4.1" + }, + "matches_declared_set": true, + "mismatched": {}, + "note": "declared_but_not_installed is expected for distributions pinned only for older Python versions (pytest pulls exceptiongroup and tomli on Python < 3.11 only). A non-empty 'mismatched' means this run did not measure the controlled dependency set." + }, + "dut_base_sha": "389e9ff557f0c1f12996b7ebbc478689f38abdda", + "dut_verification": { + "dut_base_sha": "389e9ff557f0c1f12996b7ebbc478689f38abdda", + "dut_paths": [ + "authcontract", + "tests", + "fixtures", + ".github", + "pyproject.toml", + "README.md", + "docs/SOTA.md", + "docs/SOTA-EVIDENCE.md", + "docs/DEVELOPER-LANGUAGE.md", + "docs/CLEANROOM-VALIDATION-RUNBOOK.md" + ], + "modified_dut_files": [], + "statement": "All device-under-test paths are byte-identical to 389e9ff557f0c1f12996b7ebbc478689f38abdda; the benchmark harness changed nothing under measurement.", + "verified": true + }, + "note": "DUT_BASE_SHA is the AuthContract implementation being measured. BENCHMARK_HARNESS_SHA is the commit containing the harness that measured it \u2014 necessarily a later commit, since the harness did not exist at DUT_BASE_SHA. Reproduce from BENCHMARK_HARNESS_SHA, not from DUT_BASE_SHA." + }, + "work_order": "AC-035 / AC-035A / AC-039" +} diff --git a/benchmarks/results/AC-039-E2E-RESULTS.json b/benchmarks/results/AC-039-E2E-RESULTS.json new file mode 100644 index 0000000..0cf2dae --- /dev/null +++ b/benchmarks/results/AC-039-E2E-RESULTS.json @@ -0,0 +1,284 @@ +{ + "claim_ceiling": [ + "This benchmark measures a bounded MVP-alpha implementation on one synthetic banking specimen family.", + "It does NOT establish production readiness.", + "It does NOT establish regulatory or legal correctness.", + "It does NOT establish universal source-to-rule derivation.", + "It does NOT establish arbitrary-domain compatibility.", + "It does NOT establish security certification.", + "It does NOT establish distributed or concurrent scalability.", + "It does NOT constitute a formal proof.", + "It does NOT establish comparative superiority over any other system.", + "Latency and throughput figures are single-process, single-machine, and environment-specific." + ], + "end_to_end": { + "receipt_mutation_detail": [ + { + "detected": true, + "mutated_field": "activation_id", + "reason_code": "VEIP_RECEIPT_MISMATCH" + }, + { + "detected": true, + "mutated_field": "admission_digest", + "reason_code": "VEIP_RECEIPT_MISMATCH" + }, + { + "detected": true, + "mutated_field": "contract_digest", + "reason_code": "VEIP_RECEIPT_MISMATCH" + }, + { + "detected": true, + "mutated_field": "decision", + "reason_code": "VEIP_RECEIPT_MISMATCH" + }, + { + "detected": true, + "mutated_field": "decision_time", + "reason_code": "VEIP_RECEIPT_MISMATCH" + }, + { + "detected": true, + "mutated_field": "exact_action_digest", + "reason_code": "VEIP_RECEIPT_MISMATCH" + }, + { + "detected": true, + "mutated_field": "execution_result", + "reason_code": "VEIP_INVALID_EXECUTION_RESULT" + }, + { + "detected": true, + "mutated_field": "projection_digest", + "reason_code": "VEIP_RECEIPT_MISMATCH" + }, + { + "detected": true, + "mutated_field": "receipt_digest", + "reason_code": "VEIP_RECEIPT_MISMATCH" + }, + { + "detected": true, + "mutated_field": "runtime_fact_set_digest", + "reason_code": "VEIP_RECEIPT_MISMATCH" + }, + { + "detected": true, + "mutated_field": "activation_id (removed)", + "reason_code": "VEIP_RECEIPT_MALFORMED" + }, + { + "detected": true, + "mutated_field": "admission_digest (removed)", + "reason_code": "VEIP_RECEIPT_MALFORMED" + }, + { + "detected": true, + "mutated_field": "contract_digest (removed)", + "reason_code": "VEIP_RECEIPT_MALFORMED" + }, + { + "detected": true, + "mutated_field": "decision (removed)", + "reason_code": "VEIP_RECEIPT_MALFORMED" + }, + { + "detected": true, + "mutated_field": "decision_time (removed)", + "reason_code": "VEIP_RECEIPT_MALFORMED" + }, + { + "detected": true, + "mutated_field": "exact_action_digest (removed)", + "reason_code": "VEIP_RECEIPT_MALFORMED" + }, + { + "detected": true, + "mutated_field": "execution_result (removed)", + "reason_code": "VEIP_RECEIPT_MALFORMED" + }, + { + "detected": true, + "mutated_field": "projection_digest (removed)", + "reason_code": "VEIP_RECEIPT_MALFORMED" + }, + { + "detected": true, + "mutated_field": "receipt_digest (removed)", + "reason_code": "VEIP_RECEIPT_MALFORMED" + }, + { + "detected": true, + "mutated_field": "runtime_fact_set_digest (removed)", + "reason_code": "VEIP_RECEIPT_MALFORMED" + } + ], + "specimens": [ + { + "category": "happy_path", + "description": "Happy path: valid contract, valid facts, permitted action", + "expected_result": "ALLOW", + "failure_stage": null, + "notes": "", + "observed_result": "ALLOW", + "pass_fail": "PASS", + "reason_code": "OK", + "receipt_generated": true, + "receipt_verified": true, + "specimen": "E2E-01" + }, + { + "category": "stale_fact", + "description": "Stale runtime fact: freshness window exceeded", + "expected_result": "REFUSE", + "failure_stage": "authorization_decision", + "notes": "", + "observed_result": "REFUSE", + "pass_fail": "PASS", + "reason_code": "RUN_FACT_STALE", + "receipt_generated": false, + "receipt_verified": null, + "specimen": "E2E-02" + }, + { + "category": "malformed_contract", + "description": "Malformed contract artifact", + "expected_result": "REFUSE", + "failure_stage": "authorization_decision", + "notes": "", + "observed_result": "REFUSE", + "pass_fail": "PASS", + "reason_code": "RUN_DOMAIN_ESCAPE", + "receipt_generated": false, + "receipt_verified": null, + "specimen": "E2E-03" + }, + { + "category": "unsupported_domain", + "description": "Action outside the declared projection domain", + "expected_result": "REFUSE", + "failure_stage": "authorization_decision", + "notes": "", + "observed_result": "REFUSE", + "pass_fail": "PASS", + "reason_code": "RUN_UNCLASSIFIED_ACTION", + "receipt_generated": false, + "receipt_verified": null, + "specimen": "E2E-04" + }, + { + "category": "source_version_mutation", + "description": "Source/version material changed with stale (non-recomputed) digest binding", + "expected_result": "REFUSE", + "failure_stage": "digest_binding", + "notes": "contract.identity.version altered; activation/admission/proof digests left stale", + "observed_result": "REFUSE", + "pass_fail": "PASS", + "reason_code": "AC_DIGEST", + "receipt_generated": false, + "receipt_verified": null, + "specimen": "E2E-06" + }, + { + "category": "receipt_mutation", + "description": "Receipt mutation and truncation across every protected field", + "expected_result": "REFUSE", + "failure_stage": "receipt_verification", + "notes": "20 mutation/truncation variants tested; 20 detected", + "observed_result": "REFUSE", + "pass_fail": "PASS", + "reason_code": "VEIP_RECEIPT_MISMATCH/MALFORMED", + "receipt_generated": true, + "receipt_verified": false, + "specimen": "E2E-05" + }, + { + "category": "deterministic_replay", + "description": "Deterministic replay, 100 executions of the identical specimen", + "expected_result": "ALLOW", + "failure_stage": null, + "notes": "all protected receipt fields byte-identical across replays", + "observed_result": "ALLOW", + "pass_fail": "PASS", + "reason_code": "OK", + "receipt_generated": true, + "receipt_verified": true, + "specimen": "E2E-07" + } + ], + "totals": { + "failed": 0, + "passed": 7, + "total": 7 + } + }, + "environment": { + "commit_sha": "389e9ff557f0c1f12996b7ebbc478689f38abdda", + "dependency_versions": { + "pytest": "9.1.1", + "rfc8785": "0.1.4" + }, + "git_status_clean": false, + "machine": "x86_64", + "platform": "Linux-6.18.44-fc-v21-x86_64-with-glibc2.39", + "processor": "x86_64", + "python_implementation": "CPython", + "python_version": "3.11.15", + "tree_sha": "edd3b56bc794720672ce9853e8173e1a2b446e17" + }, + "generated_at_utc": "2026-08-24T21:23:53Z", + "provenance": { + "benchmark_harness_sha": "389e9ff557f0c1f12996b7ebbc478689f38abdda", + "dependency_identity": { + "constraints_file": "constraints.txt", + "declared": { + "exceptiongroup": "1.3.1", + "iniconfig": "2.3.0", + "packaging": "25.0", + "pluggy": "1.6.0", + "pygments": "2.21.0", + "pytest": "9.1.1", + "rfc8785": "0.1.4", + "tomli": "2.4.1" + }, + "declared_but_not_installed": [ + "exceptiongroup" + ], + "installed_for_declared": { + "exceptiongroup": null, + "iniconfig": "2.3.0", + "packaging": "25.0", + "pluggy": "1.6.0", + "pygments": "2.21.0", + "pytest": "9.1.1", + "rfc8785": "0.1.4", + "tomli": "2.4.1" + }, + "matches_declared_set": true, + "mismatched": {}, + "note": "declared_but_not_installed is expected for distributions pinned only for older Python versions (pytest pulls exceptiongroup and tomli on Python < 3.11 only). A non-empty 'mismatched' means this run did not measure the controlled dependency set." + }, + "dut_base_sha": "389e9ff557f0c1f12996b7ebbc478689f38abdda", + "dut_verification": { + "dut_base_sha": "389e9ff557f0c1f12996b7ebbc478689f38abdda", + "dut_paths": [ + "authcontract", + "tests", + "fixtures", + ".github", + "pyproject.toml", + "README.md", + "docs/SOTA.md", + "docs/SOTA-EVIDENCE.md", + "docs/DEVELOPER-LANGUAGE.md", + "docs/CLEANROOM-VALIDATION-RUNBOOK.md" + ], + "modified_dut_files": [], + "statement": "All device-under-test paths are byte-identical to 389e9ff557f0c1f12996b7ebbc478689f38abdda; the benchmark harness changed nothing under measurement.", + "verified": true + }, + "note": "DUT_BASE_SHA is the AuthContract implementation being measured. BENCHMARK_HARNESS_SHA is the commit containing the harness that measured it \u2014 necessarily a later commit, since the harness did not exist at DUT_BASE_SHA. Reproduce from BENCHMARK_HARNESS_SHA, not from DUT_BASE_SHA." + }, + "work_order": "AC-035 / AC-035A / AC-039" +} diff --git a/benchmarks/results/AC-039-PERFORMANCE.json b/benchmarks/results/AC-039-PERFORMANCE.json new file mode 100644 index 0000000..27a8ef6 --- /dev/null +++ b/benchmarks/results/AC-039-PERFORMANCE.json @@ -0,0 +1,550 @@ +{ + "claim_ceiling": [ + "This benchmark measures a bounded MVP-alpha implementation on one synthetic banking specimen family.", + "It does NOT establish production readiness.", + "It does NOT establish regulatory or legal correctness.", + "It does NOT establish universal source-to-rule derivation.", + "It does NOT establish arbitrary-domain compatibility.", + "It does NOT establish security certification.", + "It does NOT establish distributed or concurrent scalability.", + "It does NOT constitute a formal proof.", + "It does NOT establish comparative superiority over any other system.", + "Latency and throughput figures are single-process, single-machine, and environment-specific." + ], + "environment": { + "commit_sha": "389e9ff557f0c1f12996b7ebbc478689f38abdda", + "dependency_versions": { + "pytest": "9.1.1", + "rfc8785": "0.1.4" + }, + "git_status_clean": false, + "machine": "x86_64", + "platform": "Linux-6.18.44-fc-v21-x86_64-with-glibc2.39", + "processor": "x86_64", + "python_implementation": "CPython", + "python_version": "3.11.15", + "tree_sha": "edd3b56bc794720672ce9853e8173e1a2b446e17" + }, + "generated_at_utc": "2026-08-24T21:23:53Z", + "performance": { + "cold_vs_warm": { + "complete_end_to_end_cold": { + "single_cold_execution": 618.815, + "unit": "microseconds" + }, + "note": "Single un-warmed execution in an already-imported interpreter. The dominant one-time cost is interpreter startup and module import, reported separately in the resource profile." + }, + "latency_derived_rate": { + "caveat": "This is arithmetic, not measurement: it assumes zero loop overhead and no drift under continuous operation. Compare against observed_sustained_throughput below; where they disagree, the observed figure is the real one.", + "complete_e2e_transactions_per_second": 1657.9, + "decisions_per_second": 3506.3, + "method": "LATENCY-DERIVED RATE \u2014 reciprocal of warm mean latency, NOT an observed rate", + "receipt_verifications_per_second": 3342.8, + "receipts_per_second": 3506.3, + "receipts_per_second_caveat": "Receipt emission is not separately callable at this commit: run_specimen decides and emits in one pass, so decisions/sec and receipts/sec are the same measurement reported twice, not two independent figures." + }, + "observed_sustained_throughput": { + "complete_end_to_end": { + "max_ops_per_second": 1710.8, + "measurement_seconds_per_trial": 5.0, + "median_ops_per_second": 1697.5, + "method": "observed sustained rate \u2014 continuous single-threaded loop over a fixed window", + "min_ops_per_second": 1688.4, + "total_operations": 25486, + "trial_detail": [ + { + "elapsed_seconds": 5.0005, + "operations": 8555, + "operations_per_second": 1710.8, + "trial": 1 + }, + { + "elapsed_seconds": 5.0005, + "operations": 8443, + "operations_per_second": 1688.4, + "trial": 2 + }, + { + "elapsed_seconds": 5.0004, + "operations": 8488, + "operations_per_second": 1697.5, + "trial": 3 + } + ], + "trials": 3, + "warmup_seconds": 1.0 + }, + "decision_and_receipt": { + "max_ops_per_second": 3622.6, + "measurement_seconds_per_trial": 5.0, + "median_ops_per_second": 3507.2, + "method": "observed sustained rate \u2014 continuous single-threaded loop over a fixed window", + "min_ops_per_second": 3483.2, + "total_operations": 53068, + "trial_detail": [ + { + "elapsed_seconds": 5.0002, + "operations": 17537, + "operations_per_second": 3507.2, + "trial": 1 + }, + { + "elapsed_seconds": 5.0003, + "operations": 18114, + "operations_per_second": 3622.6, + "trial": 2 + }, + { + "elapsed_seconds": 5.0002, + "operations": 17417, + "operations_per_second": 3483.2, + "trial": 3 + } + ], + "trials": 3, + "warmup_seconds": 1.0 + }, + "note": "Observed sustained rates: continuous single-process, single-threaded loops over fixed wall-clock windows. No concurrency. Not a distributed or multi-core claim.", + "receipt_verification": { + "max_ops_per_second": 3518.3, + "measurement_seconds_per_trial": 5.0, + "median_ops_per_second": 3434.8, + "method": "observed sustained rate \u2014 continuous single-threaded loop over a fixed window", + "min_ops_per_second": 3429.7, + "total_operations": 51915, + "trial_detail": [ + { + "elapsed_seconds": 5.0, + "operations": 17174, + "operations_per_second": 3434.8, + "trial": 1 + }, + { + "elapsed_seconds": 5.0002, + "operations": 17592, + "operations_per_second": 3518.3, + "trial": 2 + }, + { + "elapsed_seconds": 5.0001, + "operations": 17149, + "operations_per_second": 3429.7, + "trial": 3 + } + ], + "trials": 3, + "warmup_seconds": 1.0 + } + }, + "stages": { + "action_check": { + "summary": { + "inner_batch": 20, + "max": 12.425, + "mean": 5.239, + "min": 4.673, + "n": 2000, + "p50": 4.813, + "p95": 7.246, + "p99": 8.45, + "stdev": 0.942, + "unit": "microseconds", + "warmup_discarded": 50 + }, + "what": "check_action: validate a proposed action against the projection" + }, + "canonical_digest": { + "summary": { + "inner_batch": 20, + "max": 147.631, + "mean": 86.735, + "min": 78.874, + "n": 2000, + "p50": 85.76, + "p95": 95.669, + "p99": 100.828, + "stdev": 4.956, + "unit": "microseconds", + "warmup_discarded": 50 + }, + "what": "contract_digest: JCS canonicalization plus SHA-256" + }, + "canonicalization": { + "summary": { + "inner_batch": 20, + "max": 185.585, + "mean": 83.205, + "min": 75.601, + "n": 2000, + "p50": 80.717, + "p95": 96.402, + "p99": 129.943, + "stdev": 9.388, + "unit": "microseconds", + "warmup_discarded": 50 + }, + "what": "canonical_bytes: RFC 8785 JCS serialization of the contract" + }, + "complete_end_to_end": { + "summary": { + "inner_batch": 1, + "max": 1040.597, + "mean": 603.158, + "min": 516.93, + "n": 1000, + "p50": 589.267, + "p95": 735.219, + "p99": 850.597, + "stdev": 66.946, + "unit": "microseconds", + "warmup_discarded": 50 + }, + "what": "parse \u2192 decide \u2192 emit receipt \u2192 independently verify receipt" + }, + "contract_parse": { + "summary": { + "inner_batch": 20, + "max": 23.172, + "mean": 10.067, + "min": 8.947, + "n": 2000, + "p50": 9.071, + "p95": 14.068, + "p99": 17.049, + "stdev": 1.859, + "unit": "microseconds", + "warmup_discarded": 50 + }, + "what": "json.loads of the raw contract artifact text" + }, + "decision_and_receipt": { + "summary": { + "inner_batch": 1, + "max": 530.067, + "mean": 285.197, + "min": 245.726, + "n": 1000, + "p50": 275.212, + "p95": 346.964, + "p99": 431.608, + "stdev": 38.448, + "unit": "microseconds", + "warmup_discarded": 50 + }, + "what": "run_specimen: full orchestration \u2014 parse bundle, project, admit facts, decide, and emit the receipt. Supersets projection and action_check." + }, + "projection": { + "summary": { + "inner_batch": 20, + "max": 1112.742, + "mean": 103.709, + "min": 88.685, + "n": 2000, + "p50": 99.47, + "p95": 128.776, + "p99": 171.673, + "stdev": 26.99, + "unit": "microseconds", + "warmup_discarded": 50 + }, + "what": "project: build the deterministic action-domain projection" + }, + "projection_digest": { + "summary": { + "inner_batch": 20, + "max": 98.757, + "mean": 45.928, + "min": 39.865, + "n": 2000, + "p50": 44.252, + "p95": 56.455, + "p99": 73.867, + "stdev": 6.196, + "unit": "microseconds", + "warmup_discarded": 50 + }, + "what": "projection_digest over the realized projection" + }, + "receipt_verification": { + "summary": { + "inner_batch": 1, + "max": 608.38, + "mean": 299.147, + "min": 249.519, + "n": 1000, + "p50": 284.905, + "p95": 411.822, + "p99": 508.827, + "stdev": 53.877, + "unit": "microseconds", + "warmup_discarded": 50 + }, + "what": "verify_receipt: independently recompute every binding from raw inputs and compare. Internally re-runs the full decision path." + }, + "validation_and_binding": { + "summary": { + "inner_batch": 20, + "max": 575.013, + "mean": 89.437, + "min": 79.603, + "n": 2000, + "p50": 86.73, + "p95": 105.186, + "p99": 135.207, + "stdev": 14.456, + "unit": "microseconds", + "warmup_discarded": 50 + }, + "what": "verify_artifact: digest-scope validation plus sibling binding agreement" + } + } + }, + "provenance": { + "benchmark_harness_sha": "389e9ff557f0c1f12996b7ebbc478689f38abdda", + "dependency_identity": { + "constraints_file": "constraints.txt", + "declared": { + "exceptiongroup": "1.3.1", + "iniconfig": "2.3.0", + "packaging": "25.0", + "pluggy": "1.6.0", + "pygments": "2.21.0", + "pytest": "9.1.1", + "rfc8785": "0.1.4", + "tomli": "2.4.1" + }, + "declared_but_not_installed": [ + "exceptiongroup" + ], + "installed_for_declared": { + "exceptiongroup": null, + "iniconfig": "2.3.0", + "packaging": "25.0", + "pluggy": "1.6.0", + "pygments": "2.21.0", + "pytest": "9.1.1", + "rfc8785": "0.1.4", + "tomli": "2.4.1" + }, + "matches_declared_set": true, + "mismatched": {}, + "note": "declared_but_not_installed is expected for distributions pinned only for older Python versions (pytest pulls exceptiongroup and tomli on Python < 3.11 only). A non-empty 'mismatched' means this run did not measure the controlled dependency set." + }, + "dut_base_sha": "389e9ff557f0c1f12996b7ebbc478689f38abdda", + "dut_verification": { + "dut_base_sha": "389e9ff557f0c1f12996b7ebbc478689f38abdda", + "dut_paths": [ + "authcontract", + "tests", + "fixtures", + ".github", + "pyproject.toml", + "README.md", + "docs/SOTA.md", + "docs/SOTA-EVIDENCE.md", + "docs/DEVELOPER-LANGUAGE.md", + "docs/CLEANROOM-VALIDATION-RUNBOOK.md" + ], + "modified_dut_files": [], + "statement": "All device-under-test paths are byte-identical to 389e9ff557f0c1f12996b7ebbc478689f38abdda; the benchmark harness changed nothing under measurement.", + "verified": true + }, + "note": "DUT_BASE_SHA is the AuthContract implementation being measured. BENCHMARK_HARNESS_SHA is the commit containing the harness that measured it \u2014 necessarily a later commit, since the harness did not exist at DUT_BASE_SHA. Reproduce from BENCHMARK_HARNESS_SHA, not from DUT_BASE_SHA." + }, + "scaling": { + "declared_action_scaling": { + "dimension": "count of declared mediated actions in the projection domain", + "levels": [ + { + "decision": "ALLOW", + "declared_actions": 1, + "latency": { + "inner_batch": 1, + "max": 555.503, + "mean": 282.567, + "min": 242.94, + "n": 400, + "p50": 259.539, + "p95": 366.779, + "p99": 459.906, + "stdev": 46.482, + "unit": "microseconds", + "warmup_discarded": 10 + }, + "peak_traced_memory": { + "peak_traced_bytes": 6214, + "peak_traced_kib": 6.07 + }, + "reason_code": "OK" + }, + { + "decision": "ALLOW", + "declared_actions": 10, + "latency": { + "inner_batch": 1, + "max": 1851.274, + "mean": 1019.588, + "min": 838.729, + "n": 400, + "p50": 938.469, + "p95": 1547.657, + "p99": 1720.398, + "stdev": 206.297, + "unit": "microseconds", + "warmup_discarded": 10 + }, + "peak_traced_memory": { + "peak_traced_bytes": 9621, + "peak_traced_kib": 9.4 + }, + "reason_code": "OK" + }, + { + "decision": "ALLOW", + "declared_actions": 100, + "latency": { + "inner_batch": 1, + "max": 13583.084, + "mean": 7452.599, + "min": 6918.549, + "n": 400, + "p50": 7268.447, + "p95": 8621.924, + "p99": 10132.957, + "stdev": 689.625, + "unit": "microseconds", + "warmup_discarded": 10 + }, + "peak_traced_memory": { + "peak_traced_bytes": 41774, + "peak_traced_kib": 40.79 + }, + "reason_code": "OK" + }, + { + "decision": "ALLOW", + "declared_actions": 1000, + "latency": { + "inner_batch": 1, + "max": 90343.236, + "mean": 70963.27, + "min": 68665.692, + "n": 60, + "p50": 70124.992, + "p95": 76243.453, + "p99": 90343.236, + "stdev": 3108.94, + "unit": "microseconds", + "warmup_discarded": 10 + }, + "peak_traced_memory": { + "peak_traced_bytes": 374877, + "peak_traced_kib": 366.09 + }, + "reason_code": "OK" + } + ], + "observed_shape": "approximately linear (size x1000 -> time x251.1)" + }, + "not_evaluated": { + "concurrent_or_distributed_throughput": "NOT EVALUATED \u2014 no concurrency or distribution layer exists at this commit.", + "multi_contract_corpora": "NOT EVALUATED \u2014 the implementation evaluates one artifact per invocation; there is no multi-contract registry or cross-contract selection path at this commit whose scaling could be measured without inventing architecture.", + "persistent_storage_scaling": "NOT EVALUATED \u2014 the runtime is stateless over in-memory inputs; there is no storage backend to scale." + }, + "required_fact_scaling": { + "dimension": "count of required facts (contract) matched by supplied facts (bundle)", + "levels": [ + { + "decision": "ALLOW", + "latency": { + "inner_batch": 1, + "max": 1720.263, + "mean": 1027.456, + "min": 741.874, + "n": 200, + "p50": 912.574, + "p95": 1516.097, + "p99": 1688.2, + "stdev": 254.376, + "unit": "microseconds", + "warmup_discarded": 3 + }, + "peak_traced_memory": { + "peak_traced_bytes": 16262, + "peak_traced_kib": 15.88 + }, + "reason_code": "OK", + "required_facts": 10 + }, + { + "decision": "ALLOW", + "latency": { + "inner_batch": 1, + "max": 9283.612, + "mean": 5886.455, + "min": 5330.457, + "n": 200, + "p50": 5661.176, + "p95": 7960.566, + "p99": 8945.965, + "stdev": 780.109, + "unit": "microseconds", + "warmup_discarded": 3 + }, + "peak_traced_memory": { + "peak_traced_bytes": 143745, + "peak_traced_kib": 140.38 + }, + "reason_code": "OK", + "required_facts": 100 + }, + { + "decision": "ALLOW", + "latency": { + "inner_batch": 1, + "max": 64194.981, + "mean": 55691.113, + "min": 53284.464, + "n": 40, + "p50": 54623.881, + "p95": 59947.16, + "p99": 64194.981, + "stdev": 2350.238, + "unit": "microseconds", + "warmup_discarded": 3 + }, + "peak_traced_memory": { + "peak_traced_bytes": 1365487, + "peak_traced_kib": 1333.48 + }, + "reason_code": "OK", + "required_facts": 1000 + }, + { + "decision": "ALLOW", + "latency": { + "inner_batch": 1, + "max": 577010.912, + "mean": 556680.78, + "min": 536823.525, + "n": 5, + "p50": 558465.218, + "p95": 577010.912, + "p99": 577010.912, + "stdev": 15886.441, + "unit": "microseconds", + "warmup_discarded": 3 + }, + "peak_traced_memory": { + "peak_traced_bytes": 13789114, + "peak_traced_kib": 13465.93 + }, + "reason_code": "OK", + "required_facts": 10000 + } + ], + "observed_shape": "approximately linear (size x1000 -> time x541.8)" + } + }, + "work_order": "AC-035 / AC-035A / AC-039" +} diff --git a/benchmarks/results/AC-039-RESOURCE-PROFILE.json b/benchmarks/results/AC-039-RESOURCE-PROFILE.json new file mode 100644 index 0000000..8b73c00 --- /dev/null +++ b/benchmarks/results/AC-039-RESOURCE-PROFILE.json @@ -0,0 +1,124 @@ +{ + "claim_ceiling": [ + "This benchmark measures a bounded MVP-alpha implementation on one synthetic banking specimen family.", + "It does NOT establish production readiness.", + "It does NOT establish regulatory or legal correctness.", + "It does NOT establish universal source-to-rule derivation.", + "It does NOT establish arbitrary-domain compatibility.", + "It does NOT establish security certification.", + "It does NOT establish distributed or concurrent scalability.", + "It does NOT constitute a formal proof.", + "It does NOT establish comparative superiority over any other system.", + "Latency and throughput figures are single-process, single-machine, and environment-specific." + ], + "environment": { + "commit_sha": "389e9ff557f0c1f12996b7ebbc478689f38abdda", + "dependency_versions": { + "pytest": "9.1.1", + "rfc8785": "0.1.4" + }, + "git_status_clean": false, + "machine": "x86_64", + "platform": "Linux-6.18.44-fc-v21-x86_64-with-glibc2.39", + "processor": "x86_64", + "python_implementation": "CPython", + "python_version": "3.11.15", + "tree_sha": "edd3b56bc794720672ce9853e8173e1a2b446e17" + }, + "generated_at_utc": "2026-08-24T21:23:53Z", + "provenance": { + "benchmark_harness_sha": "389e9ff557f0c1f12996b7ebbc478689f38abdda", + "dependency_identity": { + "constraints_file": "constraints.txt", + "declared": { + "exceptiongroup": "1.3.1", + "iniconfig": "2.3.0", + "packaging": "25.0", + "pluggy": "1.6.0", + "pygments": "2.21.0", + "pytest": "9.1.1", + "rfc8785": "0.1.4", + "tomli": "2.4.1" + }, + "declared_but_not_installed": [ + "exceptiongroup" + ], + "installed_for_declared": { + "exceptiongroup": null, + "iniconfig": "2.3.0", + "packaging": "25.0", + "pluggy": "1.6.0", + "pygments": "2.21.0", + "pytest": "9.1.1", + "rfc8785": "0.1.4", + "tomli": "2.4.1" + }, + "matches_declared_set": true, + "mismatched": {}, + "note": "declared_but_not_installed is expected for distributions pinned only for older Python versions (pytest pulls exceptiongroup and tomli on Python < 3.11 only). A non-empty 'mismatched' means this run did not measure the controlled dependency set." + }, + "dut_base_sha": "389e9ff557f0c1f12996b7ebbc478689f38abdda", + "dut_verification": { + "dut_base_sha": "389e9ff557f0c1f12996b7ebbc478689f38abdda", + "dut_paths": [ + "authcontract", + "tests", + "fixtures", + ".github", + "pyproject.toml", + "README.md", + "docs/SOTA.md", + "docs/SOTA-EVIDENCE.md", + "docs/DEVELOPER-LANGUAGE.md", + "docs/CLEANROOM-VALIDATION-RUNBOOK.md" + ], + "modified_dut_files": [], + "statement": "All device-under-test paths are byte-identical to 389e9ff557f0c1f12996b7ebbc478689f38abdda; the benchmark harness changed nothing under measurement.", + "verified": true + }, + "note": "DUT_BASE_SHA is the AuthContract implementation being measured. BENCHMARK_HARNESS_SHA is the commit containing the harness that measured it \u2014 necessarily a later commit, since the harness did not exist at DUT_BASE_SHA. Reproduce from BENCHMARK_HARNESS_SHA, not from DUT_BASE_SHA." + }, + "resources": { + "artifact_sizes": { + "action": 148, + "canonical_contract_bytes": 773, + "contract_artifact": 1179, + "contract_body_only": 773, + "fact_bundle": 675, + "projection": 479, + "receipt": 715, + "unit": "bytes (compact JSON encoding)" + }, + "peak_memory": { + "note": "tracemalloc peak attributable to one execution; excludes interpreter baseline", + "single_decision": { + "peak_traced_bytes": 6214, + "peak_traced_kib": 6.07 + }, + "single_verification": { + "peak_traced_bytes": 6334, + "peak_traced_kib": 6.19 + } + }, + "process_max_rss": { + "note": "whole-process peak RSS at end of benchmark run, including interpreter and harness", + "unit": "kilobytes", + "value": 67708 + }, + "process_startup": { + "max": 71.91, + "mean": 64.35, + "min": 60.08, + "samples": [ + 60.58, + 71.91, + 67.66, + 60.08, + 61.52 + ], + "unit": "milliseconds", + "what": "python -c 'import authcontract.veip', out-of-process, 5 samples" + } + }, + "work_order": "AC-035 / AC-035A / AC-039" +} diff --git a/benchmarks/run_benchmarks.py b/benchmarks/run_benchmarks.py index bc9dfb6..9cc0eee 100644 --- a/benchmarks/run_benchmarks.py +++ b/benchmarks/run_benchmarks.py @@ -28,6 +28,7 @@ from harness import ( # noqa: E402 REPO_ROOT, + capture_dependency_identity, capture_environment, latency_derived_rate, load_fixture, @@ -54,7 +55,16 @@ # DUT_BASE_SHA identifies the product; the harness SHA is captured at runtime. # `verify_dut_unchanged` proves the harness commit modified nothing under # measurement, which is what makes the two safely comparable. -DUT_BASE_SHA = "e4e1a97509df1a66c44b090c0a0ca0a03907f4dc" +# AC-039 rebound this to the commit that introduced the controlled dependency +# set. The AC-035A results measured a DIFFERENT dependency environment; they are +# preserved unchanged under their own AC-035 result identity and must not be +# presented as measurements of this one. +DUT_BASE_SHA = "389e9ff557f0c1f12996b7ebbc478689f38abdda" + +# Result-set identity. Results are written under this prefix so a new +# measurement never overwrites the provenance of an earlier one. +RESULT_SET_ID = "AC-039" +WORK_ORDER = "AC-035 / AC-035A / AC-039" # Sustained-throughput measurement window. THROUGHPUT_TRIALS = 3 @@ -852,13 +862,14 @@ def main() -> int: "Reproduce from BENCHMARK_HARNESS_SHA, not from DUT_BASE_SHA." ), "dut_verification": verify_dut_unchanged(DUT_BASE_SHA), + "dependency_identity": capture_dependency_identity(), } if not provenance["dut_verification"]["verified"]: print("REFUSING TO RUN:", provenance["dut_verification"]["statement"], file=sys.stderr) return 2 - print("AC-035 benchmark") + print(f"{RESULT_SET_ID} benchmark") print(" DUT :", DUT_BASE_SHA[:12], "(verified unchanged)") print(" harness :", environment["commit_sha"][:12]) print(" phase: end-to-end + correctness matrix") @@ -875,21 +886,21 @@ def main() -> int: resources = phase_resources() common = { - "work_order": "AC-035 / AC-035A", + "work_order": WORK_ORDER, "provenance": provenance, "environment": environment, "claim_ceiling": CLAIM_CEILING, "generated_at_utc": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), } - write_result("AC-035-E2E-RESULTS.json", {**common, "end_to_end": e2e}) - write_result("AC-035-PERFORMANCE.json", {**common, "performance": performance, "scaling": scale}) + write_result(f"{RESULT_SET_ID}-E2E-RESULTS.json", {**common, "end_to_end": e2e}) + write_result(f"{RESULT_SET_ID}-PERFORMANCE.json", {**common, "performance": performance, "scaling": scale}) write_result( - "AC-035-CORRECTNESS-MATRIX.json", + f"{RESULT_SET_ID}-CORRECTNESS-MATRIX.json", {**common, "end_to_end_matrix": e2e["specimens"], "adversarial_matrix": adversarial}, ) - write_result("AC-035-DETERMINISM.json", {**common, "determinism": determinism}) - write_result("AC-035-RESOURCE-PROFILE.json", {**common, "resources": resources}) + write_result(f"{RESULT_SET_ID}-DETERMINISM.json", {**common, "determinism": determinism}) + write_result(f"{RESULT_SET_ID}-RESOURCE-PROFILE.json", {**common, "resources": resources}) e2e_totals = e2e["totals"] adv_totals = adversarial["totals"] diff --git a/constraints.txt b/constraints.txt new file mode 100644 index 0000000..6920c64 --- /dev/null +++ b/constraints.txt @@ -0,0 +1,61 @@ +# Controlled dependency set for AuthContract (AC-039). +# +# WHAT THIS IS +# ------------ +# The exact resolved versions of every third-party distribution the project and +# its test extra pull in. CI installs with `-c constraints.txt`, so a clean +# install reproduces this set rather than whatever the resolver happens to pick +# on the day. Benchmark and correctness results are bound to these versions. +# +# WHAT THIS IS NOT +# ---------------- +# This is NOT hash-pinning and NOT byte-for-byte reproducible packaging. The +# project is installed editable (`pip install -e .`), and pip's +# `--require-hashes` mode cannot be combined with an editable install, so hash +# pinning is unavailable for this install shape. No release artifact, wheel, or +# sdist is published from this repository, so there is nothing further downstream +# to attest. See docs/VERSIONING.md. +# +# UPDATE PROCESS +# -------------- +# 1. Regenerate the closure for the OLDEST supported Python (3.10): +# pip install --dry-run --report report.json --only-binary=:all: \ +# --python-version 3.10 --target /tmp/discard "rfc8785>=0.1.2" "pytest>=7.0" +# 2. Rewrite the pins below from that report. +# 3. Run the full suite and falsify.py on every supported Python version. +# 4. Re-run benchmarks/run_benchmarks.py with DUT_BASE_SHA updated to the +# commit carrying the change, and record a NEW result set. Do not present +# an older result set as a measurement of a changed dependency environment. +# +# ADVISORY COVERAGE +# ----------------- +# Every pin here must be resolvable by the advisory service the Security +# workflow queries; that workflow FAILS if any pin comes back unaudited, so +# a version with no advisory coverage cannot pass as "no findings". +# `packaging` is held at 25.0 for exactly this reason: 26.3 resolves and +# installs, but the advisory service returns no record for it, which would +# make it silently unauditable. pytest requires only `packaging>=22`, so +# 25.0 is a fully supported choice and the stricter one. +# +# Floors stay declared in pyproject.toml; this file fixes what those floors +# resolve to. Changing a floor without regenerating this file is dependency +# drift and is what this file exists to prevent. + +# --- runtime --- +rfc8785==0.1.4 + +# --- test extra --- +pytest==9.1.1 + +# --- transitive, pulled in by pytest --- +iniconfig==2.3.0 +packaging==25.0 +pluggy==1.6.0 +Pygments==2.21.0 + +# --- transitive, installed only on Python < 3.11 (pytest environment markers) --- +# Pinned even though they are absent on 3.12: a constraint on a distribution +# that is not installed is inert, and leaving them unpinned would mean the +# 3.10 job silently resolves a different set than the 3.12 job. +exceptiongroup==1.3.1 +tomli==2.4.1 diff --git a/docs/BENCHMARKS-AC-039.md b/docs/BENCHMARKS-AC-039.md new file mode 100644 index 0000000..a2c93f6 --- /dev/null +++ b/docs/BENCHMARKS-AC-039.md @@ -0,0 +1,180 @@ +# AuthContract benchmark baseline — AC-039 (current) + +**DUT_BASE_SHA (implementation measured):** `389e9ff557f0c1f12996b7ebbc478689f38abdda` +**BENCHMARK_HARNESS_SHA (harness that measured it):** `389e9ff557f0c1f12996b7ebbc478689f38abdda` +**Work order:** AC-035 / AC-035A methodology, re-measured under AC-039 +**Raw results:** [`benchmarks/results/AC-039-*.json`](../benchmarks/results/) + +> **Why this exists.** AC-039 introduced a controlled dependency set +> (`constraints.txt`), which changes the environment the implementation runs +> against. Performance and correctness are properties of the code *plus* its +> dependencies, so the earlier AC-035A figures stopped being valid measurements +> of the current environment the moment that set changed. They are preserved +> unmodified in [`docs/BENCHMARKS.md`](BENCHMARKS.md) as the historical AC-035A +> record and are **not** presented as current. This document supersedes them. + +> **Provenance.** Before measuring, the harness diffs every device-under-test +> path against `DUT_BASE_SHA` and refuses to run on any drift (exit 2). This +> run: `verified: true`, zero modified DUT files. It additionally records the +> declared-versus-installed dependency set: `matches_declared_set: true`. + +--- + +## 1. Environment + +| | | +|---|---| +| Python | 3.11.15 (CPython) | +| Platform | Linux x86_64 (glibc 2.39) | +| Dependency control | `constraints.txt`, verified against the installed set at measurement time | +| Runtime dependency | `rfc8785==0.1.4` | +| Test dependencies | `pytest==9.1.1`, `iniconfig==2.3.0`, `packaging==25.0`, `pluggy==1.6.0`, `Pygments==2.21.0`, `tomli==2.4.1` | +| Process | single, single-threaded | +| Wall time for the full run | 95.6 s | + +Figures are environment-specific. Absolute latencies will differ on other +hardware; the *shape* of the curves and the relative cost of stages are the +transferable results. + +--- + +## 2. Correctness + +| Battery | Result | +|---|---| +| End-to-end specimens | **7 / 7 passed**, 0 failed | +| Adversarial specimens | **38 / 38 passed**, 0 failed, 0 `NOT EVALUATED` | +| Regression suite | **342 passed** | +| Public falsification harness (`falsify.py`) | **5 / 5 cases matched** their declared expected disposition | + +The adversarial battery spans missing and unknown fields, malformed types, +out-of-domain values, staleness, evidence divergence, digest mutation, +substitution, and reordering. Every one fails closed. + +--- + +## 3. Latency by stage + +Microseconds. Distributions, not single timings. + +| Stage | mean | p50 | p95 | p99 | +|---|---|---|---|---| +| `action_check` | 5.24 | 4.81 | 7.25 | 8.45 | +| `contract_parse` | 10.07 | 9.07 | 14.07 | 17.05 | +| `projection_digest` | 45.93 | 44.25 | 56.46 | 73.87 | +| `canonicalization` | 83.21 | 80.72 | 96.40 | 129.94 | +| `canonical_digest` | 86.74 | 85.76 | 95.67 | 100.83 | +| `validation_and_binding` | 89.44 | 86.73 | 105.19 | 135.21 | +| `projection` | 103.71 | 99.47 | 128.78 | 171.67 | +| `decision_and_receipt` | 285.20 | 275.21 | 346.96 | 431.61 | +| `receipt_verification` | 299.15 | 284.91 | 411.82 | 508.83 | +| **`complete_end_to_end`** | **603.16** | **589.27** | **735.22** | **850.60** | + +Single un-warmed execution in an already-imported interpreter: 618.8 µs. +Out-of-process interpreter startup plus `import authcontract.veip`: 64.4 ms mean +over 5 samples — dominated by interpreter startup, not by this project. + +--- + +## 4. Throughput — two different things, reported separately + +Conflating these is the most common way a benchmark misleads, so both are given. + +**Observed sustained throughput** — a continuous single-threaded loop over a +fixed window. 3 trials × 5 s, after a 1 s warmup. This is a measurement. + +| Operation | min | median | max | total ops | +|---|---|---|---|---| +| Complete end-to-end | 1688.4 | **1697.5** | 1710.8 | 25,486 | +| Decision + receipt | 3483.2 | **3507.2** | 3622.6 | 53,068 | +| Receipt verification | 3429.7 | **3434.8** | 3518.3 | 51,915 | + +**Latency-derived rate** — the arithmetic reciprocal of warm mean latency. This +is *not* a measurement: it assumes zero loop overhead and no drift. + +| Operation | derived rate | +|---|---| +| Complete end-to-end | 1657.9 /s | +| Decision | 3506.3 /s | + +The observed end-to-end median (1697.5 /s) is in fact *higher* than the +derived figure (1657.9 /s) here, which is a useful reminder that the reciprocal +is arithmetic rather than measurement. Where the two disagree, **the observed +figure is the real one**. Units are +operations per second, single-threaded, single-process. + +**Not claimed:** distributed throughput, multi-core scaling, or throughput under +concurrency. No concurrency or distribution layer exists at this commit. + +--- + +## 5. Scaling + +Linear-to-superlinear with no observed cliff, across 1000× in two dimensions. + +**Declared mediated actions in the projection domain** (p50 µs): + +| actions | 1 | 10 | 100 | 1000 | +|---|---|---|---|---| +| p50 | 259.5 | 938.5 | 7,268.4 | 70,125.0 | + +**Required facts matched against the supplied bundle** (p50 µs, peak traced memory): + +| facts | 10 | 100 | 1000 | 10000 | +|---|---|---|---|---| +| p50 | 912.6 | 5,661.2 | 54,623.9 | 558,465.2 | +| peak | 15.9 KiB | 140.4 KiB | 1.30 MiB | 13.15 MiB | + +Every level returned `ALLOW` / `OK` — the curves measure cost, not a change in +disposition. + +**NOT EVALUATED:** concurrent or distributed throughput, and multi-contract +corpora. The implementation evaluates one artifact per invocation and there is +no multi-contract registry, so neither could be measured rather than estimated. + +--- + +## 6. Determinism + +`fully_deterministic_over_fixed_inputs: true`. + +- Contract digest: **1** distinct value across repeated execution. +- Projection: **1** distinct value. +- Valid and refused specimens: decision, reason code and all protected receipt + fields stable across replays. +- `decision_time` is stable because it binds to the fact bundle's declared + `now`, not to wall-clock time — so there is no intentionally-varying receipt + field at this commit. + +**Bounded:** this is determinism over fixed inputs in a single process. It is +**not** a claim about cross-version, cross-platform, or cross-implementation +reproducibility. None of those was tested, and closing that gap is roadmap X4. + +--- + +## 7. Resource profile + +| | | +|---|---| +| Peak traced memory, one decision | 6.07 KiB | +| Peak traced memory, one verification | 6.19 KiB | +| Whole-process peak RSS (incl. interpreter and harness) | 67,708 KB | +| Canonical contract bytes | 773 | +| Contract artifact / projection / receipt / fact bundle | 1,179 / 479 / 715 / 675 bytes | + +--- + +## 8. Claim ceiling + +These measurements establish **only** what they measured: one synthetic banking +specimen family, one machine, one operating system, one Python version, one +process, one dependency set. + +They do **not** establish production readiness; regulatory or legal correctness; +universal source-to-rule derivation; arbitrary-domain compatibility; security +certification; distributed or concurrent scalability; formal proof; or +comparative superiority over any other system. + +Independent reproduction by a party that did not author the system remains +absent. That is the binding constraint on maturity — see +[`docs/TRL-ASSESSMENT.md`](TRL-ASSESSMENT.md). diff --git a/docs/BENCHMARKS.md b/docs/BENCHMARKS.md index f74c213..f543a40 100644 --- a/docs/BENCHMARKS.md +++ b/docs/BENCHMARKS.md @@ -1,5 +1,15 @@ # AuthContract benchmark baseline +> **HISTORICAL RECORD — SUPERSEDED.** These figures were measured at +> `e4e1a975` against an **uncontrolled** dependency environment, before AC-039 +> introduced `constraints.txt`. Because performance and correctness depend on +> the dependency set as well as the code, they are **not** current measurements +> and must not be presented as such. They are preserved here unmodified as +> evidence of what was measured then. +> +> **Current measurement:** +> [`docs/BENCHMARKS-AC-039.md`](BENCHMARKS-AC-039.md). + **DUT_BASE_SHA (implementation measured):** `e4e1a97509df1a66c44b090c0a0ca0a03907f4dc` **Tree:** `9967077e7f9f9c661199c728c5a2e7fe496be07a` **BENCHMARK_HARNESS_SHA (harness that measured it):** `a7f6ba374b1362e624a3f8b912b265dd03da4cdd` diff --git a/docs/RELEASE-READINESS.md b/docs/RELEASE-READINESS.md index 7db7920..de2b8f4 100644 --- a/docs/RELEASE-READINESS.md +++ b/docs/RELEASE-READINESS.md @@ -1,5 +1,12 @@ # Release readiness record (AC-038) +> **Updated by AC-039.** The Gate A–G results below were established at +> `db1c745`. AC-039 then closed the CURRENT-SDLC v1.1 release-hardening +> requirements — Security Completeness, Supply-Chain Integrity, API/Version +> Integrity, Machine-Readable Licensing and Public Falsification. Those results, +> and the revised disposition of findings U10–U13, are in the **AC-039 +> addendum** at the end of this document. + Adjudication input for the CURRENT-SDLC public-release lifecycle. **This document records verification results. It does not certify release, and it does not declare the repository production-ready** — that adjudication is not the @@ -175,3 +182,179 @@ regulatory or legal correctness, universal source-to-rule derivation, arbitrary-domain compatibility, security certification, distributed scalability, formal correctness, independent external validation, or comparative standing against any other system. + + +--- + +# AC-039 addendum — CURRENT-SDLC v1.1 release hardening + +Everything above stands as recorded. This addendum reports what AC-039 changed +and re-verified. It records observations; it does not certify release. + +## Security completeness + +| Requirement | State | +|---|---| +| Maturity boundary stated | [`SECURITY.md`](../SECURITY.md) §1 — experimental reference implementation, TRL 4 | +| Supported versions | §2 — **`main` only**. No tag, no release, no backport branch. | +| Private reporting route | §3 — **NOT ESTABLISHED. Owner action required.** | +| Triage policy | §4 — bounded Critical / High / Medium / Low handling, explicitly not an SLA | +| "A scanner is not an audit" | §5 — stated explicitly, along with the absence of any independent security review | +| Responsible disclosure | §6 | + +**On the private reporting route.** GitHub Private Vulnerability Reporting could +not be confirmed as enabled: the repository metadata reachable from this +project's tooling does not expose that setting, so its state is *unknown*, not +*enabled*. No security contact address was invented — an address that does not +demonstrably reach someone silently swallows reports, which is worse than an +honest absence. The smallest sufficient owner action is named in `SECURITY.md` +§3: enable Private Vulnerability Reporting, then replace that section with the +resulting advisory link. + +## Supply-chain integrity + +| Control | Implementation | +|---|---| +| Dependency-advisory monitoring | [`.github/workflows/security.yml`](../.github/workflows/security.yml) — `pip-audit==2.10.1` against `constraints.txt`, on push, PR, and a weekly schedule | +| Advisory gate strictness | **No severity threshold and no ignore list.** *Any* known advisory fails the job — stricter than the HIGH/CRITICAL floor required. | +| Advisory *coverage* assertion | The job fails if any pin comes back unaudited (see finding U14) | +| Dependency version updates | [`.github/dependabot.yml`](../.github/dependabot.yml) — `pip` and `github-actions`, weekly | +| CI least privilege | Both workflows now declare `permissions: contents: read` explicitly | +| Immutable action pinning | `actions/checkout@11d5960a…` (v4.4.0), `actions/setup-python@a26af69b…` (v5.6.0) | +| Dependency identity | [`constraints.txt`](../constraints.txt); CI installs with `-c`; the benchmark records declared-vs-installed and reported `matches_declared_set: true` | + +**Not claimed:** hash-pinned or byte-for-byte reproducible installation. pip +cannot combine `--require-hashes` with an editable install, which is this +project's only supported install shape, so hash pinning is unavailable rather +than merely omitted. That limitation is stated in `constraints.txt` itself. + +**Provider-side action still required:** Dependabot *security alerts* (as +distinct from the version-update PRs `dependabot.yml` configures) are a +repository setting the owner must enable under Settings → Advanced Security. +The repository-controlled advisory gate in `security.yml` exists precisely so +the requirement does not depend on that setting. + +## Vulnerability gate result + +`pip-audit -r constraints.txt --no-deps --strict` against the exact candidate +dependency set: **0 known advisories across 8 pinned distributions.** No finding +was suppressed, and no exception was self-authorized. + +## API / version integrity + +[`docs/VERSIONING.md`](VERSIONING.md) declares the public interface surface — +six CLI commands, the two consumer flags, exit semantics, the Python entry +points, the ten receipt fields, the reason codes, the fixture-defined file +formats, and the workflow surface — and states plainly that everything else is +internal. + +Key truthful reconciliations: + +- **`0.0.1` is a never-published placeholder** that has not been incremented as + the implementation changed. Two checkouts both reporting `0.0.1` may differ. + The commit SHA is the only reliable identity. +- **Semantic Versioning is not claimed**, because it is not implemented. +- **Pre-1.0 interfaces may change**, with no deprecation period and no backports. +- The one commitment made: **reason codes will not change meaning silently under + the same version.** Scoped deliberately to *not silently* — not to *never*. + +## Machine-readable licensing + +`pyproject.toml` now declares the Trove classifier +`License :: Other/Proprietary License`. That is the ecosystem-standard +machine-readable way to state that this project is **not** open source, and it +confers no rights — it describes the existing default-copyright state rather +than creating a new one. + +No SPDX identifier was declared, because none would be true. A PEP 639 +`license = "LicenseRef-…"` expression was considered and rejected: it would +require an accompanying license text file that only the owner can author. + +**BLOCKED-OWNER-DECISION.** The exact decision needed, and nothing more: *under +what license, if any, is AuthContract offered to third parties?* Until that is +answered, no license file can be added, downstream use remains legally +impossible, and 1.0 remains unreachable. This is the single largest adoption +barrier in the repository (finding U1). + +## Public falsification + +[`falsify.py`](../falsify.py) — one command, no credentials, no network: + +```bash +python3 falsify.py +``` + +| Case | Expected | Observed | +|---|---|---| +| Valid specimen | `PASS` / `OK` / exit 0, receipt issued | **MATCH** | +| Undeclared action | `REFUSED` / `RUN_UNCLASSIFIED_ACTION` / exit 1, no receipt | **MATCH** | +| Stale runtime fact | `REFUSED` / `RUN_FACT_STALE` / exit 1, no receipt | **MATCH** | +| Untampered receipt | `PASS` / `OK` / exit 0 | **MATCH** | +| Tampered receipt binding | `REFUSED` / `VEIP_RECEIPT_MISMATCH` / exit 1 | **MATCH** | + +5 / 5 matched. A case fails on a mismatch **in either direction** — an +unexpected pass fails exactly as loudly as an unexpected refusal, which is the +half that matters for a system whose value rests on refusing correctly. The +harness exits non-zero on any mismatch and runs in CI. + +## Release-artifact provenance applicability + +Re-evaluated against current state: still **no published package, no binary, no +container, no installer, no generated SDK, and no GitHub Release artifact.** +AC-039 introduced none. + +Artifact attestation, SBOM-for-distributed-artifact, and release-digest +requirements therefore remain **NOT APPLICABLE** — there is no distributed +artifact for provenance to attach to. No package or release was created merely +to satisfy a clause that does not apply. If a distributable artifact is ever +produced, applicability changes immediately and these requirements become +mandatory. + +## Refreshed measurement + +Because the dependency environment materially changed, the AC-035A figures +stopped being measurements of the current system. The full battery was re-run: +[`docs/BENCHMARKS-AC-039.md`](BENCHMARKS-AC-039.md), raw results under +`benchmarks/results/AC-039-*.json`. The AC-035A record is preserved unmodified +in [`docs/BENCHMARKS.md`](BENCHMARKS.md) under a banner marking it superseded. + +DUT `389e9ff557f0c1f12996b7ebbc478689f38abdda`, `verified: true`, +`matches_declared_set: true`. 7/7 E2E · 38/38 adversarial · 342 tests · +determinism stable · observed sustained end-to-end throughput median 1697.5 +ops/sec (min 1688.4, max 1710.8). + +## Revised finding dispositions + +| Finding | Disposition after AC-039 | +|---|---| +| **U1** — no licence declared | **OPEN — BLOCKED-OWNER-DECISION.** Machine-readable *state* now declared; the legal choice is not the executor's to make. | +| **U5** — DUT guard covers `README.md` | **OPEN — still deliberately not worked around.** The guard fired as designed and the DUT was rebound to a new commit rather than the guard being loosened. | +| **U8** — agent-usability record is self-authored | **OPEN — inherent.** `falsify.py` now lets a third party check the dispositions without trusting the record, which narrows but does not close it. | +| **U9** — governance vocabulary in public docs | **OPEN — recorded.** Unchanged; rewording guard-pinned accepted text is a boundary decision, not an executor correction. | +| **U10** — no `SECURITY.md` / `CONTRIBUTING.md` | **CLOSED.** Both added. Neither invents a channel, CLA, SLA, or governance model. | +| **U11** — no explicit workflow `permissions:` | **CLOSED.** Both workflows declare `contents: read`. | +| **U12** — actions pinned to mutable major tags | **CLOSED.** Both pinned to immutable commit SHAs with the version retained in a comment. | +| **U13** — unpinned dependency floors | **CLOSED for identity, bounded on reproducibility.** `constraints.txt` fixes the closure and CI consumes it. Hash pinning remains unavailable for an editable install and is not claimed. | + +### New finding + +**U14 — the dependency auditor can silently skip a pin.** `pip-audit` drops a +distribution whose exact version the advisory service has no record of, reports +"No known vulnerabilities found", and exits **0** — even under `--strict`. This +was found by checking the auditor's output against the input rather than +trusting its exit code: `packaging==26.3` resolved and installed but was never +audited. + +**Disposition: CLOSED, non-suppressively.** Two changes, neither of which +weakens the gate: + +1. `security.yml` now asserts coverage — every pin in `constraints.txt` must + appear in the audit report, or the job fails. There is deliberately no + allowlist. +2. `packaging` is held at `25.0`, which *is* covered by the advisory service. + `pytest` requires only `packaging>=22`, so this is a fully supported choice + and the stricter one. + +The general lesson is recorded rather than filed away: a green scanner that was +never asked the question looks identical to a green scanner that asked and found +nothing. diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 2ed52cb..aa5dc62 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -8,7 +8,7 @@ placed under **Research / not yet established** and are explicitly not commitments. Raw evidence: [`benchmarks/results/`](../benchmarks/results/) · -Analysis: [`docs/BENCHMARKS.md`](BENCHMARKS.md) +Analysis: [`docs/BENCHMARKS-AC-039.md`](BENCHMARKS-AC-039.md) --- diff --git a/docs/TRL-ASSESSMENT.md b/docs/TRL-ASSESSMENT.md index 15b4a55..0bf936b 100644 --- a/docs/TRL-ASSESSMENT.md +++ b/docs/TRL-ASSESSMENT.md @@ -1,8 +1,11 @@ # AuthContract — TRL assessment (AC-035) -**Implementation assessed (DUT):** `e4e1a97509df1a66c44b090c0a0ca0a03907f4dc` -**Measured by harness:** `a7f6ba374b1362e624a3f8b912b265dd03da4cdd` -**Basis:** the AC-035 benchmark run only, as amended by AC-035A. Architecture documents, design intent, +**Implementation assessed (DUT):** `389e9ff557f0c1f12996b7ebbc478689f38abdda` +**Measured by harness:** `389e9ff557f0c1f12996b7ebbc478689f38abdda` +**Basis:** the AC-039 re-measurement, which re-ran the AC-035/AC-035A battery +against a controlled dependency set. The earlier AC-035A figures measured a +different dependency environment and are retained as history, not as current +evidence. Architecture documents, design intent, and roadmap items are explicitly **not** counted as evidence. --- @@ -104,7 +107,7 @@ unmodified (L4). ## Claim ceiling This assessment is bounded by the measurements in -[`docs/BENCHMARKS.md`](BENCHMARKS.md). It does not establish production +[`docs/BENCHMARKS-AC-039.md`](BENCHMARKS-AC-039.md). It does not establish production readiness, regulatory or legal correctness, universal source-to-rule derivation, arbitrary-domain compatibility, security certification, distributed scalability, formal correctness, or comparative standing against any other diff --git a/docs/VERSIONING.md b/docs/VERSIONING.md new file mode 100644 index 0000000..5532186 --- /dev/null +++ b/docs/VERSIONING.md @@ -0,0 +1,158 @@ +# Versioning and public interface policy (AC-039) + +What a downstream user or agent may depend on, and what may change without +warning. Written so nobody has to infer stability from silence. + +--- + +## Current version status + +| | | +|---|---| +| Package version | `0.0.1` (`pyproject.toml`) | +| Status | **pre-1.0, experimental** | +| Maturity | TRL 4 — see [`docs/TRL-ASSESSMENT.md`](TRL-ASSESSMENT.md) | +| Released versions | **None.** No tag, no GitHub Release, no published package. | +| Supported version | `main` only — see [`SECURITY.md`](../SECURITY.md) §2 | + +**`0.0.1` is not a release number.** It is a placeholder that has never been +published anywhere, and it has not been incremented as the implementation +changed. Two checkouts both reporting `0.0.1` may differ substantially. **The +commit SHA, not the version string, is the only reliable identity for this +project today.** Pin a commit if you depend on behaviour. + +**Semantic Versioning is not claimed.** This project does not currently +implement SemVer: version numbers are not bumped per change, so they carry none +of the guarantees SemVer attaches to major/minor/patch. Do not treat the version +string as a compatibility signal. Adopting SemVer is a 1.0 question, recorded +below, not a present-tense claim. + +--- + +## Declared public interface surface + +These are the surfaces this repository documents and tests, and therefore the +only ones a downstream consumer should build against. Anything not listed here +is internal, regardless of whether Python's import system will let you reach it. + +### 1. CLI commands — six, exactly + +``` +authcontract verify +authcontract project +authcontract check-action +authcontract git-gate [--repo REPO] +authcontract run-specimen --execution-result +authcontract verify-receipt +``` + +### 2. CLI flags intended for consumers + +| Flag | Command | Accepted values | +|---|---|---| +| `--execution-result` | `run-specimen` | `NOT_EXECUTED`, `SIMULATED_SUCCESS`, `SIMULATED_FAILURE` | +| `--repo` | `git-gate` | path to the Git repository to verify merge composition against | + +### 3. Output and exit semantics + +- Every command emits **single-line JSON on stdout**. +- Exit `0` = PASS / ALLOW. Exit `1` = REFUSED, or an error condition. +- Branch on the JSON `status` and `reason_code`. `message` is human-readable + prose and is **not** contractual at any version — do not parse it. + +### 4. Python entry points + +| Module | Public names | +|---|---| +| `authcontract.veip` | `run_specimen`, `verify_receipt` | +| `authcontract.digest` | `contract_digest`, `verify_artifact`, `canonical_bytes` | +| `authcontract.projection` | `project`, `check_action`, `projection_digest` | + +`run_specimen` returns a `RunResult` (`decision`, `reason_code`, `message`, +`receipt`); `verify_receipt` returns a `VerifyResult` (`status`, `reason_code`, +`message`). **Refusals are return values, not exceptions.** + +### 5. Receipt fields + +Ten bound fields, issued on ALLOW only: `activation_id`, `contract_digest`, +`projection_digest`, `runtime_fact_set_digest`, `exact_action_digest`, +`admission_digest`, `decision`, `execution_result`, `decision_time`, +`receipt_digest`. + +### 6. Reason codes + +The machine-facing identifiers listed in [`AGENTS.md`](../AGENTS.md) §10. See +the policy below — they are the intended programmatic signal, with an explicitly +bounded stability commitment. + +### 7. File-format surfaces + +The `.ac` / JSON artifact, action, and runtime-fact shapes consumed by the +commands above, as exercised by the committed fixtures in `fixtures/`. There is +no published schema document; the fixtures and the tests are the specification. + +### 8. Workflow-facing surface + +`authcontract git-gate` and the context JSON it consumes +(`.github/workflows/authcontract-gate.yml` shows the exact shape). The presence +of that workflow does **not** mean GitHub requires it — enforcement is branch +protection, which is separate repository configuration. + +### Not public + +Everything else, including `authcontract.facts`, `authcontract.git_gate` +internals, `authcontract.cli` internals, module-private helpers, exception +class hierarchies, benchmark harness code, and the exact wording of any +`message`. + +--- + +## Change policy before 1.0 + +**Pre-1.0 interfaces may change, including in breaking ways, without a major +version bump.** That is what pre-1.0 means here, stated plainly rather than +left to be discovered. + +What this project *does* commit to before 1.0: + +1. **Reason codes will not change meaning silently under the same version.** + If a reason code is removed, renamed, or has its semantics changed, that + change is recorded in the commit message and in `AGENTS.md` §10. A code + never quietly starts meaning something else. This is the strongest interface + commitment in the project, and it is deliberately scoped to *not silently* — + it is not a promise that codes never change. + +2. **A breaking change to any declared surface above carries a release note.** + Concretely: the commit that makes it says what broke and what to do instead. + With no releases to attach notes to, the commit message is the release note. + +3. **The claim ceiling is never weakened to accommodate a change.** If an + interface change would make a documented limitation untrue in the + *optimistic* direction, the limitation is re-verified, not deleted. + +What this project explicitly does **not** commit to before 1.0: + +- No cross-version stability guarantee of any kind beyond point 1 above. There + is no versioning or pinning mechanism that would make such a guarantee + checkable, and a guarantee nobody can check is not a guarantee. +- No deprecation period. A pre-1.0 interface may be removed in the same commit + that replaces it. +- No backports. Fixes land on `main` only. + +--- + +## What 1.0 would require + +Recorded so "pre-1.0" is a stage with an exit condition rather than an +indefinite disclaimer. All of these are absent today: + +- An explicit versioning scheme actually implemented and applied per change — + SemVer or a documented alternative. +- Tags and releases, so a version string identifies a specific artifact. +- A stated deprecation policy with a real notice period. +- A license, so downstream use is legally possible at all + ([`README.md`](../README.md) § License). +- The maturity conditions in [`docs/TRL-ASSESSMENT.md`](TRL-ASSESSMENT.md), + including independent external reproduction. + +Until then: **pin the commit SHA.** diff --git a/falsify.py b/falsify.py new file mode 100644 index 0000000..52fe2b9 --- /dev/null +++ b/falsify.py @@ -0,0 +1,209 @@ +#!/usr/bin/env python3 +"""Public falsification harness for AuthContract. + +Run it: + + python3 falsify.py + +Anyone can run this against a clean clone with no credentials, no services and +no network. It exists so an outsider can try to *break* the claims rather than +only reproduce the happy path. + +Each case declares the disposition AuthContract is expected to reach. The +harness runs the real CLI — the same commands a developer would type — and +compares what actually happened against that expectation. **A case fails if the +observed disposition differs from the expected one in either direction:** a +refusal that was supposed to pass fails, and a pass that was supposed to refuse +fails just as loudly. The second direction is the one that matters. A system +that fails closed is only trustworthy if you can watch it refuse. + +Exit code is 0 only if every case matched. Any mismatch exits 1. + +What this establishes: that the documented dispositions are the observed ones +for this bounded public set, on your machine, at this commit. What it does not +establish: production readiness, security certification, absence of defects, or +behaviour beyond these committed specimens. See docs/RELEASE-READINESS.md. +""" + +from __future__ import annotations + +import json +import subprocess +import sys +import tempfile +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parent +FIXTURES = REPO_ROOT / "fixtures" + +SPECIMEN = FIXTURES / "banking_payment_specimen.json" +ACTION_VALID = FIXTURES / "actions" / "send_payment_valid.json" +ACTION_UNDECLARED = FIXTURES / "actions" / "send_payment_unknown_action_type.json" +FACTS_VALID = FIXTURES / "runtime" / "facts_valid.json" +FACTS_STALE = FIXTURES / "runtime" / "facts_stale.json" +RECEIPT_VALID = FIXTURES / "runtime" / "receipt_valid.json" + + +def run_cli(args: list[str]) -> tuple[int, dict]: + """Invoke the installed CLI and parse its single-line JSON stdout.""" + completed = subprocess.run( + [sys.executable, "-m", "authcontract.cli", *args], + cwd=REPO_ROOT, + capture_output=True, + text=True, + ) + stdout = completed.stdout.strip() + if not stdout: + return completed.returncode, { + "status": "", + "stderr": completed.stderr.strip()[:400], + } + try: + return completed.returncode, json.loads(stdout) + except json.JSONDecodeError: + return completed.returncode, {"status": "", "raw": stdout[:400]} + + +def tampered_receipt_path(tmpdir: Path) -> Path: + """A receipt with one bound value altered, and nothing else touched. + + The point is not that the file is malformed — it is well-formed. The point + is that a bound digest no longer agrees with what the raw inputs recompute + to, which is precisely what receipt verification is for. + """ + receipt = json.loads(RECEIPT_VALID.read_text()) + receipt["contract_digest"] = "sha256:" + "0" * 64 + path = tmpdir / "receipt_tampered.json" + path.write_text(json.dumps(receipt)) + return path + + +def build_cases(tmpdir: Path) -> list[dict]: + return [ + { + "name": "valid specimen is allowed", + "why": "the whole implemented chain must reach ALLOW on a compliant request", + "args": [ + "run-specimen", str(SPECIMEN), str(ACTION_VALID), str(FACTS_VALID), + "--execution-result", "SIMULATED_SUCCESS", + ], + "expect_status": "PASS", + "expect_reason": "OK", + "expect_exit": 0, + "expect_receipt": True, + }, + { + "name": "undeclared action is refused", + "why": "an action the rule never granted authority for must not be improvised into a permission", + "args": [ + "run-specimen", str(SPECIMEN), str(ACTION_UNDECLARED), str(FACTS_VALID), + "--execution-result", "SIMULATED_SUCCESS", + ], + "expect_status": "REFUSED", + "expect_reason": "RUN_UNCLASSIFIED_ACTION", + "expect_exit": 1, + "expect_receipt": False, + }, + { + "name": "stale runtime fact is refused", + "why": "evidence that existed but is too old must not be treated as current permission", + "args": [ + "run-specimen", str(SPECIMEN), str(ACTION_VALID), str(FACTS_STALE), + "--execution-result", "SIMULATED_SUCCESS", + ], + "expect_status": "REFUSED", + "expect_reason": "RUN_FACT_STALE", + "expect_exit": 1, + "expect_receipt": False, + }, + { + "name": "untampered receipt verifies", + "why": "verification must accept a receipt whose bindings genuinely recompute", + "args": [ + "verify-receipt", str(RECEIPT_VALID), str(SPECIMEN), + str(ACTION_VALID), str(FACTS_VALID), + ], + "expect_status": "PASS", + "expect_reason": "OK", + "expect_exit": 0, + "expect_receipt": None, + }, + { + "name": "tampered receipt binding is detected", + "why": "a receipt is only evidence if altering a bound value is caught by recomputation", + "args": [ + "verify-receipt", str(tampered_receipt_path(tmpdir)), str(SPECIMEN), + str(ACTION_VALID), str(FACTS_VALID), + ], + "expect_status": "REFUSED", + "expect_reason": "VEIP_RECEIPT_MISMATCH", + "expect_exit": 1, + "expect_receipt": None, + }, + ] + + +def check(case: dict) -> tuple[bool, list[str]]: + exit_code, payload = run_cli(case["args"]) + status = payload.get("status") + reason = payload.get("reason_code") + + problems = [] + if status != case["expect_status"]: + problems.append(f"status: expected {case['expect_status']!r}, observed {status!r}") + if reason != case["expect_reason"]: + problems.append(f"reason_code: expected {case['expect_reason']!r}, observed {reason!r}") + if exit_code != case["expect_exit"]: + problems.append(f"exit code: expected {case['expect_exit']}, observed {exit_code}") + if case["expect_receipt"] is not None: + has_receipt = isinstance(payload.get("receipt"), dict) + if has_receipt != case["expect_receipt"]: + expected = "a receipt" if case["expect_receipt"] else "no receipt" + observed = "a receipt" if has_receipt else "no receipt" + problems.append(f"receipt: expected {expected}, observed {observed}") + + print(f" observed: status={status} reason_code={reason} exit={exit_code}") + return not problems, problems + + +def main() -> int: + print("AuthContract public falsification harness") + print(f"repository: {REPO_ROOT}") + print("Each case below states what SHOULD happen. A mismatch in either") + print("direction — an unexpected refusal or an unexpected pass — fails.\n") + + with tempfile.TemporaryDirectory() as tmp: + cases = build_cases(Path(tmp)) + failures = [] + for index, case in enumerate(cases, start=1): + print(f"[{index}/{len(cases)}] {case['name']}") + print(f" expects: {case['expect_status']} / {case['expect_reason']} / exit {case['expect_exit']}") + print(f" why: {case['why']}") + ok, problems = check(case) + if ok: + print(" result: MATCH\n") + else: + failures.append((case["name"], problems)) + print(" result: MISMATCH") + for problem in problems: + print(f" - {problem}") + print() + + print("-" * 68) + if failures: + print(f"FALSIFIED: {len(failures)} of {len(cases)} cases did not match expectation.") + for name, problems in failures: + print(f" {name}") + for problem in problems: + print(f" - {problem}") + print("\nThis is a real result, not a harness bug to be worked around.") + return 1 + + print(f"All {len(cases)} cases matched their expected disposition.") + print("This establishes the documented dispositions for this bounded public") + print("set at this commit. It is not an audit and not proof of correctness.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/pyproject.toml b/pyproject.toml index d80c1bc..49484b8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,6 +5,26 @@ description = "AuthContract reference implementation — canonical digest and ru requires-python = ">=3.10" dependencies = ["rfc8785>=0.1.2"] +# Machine-readable licensing state (AC-039). +# +# This project has NO license grant. Default copyright applies and no usage, +# redistribution, or derivative rights are conferred. The Trove classifier below +# is the ecosystem-standard machine-readable way to say exactly that: it marks +# the project as proprietary — i.e. NOT open source — without inventing rights +# that the owner has not granted. +# +# Deliberately NOT declared: any SPDX identifier (MIT/Apache-2.0/BSD/etc.). +# Using one would be false. A PEP 639 `license = "LicenseRef-..."` expression +# would also require an accompanying license text file, which only the owner can +# author. Selecting an actual license remains an open owner decision — see +# docs/VERSIONING.md and README.md. +classifiers = [ + "License :: Other/Proprietary License", + "Development Status :: 2 - Pre-Alpha", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.12", +] + [project.scripts] authcontract = "authcontract.cli:main"