diff --git a/.github/ISSUE_TEMPLATE/bug.yml b/.github/ISSUE_TEMPLATE/bug.yml new file mode 100644 index 0000000..887144d --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug.yml @@ -0,0 +1,92 @@ +name: Bug report +description: Report a reproducible Breakcheck defect +title: "[Bug]: " +labels: + - bug +body: + - type: markdown + attributes: + value: | + Before posting, remove private paths, private source, literals, outputs, credentials, and `.breakcheck/` artifacts from every field and attachment. + - type: input + id: version_breakcheck + attributes: + label: Breakcheck version + description: Which Breakcheck version produced this result? + placeholder: 2.0.0 + validations: + required: true + - type: input + id: python_version + attributes: + label: Python version + description: Include the exact supported Python version. + placeholder: 3.12.4 + validations: + required: true + - type: dropdown + id: operating_system + attributes: + label: Operating system + options: + - Linux + - macOS + validations: + required: true + - type: dropdown + id: mode + attributes: + label: Breakcheck mode + description: Which workflow produced the defect? + options: + - Dependency comparison + - Revision diff + - Baseline freeze + - Claim attestation + - Demo or capabilities + - Other + validations: + required: true + - type: textarea + id: sanitized_context + attributes: + label: Sanitized comparison context + description: If relevant, name safe package versions, revision refs, target symbols, or claim type. Do not paste private source or repository details. + - type: textarea + id: sanitized_command + attributes: + label: Sanitized command + description: Paste the command after removing private paths, credentials, literals, and outputs. + render: shell + validations: + required: true + - type: textarea + id: reproduction_steps + attributes: + label: Reproduction steps + description: List the smallest safe steps that reproduce the issue. + validations: + required: true + - type: textarea + id: observed_behavior + attributes: + label: Observed behavior + description: What did Breakcheck report or do? + validations: + required: true + - type: textarea + id: expected_behavior + attributes: + label: Expected behavior + description: What should Breakcheck have reported or done? + validations: + required: true + - type: checkboxes + id: privacy_confirmation + attributes: + label: Privacy and security confirmation + options: + - label: I removed private paths, private source, literals, outputs, credentials, and `.breakcheck/` artifacts before posting. + required: true + validations: + required: true diff --git a/.github/ISSUE_TEMPLATE/compatibility-case.yml b/.github/ISSUE_TEMPLATE/compatibility-case.yml new file mode 100644 index 0000000..ac4dbdc --- /dev/null +++ b/.github/ISSUE_TEMPLATE/compatibility-case.yml @@ -0,0 +1,94 @@ +name: Compatibility case +description: Report a dependency upgrade compatibility case +title: "[Compatibility]: " +labels: + - compatibility +body: + - type: markdown + attributes: + value: | + Before posting, remove private paths, private source, literals, outputs, credentials, and `.breakcheck/` artifacts from every field and attachment. + - type: input + id: version_breakcheck + attributes: + label: Breakcheck version + description: Which Breakcheck version produced this result? + placeholder: 2.0.0 + validations: + required: true + - type: input + id: python_version + attributes: + label: Python version + description: Include the exact supported Python version. + placeholder: 3.12.4 + validations: + required: true + - type: dropdown + id: operating_system + attributes: + label: Operating system + options: + - Linux + - macOS + validations: + required: true + - type: input + id: package_name + attributes: + label: Package name + description: Name of the dependency being compared. + validations: + required: true + - type: input + id: current_version + attributes: + label: Current package version + description: Version currently installed in the comparison environment. + validations: + required: true + - type: input + id: proposed_version + attributes: + label: Proposed package version + description: Version being evaluated for the upgrade. + validations: + required: true + - type: textarea + id: sanitized_command + attributes: + label: Sanitized command + description: Paste the command after removing private paths, credentials, literals, and outputs. + render: shell + validations: + required: true + - type: textarea + id: reproduction_steps + attributes: + label: Reproduction steps + description: List the smallest safe steps that reproduce the compatibility case. + validations: + required: true + - type: textarea + id: observed_behavior + attributes: + label: Observed behavior + description: What changed between the current and proposed versions? + validations: + required: true + - type: textarea + id: expected_behavior + attributes: + label: Expected behavior + description: What compatibility behavior did you expect? + validations: + required: true + - type: checkboxes + id: privacy_confirmation + attributes: + label: Privacy and security confirmation + options: + - label: I removed private paths, private source, literals, outputs, credentials, and `.breakcheck/` artifacts before posting. + required: true + validations: + required: true diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000..e5e56b0 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,8 @@ +blank_issues_enabled: false +contact_links: + - name: Private security advisory + url: https://github.com/lovettsendit/breakcheck/security/advisories/new + about: Report security vulnerabilities privately. + - name: Read the README + url: https://github.com/lovettsendit/breakcheck#readme + about: Review scope, limitations, and usage before opening an issue. diff --git a/.github/ISSUE_TEMPLATE/feature.yml b/.github/ISSUE_TEMPLATE/feature.yml new file mode 100644 index 0000000..7d8c93e --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature.yml @@ -0,0 +1,40 @@ +name: Feature request +description: Suggest a focused improvement to Breakcheck +title: "[Feature]: " +labels: + - enhancement +body: + - type: markdown + attributes: + value: | + Before posting, remove private paths, private source, literals, outputs, credentials, and `.breakcheck/` artifacts from every field and attachment. + - type: textarea + id: problem + attributes: + label: Problem to solve + description: What user problem or workflow gap should this address? + validations: + required: true + - type: textarea + id: proposed_solution + attributes: + label: Proposed solution + description: Describe the smallest useful change. + validations: + required: true + - type: textarea + id: alternatives + attributes: + label: Alternatives considered + description: What other approaches or workarounds have you considered? + validations: + required: true + - type: checkboxes + id: privacy_confirmation + attributes: + label: Privacy and security confirmation + options: + - label: I removed private paths, private source, literals, outputs, credentials, and `.breakcheck/` artifacts before posting. + required: true + validations: + required: true diff --git a/.github/ISSUE_TEMPLATE/revision-claim-case.yml b/.github/ISSUE_TEMPLATE/revision-claim-case.yml new file mode 100644 index 0000000..8a7fb17 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/revision-claim-case.yml @@ -0,0 +1,116 @@ +name: Revision or claim verification case +description: Report a sanitized diff, freeze, or attest result that needs investigation +title: "[Verification]: " +labels: + - verification +body: + - type: markdown + attributes: + value: | + Before posting, remove private paths, private source, literals, outputs, credentials, and `.breakcheck/` artifacts from every field and attachment. Report verdicts exactly: `CLAIM_UNVERIFIABLE` is not success, and `CHANGED` does not mean bad. + - type: input + id: version_breakcheck + attributes: + label: Breakcheck version + description: Which Breakcheck version produced this result? + placeholder: 2.0.0 + validations: + required: true + - type: input + id: python_version + attributes: + label: Python version + description: Include the exact supported Python version. + placeholder: 3.12.4 + validations: + required: true + - type: dropdown + id: operating_system + attributes: + label: Operating system + options: + - Linux + - macOS + validations: + required: true + - type: dropdown + id: mode + attributes: + label: Verification mode + options: + - Revision diff + - Baseline freeze + - Claim attestation + validations: + required: true + - type: input + id: base_revision + attributes: + label: Sanitized base revision + description: Provide a public commit ID or a safe placeholder that still reproduces the case. + validations: + required: true + - type: input + id: head_revision + attributes: + label: Sanitized head revision or baseline + description: Provide a public commit ID, worktree, or baseline identifier without private paths. + validations: + required: true + - type: textarea + id: target_symbols + attributes: + label: Sanitized target symbols + description: List only public or safely renamed module and symbol names. + validations: + required: true + - type: dropdown + id: fixture_source + attributes: + label: Fixture source + description: State which revision supplied the fixtures used for verification. + options: + - Base revision + - Head revision + - Explicit fixture file + - No fixtures + validations: + required: true + - type: textarea + id: sanitized_command + attributes: + label: Sanitized command + description: Paste the command after removing private paths, credentials, literals, and outputs. + render: shell + validations: + required: true + - type: textarea + id: reproduction_steps + attributes: + label: Reproduction steps + description: List the smallest safe steps that reproduce the verification case. + validations: + required: true + - type: textarea + id: observed_behavior + attributes: + label: Observed verdicts + description: Include exact sanitized verdict names such as CLAIM_VERIFIED, CLAIM_REFUTED, CLAIM_UNVERIFIABLE, or CLAIM_OUT_OF_SCOPE and the reported coverage. + validations: + required: true + - type: textarea + id: expected_behavior + attributes: + label: Expected behavior + description: Explain the expected comparison or refusal without asserting that intentional change is a defect. + validations: + required: true + - type: checkboxes + id: privacy_confirmation + attributes: + label: Privacy and security confirmation + options: + - label: I removed private paths, private source, literals, outputs, credentials, and `.breakcheck/` artifacts before posting. + required: true + validations: + required: true diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000..b3f39bc --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,36 @@ +## Linked issue + + + +## Smallest change + +- [ ] This pull request is the smallest change that solves the linked issue. + +## Regression test evidence + +- [ ] I added or updated a regression test where behavior changed. +- [ ] I ran the relevant tests and included the command and result below. + +Test command and result: + +## Preserved counterexample + +- [ ] Existing counterexamples and refusal behavior remain preserved. +- [ ] If this changes a counterexample, I documented why and included evidence. + +## Privacy/security + +- [ ] I removed private paths, private source, literals, outputs, credentials, and `.breakcheck/` artifacts from the pull request. +- [ ] I considered security and privacy impact. + +## Docs/changelog + +- [ ] I updated the documentation and changelog when user-visible behavior changed. +- [ ] No documentation or changelog update is needed, because: + +## Breaking/release assessment + +- [ ] This is not a breaking change. +- [ ] I assessed release impact, including versioning and migration notes where applicable. + +Breaking/release assessment: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e1d947b..793a71f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,7 +1,8 @@ -name: ci +name: Breakcheck CI on: push: + branches: [main] pull_request: permissions: @@ -9,7 +10,9 @@ permissions: jobs: test: + name: test (${{ matrix.os }}, ${{ matrix.python-version }}) runs-on: ${{ matrix.os }} + timeout-minutes: 15 strategy: fail-fast: false matrix: @@ -57,3 +60,24 @@ jobs: - name: Check CLI help working-directory: ${{ runner.temp }} run: "$RUNNER_TEMP/breakcheck-wheel/bin/breakcheck --help" + + acceptance: + name: demo and CLI acceptance + needs: test + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Check out source + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - name: Set up Python + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: "3.13" + - name: Install the public package + run: python -m pip install --no-deps . + - name: Check machine-readable capabilities + working-directory: ${{ runner.temp }} + run: breakcheck --capabilities --json + - name: Run the offline changed-behavior demo + working-directory: ${{ runner.temp }} + run: breakcheck demo --output-root "$RUNNER_TEMP/breakcheck-demo" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..a7caf64 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,166 @@ +name: Publish Breakcheck + +on: + release: + types: [published] + +permissions: {} + +concurrency: + group: pypi-${{ github.event.release.tag_name }} + cancel-in-progress: false + +jobs: + validate: + name: build and validate exact tagged source + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + outputs: + wheel_sha256: ${{ steps.assets.outputs.wheel_sha256 }} + sdist_sha256: ${{ steps.assets.outputs.sdist_sha256 }} + steps: + - name: Check out the exact release tag + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + ref: ${{ github.event.release.tag_name }} + persist-credentials: false + - name: Set up Python + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: "3.13" + - name: Refuse a mismatched release identity + env: + RELEASE_DRAFT: ${{ github.event.release.draft }} + RELEASE_PRERELEASE: ${{ github.event.release.prerelease }} + RELEASE_TAG: ${{ github.event.release.tag_name }} + run: | + set -euo pipefail + if [ "$RELEASE_DRAFT" = true ] || [ "$RELEASE_PRERELEASE" = true ]; then + echo "refusing draft or prerelease" + exit 1 + fi + if [ "$RELEASE_TAG" != "v2.0.0" ]; then + echo "release tag does not match project version" + exit 1 + fi + test "$(git describe --exact-match --tags HEAD)" = "$RELEASE_TAG" + python - <<'PY' + import pathlib + import tomllib + + project = tomllib.loads(pathlib.Path("pyproject.toml").read_text(encoding="utf-8"))["project"] + if project["name"] != "breakcheck" or project["version"] != "2.0.0": + raise SystemExit("project metadata does not match v2.0.0") + if project.get("dependencies") != []: + raise SystemExit("release unexpectedly has runtime dependencies") + PY + - name: Install isolated build and validation tools + run: python -m pip install --disable-pip-version-check "build>=1.2,<2" "twine>=6,<7" + - name: Build the exact tagged source + id: assets + run: | + set -euo pipefail + if [ -n "$(git status --porcelain)" ]; then + echo "checked-out release source is not clean" + exit 1 + fi + export SOURCE_DATE_EPOCH="$(git show -s --format=%ct HEAD)" + python -m build --outdir dist . + python - <<'PY' + from pathlib import Path + + expected = { + "breakcheck-2.0.0-py3-none-any.whl", + "breakcheck-2.0.0.tar.gz", + } + actual = {path.name for path in Path("dist").iterdir() if path.is_file()} + if actual != expected: + raise SystemExit(f"unexpected distribution inventory: {sorted(actual)}") + PY + echo "wheel_sha256=$(sha256sum dist/breakcheck-2.0.0-py3-none-any.whl | awk '{print $1}')" >> "$GITHUB_OUTPUT" + echo "sdist_sha256=$(sha256sum dist/breakcheck-2.0.0.tar.gz | awk '{print $1}')" >> "$GITHUB_OUTPUT" + - name: Validate metadata, inventory, privacy, and installed CLI + run: | + set -euo pipefail + python -m twine check --strict dist/* + bash scripts/scan_artifacts.sh dist/breakcheck-2.0.0-py3-none-any.whl + bash scripts/scan_artifacts.sh dist/breakcheck-2.0.0.tar.gz + python - <<'PY' + import email + import tarfile + import zipfile + + wheel = "dist/breakcheck-2.0.0-py3-none-any.whl" + sdist = "dist/breakcheck-2.0.0.tar.gz" + with zipfile.ZipFile(wheel) as archive: + metadata_name = next(name for name in archive.namelist() if name.endswith(".dist-info/METADATA")) + metadata = email.message_from_bytes(archive.read(metadata_name)) + if metadata["Name"] != "breakcheck" or metadata["Version"] != "2.0.0": + raise SystemExit("wheel identity mismatch") + with tarfile.open(sdist, "r:gz") as archive: + names = set(archive.getnames()) + required = { + "breakcheck-2.0.0/README.md", + "breakcheck-2.0.0/SECURITY.md", + "breakcheck-2.0.0/SKILL.md", + "breakcheck-2.0.0/examples/github-actions.yml", + "breakcheck-2.0.0/scripts/scan_artifacts.sh", + } + if not required <= names: + raise SystemExit("source distribution inventory incomplete") + PY + python -m venv "$RUNNER_TEMP/breakcheck-release-smoke" + "$RUNNER_TEMP/breakcheck-release-smoke/bin/python" -m pip install --no-deps dist/breakcheck-2.0.0-py3-none-any.whl + ( + cd "$RUNNER_TEMP" + "$RUNNER_TEMP/breakcheck-release-smoke/bin/breakcheck" --help + "$RUNNER_TEMP/breakcheck-release-smoke/bin/breakcheck" --capabilities --json + "$RUNNER_TEMP/breakcheck-release-smoke/bin/breakcheck" demo --output-root "$RUNNER_TEMP/release-demo" + ) + - name: Transfer only validated distributions + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: breakcheck-2.0.0-distributions + path: | + dist/breakcheck-2.0.0-py3-none-any.whl + dist/breakcheck-2.0.0.tar.gz + if-no-files-found: error + include-hidden-files: false + retention-days: 1 + + publish: + name: publish validated distributions to PyPI + needs: validate + runs-on: ubuntu-latest + timeout-minutes: 10 + environment: + name: pypi + url: https://pypi.org/p/breakcheck + permissions: + id-token: write + steps: + - name: Receive validated distributions + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: breakcheck-2.0.0-distributions + path: dist + digest-mismatch: error + - name: Recheck transferred artifact identity + env: + EXPECTED_WHEEL_SHA256: ${{ needs.validate.outputs.wheel_sha256 }} + EXPECTED_SDIST_SHA256: ${{ needs.validate.outputs.sdist_sha256 }} + run: | + set -euo pipefail + test "$(find dist -maxdepth 1 -type f | wc -l | tr -d ' ')" = 2 + test "$(sha256sum dist/breakcheck-2.0.0-py3-none-any.whl | awk '{print $1}')" = "$EXPECTED_WHEEL_SHA256" + test "$(sha256sum dist/breakcheck-2.0.0.tar.gz | awk '{print $1}')" = "$EXPECTED_SDIST_SHA256" + - name: Publish with Trusted Publishing and PEP 740 attestations + uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 + with: + packages-dir: dist/ + verify-metadata: true + skip-existing: false + print-hash: true + attestations: true diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..6a2475a --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,5 @@ +# Automated Change Verification + +Automated coding tools working in this repository should follow [SKILL.md](SKILL.md) whenever a dependency version changes or a code change is described as behavior-preserving. + +The required workflow keeps fixture proposals separate from deterministic replay. Never weaken coverage or separation policy, edit generated evidence, or describe an unexercised result as safe. diff --git a/CHANGELOG.md b/CHANGELOG.md index 3cdcb40..f993215 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,18 @@ All notable changes to Breakcheck are documented here. +## 2.0.0 - 2026-08-26 + +- Added PyPI-ready project metadata and trusted release automation. +- Added a one-command changed-behavior demonstration that verifies its report and evidence before success. +- Added structured GitHub issue and pull-request intake. +- Limited push CI to main while retaining pull-request coverage. +- Added bounded static folding, module-constant resolution, and safe nested calls. +- Added operator-reviewed fixtures, projections, coverage diagnostics, and fixture suggestions. +- Added behavioral baselines, cross-revision comparison, and behavior-preservation claims. +- Kept unchanged sibling functions out of the changed-symbol set when another definition shifts their source locations. +- Added closed schema-2 reports, replay witnesses, provenance, and machine-readable capabilities. + ## 1.0.1 - 2026-08-26 - Preserve exact static submodule imports and aliases when replaying literal calls. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3577767..9cb42b6 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -21,8 +21,41 @@ Run the test suite on Python 3.10 through 3.13 when changing runtime behavior. L - Preserve deterministic ordering and canonical JSON behavior. - Never turn malformed, missing, dynamic, or unstable evidence into an `IDENTICAL` verdict. - Keep the local-wheelhouse and no-network defaults unchanged. +- Preserve fixture authorship, source revision, argument provenance, and strict separation in every new replay path. +- Keep schema changes closed, versioned, canonical, and backward-readable where documented. +- Preserve detached-worktree cleanup and never mutate a user's checkout, index, or stash. - Do not include private source, generated `.breakcheck/` evidence, credentials, or machine-specific paths. +## Product effort budget + +Breakcheck is designed to add almost no manual work to an upgrade or review: + +- a fresh repository reaches its first report with one command in under five minutes; +- CI integration is one workflow file with fewer than 20 non-comment lines; +- a rerun after a code change is one command with no new input; +- fixture suggestions require no manual authoring before a human reviews the diff. + +A feature that cannot meet this default-path budget must remain explicitly optional. +Contributions must preserve noninteractive execution and zero runtime dependencies. + +Fail-closed behavior is mechanical: every refusal code needs a producing test, broad +adapter exceptions cannot silently continue, fixed identical and changed cases must +retain their verdicts, and every exercised observation must carry declared provenance. + +## Behavioral invariants + +Changes must preserve all of these: + +- `IDENTICAL` means both admitted observations were replayed and compared equal. +- `NOT_EXERCISED` and `CLAIM_UNVERIFIABLE` are never summarized as safe. +- Every new execution path runs twice and refuses nondeterministic observations. +- Scanning never imports or evaluates repository or dependency code. +- Fixture and claim inputs use closed, bounded formats and fail on stale or unmatched bindings. +- Machine-readable artifacts remain noninteractive, deterministically ordered, and free of hidden fallback behavior. +- Runtime package dependencies remain empty unless a reviewed design demonstrates that the benefit outweighs the embeddability and supply-chain cost. + +Add or update the fixed identical, changed, refusal, provenance, artifact-verification, and installed-CLI tests when changing these behaviors. + ## Pull requests Describe the user-visible failure, the smallest correcting change, and the tests that prove both the failure and the preserved behavior. Keep unrelated refactoring out of the same pull request. diff --git a/LICENSE b/LICENSE index b577799..1f382a9 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2026 breakcheck contributors +Copyright (c) 2026 ViDale Lovett and contributors Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/MANIFEST.in b/MANIFEST.in index eb057fa..f78ee70 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -3,10 +3,17 @@ include CONTRIBUTING.md include LICENSE include README.md include SECURITY.md +include SKILL.md +include AGENTS.md recursive-include release_evidence *.json recursive-include src/breakcheck *.py recursive-include tests *.py +graft docs +graft examples +graft scripts global-exclude *.py[cod] global-exclude .DS_Store global-exclude ._* prune .breakcheck +prune build +prune dist diff --git a/README.md b/README.md index 967acff..489b2ee 100644 --- a/README.md +++ b/README.md @@ -1,151 +1,351 @@ # Breakcheck -Breakcheck is a deterministic behavioral compatibility checker for Python package upgrades. It finds statically discoverable calls your repository makes to a dependency, replays literal calls against exact current and proposed wheels in isolated environments, and reports whether the observed behavior changed. +Dependabot tells you a new version exists. Breakcheck tells you whether the calls your code actually makes behave differently. -Breakcheck does not update dependencies and does not guess whether a change is safe. It produces bounded evidence for a maintainer or CI system to judge. +[![CI](https://github.com/lovettsendit/breakcheck/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/lovettsendit/breakcheck/actions/workflows/ci.yml) +[![PyPI](https://img.shields.io/pypi/v/breakcheck)](https://pypi.org/project/breakcheck/) +[![Python](https://img.shields.io/pypi/pyversions/breakcheck)](https://pypi.org/project/breakcheck/) +[![License](https://img.shields.io/github/license/lovettsendit/breakcheck)](LICENSE) -## Quick start +Breakcheck is a deterministic behavioral comparison tool for Python. It answers two bounded questions: -Breakcheck evaluates one dependency upgrade in an existing Python repository. Start from the repository root, with the currently used dependency installed: +- Did a dependency upgrade change the behavior of supported calls already present in this repository? +- Did a code revision change the behavior of fixture-bound functions that were expected to remain stable? + +Breakcheck compares observations. It does not guess intent, declare code correct, replace a test suite, or decide whether a change should ship. Calls it cannot lawfully exercise are reported as `NOT_EXERCISED`, never counted as safe. + +![Breakcheck report preview](docs/assets/breakcheck-social-preview.png) + +## Install ```console -python -m pip install /path/to/breakcheck-1.0.1-py3-none-any.whl +python -m pip install breakcheck +``` + +Breakcheck supports Python 3.10 through 3.13 on Linux and macOS and has no runtime package dependencies. + +## See a real result in one command + +```console +breakcheck demo --output-root .breakcheck/demo +``` + +The offline demo builds two small local package wheels, runs the installed CLI, detects a real behavioral difference, writes a report and evidence bundle, and verifies both. It does not contact a package index. + +For the longer `packaging` 21.3 to 22.0 demonstration used by the release tests, run: + +```console +sh examples/run_demo.sh +``` + +## Where Breakcheck fits + +| Neighbor | Question it answers | Breakcheck's question | +| --- | --- | --- | +| Unit and integration tests | Does known expected behavior still pass? | Did these two observations differ? | +| Snapshot testing | Does output match an approved snapshot? | Did this specific dependency or revision change behavior? | +| Static analysis | Does this code match a rule or risk pattern? | Does supported code behave differently when replayed? | +| Traffic replay | What did recorded production requests do? | What can be compared without production traffic? | +| AI-generated tests | What should this code do, as inferred by a model? | What changed, without guessing intent? | + +Breakcheck complements these tools. Its useful property is independence: a coding tool may propose a change or fixture, while deterministic replay and comparison decide the observed result. + +## Dependency upgrades + +Breakcheck holds your repository constant and varies one dependency: + +```text +same repository calls x current wheel vs proposed wheel -> behavioral difference +``` + +### Quick start + +Start at the root of the repository you want to analyze. Install the current dependency version in the interpreter running Breakcheck, then prepare an explicit local wheelhouse: + +```console +python -m pip install 'attrs==23.2.0' mkdir -p wheelhouse .breakcheck/results python -m pip download --only-binary=:all: --dest wheelhouse 'attrs==23.2.0' python -m pip download --only-binary=:all: --dest wheelhouse 'attrs==24.2.0' +``` + +Download exact versions in separate commands. Asking `pip download` to resolve two versions of the same distribution in one command can produce a dependency-resolution error. + +Run the comparison: + +```console breakcheck attrs@24.2.0 \ --wheelhouse wheelhouse \ - --runtime-root .breakcheck/runtime-001 \ --output .breakcheck/results/report.json \ --evidence .breakcheck/results/evidence.json \ + --coverage-report .breakcheck/results/coverage.json \ --json \ --ci ``` -Replace `attrs` and the versions with the dependency you are evaluating. Breakcheck expects the current version to be installed in the Python interpreter running the command and both exact wheels, plus their installation dependencies, to be present in `wheelhouse`. +Breakcheck creates fresh replay environments, installs only from the supplied wheelhouse, runs admitted calls twice in each environment, and compares normalized observations. Environments created under the default temporary runtime are removed when the command finishes, including after a failure. Supplying `--runtime-root` preserves that explicit runtime for operator-managed retention and cleanup. -## Why use it? +Treat every wheel as executable input. Obtain wheels from a source you trust and preserve expected hashes in your own supply-chain records. -Version-update tools can tell you that a newer dependency exists. Breakcheck answers a different question: **does the newer version change behavior that this repository actually exercises?** +### Supported call surface -- Uses call sites found in your repository instead of a generic API inventory. -- Runs entirely from an explicit local wheelhouse with no network fallback. -- Produces deterministically ordered JSON, human output, and replay witnesses. -- Distinguishes changed behavior, identical behavior, and calls it could not exercise. -- Refuses malformed, dynamic, missing, or unstable evidence instead of treating it as success. +Breakcheck is strongest on pure, value-in/value-out library calls: parsing, serialization, validation, encoding, schema coercion, and deterministic numeric or string transformations. -## Where Breakcheck fits +The default static path supports: -Breakcheck is a pre-merge check for a specific Python dependency update. It complements, rather than replaces, your existing tests: +- direct imports and statically attributable package calls; +- Python literal arguments; +- bounded constant expressions and f-strings; +- safe, single-assignment module constants; +- nested calls to the target package or a small fixed allowlist of pure standard-library modules; +- normalizable values, exceptions, mappings with string keys, sequences, sets, finite numbers, strings, and bytes. -1. A maintainer, dependency bot, or coding tool proposes a version change. -2. Breakcheck finds supported calls to that dependency in the repository. -3. It runs those calls against exact old and new wheels in separate environments. -4. It records identical, changed, and not-exercised results with replay evidence. -5. A maintainer reviews the changes together with the project's ordinary tests, security checks, and release process. +It does not pretend to exercise network clients, arbitrary filesystem work, stateful object graphs, dynamic dispatch, unbounded computation, or rich return objects without an explicit projection. Those cases remain visible in the coverage report. -This is useful when reviewing automated dependency updates, preparing a framework migration, or investigating whether a version bump changes literal API calls already present in a codebase. +### Coverage diagnostics -## Using Breakcheck with AI-assisted development +Every discovered candidate reaches exactly one terminal bucket: + +- `EXERCISED` +- `G1_NOT_DISCOVERABLE` +- `G2_NONLITERAL` +- `G3_UNNORMALIZABLE` +- `G4_IMPURE` -AI coding tools can propose dependency updates and help interpret a Breakcheck report, but Breakcheck remains the deterministic measurement step: +`--coverage-report` writes the machine-readable candidate inventory, reason codes, reason details, and argument provenance. A run that exercises nothing exits nonzero unless a human explicitly passes `--allow-empty`; that choice is recorded in the artifact. -1. Let the coding tool propose the dependency change without merging it. -2. Run Breakcheck locally or in CI against trusted, exact wheels. -3. Inspect and sanitize the report before sharing it with an external AI service; reports may contain source locations, literal arguments, outputs, and local paths. -4. Ask the coding tool to address specific `CHANGED` or `NOT_EXERCISED` findings. -5. Rerun Breakcheck and the project's normal test suite. -6. Require human review before accepting the upgrade. +### Generate fixture suggestions -Breakcheck does not decide whether an upgrade should ship, and AI-generated explanations do not override its recorded observations. +When source arguments cannot be resolved statically, generate a reviewable skeleton without creating replay environments: -## Supported scope +```console +breakcheck attrs@24.2.0 \ + --suggest-fixtures breakcheck.fixtures.toml +``` -- Python 3.10 through 3.13. -- Linux and macOS. -- Python package calls discoverable through static imports. -- Calls whose positional and keyword arguments are Python literals. -- Trusted package wheels supplied by the operator. +The generated file identifies each unresolved call by repository-relative file, line, column, API, and nearby source. A human or coding tool fills in concrete expressions and changes `fixture_authored_by` from `unknown` to `human` or `agent`. + +Example: + +```toml +schema_version = 1 + +[[binding]] +fixture_authored_by = "agent" +file = "src/app/serialize.py" +line = 42 +column = 8 +api = "attrs.asdict" +args = ["Point(1, 2)"] +kwargs = {} +setup = """ +import attrs +@attrs.define +class Point: + x: int + y: int +""" +``` -Dynamic dispatch, runtime-computed arguments, setup-dependent object state, and calls hidden behind unsupported indirection are reported as not exercised. Breakcheck is not a security sandbox and must not execute untrusted package code on a sensitive host. +Review the fixture diff, then run: -## What Breakcheck does not prove +```console +breakcheck attrs@24.2.0 \ + --wheelhouse wheelhouse \ + --fixtures breakcheck.fixtures.toml \ + --fixture-policy allow \ + --output .breakcheck/results/report.json \ + --evidence .breakcheck/results/evidence.json \ + --coverage-report .breakcheck/results/coverage.json \ + --ci +``` + +Fixture expressions and setup code execute inside the same best-effort isolation as dependency calls. They are trusted executable input, not data-only configuration. + +### Project rich return values -An exit status of `0` means the supported calls Breakcheck exercised met the configured coverage threshold and produced no changed observations. It does not establish that every dependency API is compatible, that unexercised application paths are safe, or that a package is secure. Keep unit, integration, security, and platform tests in the release process. +A fixture may reduce a rich result to a stable, normalizable value: -## Install for development +```toml +[[binding]] +fixture_authored_by = "human" +file = "src/app/load.py" +line = 12 +column = 8 +api = "pandas.read_csv" +args = ["io.StringIO('a,b\\n1,2\\n')"] +kwargs = {} +setup = "import io" +projection = "(list(outcome.columns), outcome.shape, outcome.to_dict('list'))" +``` + +Projected results use explicit verdicts: `IDENTICAL_UNDER_PROJECTION` and `CHANGED_UNDER_PROJECTION`. They never imply that the complete rich object was identical. + +## Code revisions + +Breakcheck can also hold the environment constant and vary repository code: + +```text +same fixture and environment x base revision vs head revision -> behavioral difference +``` + +This mode is for claims such as "this refactor preserves behavior." It cannot establish absolute correctness, judge intentional changes as bad, or verify new functions that have no baseline. + +Revision commands create detached Git worktrees under an absent runtime path. They never check out over the user's working tree, touch the index, or stash changes. Worktrees are removed on success and on handled failure. -From a source checkout: +### Bind functions to inputs + +Revision fixtures use the same file format, but point at function definitions: + +```toml +schema_version = 1 + +[[binding]] +fixture_authored_by = "human" +file = "src/app/pricing.py" +line = 10 +column = 0 +api = "app.pricing.compute_total" +args = ["100", "0.15"] +kwargs = {} +``` + +For strict separation, commit fixtures against the base revision before making the code change. Breakcheck records fixture authorship, fixture hash, source revision, and whether the fixture predates the change. + +### Capture a baseline ```console -python -m venv .venv -source .venv/bin/activate -python -m pip install --upgrade pip -python -m pip install -e '.[test]' -breakcheck --help -python -m pytest -q +breakcheck freeze \ + --revision HEAD \ + --fixtures breakcheck.fixtures.toml \ + --output .breakcheck/baseline.json ``` -To build and install the distributable wheel: +`freeze` records repeated normalized observations, source-tree identity, fixture identity, Python version, and platform. Dirty working trees are refused because silently omitting uncommitted changes would produce a misleading baseline. + +### Compare two revisions + +Compare explicit revisions: ```console -python -m pip install 'build>=1.2,<2' -python -m build -python -m pip install dist/breakcheck-1.0.1-py3-none-any.whl -breakcheck --help +breakcheck diff \ + --base main \ + --head feature/refactor \ + --fixtures breakcheck.fixtures.toml \ + --strict-separation \ + --output .breakcheck/revision-report.json \ + --evidence .breakcheck/revision-evidence.json ``` -## Prepare an offline wheelhouse +Or compare against a verified baseline artifact: -Breakcheck requires: +```console +breakcheck diff \ + --baseline .breakcheck/baseline.json \ + --head HEAD \ + --fixtures breakcheck.fixtures.toml \ + --strict-separation \ + --output .breakcheck/revision-report.json \ + --evidence .breakcheck/revision-evidence.json +``` -1. The current distribution to be installed in the Python interpreter running `breakcheck`. -2. An exact wheel for the installed version. -3. An exact wheel for the proposed version. -4. Wheels for any dependencies that pip must install into either replay environment. +By default, Breakcheck selects top-level and class-level functions whose normalized signatures, bodies, or relevant non-callable module or class context changed. It also surfaces additions, removals, ambiguous definitions, and signature drift so structural changes cannot disappear from the comparison. Use a repeatable `--target module.path:symbol` to select targets explicitly. Missing baselines, import asymmetry, and unexercised targets remain distinct fail-closed results. -For example, if the repository currently uses `attrs==23.2.0` and you want to evaluate `attrs==24.2.0`: +To disclose fixture retuning, pass an earlier verified revision report: ```console -python -m pip install 'attrs==23.2.0' -mkdir -p wheelhouse -python -m pip download --only-binary=:all: --dest wheelhouse 'attrs==23.2.0' -python -m pip download --only-binary=:all: --dest wheelhouse 'attrs==24.2.0' +breakcheck diff \ + --base main \ + --head HEAD \ + --fixtures breakcheck.fixtures.toml \ + --previous-report .breakcheck/previous-revision-report.json \ + --output .breakcheck/revision-report.json ``` -Treat every wheel in the wheelhouse as executable input. Obtain wheels from a source you trust and preserve their hashes in your own supply-chain records. +When a changed fixture turns a prior `CHANGED` result into `IDENTICAL` for the same base and target, the new report emits `FIXTURE_REVISED_AFTER_FAILURE`. The event is visible and integrity-bound but does not automatically block the run because the original fixture may have been wrong. Breakcheck never infers this history from a mutable local cache; the prior report must be supplied explicitly. + +### Adjudicate a preservation claim + +A claim file lists exactly what the change asserts it preserved: -## Run a comparison +```toml +schema_version = 1 +claim = "behavior_preserved" +base_revision = "0123456789abcdef0123456789abcdef01234567" -Run Breakcheck from the root of the repository to analyze: +[[target]] +symbol = "app.pricing:compute_total" +``` + +Run: ```console -mkdir -p .breakcheck/results -breakcheck attrs@24.2.0 \ - --wheelhouse wheelhouse \ - --runtime-root .breakcheck/runtime-001 \ - --output .breakcheck/results/report.json \ - --evidence .breakcheck/results/evidence.json \ - --json \ - --ci +breakcheck attest \ + --head HEAD \ + --claim breakcheck.claim.toml \ + --fixtures breakcheck.fixtures.toml \ + --output .breakcheck/claim-report.json \ + --evidence .breakcheck/claim-evidence.json ``` -The target grammar is `@`. Use a fresh, absent `--runtime-root` for every run. If `--runtime-root` is omitted, Breakcheck creates a temporary runtime directory and records its path in the evidence. +The strict defaults require fixtures to predate the head revision and refuse to treat an unverifiable claim as success. + +Claim dispositions are: + +- `CLAIM_VERIFIED`: exercised and identical; +- `CLAIM_REFUTED`: exercised and changed; +- `CLAIM_UNVERIFIABLE`: not lawfully exercised; +- `CLAIM_OUT_OF_SCOPE`: the change touched symbols omitted from the claim. + +## Using Breakcheck with AI-assisted development + +Breakcheck provides a CLI and deterministic JSON contract; it does not require a model-specific server or protocol. + +A productive division of labor is: + +1. A coding agent proposes a dependency or code change. +2. Breakcheck identifies admitted and unresolved targets. +3. The agent may propose fixture values for unresolved targets. +4. A human reviews the fixture diff. +5. Breakcheck performs repeated replay, normalization, comparison, provenance recording, and refusal. +6. The agent reports `CHANGED`, `NOT_EXERCISED`, `CLAIM_REFUTED`, `CLAIM_UNVERIFIABLE`, and `CLAIM_OUT_OF_SCOPE` results verbatim for human review. + +The agent must not edit reports, evidence, baselines, verdict logic, or a fixture after seeing an unfavorable result. It must not lower coverage or separation policy to obtain a passing run. Repository-local instructions are provided in [SKILL.md](SKILL.md). -### CI exit statuses +Before sending any generated artifact to an external AI service, inspect and sanitize the report and evidence because they may contain repository code, arguments, outputs, and source locations. + +This separation matters when software, tests, and review may all involve probabilistic tools: fixture proposals can be reviewed, while the replay and comparison outcome is deterministic and independently verifiable. + +## Result semantics and exit codes + +### Dependency and revision comparison | Exit | Meaning | | ---: | --- | -| `0` | Coverage is at least 80 percent and no exercised behavior changed. | -| `2` | The request or its evidence was refused. | +| `0` | The configured coverage requirement passed and no exercised behavior changed. | +| `2` | The request, input, or evidence was refused. | | `3` | At least one exercised behavior changed. | -| `4` | Exercised coverage is below 80 percent. | +| `4` | Exercised coverage was below the configured minimum. | -An exit of `0` is evidence about the supported, exercised calls only. It is not a claim that an entire dependency is universally compatible. +Revision comparison always uses these semantic exit codes. Dependency comparison uses changed and coverage exit codes when `--ci` is present; without `--ci`, findings remain visible in the output but a changed result is informational. A dependency run that exercises nothing still exits nonzero unless `--allow-empty` was explicitly recorded. -## Verify persisted evidence +### Claim attestation + +| Exit | Meaning | +| ---: | --- | +| `0` | All claims admitted by the selected policy passed. Under the default strict policy, every claim was verified. | +| `1` | At least one claim was refuted. | +| `2` | At least one claim was unverifiable under strict policy. | +| `3` | Out-of-scope changed symbols were detected. | + +An exit of `0` is evidence about admitted, exercised targets only. It is not a universal compatibility, correctness, or security claim. +Using `--no-strict` makes claim attestation advisory: unverifiable claims remain explicit in the report but do not by themselves make the command fail. Automated agents must not weaken the strict defaults. -Verification rechecks report integrity, finding and witness identities, observation hashes, and the recorded replay environments: +## What Breakcheck does not prove + +Breakcheck reports observed differences across the inputs it lawfully exercises. It does not prove that either side is correct, that unexercised paths are safe, that a dependency is secure, or that every platform behaves identically. A `CHANGED` result may describe an intentional improvement. Breakcheck does not decide whether an upgrade should ship; maintainers make that decision alongside ordinary unit, integration, security, and platform testing. + +## Verify persisted evidence ```console breakcheck \ @@ -153,38 +353,66 @@ breakcheck \ --evidence .breakcheck/results/evidence.json ``` -Successful verification prints `VERIFIED`. Verification requires both recorded environment roots to remain present and byte-identical. Once verification and retention obligations are complete, delete `.breakcheck/runtime-001` yourself to recover disk space. +Successful verification prints `VERIFIED`. Schema-2 verification checks the artifact envelope, payload hash, finding and witness identities, observation hashes, repeat hashes, replay-source hashes, projection hashes, provenance, ordering, counts, and report/evidence binding. Schema-1 reports from Breakcheck 1.x remain readable. -The hashes provide integrity and self-consistency checks. They are not signatures and do not prove who created an evidence bundle. Anyone who can replace an entire report and evidence bundle can compute new hashes. +Hashes provide integrity and self-consistency, not identity or secrecy. They do not make your computer a server, open a network port, or reveal the original content from the digest alone. -## Reading results +## Machine-readable integration -Each finding has one of three verdicts: +Discover supported platforms, Python versions, schemas, and features without prompts: -- `IDENTICAL`: the normalized observations matched. -- `CHANGED`: the normalized observations differed and should be reviewed. -- `NOT_EXERCISED`: Breakcheck could not lawfully replay the call; the reason code explains why. +```console +breakcheck --capabilities --json +``` -Reports include deterministic finding IDs, call-site locations, normalized old and new observations, suggested maintainer actions, and replay witnesses for exercised findings. +All commands are noninteractive. Reports use a versioned schema, canonical JSON, stable identities, explicit provenance, and documented exit codes so CI and coding tools do not need to parse human prose. + +The repository includes a compact workflow example at [examples/github-actions.yml](examples/github-actions.yml). Copy it into your project as `.github/workflows/breakcheck.yml`, then adapt the package versions and command to your dependency update process. ## Security and privacy -Breakcheck launches package code with a scrubbed environment, no shell, bounded resources, a fresh working directory, and a Python socket audit guard. These are containment measures, not a security boundary. Native extensions, subprocesses, and code with the caller's operating-system permissions can still affect the host. +Breakcheck executes trusted package and repository code with a scrubbed environment, no shell, bounded resources, a fresh working directory, and a Python socket audit guard. These are containment measures, not a security boundary. Native extensions, subprocesses, import-time code, or code with the caller's operating-system permissions can still affect the host. -Generated reports and evidence can contain: +Reports and evidence may contain: -- repository-relative source locations and literal call snippets; -- literal arguments; +- repository-relative source locations and replay source; +- literal or fixture arguments and setup code; - normalized return values and exception messages; -- captured standard output and standard error; -- absolute replay-environment paths. +- captured output; +- source, fixture, environment, and observation hashes. + +Inspect artifacts before sharing them with another person or an external service. Do not commit `.breakcheck/` output from a private repository. See [SECURITY.md](SECURITY.md) for the complete trust model. -Inspect and sanitize generated artifacts before sharing them. Do not commit `.breakcheck/` output from private repositories. See [SECURITY.md](SECURITY.md) for the complete trust model and vulnerability-reporting process. +## Release evidence -## Reproducibility evidence +The repository includes bounded release measurements under `release_evidence/`: -The repository contains release evidence for a 20-package battery and a 16,002-call-site scale exercise under `release_evidence/`. Those records are release measurements, not substitutes for running Breakcheck against your own repository and wheels. +- a 20-package dependency battery; +- a 16,002-call-site scale exercise; +- a three-repository fixture-loop viability run. -## Contributing +The fixture run records 49 generated bindings across three unrelated public repositories. In that bounded pre-release evaluation, one automated fixture-authoring pass produced 49 valid, executable, deterministic bindings and increased aggregate exercised calls from 1 to 50 without manual fixture edits. The included record preserves aggregate counts and fixture digests, but not enough source coordinates to serve as an independently reproducible benchmark. It is a release measurement, not a claim that the fixtures exhaust each repository's behavior. + +## Development + +```console +git clone https://github.com/lovettsendit/breakcheck.git +cd breakcheck +python -m venv .venv +source .venv/bin/activate +python -m pip install --upgrade pip +python -m pip install -e '.[test]' +python -m pytest -q +breakcheck --help +``` + +To build release artifacts: + +```console +python -m pip install 'build>=1.2,<2' +python -m build +python -m pip install dist/breakcheck-2.0.0-py3-none-any.whl +breakcheck --capabilities --json +``` -See [CONTRIBUTING.md](CONTRIBUTING.md). Breakcheck is licensed under the [MIT License](LICENSE). +See [CONTRIBUTING.md](CONTRIBUTING.md) before proposing changes. Breakcheck is licensed under the [MIT License](LICENSE). diff --git a/SECURITY.md b/SECURITY.md index e307a25..7b0b840 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -2,7 +2,7 @@ ## Supported versions -Security fixes are provided for the latest `1.x` release. +Security fixes are provided for the latest `2.x` release. ## Trust model @@ -12,6 +12,16 @@ Replay processes receive a scrubbed environment, a fresh working directory, reso Do not use Breakcheck to execute an untrusted wheel on a sensitive machine. Use a disposable virtual machine or comparably isolated host when package provenance is uncertain. +## Repository code, fixtures, and revisions + +Dependency wheels, imported repository modules, fixture expressions, fixture setup, and fixture projections are executable input. Review them before running Breakcheck. + +Revision comparison materializes committed source into detached Git worktrees. It does not mutate the active checkout, index, or stash. Worktree confinement and cleanup reduce accidental interference; they do not make imported application code safe. Import-time code can read files, inspect permitted environment state, start subprocesses, or exercise native extensions with the invoking user's permissions. + +Fixture files are read only from repository-relative paths. Expressions are parsed before replay, bounded in size, and executed only inside the replay process. A projection limits what is compared; it does not prevent the underlying call from executing. + +Run Breakcheck only on repositories, revisions, wheels, fixtures, and claim files you trust. Use a disposable host for code whose provenance is uncertain. + ## Wheelhouse integrity - Breakcheck never falls back to a network package index. @@ -21,9 +31,11 @@ Do not use Breakcheck to execute an untrusted wheel on a sensitive machine. Use ## Reports and evidence -Generated artifacts may contain source snippets, literal arguments, returned values, exception messages, stdout, stderr, usernames embedded in paths, and absolute replay-environment paths. Treat reports and witnesses with the same confidentiality as the source repository being analyzed. +Generated artifacts may contain repository-relative paths, replay source, literal or fixture arguments, setup and projection expressions, returned values, exception messages, captured output, revision identities, and hashes derived from source or environments. Treat reports, baselines, claims, coverage records, and witnesses with the same confidentiality as the source repository being analyzed. + +Breakcheck hashes reports, witnesses, finding identities, observations, replay source, fixtures, revisions, and environment artifacts to detect accidental or partial alteration. These hashes are integrity checks, not authenticated signatures, encryption, or remote connectivity. Creating or publishing a hash does not make a computer a server and does not expose the original content from the digest alone. A party able to replace the complete bundle can generate a new internally consistent bundle. -Breakcheck hashes reports, witnesses, finding identities, observations, and replay-environment artifacts to detect accidental or partial alteration. These hashes are integrity checks, not authenticated signatures. A party able to replace the complete bundle can generate a new internally consistent bundle. +Inspect generated artifacts before sending them to an external service. Do not commit `.breakcheck/` output from a private repository. ## Reporting a vulnerability diff --git a/SKILL.md b/SKILL.md new file mode 100644 index 0000000..c58823c --- /dev/null +++ b/SKILL.md @@ -0,0 +1,81 @@ +--- +name: verifying-python-changes-with-breakcheck +description: Use when a Python dependency version changes or when a code change claims to preserve behavior. +--- + +# Verifying Python Changes with Breakcheck + +## Purpose + +Use Breakcheck as the deterministic measurement step after proposing a dependency upgrade or a behavior-preserving code change. A coding agent may propose inputs and interpret results; Breakcheck owns replay, comparison, refusal, and evidence. + +## Dependency upgrades + +Run from the repository root after preparing exact trusted wheels in a local wheelhouse: + +```console +breakcheck PACKAGE@NEW_VERSION \ + --wheelhouse wheelhouse \ + --output .breakcheck/report.json \ + --evidence .breakcheck/evidence.json \ + --coverage-report .breakcheck/coverage.json \ + --json --ci +``` + +If coverage is limited by unresolved arguments, generate proposals with: + +```console +breakcheck PACKAGE@NEW_VERSION --suggest-fixtures breakcheck.fixtures.toml +``` + +Fill only fixtures that can be justified from repository context, mark `fixture_authored_by = "agent"`, and present the fixture diff for human review before replay. + +## Behavior-preserving code changes + +Fixtures must be authored, reviewed, and committed against the base revision before the target code changes. Use this sequence: + +1. Run `breakcheck freeze` for the affected symbols and retain the verified baseline. +2. Make the behavior-preserving code change without altering its fixtures or baseline. +3. Create a claim file listing every symbol the change is intended to preserve. +4. Run `breakcheck attest` against the changed revision. +5. Report every disposition verbatim to the human, including all unverifiable and out-of-scope counts. + +An explicit comparison between committed revisions is also available: + +```console +breakcheck diff \ + --base BASE_REVISION --head HEAD_REVISION \ + --fixtures breakcheck.fixtures.toml \ + --strict-separation \ + --output .breakcheck/revision-report.json \ + --evidence .breakcheck/revision-evidence.json +``` + +For an explicit preservation claim, create a reviewed `breakcheck.claim.toml` and run: + +```console +breakcheck attest \ + --head HEAD_REVISION \ + --claim breakcheck.claim.toml \ + --fixtures breakcheck.fixtures.toml \ + --output .breakcheck/claim-report.json \ + --evidence .breakcheck/claim-evidence.json +``` + +## Result contract + +- `IDENTICAL` or `CLAIM_VERIFIED`: exercised evidence matched within the stated scope. +- `CHANGED` or `CLAIM_REFUTED`: surface the finding to the human; do not silently resolve it. +- `NOT_EXERCISED` or `CLAIM_UNVERIFIABLE`: report the count and reason explicitly; never describe it as safe. +- `CLAIM_OUT_OF_SCOPE`: report the omitted changed symbols. + +## Required separation + +- Never edit `report.json`, `evidence.json`, or `baseline.json`. +- Never author or modify a fixture for a target after changing that target's code; fixtures used to verify preservation must predate the change. +- Never modify a fixture after seeing `CHANGED` for that binding in the same session. +- Never pass `--allow-empty`, lower `--min-coverage`, disable strict policy, or weaken fixture separation to obtain a passing result. +- Never modify verdict or verification logic as part of the change being verified. +- Inspect and sanitize artifacts before sending them to an external service; they may contain source locations, replay source, arguments, setup, projections, and observed values. + +Report the exact Breakcheck command, exit code, verdict counts, and artifact paths alongside the ordinary project test results. diff --git a/docs/assets/breakcheck-social-preview.png b/docs/assets/breakcheck-social-preview.png new file mode 100644 index 0000000..7833904 Binary files /dev/null and b/docs/assets/breakcheck-social-preview.png differ diff --git a/examples/github-actions.yml b/examples/github-actions.yml new file mode 100644 index 0000000..020c369 --- /dev/null +++ b/examples/github-actions.yml @@ -0,0 +1,17 @@ +name: Breakcheck dependency comparison +on: pull_request +permissions: + contents: read +env: {DEPENDENCY: attrs, CURRENT_VERSION: 23.2.0, PROPOSED_VERSION: 24.2.0} +jobs: + compare: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 + with: {python-version: "3.13"} + - run: python -m pip install "breakcheck==2.0.0" "$DEPENDENCY==$CURRENT_VERSION" + - run: mkdir wheelhouse + - run: 'python -m pip download --only-binary=:all: -d wheelhouse "$DEPENDENCY==$CURRENT_VERSION"' + - run: 'python -m pip download --only-binary=:all: -d wheelhouse "$DEPENDENCY==$PROPOSED_VERSION"' + - run: breakcheck "$DEPENDENCY@$PROPOSED_VERSION" --wheelhouse wheelhouse --ci diff --git a/examples/packaging-change/app.py b/examples/packaging-change/app.py new file mode 100644 index 0000000..7c3ff3b --- /dev/null +++ b/examples/packaging-change/app.py @@ -0,0 +1,3 @@ +from packaging.utils import canonicalize_version + +canonicalize_version('1.0.0', strip_trailing_zero=False) diff --git a/examples/run_demo.sh b/examples/run_demo.sh new file mode 100644 index 0000000..1921dbb --- /dev/null +++ b/examples/run_demo.sh @@ -0,0 +1,147 @@ +#!/bin/sh +set -eu + +PYTHON="${PYTHON:-python3}" +KEEP="${BREAKCHECK_DEMO_KEEP:-0}" +CHECKOUT=$(CDPATH= cd "$(dirname "$0")/.." && pwd) +EXAMPLE_ROOT="$CHECKOUT/examples/packaging-change" +DEMO_ROOT=$(mktemp -d "${TMPDIR:-/tmp}/breakcheck-demo.XXXXXX") +DEMO_ROOT=$(CDPATH= cd "$DEMO_ROOT" && pwd -P) +TOOL_SITE="$DEMO_ROOT/tool-site" +RUNTIME_ROOT="$DEMO_ROOT/runtime" +RESULTS_ROOT="$DEMO_ROOT/results" +REPORT_PATH="$RESULTS_ROOT/report.json" +EVIDENCE_PATH="$RESULTS_ROOT/evidence.json" + +cleanup() { + status=$? + trap - 0 HUP INT TERM + if [ "$KEEP" = "1" ]; then + printf '%s\n' "DEMO_ROOT=$DEMO_ROOT" + else + rm -rf "$DEMO_ROOT" + fi + exit "$status" +} +trap cleanup 0 HUP INT TERM + +"$PYTHON" -c 'import sys; raise SystemExit(0 if (3, 10) <= sys.version_info[:2] < (3, 14) else 1)' || { + printf '%s\n' "BREAKCHECK_DEMO_REFUSED: Python 3.10 through 3.13 is required" >&2 + exit 2 +} + +if [ -n "${BREAKCHECK_DEMO_WHEELHOUSE:-}" ]; then + WHEELHOUSE=$BREAKCHECK_DEMO_WHEELHOUSE + if [ ! -d "$WHEELHOUSE" ]; then + printf '%s\n' "BREAKCHECK_DEMO_REFUSED: wheelhouse does not exist: $WHEELHOUSE" >&2 + exit 2 + fi +else + WHEELHOUSE="$DEMO_ROOT/wheelhouse" + mkdir -p "$WHEELHOUSE" + "$PYTHON" -m pip download --only-binary=:all: --no-deps --dest "$WHEELHOUSE" 'packaging==21.3' + "$PYTHON" -m pip download --only-binary=:all: --no-deps --dest "$WHEELHOUSE" 'packaging==22.0' + "$PYTHON" -m pip download --only-binary=:all: --no-deps --dest "$WHEELHOUSE" 'pyparsing==3.3.2' +fi + +WHEEL_21="$WHEELHOUSE/packaging-21.3-py3-none-any.whl" +WHEEL_22="$WHEELHOUSE/packaging-22.0-py3-none-any.whl" +PYPARSING_WHEEL="$WHEELHOUSE/pyparsing-3.3.2-py3-none-any.whl" +[ -f "$WHEEL_21" ] && [ -f "$WHEEL_22" ] || { + printf '%s\n' "BREAKCHECK_DEMO_REFUSED: exact packaging wheels are required" >&2 + exit 2 +} + +verify_sha256() { + expected=$1 + artifact=$2 + actual=$("$PYTHON" - "$artifact" <<'PY' +import hashlib +import sys +print(hashlib.sha256(open(sys.argv[1], 'rb').read()).hexdigest()) +PY +) + [ "$actual" = "$expected" ] || { + printf '%s\n' "BREAKCHECK_DEMO_REFUSED: unexpected wheel digest: $artifact" >&2 + exit 2 + } +} + +if [ -z "${BREAKCHECK_DEMO_WHEELHOUSE:-}" ]; then + verify_sha256 ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522 "$WHEEL_21" + verify_sha256 957e2148ba0e1a3b282772e791ef1d8083648bc131c8ab0c1feba110ce1146c3 "$WHEEL_22" + [ -f "$PYPARSING_WHEEL" ] || { + printf '%s\n' "BREAKCHECK_DEMO_REFUSED: exact pyparsing wheel is required" >&2 + exit 2 + } + verify_sha256 850ba148bd908d7e2411587e247a1e4f0327839c40e2e5e6d05a007ecc69911d "$PYPARSING_WHEEL" + unexpected=$(find "$WHEELHOUSE" -type f ! -name 'packaging-21.3-py3-none-any.whl' ! -name 'packaging-22.0-py3-none-any.whl' ! -name 'pyparsing-3.3.2-py3-none-any.whl' -print -quit) + [ -z "$unexpected" ] || { + printf '%s\n' "BREAKCHECK_DEMO_REFUSED: unexpected wheelhouse file: $unexpected" >&2 + exit 2 + } +fi + +TOOL_PYTHON="$PYTHON" +if "$PYTHON" -c 'import breakcheck; raise SystemExit(0 if breakcheck.__version__ == "2.0.0" else 1)' 2>/dev/null; then + BREAKCHECK_IMPORT_ROOT=$("$PYTHON" -c 'from pathlib import Path; import breakcheck; print(Path(breakcheck.__file__).resolve().parent.parent)') +else + BREAKCHECK_IMPORT_ROOT="$CHECKOUT/src" +fi +mkdir -p "$TOOL_SITE" +"$PYTHON" -m pip install --no-index --no-deps --target "$TOOL_SITE" "$WHEEL_21" +mkdir -p "$RESULTS_ROOT" + +run_breakcheck() { + PYTHONPATH= "$TOOL_PYTHON" - "$TOOL_SITE" "$BREAKCHECK_IMPORT_ROOT" "$@" <<'PY' +import runpy +import sys + +tool_site, import_root, *arguments = sys.argv[1:] +sys.path[:0] = [tool_site, import_root] +sys.argv = ["breakcheck", *arguments] +runpy.run_module("breakcheck", run_name="__main__") +PY +} + +set +e +( + cd "$EXAMPLE_ROOT" + run_breakcheck packaging@22.0 \ + --wheelhouse "$WHEELHOUSE" \ + --runtime-root "$RUNTIME_ROOT" \ + --output "$REPORT_PATH" \ + --evidence "$EVIDENCE_PATH" \ + --json \ + --ci +) +BREAKCHECK_EXIT=$? +set -e +printf '%s\n' "BREAKCHECK_EXIT=$BREAKCHECK_EXIT" +[ "$BREAKCHECK_EXIT" -eq 3 ] || { + printf '%s\n' "BREAKCHECK_DEMO_REFUSED: expected exit 3" >&2 + exit 2 +} + +"$TOOL_PYTHON" - "$REPORT_PATH" <<'PY' +import json +import sys + +report = json.load(open(sys.argv[1], encoding="utf-8")) +if report.get("schema_version") != 2 or report.get("artifact_kind") != "dependency_report": + raise SystemExit("BREAKCHECK_DEMO_REFUSED: expected schema-2 dependency report") +payload = report.get("payload", {}) +findings = payload.get("findings") +summary = payload.get("summary") +if not isinstance(findings, list) or len(findings) != 1: + raise SystemExit("BREAKCHECK_DEMO_REFUSED: expected exactly one finding") +if findings[0].get("verdict") != "CHANGED": + raise SystemExit("BREAKCHECK_DEMO_REFUSED: expected CHANGED finding") +if not isinstance(summary, dict) or summary.get("changed") != 1: + raise SystemExit("BREAKCHECK_DEMO_REFUSED: expected summary.changed == 1") +PY + +run_breakcheck --verify "$REPORT_PATH" --evidence "$EVIDENCE_PATH" +printf '%s\n' "REPORT_PATH=$REPORT_PATH" +printf '%s\n' "EVIDENCE_PATH=$EVIDENCE_PATH" +printf '%s\n' "DEMO_VERDICT=PASS" diff --git a/pyproject.toml b/pyproject.toml index 45734ae..09a540e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,11 +4,11 @@ build-backend = "setuptools.build_meta" [project] name = "breakcheck" -version = "1.0.1" -description = "Offline behavioral compatibility checks for Python dependency upgrades." +version = "2.0.0" +description = "Deterministic behavioral comparison for Python dependency and code changes." requires-python = ">=3.10,<3.14" dependencies = [] -authors = [{name = "breakcheck contributors"}] +authors = [{name = "ViDale Lovett"}] license = "MIT" license-files = ["LICENSE"] readme = "README.md" @@ -20,6 +20,7 @@ keywords = [ "deterministic", "offline", "python", + "refactoring", "regression-testing", "testing", "upgrades", @@ -53,6 +54,12 @@ dev = [ [project.scripts] "breakcheck" = "breakcheck.cli:main" +[project.urls] +Homepage = "https://github.com/lovettsendit/breakcheck" +Repository = "https://github.com/lovettsendit/breakcheck" +Issues = "https://github.com/lovettsendit/breakcheck/issues" +Changelog = "https://github.com/lovettsendit/breakcheck/blob/main/CHANGELOG.md" + [tool.setuptools.packages.find] where = ["src"] diff --git a/release_evidence/fixture-viability.json b/release_evidence/fixture-viability.json new file mode 100644 index 0000000..71ddb38 --- /dev/null +++ b/release_evidence/fixture-viability.json @@ -0,0 +1,57 @@ +{ + "aggregate": { + "authored": 49, + "deterministic": 49, + "executable": 49, + "exercised_after": 50, + "exercised_before": 1, + "fixture_yield_percent": 100.0, + "human_minutes": 0.0, + "offered": 49, + "valid": 49 + }, + "method": { + "fixture_edits_after_verdict": 0, + "repositories": 3, + "repository_authorship": "independent public projects", + "runs_per_binding": 2, + "wheel_policy": "local-only" + }, + "repositories": [ + { + "authored": 9, + "deterministic": 9, + "executable": 9, + "exercised_after": 9, + "exercised_before": 0, + "fixture_sha256": "5f17b6a8971c7574219bfa53f981164e360150b31ee380948ff73813052324ec", + "name": "Black", + "offered": 9, + "valid": 9 + }, + { + "authored": 37, + "deterministic": 37, + "executable": 37, + "exercised_after": 38, + "exercised_before": 1, + "fixture_sha256": "385a6f4ee479c07fe8f803d12755a138bb59c210f6490e639a6a83c685d1e1da", + "name": "Rich CLI", + "offered": 37, + "valid": 37 + }, + { + "authored": 3, + "deterministic": 3, + "executable": 3, + "exercised_after": 3, + "exercised_before": 0, + "fixture_sha256": "fc5c3a2585b9dba059f56893e9c0d931e63f4814e562e8a8693a4baa1aa64c3e", + "name": "Flask", + "offered": 3, + "valid": 3 + } + ], + "schema_version": 1, + "verdict": "PASS" +} diff --git a/scripts/scan_artifacts.sh b/scripts/scan_artifacts.sh new file mode 100644 index 0000000..69743ef --- /dev/null +++ b/scripts/scan_artifacts.sh @@ -0,0 +1,277 @@ +#!/usr/bin/env bash +# Read-only release hygiene scan for a checkout, wheel, or source distribution. +set -u + +if [ "$#" -ne 1 ]; then + echo "usage: $0 " >&2 + exit 2 +fi + +python_bin=${PYTHON:-python3} +exec "$python_bin" - "$1" <<'PY' +from __future__ import annotations + +import re +import stat +import subprocess +import sys +import tarfile +import zipfile +from pathlib import Path, PurePosixPath + + +MAX_MEMBER_BYTES = 32 * 1024 * 1024 +MAX_TOTAL_BYTES = 128 * 1024 * 1024 +findings: list[str] = [] + + +def forbidden(message: str) -> None: + findings.append(message) + + +local_path_patterns = ( + re.compile( + rb"(?m)(?:^|[\s\"'=:(])/(?:Users|home|Volumes)/[^\s\"'<>]+" + ), + re.compile( + rb"(?mi)(?:^|[\s\"'=:(])[A-Z]:\\Users\\[^\s\"'<>]+" + ), +) +email_pattern = re.compile( + rb"(? str | None: + if not name or "\x00" in name or "\\" in name: + forbidden(f"unsafe archive path: {name!r}") + return None + raw_parts = name.rstrip("/").split("/") + path = PurePosixPath(name) + if path.is_absolute() or any(part in ("", ".", "..") for part in raw_parts): + forbidden(f"unsafe archive path: {name}") + return None + return str(path) + + +def scan_name(relative: str) -> None: + parts = PurePosixPath(relative).parts + lowered = [part.casefold() for part in parts] + if any( + part == ".git" + or part == ".ds_store" + or part.startswith("._") + for part in lowered + ): + forbidden(f"local or macOS path: {relative}") + return + + +def scan_content(data: bytes, relative: str) -> None: + if not data or b"\x00" in data[:4096]: + return + try: + data.decode("utf-8") + except UnicodeDecodeError: + return + for pattern in local_path_patterns: + if pattern.search(data): + forbidden(f"local filesystem path: {relative}") + break + for match in email_pattern.finditer(data): + if not noreply_pattern.fullmatch(match.group(0)): + forbidden(f"personal email content: {relative}") + break + for pattern in credential_patterns: + if pattern.search(data): + forbidden(f"credential-shaped content: {relative}") + break + + +def scan_zip(path: Path) -> None: + total = 0 + seen: set[str] = set() + try: + with zipfile.ZipFile(path) as archive: + for info in archive.infolist(): + relative = checked_relative(info.filename) + if relative is None: + continue + if relative in seen: + forbidden(f"duplicate archive path: {relative}") + continue + seen.add(relative) + scan_name(relative) + mode = info.external_attr >> 16 + file_type = stat.S_IFMT(mode) + if file_type == stat.S_IFLNK or file_type not in (0, stat.S_IFREG, stat.S_IFDIR): + forbidden(f"archive link or special file: {relative}") + continue + if info.is_dir(): + continue + total += info.file_size + if info.file_size > MAX_MEMBER_BYTES or total > MAX_TOTAL_BYTES: + forbidden(f"archive size limit exceeded: {relative}") + continue + scan_content(archive.read(info), relative) + except (OSError, zipfile.BadZipFile, RuntimeError) as exc: + forbidden(f"unreadable zip archive: {exc}") + + +def scan_tar(path: Path) -> None: + total = 0 + seen: set[str] = set() + try: + with tarfile.open(path, "r:*") as archive: + for member in archive: + relative = checked_relative(member.name) + if relative is None: + continue + if relative in seen: + forbidden(f"duplicate archive path: {relative}") + continue + seen.add(relative) + scan_name(relative) + if member.isdir(): + continue + if not member.isfile(): + forbidden(f"archive link or special file: {relative}") + continue + total += member.size + if member.size > MAX_MEMBER_BYTES or total > MAX_TOTAL_BYTES: + forbidden(f"archive size limit exceeded: {relative}") + continue + stream = archive.extractfile(member) + if stream is None: + forbidden(f"unreadable archive member: {relative}") + continue + scan_content(stream.read(MAX_MEMBER_BYTES + 1), relative) + except (OSError, tarfile.TarError) as exc: + forbidden(f"unreadable tar archive: {exc}") + + +def git_output(root: Path, *args: str) -> bytes: + return subprocess.run( + ["git", "-C", str(root), *args], + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ).stdout + + +def scan_directory(path: Path) -> None: + try: + repository = Path( + git_output(path, "rev-parse", "--show-toplevel").decode().strip() + ) + except (OSError, subprocess.CalledProcessError, UnicodeDecodeError): + repository = None + + if repository is None: + candidates = sorted( + item for item in path.rglob("*") if ".git" not in item.parts + ) + base = path + else: + tracked = git_output(repository, "ls-files", "-z").split(b"\0") + untracked = git_output( + repository, "ls-files", "--others", "--exclude-standard", "-z" + ).split(b"\0") + names = sorted({name.decode() for name in tracked + untracked if name}) + candidates = [repository / name for name in names] + base = repository + + for item in candidates: + try: + relative = item.relative_to(base).as_posix() + except ValueError: + forbidden(f"path escaped scan root: {item}") + continue + scan_name(relative) + if item.is_symlink(): + forbidden(f"symlink in release tree: {relative}") + elif item.is_file(): + try: + size = item.stat().st_size + if size > MAX_MEMBER_BYTES: + forbidden(f"file size limit exceeded: {relative}") + else: + scan_content(item.read_bytes(), relative) + except OSError as exc: + forbidden(f"unreadable file: {relative}: {exc}") + + if repository is not None: + try: + metadata_emails = git_output( + repository, "log", "--all", "--format=%ae%n%ce" + ).splitlines() + except (OSError, subprocess.CalledProcessError) as exc: + forbidden(f"unable to inspect git metadata: {exc}") + else: + for address in metadata_emails: + if address and not noreply_pattern.fullmatch(address): + forbidden("personal email in git metadata") + break + try: + patch = git_output(repository, "log", "--format=", "--all", "-p", "--no-ext-diff") + except (OSError, subprocess.CalledProcessError) as exc: + forbidden(f"unable to inspect git history: {exc}") + else: + changed = b"\n".join( + line[1:] + for line in patch.splitlines() + if line[:1] in (b"+", b"-") + and not line.startswith((b"+++", b"---")) + ) + before = len(findings) + scan_content(changed, "git history") + if len(findings) > before: + findings[before:] = ["sensitive content in git history"] + + +def main() -> int: + if len(sys.argv) != 2: + print("scanner argument error", file=sys.stderr) + return 2 + input_path = Path(sys.argv[1]) + if not input_path.exists() and not input_path.is_symlink(): + print(f"artifact not found: {input_path}", file=sys.stderr) + return 2 + if input_path.is_symlink(): + forbidden(f"symlink input refused: {input_path}") + elif input_path.is_dir(): + scan_directory(input_path) + elif input_path.name.endswith((".whl", ".zip")): + scan_zip(input_path) + elif input_path.name.endswith((".tar.gz", ".tgz")): + scan_tar(input_path) + elif input_path.is_file(): + scan_name(input_path.name) + scan_content(input_path.read_bytes(), input_path.name) + else: + forbidden(f"unsupported input: {input_path}") + + for finding in dict.fromkeys(findings): + print(f"FORBIDDEN: {finding}") + if findings: + return 1 + print(f"artifact scan clean: {input_path}") + return 0 + + +raise SystemExit(main()) +PY diff --git a/src/breakcheck/__init__.py b/src/breakcheck/__init__.py index 26d0bef..8fc7dc1 100644 --- a/src/breakcheck/__init__.py +++ b/src/breakcheck/__init__.py @@ -1,4 +1,6 @@ 'Public command surface.' +__version__ = "2.0.0" + from .cli import main -__all__ = ('main',) +__all__ = ('__version__', 'main') diff --git a/src/breakcheck/adapters/python/coverage.py b/src/breakcheck/adapters/python/coverage.py new file mode 100644 index 0000000..88e9320 --- /dev/null +++ b/src/breakcheck/adapters/python/coverage.py @@ -0,0 +1,211 @@ +"""Deterministic identities and terminal records for coverage accounting. + +This module is intentionally independent of the command-line integration. It +defines the closed vocabulary and canonical records that later pipeline stages +can consume without changing the legacy scanner call-site schema. +""" + +from __future__ import annotations + +import hashlib +import json +from collections.abc import Iterable, Mapping + + +TERMINAL_BUCKETS = ( + "EXERCISED", + "G1_NOT_DISCOVERABLE", + "G2_NONLITERAL", + "G3_UNNORMALIZABLE", + "G4_IMPURE", +) + +PROVENANCE_ORDER = ( + "SOURCE_LITERAL", + "SOURCE_FOLDED", + "SOURCE_MODULE_CONSTANT", + "SOURCE_NESTED_CALL", + "OPERATOR_FIXTURE", + "RUNTIME_CAPTURE", +) + +_PROVENANCE_POSITION = { + value: position for position, value in enumerate(PROVENANCE_ORDER) +} + + +def _canonical_bytes(value: object) -> bytes: + return json.dumps( + value, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=True, + allow_nan=False, + ).encode("utf-8") + + +def _nonempty_text(value: object, refusal: str) -> str: + if not isinstance(value, str) or not value: + raise ValueError(refusal) + return value + + +def _location(*, api: object, file: object, line: object, column: object) -> dict: + observed_api = _nonempty_text(api, "CANDIDATE_API_REFUSED") + observed_file = _nonempty_text(file, "CANDIDATE_FILE_REFUSED") + if type(line) is not int or line < 1: + raise ValueError("CANDIDATE_LINE_REFUSED") + if type(column) is not int or column < 0: + raise ValueError("CANDIDATE_COLUMN_REFUSED") + return { + "api": observed_api, + "file": observed_file, + "line": line, + "column": column, + } + + +def candidate_id(*, api: object, file: object, line: object, column: object) -> str: + """Return the stable identity of one statically attributable call site.""" + + payload = { + "kind": "python_dependency_call", + "location": _location(api=api, file=file, line=line, column=column), + "schema_version": 1, + } + return hashlib.sha256(_canonical_bytes(payload)).hexdigest() + + +def make_candidate(*, api: object, file: object, line: object, column: object) -> dict: + location = _location(api=api, file=file, line=line, column=column) + return { + "candidate_id": candidate_id(**location), + **location, + } + + +def order_provenance(values: Iterable[str]) -> tuple[str, ...]: + try: + observed = set(values) + except TypeError as exc: + raise ValueError("PROVENANCE_REFUSED") from exc + if not observed or any(value not in _PROVENANCE_POSITION for value in observed): + raise ValueError("PROVENANCE_REFUSED") + return tuple(sorted(observed, key=_PROVENANCE_POSITION.__getitem__)) + + +def _validate_candidate(candidate: Mapping[str, object]) -> dict: + if not isinstance(candidate, Mapping) or set(candidate) != { + "candidate_id", + "api", + "file", + "line", + "column", + }: + raise ValueError("CANDIDATE_SCHEMA_REFUSED") + location = _location( + api=candidate["api"], + file=candidate["file"], + line=candidate["line"], + column=candidate["column"], + ) + expected = candidate_id(**location) + if candidate["candidate_id"] != expected: + raise ValueError("CANDIDATE_IDENTITY_REFUSED") + return {"candidate_id": expected, **location} + + +def terminal_record( + candidate: Mapping[str, object], + bucket: str, + *, + reason_code: str | None = None, + reason_detail: str | None = None, + raw_type: str | None = None, + environment: str | None = None, + provenance: Iterable[str], +) -> dict: + """Bind one candidate to exactly one terminal coverage disposition.""" + + observed = _validate_candidate(candidate) + if bucket not in TERMINAL_BUCKETS: + raise ValueError("COVERAGE_BUCKET_REFUSED") + if bucket == "EXERCISED": + if any(value is not None for value in (reason_code, reason_detail, raw_type, environment)): + raise ValueError("EXERCISED_DETAIL_REFUSED") + elif reason_code is None: + raise ValueError("COVERAGE_REASON_REQUIRED") + row = { + **observed, + "bucket": bucket, + "provenance": list(order_provenance(provenance)), + } + for name, value in ( + ("reason_code", reason_code), + ("reason_detail", reason_detail), + ("raw_type", raw_type), + ("environment", environment), + ): + if value is not None: + row[name] = _nonempty_text(value, "COVERAGE_DETAIL_REFUSED") + return row + + +def finalize_terminal_records(records: Iterable[Mapping[str, object]]) -> list[dict]: + """Validate uniqueness and return canonical candidate-id order.""" + + observed = [] + identities = set() + for record in records: + if not isinstance(record, Mapping): + raise ValueError("CANDIDATE_TERMINAL_SCHEMA_REFUSED") + candidate = { + key: record[key] + for key in ("candidate_id", "api", "file", "line", "column") + if key in record + } + validated = _validate_candidate(candidate) + identity = validated["candidate_id"] + if identity in identities: + raise ValueError("CANDIDATE_TERMINAL_DUPLICATE") + identities.add(identity) + bucket = record.get("bucket") + if bucket not in TERMINAL_BUCKETS: + raise ValueError("COVERAGE_BUCKET_REFUSED") + allowed = { + "candidate_id", + "api", + "file", + "line", + "column", + "bucket", + "provenance", + "reason_code", + "reason_detail", + "raw_type", + "environment", + } + if not set(record).issubset(allowed): + raise ValueError("CANDIDATE_TERMINAL_SCHEMA_REFUSED") + rebuilt = terminal_record( + validated, + bucket, + reason_code=record.get("reason_code"), + reason_detail=record.get("reason_detail"), + raw_type=record.get("raw_type"), + environment=record.get("environment"), + provenance=record.get("provenance", ()), + ) + if rebuilt != dict(record): + raise ValueError("CANDIDATE_TERMINAL_CANONICAL_REFUSED") + observed.append(rebuilt) + return sorted(observed, key=lambda row: row["candidate_id"]) + + +def count_terminal_records(records: Iterable[Mapping[str, object]]) -> dict: + finalized = finalize_terminal_records(records) + counts = {bucket: 0 for bucket in TERMINAL_BUCKETS} + for record in finalized: + counts[record["bucket"]] += 1 + counts["total"] = len(finalized) + return counts diff --git a/src/breakcheck/adapters/python/envs.py b/src/breakcheck/adapters/python/envs.py index cb04a56..c78776f 100644 --- a/src/breakcheck/adapters/python/envs.py +++ b/src/breakcheck/adapters/python/envs.py @@ -192,8 +192,8 @@ def env_fingerprint(*, environment=None): for distribution in importlib.metadata.distributions(path=sorted(paths)): rows.append((str(distribution.metadata.get("Name", "")), str(distribution.version))) - except Exception: - rows = [] + except Exception as exc: + raise ValueError("ENVIRONMENT_FINGERPRINT_REFUSED") from exc rows = sorted(rows) payload = "\n".join("%s==%s" % row for row in rows).encode("utf-8") return hashlib.sha256(payload).hexdigest() diff --git a/src/breakcheck/adapters/python/equality.py b/src/breakcheck/adapters/python/equality.py index eaeee9e..04ff367 100644 --- a/src/breakcheck/adapters/python/equality.py +++ b/src/breakcheck/adapters/python/equality.py @@ -3,6 +3,8 @@ import math from collections.abc import Mapping +from .normalization import tagged_value_kind, tagged_value_payload + class PythonEqualityRules: def __init__(self, rtol=1e-09, atol=0): @@ -37,6 +39,12 @@ def _field(value, name): def _summary(value): + tagged_kind = tagged_value_kind(value) + if tagged_kind is not None: + payload = tagged_value_payload(value) + if tagged_kind == "bytes": + return "bytes:" + str(payload) + return tagged_kind + ":" + str(_stable_key(payload))[:180] if value is None: return "NoneType:None" if type(value) is bool: @@ -59,6 +67,22 @@ def _summary(value): def _stable_key(value): + tagged_kind = tagged_value_kind(value) + if tagged_kind is not None: + payload = tagged_value_payload(value) + if tagged_kind == "bytes": + return ("bytes", payload) + if tagged_kind in {"set", "frozenset"}: + return ( + tagged_kind, + tuple(sorted(_stable_key(item) for item in payload)), + ) + if tagged_kind == "mapping": + return ( + "mapping", + tuple((key, _stable_key(item)) for key, item in payload), + ) + return (tagged_kind, tuple(_stable_key(item) for item in payload)) if value is None: return ("none",) if type(value) is bool: @@ -111,6 +135,9 @@ def _result(verdict, reason, path, old_summary, new_summary, policy): def _kind(value): + tagged_kind = tagged_value_kind(value) + if tagged_kind is not None: + return tagged_kind if type(value) is bool: return "bool" if type(value) is int: @@ -140,9 +167,15 @@ def _compare_values(left, right, path, rules): equal = float_eq(left, right, rules) return equal, "EQUAL" if equal else "FLOAT_MISMATCH", path, left, right, _FLOAT_POLICY if left_kind in {"bool", "int", "str", "bytes", "NoneType"}: + if left_kind == "bytes" and tagged_value_kind(left) == "bytes": + left = tagged_value_payload(left) + right = tagged_value_payload(right) equal = left == right return equal, "EQUAL" if equal else "VALUE_MISMATCH", path, left, right, "canonical_json_strict" if left_kind == "mapping": + if tagged_value_kind(left) == "mapping": + left = dict(tagged_value_payload(left)) + right = dict(tagged_value_payload(right)) left_keys = set(left) right_keys = set(right) for key in sorted(left_keys | right_keys): @@ -165,10 +198,42 @@ def _compare_values(left, right, path, rules): return False, reason, mismatch, old, new, "sequence_order_significant" return True, "EQUAL", path, left, right, "sequence_order_significant" if left_kind == "set": - left_values = sorted(_stable_key(item) for item in left) - right_values = sorted(_stable_key(item) for item in right) + left_items = ( + tagged_value_payload(left) + if tagged_value_kind(left) == "set" + else left + ) + right_items = ( + tagged_value_payload(right) + if tagged_value_kind(right) == "set" + else right + ) + left_values = sorted(_stable_key(item) for item in left_items) + right_values = sorted(_stable_key(item) for item in right_items) equal = left_values == right_values return equal, "EQUAL" if equal else "VALUE_MISMATCH", path, left, right, "unordered_set_canonical" + if left_kind == "frozenset": + left_values = sorted( + _stable_key(item) for item in tagged_value_payload(left) + ) + right_values = sorted( + _stable_key(item) for item in tagged_value_payload(right) + ) + equal = left_values == right_values + return equal, "EQUAL" if equal else "VALUE_MISMATCH", path, left, right, "unordered_frozenset_canonical" + if left_kind == "tuple": + left_values = tagged_value_payload(left) + right_values = tagged_value_payload(right) + if len(left_values) != len(right_values): + return False, "LENGTH_MISMATCH", path, left, right, "tuple_length_exact" + for index, (left_item, right_item) in enumerate(zip(left_values, right_values)): + child = _pointer(path, index) + equal, reason, mismatch, old, new, _policy = _compare_values( + left_item, right_item, child, rules + ) + if not equal: + return False, reason, mismatch, old, new, "tuple_order_significant" + return True, "EQUAL", path, left, right, "tuple_order_significant" equal = left == right return equal, "EQUAL" if equal else "VALUE_MISMATCH", path, left, right, "canonical_json_strict" diff --git a/src/breakcheck/adapters/python/executor.py b/src/breakcheck/adapters/python/executor.py index ca58a92..3b34219 100644 --- a/src/breakcheck/adapters/python/executor.py +++ b/src/breakcheck/adapters/python/executor.py @@ -8,6 +8,12 @@ import threading import time +from breakcheck.adapters.python import protocol +from breakcheck.adapters.python.normalization import ( + normalize_protocol_packet, + observation_identity, +) + try: import resource except ImportError: @@ -84,11 +90,54 @@ def _environment_runtime(environment): def _scrub_environment_root(data, environment_root): + replacements = () if environment_root is None else ( + (environment_root, ""), + ) + return _scrub_roots_bytes(data, replacements) + + +def _scrub_roots_bytes(data, replacements): value = bytes(data or b"") - if environment_root is None: + encoded = [] + for root, marker in replacements: + if root is None: + continue + raw = os.path.abspath(os.fspath(root)) + real = os.path.realpath(raw) + for candidate in (raw, real): + pair = (os.fsencode(candidate), str(marker).encode("ascii")) + if pair[0] and pair not in encoded: + encoded.append(pair) + for needle, marker in sorted(encoded, key=lambda row: len(row[0]), reverse=True): + value = value.replace(needle, marker) + return value + + +def _scrub_packet(packet, replacements): + encoded_replacements = [] + for root, marker in replacements: + if root is None: + continue + raw = os.path.abspath(os.fspath(root)) + real = os.path.realpath(raw) + for candidate in (raw, real): + pair = (candidate, str(marker)) + if pair[0] and pair not in encoded_replacements: + encoded_replacements.append(pair) + encoded_replacements.sort(key=lambda row: len(row[0]), reverse=True) + + def scrub(value): + if type(value) is str: + for needle, marker in encoded_replacements: + value = value.replace(needle, marker) + return value + if type(value) is list: + return [scrub(item) for item in value] + if type(value) is dict: + return {key: scrub(item) for key, item in value.items()} return value - encoded = os.fsencode(environment_root) - return value.replace(encoded, b"") + + return scrub(packet) def _source(value): @@ -135,24 +184,55 @@ def _kill_process_group(process): def _run_in_process_group(command, *, cwd, env, timeout_seconds, - max_output_bytes, preexec_fn): - process = subprocess.Popen( - command, - cwd=cwd, - env=env, - stdin=subprocess.DEVNULL, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - shell=False, - preexec_fn=preexec_fn, - start_new_session=True, - ) + max_output_bytes, preexec_fn, pass_fds=(), + parent_close_fds=(), protocol_read_fd=None, + max_protocol_bytes=1048576): + try: + process = subprocess.Popen( + command, + cwd=cwd, + env=env, + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + shell=False, + preexec_fn=preexec_fn, + start_new_session=True, + pass_fds=tuple(pass_fds), + ) + except BaseException: + for descriptor in parent_close_fds: + try: + os.close(descriptor) + except OSError: + pass + if protocol_read_fd is not None: + try: + os.close(protocol_read_fd) + except OSError: + pass + raise + for descriptor in parent_close_fds: + try: + os.close(descriptor) + except OSError: + pass stdout = _BoundedStream(process.stdout, max_output_bytes) stderr = _BoundedStream(process.stderr, max_output_bytes) + protocol_stream = ( + os.fdopen(protocol_read_fd, "rb", buffering=0) + if protocol_read_fd is not None else None + ) + protocol_output = ( + _BoundedStream(protocol_stream, max_protocol_bytes) + if protocol_stream is not None else None + ) threads = [ threading.Thread(target=stdout.drain, daemon=True), threading.Thread(target=stderr.drain, daemon=True), ] + if protocol_output is not None: + threads.append(threading.Thread(target=protocol_output.drain, daemon=True)) for thread in threads: thread.start() timed_out = False @@ -164,7 +244,7 @@ def _run_in_process_group(command, *, cwd, env, timeout_seconds, finally: for thread in threads: thread.join(timeout=2.0) - for stream in (process.stdout, process.stderr): + for stream in (process.stdout, process.stderr, protocol_stream): if stream is not None: stream.close() for thread in threads: @@ -175,6 +255,12 @@ def _run_in_process_group(command, *, cwd, env, timeout_seconds, "stderr": bytes(stderr.data), "timed_out": timed_out, "output_limited": stdout.limited or stderr.limited, + "protocol": ( + bytes(protocol_output.data) if protocol_output is not None else b"" + ), + "protocol_limited": ( + bool(protocol_output.limited) if protocol_output is not None else False + ), } @@ -212,6 +298,8 @@ def _run_with_injected_runner(runner, command, *, cwd, env, timeout_seconds, "output_limited": ( len(stdout) > max_output_bytes or len(stderr) > max_output_bytes ), + "protocol": bytes(getattr(result, "protocol", b"") or b""), + "protocol_limited": False, } @@ -259,6 +347,242 @@ def run_snippet_isolated(*, snippet=None, snippet_source=None, code=None, enviro } +def _validated_sys_path_prefixes(values): + if values is None: + return () + if isinstance(values, (str, bytes, os.PathLike)): + raise ValueError("SYS_PATH_PREFIX_REFUSED") + result = [] + for value in values: + path = os.path.abspath(os.fspath(value)) + if ( + path != os.fspath(value) + or not os.path.isdir(path) + or os.path.islink(path) + ): + raise ValueError("SYS_PATH_PREFIX_REFUSED") + real = os.path.realpath(path) + if real != path or path in result: + raise ValueError("SYS_PATH_PREFIX_REFUSED") + result.append(path) + return tuple(result) + + +def _path_replacements(environment_root, sys_path_prefixes): + rows = [] + if environment_root is not None: + rows.append((environment_root, "")) + for index, root in enumerate(sys_path_prefixes, start=1): + marker = "" if index == 1 else f"" + rows.append((root, marker)) + return tuple(rows) + + +def _typed_result(packet, *, result, stdout, stderr, elapsed_ms): + observation = normalize_protocol_packet(packet) + return { + "status": packet["status"], + "observation": observation, + "reason_code": packet["reason_code"], + "raw_type": packet["raw_type"], + "stdout": stdout, + "stderr": stderr, + "returncode": result.get("returncode"), + "elapsed_ms": elapsed_ms, + } + + +def _typed_refusal(status, reason_code, *, result, stdout, stderr, elapsed_ms): + packet = protocol.status_packet(status, reason_code) + return _typed_result( + packet, + result=result, + stdout=stdout, + stderr=stderr, + elapsed_ms=elapsed_ms, + ) + + +def run_typed_snippet_isolated( + *, + snippet=None, + snippet_source=None, + code=None, + environment=None, + timeout_seconds=30.0, + max_output_bytes=1048576, + max_protocol_bytes=1048576, + sys_path_prefixes=(), + runner=None, +): + """Execute a snippet with a separate, framed observation channel. + + Normal stdout and stderr remain diagnostics. Only the private inherited pipe + can carry the typed result, so printed package output cannot be mistaken for an + observation. + """ + if sys.platform not in ("darwin", "linux"): + raise RuntimeError("PLATFORM_REFUSED") + source = _source( + snippet_source if snippet_source is not None else + code if code is not None else snippet + ) + if not source: + raise ValueError("snippet_source") + if ( + timeout_seconds <= 0 + or max_output_bytes < 1 + or type(max_protocol_bytes) is not int + or max_protocol_bytes < 256 + ): + raise ValueError("limits") + prefixes = _validated_sys_path_prefixes(sys_path_prefixes) + executable, environment_extra, environment_root = _environment_runtime(environment) + replacements = _path_replacements(environment_root, prefixes) + + with tempfile.TemporaryDirectory(prefix="isolated-runtime-") as fresh_cwd: + started = time.monotonic() + kwargs = { + "cwd": fresh_cwd, + "env": _safe_environment(environment_extra), + "timeout_seconds": timeout_seconds, + "max_output_bytes": max_output_bytes, + "preexec_fn": _resource_hook(), + } + if runner is None: + protocol_read_fd, protocol_write_fd = os.pipe() + child = protocol.child_source( + source, + protocol_fd=protocol_write_fd, + sys_path_prefixes=prefixes, + max_protocol_bytes=max_protocol_bytes, + ) + command = [executable, "-I", "-c", child] + result = _run_in_process_group( + command, + pass_fds=(protocol_write_fd,), + parent_close_fds=(protocol_write_fd,), + protocol_read_fd=protocol_read_fd, + max_protocol_bytes=max_protocol_bytes, + **kwargs, + ) + else: + child = protocol.child_source( + source, + protocol_fd=3, + sys_path_prefixes=prefixes, + max_protocol_bytes=max_protocol_bytes, + ) + command = [executable, "-I", "-c", child] + result = _run_with_injected_runner( + runner, + command, + max_output_bytes=max_output_bytes, + **{key: value for key, value in kwargs.items() + if key != "max_output_bytes"}, + ) + + stdout = _scrub_roots_bytes(result.get("stdout", b""), replacements) + stderr = _scrub_roots_bytes(result.get("stderr", b""), replacements) + elapsed_ms = ( + None if result.get("timed_out") + else int((time.monotonic() - started) * 1000) + ) + if result.get("timed_out"): + return _typed_refusal( + protocol.TIMEOUT, + "EXECUTION_TIMEOUT", + result=result, + stdout=stdout, + stderr=stderr, + elapsed_ms=None, + ) + if result.get("output_limited"): + return _typed_refusal( + protocol.OUTPUT_LIMIT_REFUSED, + "OUTPUT_LIMIT_REFUSED", + result=result, + stdout=stdout, + stderr=stderr, + elapsed_ms=elapsed_ms, + ) + if result.get("protocol_limited"): + return _typed_refusal( + protocol.PROTOCOL_REFUSED, + "PROTOCOL_SIZE_REFUSED", + result=result, + stdout=stdout, + stderr=stderr, + elapsed_ms=elapsed_ms, + ) + if result.get("returncode") != 0: + return _typed_refusal( + protocol.PROTOCOL_REFUSED, + "PROTOCOL_REFUSED", + result=result, + stdout=stdout, + stderr=stderr, + elapsed_ms=elapsed_ms, + ) + try: + packet = protocol.decode_packet( + result.get("protocol", b""), max_bytes=max_protocol_bytes + ) + except ValueError: + return _typed_refusal( + protocol.PROTOCOL_REFUSED, + "PROTOCOL_REFUSED", + result=result, + stdout=stdout, + stderr=stderr, + elapsed_ms=elapsed_ms, + ) + packet = _scrub_packet(packet, replacements) + return _typed_result( + packet, + result=result, + stdout=stdout, + stderr=stderr, + elapsed_ms=elapsed_ms, + ) + + +def _repeat_identity(result): + return ( + result["status"], + result["reason_code"], + result["raw_type"], + observation_identity(result["observation"]), + ) + + +def run_repeated_typed_snippet_isolated(*, runs=2, **kwargs): + """Repeat a typed execution and admit only byte-identical observations.""" + if type(runs) is not int or runs != 2: + raise ValueError("REPEAT_COUNT_REFUSED") + observed = [run_typed_snippet_isolated(**kwargs) for _ in range(runs)] + repeatable = _repeat_identity(observed[0]) == _repeat_identity(observed[1]) + if not repeatable: + return { + "runs": observed, + "repeatable": False, + "status": protocol.PROTOCOL_REFUSED, + "reason_code": "NONDETERMINISTIC_OBSERVATION", + "observation": None, + } + first = observed[0] + return { + "runs": observed, + "repeatable": True, + "status": first["status"], + "reason_code": first["reason_code"], + "observation": first["observation"], + } + + +run_typed_snippet_repeated = run_repeated_typed_snippet_isolated + + class PythonExecutor: def __init__(self, runner=None): self.runner = runner @@ -278,3 +602,30 @@ def execute(self, *, snippet=None, snippet_source=None, code=None, environment=N environment=environment, timeout_seconds=timeout_seconds, max_output_bytes=max_output_bytes, ) + + def run_typed( + self, + *, + snippet=None, + snippet_source=None, + code=None, + environment=None, + timeout_seconds=30.0, + max_output_bytes=1048576, + max_protocol_bytes=1048576, + sys_path_prefixes=(), + ): + return run_typed_snippet_isolated( + snippet=snippet, + snippet_source=snippet_source, + code=code, + environment=environment, + timeout_seconds=timeout_seconds, + max_output_bytes=max_output_bytes, + max_protocol_bytes=max_protocol_bytes, + sys_path_prefixes=sys_path_prefixes, + runner=self.runner, + ) + + def run_repeated_typed(self, **kwargs): + return run_repeated_typed_snippet_isolated(runner=self.runner, **kwargs) diff --git a/src/breakcheck/adapters/python/fixtures.py b/src/breakcheck/adapters/python/fixtures.py new file mode 100644 index 0000000..0352188 --- /dev/null +++ b/src/breakcheck/adapters/python/fixtures.py @@ -0,0 +1,728 @@ +from __future__ import annotations + +import ast +from dataclasses import dataclass +import hashlib +import json +import math +import os +from pathlib import Path, PurePosixPath +import re +import stat +from typing import Iterable, Mapping, Sequence + + +__all__ = ( + "FixtureBinding", + "FixtureFile", + "FixtureRefusal", + "REFUSAL_CODES", + "coverage_delta", + "deterministic", + "executable", + "fixture_yield", + "human_minutes", + "load_fixture_file", + "render_fixture_source", + "resolve_fixture_policy", + "suggest_fixtures", + "valid", +) + + +REFUSAL_CODES = frozenset( + { + "FIXTURE_AMBIGUOUS_REFUSED", + "FIXTURE_ARGUMENT_CAP_REFUSED", + "FIXTURE_AUTHOR_REFUSED", + "FIXTURE_BINDING_CAP_REFUSED", + "FIXTURE_BINDING_PATH_REFUSED", + "FIXTURE_DUPLICATE_BINDING_REFUSED", + "FIXTURE_DUPLICATE_FIELD_REFUSED", + "FIXTURE_ENCODING_REFUSED", + "FIXTURE_EXPRESSION_REFUSED", + "FIXTURE_FILE_CAP_REFUSED", + "FIXTURE_INVENTORY_REFUSED", + "FIXTURE_METRIC_REFUSED", + "FIXTURE_PATH_REFUSED", + "FIXTURE_POLICY_FORBID", + "FIXTURE_POLICY_REFUSED", + "FIXTURE_PROJECTION_REFUSED", + "FIXTURE_RENDER_REFUSED", + "FIXTURE_REQUIRED", + "FIXTURE_SCHEMA_REFUSED", + "FIXTURE_SCHEMA_VERSION_REFUSED", + "FIXTURE_SOURCE_CAP_REFUSED", + "FIXTURE_STALE_REFUSED", + "FIXTURE_SUGGESTION_EXISTS", + "FIXTURE_SUGGESTION_WRITE_REFUSED", + "FIXTURE_SYMLINK_REFUSED", + "FIXTURE_SYNTAX_REFUSED", + "FIXTURE_UNMATCHED_REFUSED", + } +) + + +_AUTHOR_VALUES = frozenset(("human", "agent", "unknown")) +_POLICY_VALUES = frozenset(("forbid", "allow", "require")) +_TOP_LEVEL_FIELDS = frozenset(("schema_version",)) +_BINDING_REQUIRED_FIELDS = frozenset( + ("file", "line", "column", "api", "args", "kwargs", "fixture_authored_by") +) +_BINDING_OPTIONAL_FIELDS = frozenset(("setup", "projection")) +_KEY = re.compile(r"[A-Za-z_][A-Za-z0-9_]*\Z") +_API = re.compile(r"[A-Za-z_][A-Za-z0-9_]*(?:\.[A-Za-z_][A-Za-z0-9_]*)*\Z") +_INTEGER = re.compile(r"(?:0|[1-9][0-9]*)\Z") +_MAX_FILE_BYTES = 262_144 +_MAX_BINDINGS = 256 +_MAX_ARGS = 64 +_MAX_KWARGS = 64 +_MAX_EXPRESSION_BYTES = 65_536 +_MAX_SETUP_BYTES = 65_536 +_MAX_EXPRESSION_NODES = 256 +_MAX_SETUP_NODES = 1_024 +_MAX_CONTEXT_BYTES = 512 + + +class FixtureRefusal(ValueError): + """A stable fail-closed fixture refusal.""" + + def __init__(self, code: str): + self.code = code + super().__init__(code) + + +@dataclass(frozen=True) +class FixtureBinding: + file: str + line: int + column: int + api: str + args: tuple[str, ...] + kwargs: tuple[tuple[str, str], ...] + setup: str + projection: str | None + fixture_authored_by: str + binding_sha256: str + + @property + def key(self) -> tuple[str, int, int, str]: + return (self.file, self.line, self.column, self.api) + + +@dataclass(frozen=True) +class FixtureFile: + path: str + bindings: tuple[FixtureBinding, ...] + file_sha256: str + canonical_sha256: str + + +def _refuse(code: str) -> None: + if code not in REFUSAL_CODES: + raise RuntimeError("FIXTURE_REFUSAL_UNDECLARED") + raise FixtureRefusal(code) + + +def _canonical(value: object) -> bytes: + return json.dumps( + value, sort_keys=True, separators=(",", ":"), ensure_ascii=True + ).encode("utf-8") + + +def _digest(value: object) -> str: + return hashlib.sha256(_canonical(value)).hexdigest() + + +def _strip_comment(line: str) -> str: + quoted = False + escaped = False + for index, character in enumerate(line): + if escaped: + escaped = False + continue + if quoted and character == "\\": + escaped = True + continue + if character == '"': + quoted = not quoted + continue + if character == "#" and not quoted: + return line[:index] + if quoted or escaped: + _refuse("FIXTURE_SYNTAX_REFUSED") + return line + + +def _split_unquoted(value: str, delimiter: str) -> list[str]: + pieces: list[str] = [] + start = 0 + quoted = False + escaped = False + nesting = 0 + for index, character in enumerate(value): + if escaped: + escaped = False + continue + if quoted and character == "\\": + escaped = True + continue + if character == '"': + quoted = not quoted + continue + if quoted: + continue + if character in "[{(": + nesting += 1 + elif character in "]})": + nesting -= 1 + if nesting < 0: + _refuse("FIXTURE_SYNTAX_REFUSED") + elif character == delimiter and nesting == 0: + pieces.append(value[start:index]) + start = index + 1 + if quoted or escaped or nesting != 0: + _refuse("FIXTURE_SYNTAX_REFUSED") + pieces.append(value[start:]) + return pieces + + +def _parse_string(value: str) -> str: + if not value.startswith('"') or not value.endswith('"'): + _refuse("FIXTURE_SYNTAX_REFUSED") + try: + parsed = json.loads(value) + except (json.JSONDecodeError, UnicodeError): + _refuse("FIXTURE_SYNTAX_REFUSED") + if not isinstance(parsed, str): + _refuse("FIXTURE_SYNTAX_REFUSED") + return parsed + + +def _parse_string_array(value: str) -> list[str]: + if not value.startswith("[") or not value.endswith("]"): + _refuse("FIXTURE_SYNTAX_REFUSED") + body = value[1:-1].strip() + if not body: + return [] + parsed = [] + for item in _split_unquoted(body, ","): + item = item.strip() + if not item: + _refuse("FIXTURE_SYNTAX_REFUSED") + parsed.append(_parse_string(item)) + return parsed + + +def _parse_inline_string_table(value: str) -> dict[str, str]: + if not value.startswith("{") or not value.endswith("}"): + _refuse("FIXTURE_SYNTAX_REFUSED") + body = value[1:-1].strip() + if not body: + return {} + result: dict[str, str] = {} + for item in _split_unquoted(body, ","): + if "=" not in item: + _refuse("FIXTURE_SYNTAX_REFUSED") + key, raw = item.split("=", 1) + key = key.strip() + if not _KEY.fullmatch(key): + _refuse("FIXTURE_SYNTAX_REFUSED") + if key in result: + _refuse("FIXTURE_DUPLICATE_FIELD_REFUSED") + result[key] = _parse_string(raw.strip()) + return result + + +def _parse_value(field: str, raw: str) -> object: + if field in ("schema_version", "line", "column"): + if not _INTEGER.fullmatch(raw): + _refuse("FIXTURE_SYNTAX_REFUSED") + return int(raw) + if field == "args": + return _parse_string_array(raw) + if field == "kwargs": + return _parse_inline_string_table(raw) + return _parse_string(raw) + + +def _parse_document(text: str) -> tuple[dict[str, object], list[dict[str, object]]]: + top: dict[str, object] = {} + bindings: list[dict[str, object]] = [] + current: dict[str, object] = top + for raw_line in text.splitlines(): + line = _strip_comment(raw_line).strip() + if not line: + continue + if line == "[[binding]]": + if len(bindings) >= _MAX_BINDINGS: + _refuse("FIXTURE_BINDING_CAP_REFUSED") + current = {} + bindings.append(current) + continue + if line.startswith("["): + _refuse("FIXTURE_SCHEMA_REFUSED") + if "=" not in line: + _refuse("FIXTURE_SYNTAX_REFUSED") + key, raw_value = line.split("=", 1) + key = key.strip() + raw_value = raw_value.strip() + allowed = ( + _TOP_LEVEL_FIELDS + if current is top + else _BINDING_REQUIRED_FIELDS | _BINDING_OPTIONAL_FIELDS + ) + if key not in allowed: + _refuse("FIXTURE_SCHEMA_REFUSED") + if key in current: + _refuse("FIXTURE_DUPLICATE_FIELD_REFUSED") + current[key] = _parse_value(key, raw_value) + if set(top) != _TOP_LEVEL_FIELDS: + _refuse("FIXTURE_SCHEMA_REFUSED") + return top, bindings + + +def _repository_root(value: os.PathLike[str] | str) -> Path: + root = Path(value) + if root.is_symlink() or not root.is_dir(): + _refuse("FIXTURE_PATH_REFUSED") + return Path(os.path.abspath(root)) + + +def _path_under_root( + value: os.PathLike[str] | str, root: Path, *, must_exist: bool +) -> Path: + candidate = Path(value) + if not candidate.is_absolute(): + candidate = root / candidate + candidate = Path(os.path.abspath(candidate)) + try: + relative = candidate.relative_to(root) + except ValueError: + _refuse("FIXTURE_PATH_REFUSED") + current = root + for part in relative.parts: + current = current / part + if current.is_symlink(): + _refuse("FIXTURE_SYMLINK_REFUSED") + if must_exist: + try: + metadata = candidate.stat() + except OSError: + _refuse("FIXTURE_PATH_REFUSED") + if not stat.S_ISREG(metadata.st_mode): + _refuse("FIXTURE_PATH_REFUSED") + return candidate + + +def _binding_file(value: object) -> str: + if not isinstance(value, str) or not value or "\\" in value: + _refuse("FIXTURE_BINDING_PATH_REFUSED") + path = PurePosixPath(value) + if path.is_absolute() or any(part in ("", ".", "..") for part in path.parts): + _refuse("FIXTURE_BINDING_PATH_REFUSED") + normalized = path.as_posix() + if normalized != value: + _refuse("FIXTURE_BINDING_PATH_REFUSED") + return normalized + + +def _validate_source(source: str, *, setup: bool = False) -> ast.AST: + cap = _MAX_SETUP_BYTES if setup else _MAX_EXPRESSION_BYTES + if len(source.encode("utf-8")) > cap: + _refuse("FIXTURE_SOURCE_CAP_REFUSED") + try: + tree = ast.parse(source, mode="exec" if setup else "eval") + except (SyntaxError, ValueError, TypeError, MemoryError, RecursionError): + _refuse("FIXTURE_EXPRESSION_REFUSED") + node_cap = _MAX_SETUP_NODES if setup else _MAX_EXPRESSION_NODES + if sum(1 for _ in ast.walk(tree)) > node_cap: + _refuse("FIXTURE_SOURCE_CAP_REFUSED") + return tree + + +def _binding_payload(row: Mapping[str, object]) -> dict[str, object]: + if set(row) - (_BINDING_REQUIRED_FIELDS | _BINDING_OPTIONAL_FIELDS): + _refuse("FIXTURE_SCHEMA_REFUSED") + if "fixture_authored_by" not in row: + _refuse("FIXTURE_AUTHOR_REFUSED") + if not _BINDING_REQUIRED_FIELDS.issubset(row): + _refuse("FIXTURE_SCHEMA_REFUSED") + file_name = _binding_file(row["file"]) + line = row["line"] + column = row["column"] + api = row["api"] + args = row["args"] + kwargs = row["kwargs"] + setup = row.get("setup", "") + projection = row.get("projection") + fixture_authored_by = row["fixture_authored_by"] + if ( + type(line) is not int + or line < 1 + or type(column) is not int + or column < 0 + or not isinstance(api, str) + or not _API.fullmatch(api) + or not isinstance(args, list) + or not isinstance(kwargs, dict) + or not isinstance(setup, str) + or (projection is not None and not isinstance(projection, str)) + ): + _refuse("FIXTURE_SCHEMA_REFUSED") + if fixture_authored_by not in _AUTHOR_VALUES: + _refuse("FIXTURE_AUTHOR_REFUSED") + if len(args) > _MAX_ARGS or len(kwargs) > _MAX_KWARGS: + _refuse("FIXTURE_ARGUMENT_CAP_REFUSED") + for source in args: + if not isinstance(source, str): + _refuse("FIXTURE_SCHEMA_REFUSED") + _validate_source(source) + ordered_kwargs = [] + for name in sorted(kwargs): + source = kwargs[name] + if not _KEY.fullmatch(name) or not isinstance(source, str): + _refuse("FIXTURE_SCHEMA_REFUSED") + _validate_source(source) + ordered_kwargs.append((name, source)) + if setup: + _validate_source(setup, setup=True) + if projection is not None: + tree = _validate_source(projection) + if not any(isinstance(node, ast.Name) and node.id == "outcome" for node in ast.walk(tree)): + _refuse("FIXTURE_PROJECTION_REFUSED") + return { + "file": file_name, + "line": line, + "column": column, + "api": api, + "args": list(args), + "kwargs": ordered_kwargs, + "setup": setup, + "projection": projection, + "fixture_authored_by": fixture_authored_by, + } + + +def _inventory_keys( + inventory: Iterable[Mapping[str, object]], +) -> tuple[dict[tuple[str, int, int, str], int], set[tuple[str, str]]]: + counts: dict[tuple[str, int, int, str], int] = {} + nearby: set[tuple[str, str]] = set() + try: + rows = list(inventory) + except TypeError: + _refuse("FIXTURE_INVENTORY_REFUSED") + for row in rows: + if not isinstance(row, Mapping): + _refuse("FIXTURE_INVENTORY_REFUSED") + try: + file_name = _binding_file(row["file"]) + line = row["line"] + column = row["column"] + api = row["api"] + except (KeyError, TypeError): + _refuse("FIXTURE_INVENTORY_REFUSED") + if ( + type(line) is not int + or line < 1 + or type(column) is not int + or column < 0 + or not isinstance(api, str) + or not _API.fullmatch(api) + ): + _refuse("FIXTURE_INVENTORY_REFUSED") + key = (file_name, line, column, api) + counts[key] = counts.get(key, 0) + 1 + nearby.add((file_name, api)) + return counts, nearby + + +def load_fixture_file( + path: os.PathLike[str] | str, + *, + repository_root: os.PathLike[str] | str, + inventory: Iterable[Mapping[str, object]], +) -> FixtureFile: + """Parse and exactly bind a closed fixture file without executing its source.""" + + root = _repository_root(repository_root) + source_path = _path_under_root(path, root, must_exist=True) + if source_path.stat().st_size > _MAX_FILE_BYTES: + _refuse("FIXTURE_FILE_CAP_REFUSED") + try: + raw = source_path.read_bytes() + text = raw.decode("utf-8", "strict") + except (OSError, UnicodeError): + _refuse("FIXTURE_ENCODING_REFUSED") + top, rows = _parse_document(text) + if top["schema_version"] != 1: + _refuse("FIXTURE_SCHEMA_VERSION_REFUSED") + inventory_counts, nearby = _inventory_keys(inventory) + bindings: list[FixtureBinding] = [] + seen: set[tuple[str, int, int, str]] = set() + payloads: list[dict[str, object]] = [] + for row in rows: + payload = _binding_payload(row) + key = ( + str(payload["file"]), + int(payload["line"]), + int(payload["column"]), + str(payload["api"]), + ) + if key in seen: + _refuse("FIXTURE_DUPLICATE_BINDING_REFUSED") + seen.add(key) + matches = inventory_counts.get(key, 0) + if matches > 1: + _refuse("FIXTURE_AMBIGUOUS_REFUSED") + if matches == 0: + if (key[0], key[3]) in nearby: + _refuse("FIXTURE_STALE_REFUSED") + _refuse("FIXTURE_UNMATCHED_REFUSED") + binding_sha256 = _digest(payload) + payloads.append(payload) + bindings.append( + FixtureBinding( + file=key[0], + line=key[1], + column=key[2], + api=key[3], + args=tuple(str(value) for value in payload["args"]), + kwargs=tuple((str(name), str(value)) for name, value in payload["kwargs"]), + setup=str(payload["setup"]), + projection=( + None if payload["projection"] is None else str(payload["projection"]) + ), + fixture_authored_by=str(payload["fixture_authored_by"]), + binding_sha256=binding_sha256, + ) + ) + bindings.sort(key=lambda binding: binding.key) + payloads.sort( + key=lambda payload: ( + str(payload["file"]), + int(payload["line"]), + int(payload["column"]), + str(payload["api"]), + ) + ) + canonical_sha256 = _digest( + { + "schema_version": 1, + "bindings": payloads, + } + ) + return FixtureFile( + path=source_path.relative_to(root).as_posix(), + bindings=tuple(bindings), + file_sha256=hashlib.sha256(raw).hexdigest(), + canonical_sha256=canonical_sha256, + ) + + +def resolve_fixture_policy( + policy: str = "forbid", + *, + fixture_path: os.PathLike[str] | str | None, + repository_root: os.PathLike[str] | str, + inventory: Iterable[Mapping[str, object]], +) -> FixtureFile | None: + """Apply the pure forbid/allow/require policy and parse only when admitted.""" + + if policy not in _POLICY_VALUES: + _refuse("FIXTURE_POLICY_REFUSED") + root = _repository_root(repository_root) + selected: os.PathLike[str] | str | None = fixture_path + if selected is None: + default = root / "breakcheck.fixtures.toml" + if default.exists() or default.is_symlink(): + selected = default + if policy == "forbid": + if selected is not None: + _refuse("FIXTURE_POLICY_FORBID") + return None + if selected is None: + if policy == "require": + _refuse("FIXTURE_REQUIRED") + return None + return load_fixture_file(selected, repository_root=root, inventory=inventory) + + +def render_fixture_source(binding: FixtureBinding, callable_source: str) -> str: + """Render admitted source for later isolated execution; this function never runs it.""" + + if not isinstance(binding, FixtureBinding) or not _API.fullmatch(callable_source): + _refuse("FIXTURE_RENDER_REFUSED") + arguments = list(binding.args) + arguments.extend(name + "=" + source for name, source in binding.kwargs) + pieces = [] + if binding.setup: + pieces.append(binding.setup.rstrip("\n") + "\n") + pieces.append("outcome = " + callable_source + "(" + ", ".join(arguments) + ")\n") + if binding.projection is not None: + pieces.append("outcome = " + binding.projection + "\n") + return "".join(pieces) + + +def _toml_string(value: str) -> str: + return json.dumps(value, ensure_ascii=True) + + +def _context(value: object) -> str | None: + if value is None: + return None + text = str(value).replace("\r", "\\r").replace("\n", "\\n") + encoded = text.encode("utf-8") + if len(encoded) > _MAX_CONTEXT_BYTES: + encoded = encoded[:_MAX_CONTEXT_BYTES] + text = encoded.decode("utf-8", "ignore") + "..." + return "".join(character if character >= " " else "?" for character in text) + + +def suggest_fixtures( + destination: os.PathLike[str] | str, + candidates: Iterable[Mapping[str, object]], + *, + repository_root: os.PathLike[str] | str, +) -> str: + """Write deterministic fixture skeletons once, refusing overwrite races.""" + + root = _repository_root(repository_root) + output = _path_under_root(destination, root, must_exist=False) + if output.exists() or output.is_symlink(): + _refuse("FIXTURE_SUGGESTION_EXISTS") + if not output.parent.is_dir() or output.parent.is_symlink(): + _refuse("FIXTURE_PATH_REFUSED") + normalized = [] + seen = set() + for candidate in candidates: + if not isinstance(candidate, Mapping): + _refuse("FIXTURE_INVENTORY_REFUSED") + counts, _ = _inventory_keys([candidate]) + key = next(iter(counts)) + if key in seen: + _refuse("FIXTURE_AMBIGUOUS_REFUSED") + seen.add(key) + normalized.append( + { + "key": key, + "signature": _context(candidate.get("signature")), + "type_hints": _context(candidate.get("type_hints")), + "nearby_source": _context(candidate.get("nearby_source")), + } + ) + normalized.sort(key=lambda item: item["key"]) + lines = [ + "# CONFIDENTIALITY WARNING: review source context and fixture values before sharing.", + "# Fixture expressions are executable source and must be human-reviewed.", + "schema_version = 1", + ] + for item in normalized: + file_name, line, column, api = item["key"] + lines.append("") + for label in ("signature", "type_hints", "nearby_source"): + if item[label] is not None: + lines.append("# " + label + ": " + str(item[label])) + lines.extend( + [ + "[[binding]]", + 'fixture_authored_by = "unknown"', + "file = " + _toml_string(file_name), + "line = " + str(line), + "column = " + str(column), + "api = " + _toml_string(api), + "args = []", + "kwargs = {}", + ] + ) + data = ("\n".join(lines) + "\n").encode("utf-8") + flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL + if hasattr(os, "O_NOFOLLOW"): + flags |= os.O_NOFOLLOW + descriptor = None + created = False + try: + descriptor = os.open(output, flags, 0o644) + created = True + with os.fdopen(descriptor, "wb") as stream: + descriptor = None + stream.write(data) + stream.flush() + os.fsync(stream.fileno()) + except FileExistsError: + _refuse("FIXTURE_SUGGESTION_EXISTS") + except OSError: + if created: + try: + output.unlink() + except OSError: + pass + _refuse("FIXTURE_SUGGESTION_WRITE_REFUSED") + finally: + if descriptor is not None: + os.close(descriptor) + return hashlib.sha256(data).hexdigest() + + +def _percentage(numerator: object, denominator: object) -> float: + if ( + isinstance(numerator, bool) + or isinstance(denominator, bool) + or not isinstance(numerator, (int, float)) + or not isinstance(denominator, (int, float)) + or not math.isfinite(float(numerator)) + or not math.isfinite(float(denominator)) + or denominator <= 0 + or numerator < 0 + or numerator > denominator + ): + _refuse("FIXTURE_METRIC_REFUSED") + return 100.0 * float(numerator) / float(denominator) + + +def fixture_yield(matched: object, candidates: object) -> float: + return _percentage(matched, candidates) + + +def valid(valid_fixtures: object, proposed_fixtures: object) -> float: + return _percentage(valid_fixtures, proposed_fixtures) + + +def executable(executable_fixtures: object, valid_fixtures: object) -> float: + return _percentage(executable_fixtures, valid_fixtures) + + +def deterministic(deterministic_fixtures: object, executable_fixtures: object) -> float: + return _percentage(deterministic_fixtures, executable_fixtures) + + +def coverage_delta(exercised_without: object, exercised_with: object) -> float: + """Return the measured fixture coverage multiplier. + + A zero non-fixture baseline has no finite multiplier, so it is refused rather + than represented with an invented sentinel or infinity. + """ + if ( + isinstance(exercised_without, bool) + or isinstance(exercised_with, bool) + or not isinstance(exercised_without, (int, float)) + or not isinstance(exercised_with, (int, float)) + or not math.isfinite(float(exercised_without)) + or not math.isfinite(float(exercised_with)) + or float(exercised_without) <= 0 + or float(exercised_with) < 0 + ): + _refuse("FIXTURE_METRIC_REFUSED") + return float(exercised_with) / float(exercised_without) + + +def human_minutes(observed_minutes: object) -> float: + if ( + isinstance(observed_minutes, bool) + or not isinstance(observed_minutes, (int, float)) + or not math.isfinite(float(observed_minutes)) + or observed_minutes < 0 + ): + _refuse("FIXTURE_METRIC_REFUSED") + return float(observed_minutes) diff --git a/src/breakcheck/adapters/python/literals.py b/src/breakcheck/adapters/python/literals.py index 8b90a82..1367223 100644 --- a/src/breakcheck/adapters/python/literals.py +++ b/src/breakcheck/adapters/python/literals.py @@ -1,34 +1,318 @@ +from __future__ import annotations + import ast import keyword +import math +import operator +from collections.abc import Iterable, Mapping +from dataclasses import dataclass, field + +from .coverage import order_provenance + + +_REFUSAL_CODE = "NONLITERAL_ARGS" +_NODE_BUDGET = 256 +_SEQUENCE_LENGTH_CAP = 10_000 +_TEXT_LENGTH_CAP = 65_536 +_INTEGER_BIT_CAP = 65_536 +_NESTED_CALL_DEPTH_CAP = 3 +_RENDERED_SOURCE_BYTES_CAP = 1_048_576 +_VALUE_DEPTH_CAP = 64 + +_SAFE_NESTED_MODULES = frozenset( + { + "base64", + "binascii", + "decimal", + "fractions", + "hashlib", + "json", + "math", + "re", + "struct", + "textwrap", + "urllib.parse", + } +) + +_REFUSAL_DETAILS = frozenset( + { + "ATTRIBUTE_ACCESS", + "COMPREHENSION", + "FOLDABLE_EXPR", + "FOLD_REFUSED", + "LOCAL_NAME", + "MODULE_CONSTANT", + "MODULE_CONSTANT_CROSS_MODULE", + "NESTED_CALL", + "NESTED_CALL_DEPTH_EXCEEDED", + "OTHER", + "STARRED", + } +) + + +class LiteralRefusal(ValueError): + """Backward-compatible literal refusal with bounded diagnostics.""" + + family = _REFUSAL_CODE + + def __init__(self, reason_detail: str = "OTHER") -> None: + if reason_detail not in _REFUSAL_DETAILS: + raise ValueError("LITERAL_REFUSAL_DETAIL_REFUSED") + self.reason_detail = reason_detail + super().__init__(self.family) + + def as_dict(self) -> dict[str, str]: + return { + "reason_code": self.family, + "reason_detail": self.reason_detail, + } + + +@dataclass(frozen=True) +class LiftedLiteral: + value: object + provenance: tuple[str, ...] + + +@dataclass(frozen=True) +class SynthesizedSnippet: + source: str + provenance: tuple[str, ...] + + +@dataclass +class _LiftContext: + module_constants: Mapping[str, LiftedLiteral] + imported_names: frozenset[str] + visited: int = 0 + rendered_bytes: list[int] = field(default_factory=lambda: [0]) + def consume(self) -> None: + self.visited += 1 + if self.visited > _NODE_BUDGET: + _refuse("FOLD_REFUSED") -_REFUSAL_CODE = 'NONLITERAL_ARGS' +def _refuse(reason_detail: str = "OTHER") -> None: + raise LiteralRefusal(reason_detail) -def _refuse(): - raise ValueError(_REFUSAL_CODE) +def _reason_detail(node: ast.AST) -> str: + if isinstance(node, (ast.BinOp, ast.JoinedStr)): + return "FOLDABLE_EXPR" + if isinstance(node, ast.Name): + return "LOCAL_NAME" + if isinstance(node, ast.Call): + return "NESTED_CALL" + if isinstance(node, ast.Attribute): + return "ATTRIBUTE_ACCESS" + if isinstance(node, (ast.ListComp, ast.SetComp, ast.DictComp, ast.GeneratorExp)): + return "COMPREHENSION" + if isinstance(node, ast.Starred): + return "STARRED" + return "OTHER" -def _lift(node): - if isinstance(node, ast.Constant) and isinstance(node.value, (type(None), bool, int, float, str, bytes)): - return node.value - if (isinstance(node, ast.UnaryOp) and isinstance(node.op, (ast.UAdd, ast.USub)) - and isinstance(node.operand, ast.Constant) - and type(node.operand.value) in (int, float)): - return node.operand.value if isinstance(node.op, ast.UAdd) else -node.operand.value + +def _provenance(values: Iterable[str]) -> tuple[str, ...]: + observed = tuple(values) + return order_provenance(observed or ("SOURCE_LITERAL",)) + + +def _bounded_literal_value(value: object) -> None: + if type(value) is int and value.bit_length() > _INTEGER_BIT_CAP: + _refuse("FOLD_REFUSED") + if type(value) is float and not math.isfinite(value): + _refuse("FOLD_REFUSED") + if isinstance(value, (str, bytes)) and len(value) > _TEXT_LENGTH_CAP: + _refuse("FOLD_REFUSED") + if isinstance(value, (list, tuple)) and len(value) > _SEQUENCE_LENGTH_CAP: + _refuse("FOLD_REFUSED") + + +def _sequence_multiplication_length(left: object, right: object) -> int | None: + if isinstance(left, (str, bytes, list, tuple)) and type(right) is int: + return len(left) * max(right, 0) + if isinstance(right, (str, bytes, list, tuple)) and type(left) is int: + return len(right) * max(left, 0) + return None + + +def _fold_binop(node: ast.BinOp, context: _LiftContext) -> LiftedLiteral: + left = _lift_result(node.left, context) + right = _lift_result(node.right, context) + op_type = type(node.op) + operations = { + ast.Add: operator.add, + ast.Sub: operator.sub, + ast.Mult: operator.mul, + ast.Div: operator.truediv, + ast.FloorDiv: operator.floordiv, + ast.Mod: operator.mod, + ast.Pow: operator.pow, + } + operation = operations.get(op_type) + if operation is None: + _refuse("FOLD_REFUSED") + if op_type is ast.Pow and ( + type(left.value) is not int + or type(right.value) is not int + or abs(right.value) > 64 + ): + _refuse("FOLD_REFUSED") + if op_type is ast.Mult: + result_length = _sequence_multiplication_length(left.value, right.value) + if result_length is not None: + cap = ( + _TEXT_LENGTH_CAP + if isinstance(left.value, (str, bytes)) + or isinstance(right.value, (str, bytes)) + else _SEQUENCE_LENGTH_CAP + ) + if result_length > cap: + _refuse("FOLD_REFUSED") + if op_type is ast.Add: + if isinstance(left.value, (str, bytes)) and isinstance( + right.value, type(left.value) + ): + if len(left.value) + len(right.value) > _TEXT_LENGTH_CAP: + _refuse("FOLD_REFUSED") + if isinstance(left.value, (list, tuple)) and isinstance( + right.value, type(left.value) + ): + if len(left.value) + len(right.value) > _SEQUENCE_LENGTH_CAP: + _refuse("FOLD_REFUSED") + if op_type is ast.Mod and isinstance(left.value, (str, bytes)): + _refuse("FOLD_REFUSED") + try: + value = operation(left.value, right.value) + except (ArithmeticError, OverflowError, TypeError, ValueError): + _refuse("FOLD_REFUSED") + if not isinstance(value, (type(None), bool, int, float, str, bytes, list, tuple)): + _refuse("FOLD_REFUSED") + _bounded_literal_value(value) + inherited = set(left.provenance) | set(right.provenance) + inherited.discard("SOURCE_LITERAL") + inherited.add("SOURCE_FOLDED") + return LiftedLiteral(value, _provenance(inherited)) + + +def _fold_joined_string(node: ast.JoinedStr, context: _LiftContext) -> LiftedLiteral: + pieces: list[str] = [] + inherited: set[str] = set() + for item in node.values: + if isinstance(item, ast.Constant) and isinstance(item.value, str): + pieces.append(item.value) + continue + if not isinstance(item, ast.FormattedValue) or item.conversion != -1: + _refuse("FOLD_REFUSED") + if item.format_spec is not None and not ( + isinstance(item.format_spec, ast.JoinedStr) and not item.format_spec.values + ): + _refuse("FOLD_REFUSED") + try: + lifted = _lift_result(item.value, context) + except LiteralRefusal as exc: + if exc.reason_detail == "FOLD_REFUSED": + raise + _refuse("FOLDABLE_EXPR") + if type(lifted.value) not in (str, int, float, bool): + _refuse("FOLD_REFUSED") + pieces.append(str(lifted.value)) + inherited.update(lifted.provenance) + value = "".join(pieces) + _bounded_literal_value(value) + inherited.discard("SOURCE_LITERAL") + inherited.add("SOURCE_FOLDED") + return LiftedLiteral(value, _provenance(inherited)) + + +def _lift_result(node: ast.AST, context: _LiftContext) -> LiftedLiteral: + context.consume() + if isinstance(node, ast.Constant) and isinstance( + node.value, (type(None), bool, int, float, str, bytes) + ): + return LiftedLiteral(node.value, ("SOURCE_LITERAL",)) + if ( + isinstance(node, ast.UnaryOp) + and isinstance(node.op, (ast.UAdd, ast.USub)) + and isinstance(node.operand, ast.Constant) + and type(node.operand.value) in (int, float) + ): + value = node.operand.value if isinstance(node.op, ast.UAdd) else -node.operand.value + _bounded_literal_value(value) + return LiftedLiteral(value, ("SOURCE_LITERAL",)) + if isinstance(node, ast.BinOp): + return _fold_binop(node, context) + if isinstance(node, ast.JoinedStr): + return _fold_joined_string(node, context) + if isinstance(node, ast.Name): + if node.id in context.module_constants: + resolved = context.module_constants[node.id] + if not isinstance(resolved, LiftedLiteral): + _refuse("MODULE_CONSTANT") + return LiftedLiteral( + resolved.value, + _provenance((*resolved.provenance, "SOURCE_MODULE_CONSTANT")), + ) + if node.id in context.imported_names: + _refuse("MODULE_CONSTANT_CROSS_MODULE") + _refuse("LOCAL_NAME") + if isinstance(node, ast.Attribute): + root = node + while isinstance(root, ast.Attribute): + root = root.value + if isinstance(root, ast.Name) and root.id in context.imported_names: + _refuse("MODULE_CONSTANT_CROSS_MODULE") + _refuse("ATTRIBUTE_ACCESS") if isinstance(node, ast.List): - return [_lift(item) for item in node.elts] + values = [_lift_result(item, context) for item in node.elts] + provenance = _provenance(value for item in values for value in item.provenance) + return LiftedLiteral([item.value for item in values], provenance) if isinstance(node, ast.Tuple): - return tuple(_lift(item) for item in node.elts) + values = [_lift_result(item, context) for item in node.elts] + provenance = _provenance(value for item in values for value in item.provenance) + return LiftedLiteral(tuple(item.value for item in values), provenance) if isinstance(node, ast.Dict) and all(key is not None for key in node.keys): + keys = [_lift_result(key, context) for key in node.keys] + values = [_lift_result(value, context) for value in node.values] try: - return {_lift(key): _lift(value) for key, value in zip(node.keys, node.values)} - except (TypeError, ValueError): + observed = { + key.value: value.value for key, value in zip(keys, values, strict=True) + } + except TypeError: _refuse() - _refuse() + provenance = _provenance( + value for item in (*keys, *values) for value in item.provenance + ) + return LiftedLiteral(observed, provenance) + _refuse(_reason_detail(node)) + + +def _context( + module_constants: Mapping[str, LiftedLiteral] | None = None, + imported_names: Iterable[str] = (), +) -> _LiftContext: + constants = {} if module_constants is None else module_constants + if not isinstance(constants, Mapping): + raise ValueError("MODULE_CONSTANT_TABLE_REFUSED") + names = frozenset(imported_names) + if any(not isinstance(name, str) or not name.isidentifier() for name in names): + raise ValueError("IMPORTED_NAME_TABLE_REFUSED") + return _LiftContext(constants, names) + + +def _lift( + node: ast.AST, + *, + module_constants: Mapping[str, LiftedLiteral] | None = None, + imported_names: Iterable[str] = (), +) -> object: + return _lift_result(node, _context(module_constants, imported_names)).value -def _call_node(source_text): +def _call_node(source_text: str) -> ast.Call: if not isinstance(source_text, str): _refuse() try: @@ -37,58 +321,107 @@ def _call_node(source_text): _refuse() if not isinstance(node, ast.Call): _refuse() - if any(keyword.arg is None for keyword in node.keywords): - _refuse() + if any(keyword_item.arg is None for keyword_item in node.keywords): + _refuse("STARRED") + if any(isinstance(argument, ast.Starred) for argument in node.args): + _refuse("STARRED") return node -def lift_literal_args(node): +def lift_literal_args( + node: ast.AST | str, + *, + module_constants: Mapping[str, LiftedLiteral] | None = None, + imported_names: Iterable[str] = (), +) -> object: + context = _context(module_constants, imported_names) if isinstance(node, str): call = _call_node(node) - return [_lift(item) for item in call.args] - return _lift(node) + return [_lift_result(item, context).value for item in call.args] + return _lift_result(node, context).value + + +def lift_with_provenance( + node: ast.AST, + *, + module_constants: Mapping[str, LiftedLiteral] | None = None, + imported_names: Iterable[str] = (), +) -> LiftedLiteral: + return _lift_result(node, _context(module_constants, imported_names)) + +def _reserve_rendered_bytes(budget: list[int], amount: int) -> None: + budget[0] += amount + if budget[0] > _RENDERED_SOURCE_BYTES_CAP: + _refuse("FOLD_REFUSED") -def _literal_source(value): + +def _literal_source( + value: object, + budget: list[int] | None = None, + depth: int = 0, +) -> str: + if budget is None: + budget = [0] + if depth > _VALUE_DEPTH_CAP: + _refuse("FOLD_REFUSED") if value is None or isinstance(value, (bool, int, float, str, bytes)): - return repr(value) + source = repr(value) + _reserve_rendered_bytes(budget, len(source.encode("utf-8"))) + return source if isinstance(value, list): - return "[" + ", ".join(_literal_source(item) for item in value) + "]" + _reserve_rendered_bytes(budget, 2 + max(0, len(value) - 1) * 2) + return "[" + ", ".join( + _literal_source(item, budget, depth + 1) for item in value + ) + "]" if isinstance(value, tuple): - body = ", ".join(_literal_source(item) for item in value) + _reserve_rendered_bytes( + budget, + 2 + max(0, len(value) - 1) * 2 + (1 if len(value) == 1 else 0), + ) + body = ", ".join( + _literal_source(item, budget, depth + 1) for item in value + ) if len(value) == 1: body += "," return "(" + body + ")" if isinstance(value, dict): - return "{" + ", ".join(_literal_source(key) + ": " + _literal_source(item) for key, item in value.items()) + "}" + _reserve_rendered_bytes( + budget, + 2 + max(0, len(value) - 1) * 2 + len(value) * 2, + ) + return "{" + ", ".join( + _literal_source(key, budget, depth + 1) + + ": " + + _literal_source(item, budget, depth + 1) + for key, item in value.items() + ) + "}" _refuse() -def _call_name(node): - parts = [] +def _call_name(node: ast.AST) -> str: + parts: list[str] = [] current = node while isinstance(current, ast.Attribute): parts.append(current.attr) current = current.value if not isinstance(current, ast.Name): - _refuse() + _refuse("NESTED_CALL") parts.append(current.id) parts.reverse() if not all(part.isidentifier() and not keyword.iskeyword(part) for part in parts): - _refuse() + _refuse("NESTED_CALL") return ".".join(parts) -def _import_source(source): +def _import_source(source: str) -> str: if not isinstance(source, str): _refuse() try: tree = ast.parse(source, mode="exec") except (SyntaxError, ValueError, TypeError): _refuse() - if len(tree.body) != 1 or not isinstance( - tree.body[0], (ast.Import, ast.ImportFrom) - ): + if len(tree.body) != 1 or not isinstance(tree.body[0], (ast.Import, ast.ImportFrom)): _refuse() node = tree.body[0] if len(node.names) != 1 or node.names[0].name == "*": @@ -106,47 +439,185 @@ def _import_source(source): if node.level != 0 or not node.module: _refuse() module_parts = node.module.split(".") - if not all( - part.isidentifier() and not keyword.iskeyword(part) - for part in module_parts - ) or not alias.name.isidentifier() or keyword.iskeyword(alias.name): + if not all(part.isidentifier() and not keyword.iskeyword(part) for part in module_parts): + _refuse() + if not alias.name.isidentifier() or keyword.iskeyword(alias.name): _refuse() return ast.unparse(node) -def _snippet(call, import_statement): +def _import_binding(import_statement: str) -> str: + node = ast.parse(import_statement).body[0] + alias = node.names[0] + if alias.asname: + return alias.asname + if isinstance(node, ast.Import): + return alias.name.split(".", 1)[0] + return alias.name + + +def _safe_nested_module(api: str) -> str | None: + matches = [ + module + for module in _SAFE_NESTED_MODULES + if api == module or api.startswith(module + ".") + ] + return max(matches, key=len) if matches else None + + +def _render_nested_call( + call: ast.Call, + context: _LiftContext, + *, + depth: int, + target_roots: frozenset[str], +) -> tuple[str, tuple[str, ...], set[str]]: + if depth > _NESTED_CALL_DEPTH_CAP: + _refuse("NESTED_CALL_DEPTH_EXCEEDED") + if any(keyword_item.arg is None for keyword_item in call.keywords) or any( + isinstance(argument, ast.Starred) for argument in call.args + ): + _refuse("STARRED") api = _call_name(call.func) - values = [_lift(item) for item in call.args] - keywords = [(item.arg, _lift(item.value)) for item in call.keywords] - rendered = [_literal_source(item) for item in values] - rendered.extend( - name + "=" + _literal_source(value) for name, value in keywords + root = api.split(".", 1)[0] + module = _safe_nested_module(api) + if root not in target_roots and module is None: + _refuse("NESTED_CALL") + imports = set() + if root not in target_roots and module is not None: + imports.add("import " + module) + rendered: list[str] = [] + provenance: set[str] = {"SOURCE_NESTED_CALL"} + for argument in call.args: + source, observed_provenance, observed_imports = _render_argument( + argument, context, depth=depth + 1, target_roots=target_roots + ) + rendered.append(source) + provenance.update(observed_provenance) + imports.update(observed_imports) + for item in call.keywords: + source, observed_provenance, observed_imports = _render_argument( + item.value, context, depth=depth + 1, target_roots=target_roots + ) + rendered.append(item.arg + "=" + source) + provenance.update(observed_provenance) + imports.update(observed_imports) + return ( + api + "(" + ", ".join(rendered) + ")", + _provenance(provenance), + imports, ) + + +def _render_argument( + node: ast.AST, + context: _LiftContext, + *, + depth: int, + target_roots: frozenset[str], +) -> tuple[str, tuple[str, ...], set[str]]: + if isinstance(node, ast.Call): + return _render_nested_call(node, context, depth=depth, target_roots=target_roots) + lifted = _lift_result(node, context) return ( - import_statement + _literal_source(lifted.value, context.rendered_bytes), + lifted.provenance, + set(), + ) + + +def _snippet_result( + call: ast.Call, + import_statement: str, + *, + module_constants: Mapping[str, LiftedLiteral] | None, + imported_names: Iterable[str], +) -> SynthesizedSnippet: + api = _call_name(call.func) + target_import = _import_source(import_statement) + target_roots = frozenset({_import_binding(target_import), api.split(".", 1)[0]}) + context = _context(module_constants, imported_names) + imports = {target_import} + rendered: list[str] = [] + provenance: set[str] = set() + for argument in call.args: + source, observed_provenance, observed_imports = _render_argument( + argument, context, depth=1, target_roots=target_roots + ) + rendered.append(source) + provenance.update(observed_provenance) + imports.update(observed_imports) + for item in call.keywords: + source, observed_provenance, observed_imports = _render_argument( + item.value, context, depth=1, target_roots=target_roots + ) + rendered.append(item.arg + "=" + source) + provenance.update(observed_provenance) + imports.update(observed_imports) + source = ( + "\n".join(sorted(imports)) + "\n\noutcome = " + api + "(" + ", ".join(rendered) + ")\nprint(repr(outcome))\n" ) + return SynthesizedSnippet(source, _provenance(provenance)) -def synthesize_snippet(*arguments): +def synthesize_with_provenance( + *arguments: object, + module_constants: Mapping[str, LiftedLiteral] | None = None, + imported_names: Iterable[str] = (), +) -> SynthesizedSnippet: if len(arguments) == 1 and isinstance(arguments[0], str): call = _call_node(arguments[0]) api = _call_name(call.func) - root = api.split(".", 1)[0] - return _snippet(call, "import " + root) + return _snippet_result( + call, + "import " + api.split(".", 1)[0], + module_constants=module_constants, + imported_names=imported_names, + ) if ( len(arguments) == 2 and isinstance(arguments[0], str) and isinstance(arguments[1], str) ): - return _snippet(_call_node(arguments[0]), _import_source(arguments[1])) + return _snippet_result( + _call_node(arguments[0]), + arguments[1], + module_constants=module_constants, + imported_names=imported_names, + ) if len(arguments) != 2: _refuse() function_name, node = arguments - if not isinstance(function_name, str) or not function_name.isidentifier() or keyword.iskeyword(function_name): + if ( + not isinstance(function_name, str) + or not function_name.isidentifier() + or keyword.iskeyword(function_name) + or not isinstance(node, ast.AST) + ): _refuse() - return function_name + "(" + _literal_source(_lift(node)) + ")" + context = _context(module_constants, imported_names) + lifted = _lift_result(node, context) + return SynthesizedSnippet( + function_name + + "(" + + _literal_source(lifted.value, context.rendered_bytes) + + ")", + lifted.provenance, + ) + + +def synthesize_snippet( + *arguments: object, + module_constants: Mapping[str, LiftedLiteral] | None = None, + imported_names: Iterable[str] = (), +) -> str: + return synthesize_with_provenance( + *arguments, + module_constants=module_constants, + imported_names=imported_names, + ).source diff --git a/src/breakcheck/adapters/python/normalization.py b/src/breakcheck/adapters/python/normalization.py index 2bceb51..40df39b 100644 --- a/src/breakcheck/adapters/python/normalization.py +++ b/src/breakcheck/adapters/python/normalization.py @@ -5,6 +5,32 @@ from collections.abc import Mapping +TYPE_TAG = "$breakcheck_type" +VALUE_TAG = "$breakcheck_value" +TAGGED_TYPES = frozenset({"bytes", "tuple", "set", "frozenset", "mapping"}) + + +def tagged_value_kind(value): + if ( + type(value) is dict + and set(value) == {TYPE_TAG, VALUE_TAG} + and type(value[TYPE_TAG]) is str + and value[TYPE_TAG] in TAGGED_TYPES + ): + return value[TYPE_TAG] + return None + + +def tagged_value_payload(value): + if tagged_value_kind(value) is None: + raise ValueError("UNTAGGED_VALUE_REFUSED") + return value[VALUE_TAG] + + +def _tag(kind, value): + return {TYPE_TAG: kind, VALUE_TAG: value} + + def _normalized(value): if value is None: return None @@ -19,7 +45,7 @@ def _normalized(value): raise ValueError("UNSTABLE_OBSERVATION_REFUSED") return float(repr(value)) if isinstance(value, bytes): - return value.hex() + return _tag("bytes", value.hex()) if isinstance(value, BaseException): return { "exception_class": value.__class__.__name__, @@ -29,9 +55,17 @@ def _normalized(value): if any(type(key) is not str for key in value): raise ValueError("UNSTABLE_OBSERVATION_REFUSED") items = sorted(value.items(), key=lambda item: item[0]) - return {key: _normalized(item) for key, item in items} - if isinstance(value, (list, tuple)): + normalized = {key: _normalized(item) for key, item in items} + if TYPE_TAG in normalized or VALUE_TAG in normalized: + return _tag( + "mapping", + [[key, normalized[key]] for key in sorted(normalized)], + ) + return normalized + if isinstance(value, list): return [_normalized(item) for item in value] + if isinstance(value, tuple): + return _tag("tuple", [_normalized(item) for item in value]) if isinstance(value, (set, frozenset)): items = [_normalized(item) for item in value] items.sort(key=lambda item: ( @@ -44,7 +78,7 @@ def _normalized(value): allow_nan=False, ), )) - return items + return _tag("frozenset" if isinstance(value, frozenset) else "set", items) raise ValueError("UNSTABLE_OBSERVATION_REFUSED") @@ -76,9 +110,43 @@ def normalize_outcome(value): } +def normalize_protocol_packet(packet): + """Convert an already validated child packet into a comparable observation. + + Refusal statuses deliberately return ``None``. They are execution evidence, + not observations, and therefore must never reach the equality engine. + """ + from breakcheck.adapters.python import protocol + + protocol.validate_packet(packet) + status = packet["status"] + if status == protocol.VALUE: + return { + "kind": "value", + "payload": packet["payload"], + "exception_class": None, + "duration_ms": None, + } + if status == protocol.EXCEPTION: + return { + "kind": "exception", + "payload": packet["payload"], + "exception_class": packet["exception_class"], + "duration_ms": None, + } + return None + + +def observation_identity(value): + """Return deterministic bytes for same-environment repeat comparison.""" + if value is None: + return b"null" + return canonical_json(value).encode("utf-8") + + def canonical_json(value): return json.dumps( - _normalized(value), + value, sort_keys=True, separators=(",", ":"), ensure_ascii=False, diff --git a/src/breakcheck/adapters/python/protocol.py b/src/breakcheck/adapters/python/protocol.py new file mode 100644 index 0000000..7f7ee88 --- /dev/null +++ b/src/breakcheck/adapters/python/protocol.py @@ -0,0 +1,375 @@ +from __future__ import annotations + +import hashlib +import hmac +import json +import math +import re +import struct + +from .normalization import TYPE_TAG, VALUE_TAG + + +PROTOCOL_VERSION = 1 +VALUE = "VALUE" +EXCEPTION = "EXCEPTION" +UNNORMALIZABLE = "UNNORMALIZABLE" +NETWORK_REFUSED = "NETWORK_REFUSED" +TIMEOUT = "TIMEOUT" +OUTPUT_LIMIT_REFUSED = "OUTPUT_LIMIT_REFUSED" +PROTOCOL_REFUSED = "PROTOCOL_REFUSED" + +STATUSES = frozenset( + { + VALUE, + EXCEPTION, + UNNORMALIZABLE, + NETWORK_REFUSED, + TIMEOUT, + OUTPUT_LIMIT_REFUSED, + PROTOCOL_REFUSED, + } +) + +_FIELDS = frozenset( + { + "protocol_version", + "status", + "payload", + "exception_class", + "reason_code", + "raw_type", + } +) +_MAGIC = b"BRKCHK2\0" +_HEADER = struct.Struct(">8sQ") +_DIGEST_SIZE = hashlib.sha256().digest_size +_DEFAULT_MAX_PROTOCOL_BYTES = 1024 * 1024 +_MAX_DEPTH = 64 +_MAX_NODES = 10000 +_MAX_CONTAINER_ITEMS = 10000 +_MAX_TEXT_BYTES = 65536 +_NAME = re.compile(r"[A-Za-z_][A-Za-z0-9_.]{0,255}\Z") +_REASON = re.compile(r"[A-Z][A-Z0-9_]{0,127}\Z") + + +def _refuse(): + raise ValueError(PROTOCOL_REFUSED) + + +def _json_value(value, *, depth=0, budget=None): + if budget is None: + budget = [0] + budget[0] += 1 + if budget[0] > _MAX_NODES or depth > _MAX_DEPTH: + _refuse() + if value is None or type(value) in (bool, int): + return + if type(value) is float: + if not math.isfinite(value): + _refuse() + return + if type(value) is str: + if len(value.encode("utf-8")) > _MAX_TEXT_BYTES: + _refuse() + return + if type(value) is list: + if len(value) > _MAX_CONTAINER_ITEMS: + _refuse() + for item in value: + _json_value(item, depth=depth + 1, budget=budget) + return + if type(value) is dict: + if len(value) > _MAX_CONTAINER_ITEMS or any( + type(key) is not str for key in value + ): + _refuse() + for key in sorted(value): + _json_value(key, depth=depth + 1, budget=budget) + _json_value(value[key], depth=depth + 1, budget=budget) + return + _refuse() + + +def _optional_name(value): + return value is None or (type(value) is str and _NAME.fullmatch(value)) + + +def _optional_reason(value): + return value is None or (type(value) is str and _REASON.fullmatch(value)) + + +def validate_packet(packet): + if type(packet) is not dict or set(packet) != _FIELDS: + _refuse() + if packet["protocol_version"] != PROTOCOL_VERSION: + _refuse() + status = packet["status"] + if status not in STATUSES: + _refuse() + if not _optional_name(packet["exception_class"]): + _refuse() + if not _optional_reason(packet["reason_code"]): + _refuse() + if not _optional_name(packet["raw_type"]): + _refuse() + _json_value(packet["payload"]) + + payload = packet["payload"] + exception_class = packet["exception_class"] + reason_code = packet["reason_code"] + raw_type = packet["raw_type"] + if status == VALUE: + if any(value is not None for value in (exception_class, reason_code, raw_type)): + _refuse() + elif status == EXCEPTION: + if exception_class is None or type(payload) is not list: + _refuse() + if reason_code is not None or raw_type is not None: + _refuse() + else: + if payload is not None or exception_class is not None or reason_code is None: + _refuse() + if status != UNNORMALIZABLE and raw_type is not None: + _refuse() + if status == UNNORMALIZABLE and raw_type is None: + _refuse() + return packet + + +def _canonical_payload(packet): + validate_packet(packet) + return json.dumps( + packet, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + allow_nan=False, + ).encode("utf-8") + + +def encode_packet(packet, *, max_bytes=_DEFAULT_MAX_PROTOCOL_BYTES): + if type(max_bytes) is not int or max_bytes < 256: + _refuse() + payload = _canonical_payload(packet) + frame = ( + _HEADER.pack(_MAGIC, len(payload)) + + payload + + hashlib.sha256(payload).digest() + ) + if len(frame) > max_bytes: + raise ValueError("PROTOCOL_SIZE_REFUSED") + return frame + + +def _closed_object(pairs): + result = {} + for key, value in pairs: + if key in result: + _refuse() + result[key] = value + return result + + +def _reject_constant(_value): + _refuse() + + +def decode_packet(data, *, max_bytes=_DEFAULT_MAX_PROTOCOL_BYTES): + if type(max_bytes) is not int or max_bytes < 256: + _refuse() + value = bytes(data or b"") + if len(value) > max_bytes or len(value) < _HEADER.size + _DIGEST_SIZE: + _refuse() + magic, payload_size = _HEADER.unpack(value[: _HEADER.size]) + if magic != _MAGIC: + _refuse() + expected_size = _HEADER.size + payload_size + _DIGEST_SIZE + if len(value) != expected_size: + _refuse() + payload = value[_HEADER.size : _HEADER.size + payload_size] + digest = value[-_DIGEST_SIZE:] + if not hmac.compare_digest(digest, hashlib.sha256(payload).digest()): + _refuse() + try: + packet = json.loads( + payload.decode("utf-8", "strict"), + object_pairs_hook=_closed_object, + parse_constant=_reject_constant, + ) + except (UnicodeDecodeError, json.JSONDecodeError, ValueError, TypeError) as exc: + raise ValueError(PROTOCOL_REFUSED) from exc + return validate_packet(packet) + + +def status_packet(status, reason_code, *, raw_type=None): + packet = { + "protocol_version": PROTOCOL_VERSION, + "status": status, + "payload": None, + "exception_class": None, + "reason_code": reason_code, + "raw_type": raw_type, + } + return validate_packet(packet) + + +def child_source( + snippet_source, + *, + protocol_fd, + sys_path_prefixes=(), + max_protocol_bytes=_DEFAULT_MAX_PROTOCOL_BYTES, +): + if type(snippet_source) is not str or not snippet_source: + raise ValueError("snippet_source") + if type(protocol_fd) is not int or protocol_fd < 0: + _refuse() + if type(max_protocol_bytes) is not int or max_protocol_bytes < 256: + _refuse() + prefixes = tuple(str(value) for value in sys_path_prefixes) + return f'''import collections.abc as _bc_collections +import hashlib as _bc_hashlib +import json as _bc_json +import math as _bc_math +import os as _bc_os +import struct as _bc_struct +import sys as _bc_sys + +_bc_dumps = _bc_json.dumps +_bc_isfinite = _bc_math.isfinite +_bc_mapping = _bc_collections.Mapping +_bc_pack = _bc_struct.pack +_bc_sha256 = _bc_hashlib.sha256 +_bc_write = _bc_os.write +_bc_type_tag = {TYPE_TAG!r} +_bc_value_tag = {VALUE_TAG!r} + +def _bc_tag(kind, value): + return {{_bc_type_tag: kind, _bc_value_tag: value}} + +class _BreakcheckNetworkRefused(BaseException): + pass + +class _BreakcheckUnnormalizable(BaseException): + def __init__(self, raw_type): + self.raw_type = raw_type + +def _bc_audit(event, args): + if event.startswith("socket."): + _bc_network_attempted[0] = True + raise _BreakcheckNetworkRefused() + +_bc_network_attempted = [False] +_bc_sys.addaudithook(_bc_audit) +_bc_budget = [0] + +def _bc_normalize(value, depth=0): + _bc_budget[0] += 1 + if _bc_budget[0] > {_MAX_NODES!r} or depth > {_MAX_DEPTH!r}: + raise _BreakcheckUnnormalizable(type(value).__name__) + if value is None or type(value) in (bool, str, int): + if type(value) is str and len(value.encode("utf-8")) > {_MAX_TEXT_BYTES!r}: + raise _BreakcheckUnnormalizable("str") + return value + if type(value) is float: + if not _bc_isfinite(value): + raise _BreakcheckUnnormalizable("float") + return float(repr(value)) + if isinstance(value, bytes): + if len(value) > {_MAX_TEXT_BYTES!r}: + raise _BreakcheckUnnormalizable("bytes") + return _bc_tag("bytes", value.hex()) + if isinstance(value, _bc_mapping): + if len(value) > {_MAX_CONTAINER_ITEMS!r} or any(type(key) is not str for key in value): + raise _BreakcheckUnnormalizable(type(value).__name__) + normalized = {{key: _bc_normalize(value[key], depth + 1) for key in sorted(value)}} + if _bc_type_tag in normalized or _bc_value_tag in normalized: + return _bc_tag("mapping", [[key, normalized[key]] for key in sorted(normalized)]) + return normalized + if isinstance(value, list): + if len(value) > {_MAX_CONTAINER_ITEMS!r}: + raise _BreakcheckUnnormalizable(type(value).__name__) + return [_bc_normalize(item, depth + 1) for item in value] + if isinstance(value, tuple): + if len(value) > {_MAX_CONTAINER_ITEMS!r}: + raise _BreakcheckUnnormalizable(type(value).__name__) + return _bc_tag("tuple", [_bc_normalize(item, depth + 1) for item in value]) + if isinstance(value, (set, frozenset)): + if len(value) > {_MAX_CONTAINER_ITEMS!r}: + raise _BreakcheckUnnormalizable(type(value).__name__) + items = [_bc_normalize(item, depth + 1) for item in value] + items.sort(key=lambda item: (type(item).__name__, _bc_dumps(item, sort_keys=True, separators=(",", ":"), ensure_ascii=False, allow_nan=False))) + return _bc_tag("frozenset" if isinstance(value, frozenset) else "set", items) + raise _BreakcheckUnnormalizable(type(value).__name__) + +def _bc_packet(status, payload=None, exception_class=None, reason_code=None, raw_type=None): + return {{ + "protocol_version": {PROTOCOL_VERSION!r}, + "status": status, + "payload": payload, + "exception_class": exception_class, + "reason_code": reason_code, + "raw_type": raw_type, + }} + +for _bc_prefix in reversed({prefixes!r}): + _bc_sys.path.insert(0, _bc_prefix) + +_bc_scope = {{"__name__": "__main__", "__file__": ""}} +_bc_target_exception = None +try: + exec(compile({snippet_source!r}, "", "exec"), _bc_scope, _bc_scope) +except BaseException as _bc_exc: + _bc_target_exception = _bc_exc + +if _bc_network_attempted[0]: + _bc_result = _bc_packet("NETWORK_REFUSED", reason_code="NETWORK_ACCESS_REFUSED") +elif _bc_target_exception is not None: + try: + _bc_budget[0] = 0 + _bc_args = _bc_normalize(list(_bc_target_exception.args)) + _bc_result = _bc_packet("EXCEPTION", payload=_bc_args, exception_class=type(_bc_target_exception).__name__) + except BaseException: + _bc_result = _bc_packet("UNNORMALIZABLE", reason_code="UNSTABLE_OBSERVATION_REFUSED", raw_type=type(_bc_target_exception).__name__) +elif "outcome" not in _bc_scope: + _bc_result = _bc_packet("PROTOCOL_REFUSED", reason_code="OUTCOME_MISSING") +else: + try: + _bc_budget[0] = 0 + _bc_value = _bc_normalize(_bc_scope["outcome"]) + _bc_result = _bc_packet("VALUE", payload=_bc_value) + except BaseException as _bc_exc: + _bc_raw_type = getattr(_bc_exc, "raw_type", type(_bc_scope["outcome"]).__name__) + _bc_result = _bc_packet("UNNORMALIZABLE", reason_code="UNSTABLE_OBSERVATION_REFUSED", raw_type=_bc_raw_type) + +_bc_payload = _bc_dumps(_bc_result, sort_keys=True, separators=(",", ":"), ensure_ascii=False, allow_nan=False).encode("utf-8") +_bc_frame = _bc_pack(">8sQ", {_MAGIC!r}, len(_bc_payload)) + _bc_payload + _bc_sha256(_bc_payload).digest() +if len(_bc_frame) > {max_protocol_bytes!r}: + _bc_result = _bc_packet("PROTOCOL_REFUSED", reason_code="PROTOCOL_SIZE_REFUSED") + _bc_payload = _bc_dumps(_bc_result, sort_keys=True, separators=(",", ":"), ensure_ascii=False, allow_nan=False).encode("utf-8") + _bc_frame = _bc_pack(">8sQ", {_MAGIC!r}, len(_bc_payload)) + _bc_payload + _bc_sha256(_bc_payload).digest() + +_bc_view = memoryview(_bc_frame) +while _bc_view: + _bc_written = _bc_write({protocol_fd!r}, _bc_view) + _bc_view = _bc_view[_bc_written:] +''' + + +__all__ = [ + "PROTOCOL_VERSION", + "VALUE", + "EXCEPTION", + "UNNORMALIZABLE", + "NETWORK_REFUSED", + "TIMEOUT", + "OUTPUT_LIMIT_REFUSED", + "PROTOCOL_REFUSED", + "STATUSES", + "validate_packet", + "encode_packet", + "decode_packet", + "status_packet", + "child_source", +] diff --git a/src/breakcheck/adapters/python/scanner.py b/src/breakcheck/adapters/python/scanner.py index 335fe9e..ee03117 100644 --- a/src/breakcheck/adapters/python/scanner.py +++ b/src/breakcheck/adapters/python/scanner.py @@ -1,6 +1,169 @@ import ast +import math +from dataclasses import dataclass from pathlib import Path +from .coverage import make_candidate +from .literals import LiftedLiteral, LiteralRefusal, lift_with_provenance + + +@dataclass(frozen=True) +class StaticContext: + module_constants: dict[str, LiftedLiteral] + imported_names: frozenset[str] + + +def _is_deeply_immutable(value): + if type(value) in (type(None), bool, int, str, bytes): + return True + if type(value) is float: + return math.isfinite(value) + if type(value) is tuple: + return all(_is_deeply_immutable(item) for item in value) + return False + + +class _BindingInventory(ast.NodeVisitor): + def __init__(self): + self.bindings = {} + self.parameters = set() + self.declarations = set() + + def _bind(self, name): + if isinstance(name, str) and name: + self.bindings[name] = self.bindings.get(name, 0) + 1 + + def visit_Name(self, node): + if isinstance(node.ctx, (ast.Store, ast.Del)): + self._bind(node.id) + self.generic_visit(node) + + def _visit_arguments(self, arguments): + for argument in ( + *arguments.posonlyargs, + *arguments.args, + *arguments.kwonlyargs, + ): + self.parameters.add(argument.arg) + if arguments.vararg is not None: + self.parameters.add(arguments.vararg.arg) + if arguments.kwarg is not None: + self.parameters.add(arguments.kwarg.arg) + + def visit_FunctionDef(self, node): + self._bind(node.name) + self._visit_arguments(node.args) + self.generic_visit(node) + + def visit_AsyncFunctionDef(self, node): + self.visit_FunctionDef(node) + + def visit_Lambda(self, node): + self._visit_arguments(node.args) + self.generic_visit(node) + + def visit_ClassDef(self, node): + self._bind(node.name) + self.generic_visit(node) + + def visit_Import(self, node): + for alias in node.names: + self._bind(alias.asname or alias.name.split(".", 1)[0]) + + def visit_ImportFrom(self, node): + for alias in node.names: + if alias.name != "*": + self._bind(alias.asname or alias.name) + + def visit_ExceptHandler(self, node): + self._bind(node.name) + self.generic_visit(node) + + def visit_Global(self, node): + self.declarations.update(node.names) + + def visit_Nonlocal(self, node): + self.declarations.update(node.names) + + def visit_MatchAs(self, node): + self._bind(node.name) + self.generic_visit(node) + + def visit_MatchStar(self, node): + self._bind(node.name) + + def visit_MatchMapping(self, node): + self._bind(node.rest) + self.generic_visit(node) + + +def _imported_names(tree): + names = set() + for node in ast.walk(tree): + if isinstance(node, ast.Import): + for alias in node.names: + names.add(alias.asname or alias.name.split(".", 1)[0]) + elif isinstance(node, ast.ImportFrom): + for alias in node.names: + if alias.name != "*": + names.add(alias.asname or alias.name) + return frozenset(names) + + +def build_static_context(source): + if isinstance(source, str): + tree = ast.parse(source) + elif isinstance(source, ast.Module): + tree = source + else: + raise ValueError("STATIC_CONTEXT_SOURCE_REFUSED") + imported_names = _imported_names(tree) + if any( + isinstance(node, ast.Call) + and isinstance(node.func, ast.Name) + and node.func.id == "globals" + for node in ast.walk(tree) + ): + return StaticContext({}, imported_names) + inventory = _BindingInventory() + inventory.visit(tree) + candidates = {} + for node in tree.body: + if ( + isinstance(node, ast.Assign) + and node.col_offset == 0 + and len(node.targets) == 1 + and isinstance(node.targets[0], ast.Name) + ): + candidates[node.targets[0].id] = node.value + elif ( + isinstance(node, ast.AnnAssign) + and node.col_offset == 0 + and isinstance(node.target, ast.Name) + ): + if node.value is not None: + candidates[node.target.id] = node.value + constants = {} + for name in sorted(candidates): + if ( + inventory.bindings.get(name) != 1 + or name in inventory.parameters + or name in inventory.declarations + or name in imported_names + ): + continue + try: + lifted = lift_with_provenance(candidates[name]) + except LiteralRefusal: + continue + if _is_deeply_immutable(lifted.value): + constants[name] = lifted + return StaticContext(constants, imported_names) + + +def build_module_constant_table(source): + return build_static_context(source).module_constants + def _syntax_refusal(path, exc): return { @@ -18,6 +181,12 @@ def __init__(self, package_name=None): raise ValueError("PACKAGE_NAME_REFUSED") self.package_name = package_name + @staticmethod + def build_static_context(source_text): + """Return explicit, AST-only resolution context without changing scan rows.""" + + return build_static_context(source_text) + def _rooted(self, value): return value == self.package_name or value.startswith(self.package_name + ".") @@ -64,20 +233,24 @@ def extract_call_sites(self, source_text=None, file_name=None, *, source=None, p tree = ast.parse(text, filename=name) bindings, _ = scanner._imports(tree, name) rebound_at = {} + + def record_rebinding(local_name, node): + if local_name in bindings: + rebound_at[local_name] = min( + rebound_at.get(local_name, node.lineno), node.lineno + ) + for node in ast.walk(tree): - targets = [] - if isinstance(node, ast.Assign): - targets = list(node.targets) - elif isinstance(node, (ast.AnnAssign, ast.NamedExpr)): - targets = [node.target] - elif isinstance(node, (ast.For, ast.AsyncFor)): - targets = [node.target] - for target in targets: - for child in ast.walk(target): - if isinstance(child, ast.Name) and child.id in bindings: - rebound_at[child.id] = min( - rebound_at.get(child.id, node.lineno), node.lineno - ) + if isinstance(node, ast.Name) and isinstance( + node.ctx, (ast.Store, ast.Del) + ): + record_rebinding(node.id, node) + elif isinstance(node, ast.arg): + record_rebinding(node.arg, node) + elif isinstance(node, ast.ExceptHandler): + record_rebinding(node.name, node) + elif isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): + record_rebinding(node.name, node) rows = [] unsupported = [] for node in ast.walk(tree): @@ -138,6 +311,31 @@ def extract_call_sites(self, source_text=None, file_name=None, *, source=None, p scanner._last_unsupported = unsupported return rows + @staticmethod + def _candidates(call_sites, unsupported): + rows = [] + for row in call_sites: + rows.append({**make_candidate( + api=row["api"], + file=row["file"], + line=row["line"], + column=row["column"], + ), "reason_code": None}) + for row in unsupported: + if row.get("reason_code") == "SOURCE_SYNTAX_REFUSED": + continue + rows.append({**make_candidate( + api=row["api"], + file=row["file"], + line=row["line"], + column=row["column"], + ), "reason_code": row["reason_code"]}) + rows.sort(key=lambda row: ( + row["file"], row["line"], row["column"], row["api"], + row["candidate_id"], + )) + return rows + def scan(self, repository=None, package_name=None, *, source=None, path=None, package=None, repo=None): selected_package = package or package_name or self.package_name if source is not None: @@ -149,6 +347,7 @@ def scan(self, repository=None, package_name=None, *, source=None, path=None, pa "imports": [], "call_sites": [], "unsupported": [_syntax_refusal(selected_path, exc)], + "candidates": [], } scanner = self if selected_package == self.package_name else type(self)(selected_package) result = { @@ -156,6 +355,9 @@ def scan(self, repository=None, package_name=None, *, source=None, path=None, pa "call_sites": scanner.extract_call_sites(source=source, path=selected_path, package=selected_package), } result["unsupported"] = list(getattr(scanner, "_last_unsupported", ())) + result["candidates"] = scanner._candidates( + result["call_sites"], result["unsupported"] + ) return result root = Path(repository if repository is not None else repo).resolve() package = selected_package @@ -181,7 +383,12 @@ def scan(self, repository=None, package_name=None, *, source=None, path=None, pa imports.sort(key=lambda row: (row["line"], row["file"], row["module"], row["alias"])) calls.sort(key=lambda row: (row["line"], row["column"], row["file"], row["api"])) unsupported.sort(key=lambda row: (row["line"], row["column"], row["file"], row["api"])) - return {"imports": imports, "call_sites": calls, "unsupported": unsupported} + return { + "imports": imports, + "call_sites": calls, + "unsupported": unsupported, + "candidates": scanner._candidates(calls, unsupported), + } def _first(*values): diff --git a/src/breakcheck/adapters/python/symbols.py b/src/breakcheck/adapters/python/symbols.py new file mode 100644 index 0000000..907a8fd --- /dev/null +++ b/src/breakcheck/adapters/python/symbols.py @@ -0,0 +1,353 @@ +from __future__ import annotations + +import ast +from dataclasses import dataclass +import hashlib +import json +import os +from pathlib import Path +import stat +import subprocess +from typing import Iterable + + +class SymbolAnalysisRefusal(ValueError): + """A source tree could not be analyzed without guessing.""" + + def __init__(self, code: str): + super().__init__(code) + self.code = code + + +def _sha256(value: object) -> str: + payload = json.dumps(value, sort_keys=True, separators=(",", ":")).encode("utf-8") + return hashlib.sha256(payload).hexdigest() + + +def _ast_value(value: object) -> object: + if isinstance(value, ast.AST): + return ast.dump(value, annotate_fields=True, include_attributes=False) + if isinstance(value, list): + return [_ast_value(item) for item in value] + if isinstance(value, tuple): + return [_ast_value(item) for item in value] + return value + + +def _module_name(relative: Path) -> str: + parts = list(relative.with_suffix("").parts) + if parts and parts[0] == "src": + parts = parts[1:] + if parts and parts[-1] == "__init__": + parts = parts[:-1] + if not parts: + return "__init__" + return ".".join(parts) + + +def _iter_python_files(root: Path) -> Iterable[Path]: + requested = Path(root) + if requested.is_symlink() or not requested.is_dir(): + raise SymbolAnalysisRefusal("SYMBOL_ROOT_REFUSED") + resolved = requested.resolve(strict=True) + for path in sorted(resolved.rglob("*.py"), key=lambda item: item.relative_to(resolved).as_posix()): + relative = path.relative_to(resolved) + if any(part.startswith(".") or part in {"site-packages", "venv"} for part in relative.parts): + continue + if path.is_symlink() or not path.is_file(): + raise SymbolAnalysisRefusal("SYMBOL_SOURCE_SYMLINK_REFUSED") + yield path + + +def _signature_payload(node: ast.FunctionDef | ast.AsyncFunctionDef) -> object: + return { + "arguments": _ast_value(node.args), + "returns": _ast_value(node.returns), + "type_comment": node.type_comment, + "type_params": _ast_value(getattr(node, "type_params", [])), + } + + +def _behavior_payload(node: ast.FunctionDef | ast.AsyncFunctionDef) -> object: + return { + "async": isinstance(node, ast.AsyncFunctionDef), + "decorators": _ast_value(node.decorator_list), + "body": _ast_value(node.body), + } + + +def _context_payload( + module: ast.Module, + top_level: ast.stmt, + target: ast.FunctionDef | ast.AsyncFunctionDef, +) -> object: + callable_declarations = (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef) + module_context = [ + _ast_value(statement) + for statement in module.body + if statement is not top_level + and not isinstance(statement, callable_declarations) + ] + class_context: object = None + if isinstance(top_level, ast.ClassDef): + class_context = { + "bases": _ast_value(top_level.bases), + "keywords": _ast_value(top_level.keywords), + "decorators": _ast_value(top_level.decorator_list), + "type_params": _ast_value(getattr(top_level, "type_params", [])), + "body": [ + _ast_value(statement) + for statement in top_level.body + if statement is not target + and not isinstance(statement, callable_declarations) + ], + } + return {"module": module_context, "class": class_context} + + +@dataclass(frozen=True) +class SymbolDefinition: + target: str + module: str + symbol: str + kind: str + relative_path: str + line: int + column: int + signature_sha256: str + behavior_sha256: str + context_sha256: str + definition_sha256: str + + +@dataclass(frozen=True) +class SymbolInventory: + definitions: tuple[SymbolDefinition, ...] + + def by_target(self) -> dict[str, tuple[SymbolDefinition, ...]]: + grouped: dict[str, list[SymbolDefinition]] = {} + for definition in self.definitions: + grouped.setdefault(definition.target, []).append(definition) + return { + target: tuple(sorted(values, key=lambda value: (value.relative_path, value.line, value.column))) + for target, values in sorted(grouped.items()) + } + + +@dataclass(frozen=True) +class SymbolChange: + target: str + status: str + base: SymbolDefinition | None + head: SymbolDefinition | None + + +@dataclass(frozen=True) +class TrackedTreeEntry: + path: str + mode: str + kind: str + sha256: str + size: int + + +@dataclass(frozen=True) +class TrackedTreeIdentity: + sha256: str + entries: tuple[TrackedTreeEntry, ...] + regular_files: int + symlinks: int + + +def _git_output(root: Path, *arguments: str) -> bytes: + environment = { + key: os.environ[key] + for key in ("PATH", "SYSTEMROOT", "TMPDIR", "TEMP", "TMP") + if key in os.environ + } + environment.update( + { + "GIT_CONFIG_GLOBAL": os.devnull, + "GIT_CONFIG_NOSYSTEM": "1", + "GIT_CONFIG_SYSTEM": os.devnull, + "GIT_OPTIONAL_LOCKS": "0", + "GIT_TERMINAL_PROMPT": "0", + } + ) + try: + result = subprocess.run( + ["git", "-C", str(root), *arguments], + check=False, + capture_output=True, + env=environment, + shell=False, + ) + except OSError as exc: + raise SymbolAnalysisRefusal("TRACKED_TREE_REFUSED") from exc + if result.returncode != 0: + raise SymbolAnalysisRefusal("TRACKED_TREE_REFUSED") + return result.stdout + + +def tracked_tree_identity(root: Path | str) -> TrackedTreeIdentity: + requested = Path(root) + if requested.is_symlink() or not requested.is_dir(): + raise SymbolAnalysisRefusal("TRACKED_TREE_ROOT_REFUSED") + resolved = requested.resolve(strict=True) + try: + top = Path(_git_output(resolved, "rev-parse", "--show-toplevel").decode("utf-8").strip()).resolve() + except (UnicodeDecodeError, OSError) as exc: + raise SymbolAnalysisRefusal("TRACKED_TREE_ROOT_REFUSED") from exc + if top != resolved: + raise SymbolAnalysisRefusal("TRACKED_TREE_ROOT_REFUSED") + entries = [] + for record in _git_output(top, "ls-files", "--stage", "-z").split(b"\0"): + if not record: + continue + try: + metadata, raw_path = record.split(b"\t", 1) + mode, _, stage = metadata.split(b" ", 2) + relative_text = raw_path.decode("utf-8", errors="strict") + except (ValueError, UnicodeDecodeError) as exc: + raise SymbolAnalysisRefusal("TRACKED_TREE_REFUSED") from exc + relative = Path(relative_text) + if stage != b"0" or relative.is_absolute() or not relative.parts or ".." in relative.parts: + raise SymbolAnalysisRefusal("TRACKED_TREE_REFUSED") + if mode == b"160000": + raise SymbolAnalysisRefusal("TRACKED_TREE_SUBMODULE_REFUSED") + path = top / relative + try: + metadata_stat = path.lstat() + except OSError as exc: + raise SymbolAnalysisRefusal("TRACKED_TREE_REFUSED") from exc + mode_text = mode.decode("ascii", errors="strict") + if mode in (b"100644", b"100755"): + if not stat.S_ISREG(metadata_stat.st_mode) or path.is_symlink(): + raise SymbolAnalysisRefusal("TRACKED_TREE_REFUSED") + mode_text = "100755" if metadata_stat.st_mode & 0o111 else "100644" + try: + content = path.read_bytes() + except OSError as exc: + raise SymbolAnalysisRefusal("TRACKED_TREE_REFUSED") from exc + kind = "regular" + elif mode == b"120000": + if not stat.S_ISLNK(metadata_stat.st_mode): + raise SymbolAnalysisRefusal("TRACKED_TREE_REFUSED") + try: + content = os.fsencode(os.readlink(path)) + except OSError as exc: + raise SymbolAnalysisRefusal("TRACKED_TREE_REFUSED") from exc + kind = "symlink" + else: + raise SymbolAnalysisRefusal("TRACKED_TREE_MODE_REFUSED") + entries.append( + TrackedTreeEntry( + path=relative.as_posix(), + mode=mode_text, + kind=kind, + sha256=hashlib.sha256(content).hexdigest(), + size=len(content), + ) + ) + ordered = tuple(sorted(entries, key=lambda entry: entry.path)) + digest_rows = [ + { + "kind": entry.kind, + "mode": entry.mode, + "path": entry.path, + "sha256": entry.sha256, + "size": entry.size, + } + for entry in ordered + ] + return TrackedTreeIdentity( + sha256=_sha256(digest_rows), + entries=ordered, + regular_files=sum(entry.kind == "regular" for entry in ordered), + symlinks=sum(entry.kind == "symlink" for entry in ordered), + ) + + +def inventory_symbols(root: Path | str) -> SymbolInventory: + requested = Path(root) + if requested.is_symlink() or not requested.is_dir(): + raise SymbolAnalysisRefusal("SYMBOL_ROOT_REFUSED") + resolved = requested.resolve(strict=True) + definitions = [] + for path in _iter_python_files(resolved): + relative = path.relative_to(resolved) + try: + source = path.read_text(encoding="utf-8", errors="strict") + module_ast = ast.parse(source, filename=relative.as_posix(), type_comments=True) + except (OSError, UnicodeError, SyntaxError) as exc: + raise SymbolAnalysisRefusal("SYMBOL_SOURCE_SYNTAX_REFUSED") from exc + module_name = _module_name(relative) + for statement in module_ast.body: + candidates: list[tuple[str, ast.FunctionDef | ast.AsyncFunctionDef]] = [] + if isinstance(statement, (ast.FunctionDef, ast.AsyncFunctionDef)): + candidates.append((statement.name, statement)) + elif isinstance(statement, ast.ClassDef): + for member in statement.body: + if isinstance(member, (ast.FunctionDef, ast.AsyncFunctionDef)): + candidates.append((f"{statement.name}.{member.name}", member)) + for symbol, node in candidates: + signature = _signature_payload(node) + behavior = _behavior_payload(node) + context = _context_payload(module_ast, statement, node) + signature_sha256 = _sha256(signature) + behavior_sha256 = _sha256(behavior) + definition_sha256 = _sha256( + {"signature": signature_sha256, "behavior": behavior_sha256} + ) + kind_prefix = "async_" if isinstance(node, ast.AsyncFunctionDef) else "" + kind = kind_prefix + ("method" if isinstance(statement, ast.ClassDef) else "function") + definitions.append( + SymbolDefinition( + target=f"{module_name}:{symbol}", + module=module_name, + symbol=symbol, + kind=kind, + relative_path=relative.as_posix(), + line=node.lineno, + column=node.col_offset, + signature_sha256=signature_sha256, + behavior_sha256=behavior_sha256, + context_sha256=_sha256(context), + definition_sha256=definition_sha256, + ) + ) + return SymbolInventory( + definitions=tuple( + sorted(definitions, key=lambda value: (value.target, value.relative_path, value.line, value.column)) + ) + ) + + +def compare_symbol_trees(base_root: Path | str, head_root: Path | str) -> tuple[SymbolChange, ...]: + base = inventory_symbols(base_root).by_target() + head = inventory_symbols(head_root).by_target() + changes = [] + for target in sorted(set(base) | set(head)): + base_values = base.get(target, ()) + head_values = head.get(target, ()) + if len(base_values) > 1 or len(head_values) > 1: + changes.append(SymbolChange(target, "SYMBOL_AMBIGUOUS", None, None)) + continue + if not base_values: + changes.append(SymbolChange(target, "NO_BASELINE_REVISION", None, head_values[0] if len(head_values) == 1 else None)) + continue + if not head_values: + changes.append(SymbolChange(target, "SYMBOL_REMOVED", base_values[0] if len(base_values) == 1 else None, None)) + continue + base_definition = base_values[0] + head_definition = head_values[0] + if base_definition.signature_sha256 != head_definition.signature_sha256: + status = "FIXTURE_SIGNATURE_DRIFT" + elif base_definition.behavior_sha256 != head_definition.behavior_sha256: + status = "CHANGED" + elif base_definition.context_sha256 != head_definition.context_sha256: + status = "CONTEXT_CHANGED" + else: + status = "UNCHANGED" + changes.append(SymbolChange(target, status, base_definition, head_definition)) + return tuple(changes) diff --git a/src/breakcheck/adapters/python/worktrees.py b/src/breakcheck/adapters/python/worktrees.py new file mode 100644 index 0000000..3af91f1 --- /dev/null +++ b/src/breakcheck/adapters/python/worktrees.py @@ -0,0 +1,349 @@ +from __future__ import annotations + +from contextlib import contextmanager +from dataclasses import dataclass +import os +from pathlib import Path +import subprocess +from typing import Iterator + + +_GIT_CONFIG = ( + "core.hooksPath=/dev/null", + "core.fsmonitor=false", + "filter.lfs.required=false", + "filter.lfs.smudge=", + "filter.lfs.process=", + "advice.detachedHead=false", + "protocol.file.allow=never", +) + + +class WorktreeRefusal(ValueError): + """A safe revision pair could not be materialized.""" + + def __init__(self, code: str): + super().__init__(code) + self.code = code + + +def _git_environment() -> dict[str, str]: + environment = { + key: os.environ[key] + for key in ("PATH", "SYSTEMROOT", "TMPDIR", "TEMP", "TMP") + if key in os.environ + } + environment.update( + { + "GIT_ASKPASS": os.devnull, + "GIT_CONFIG_GLOBAL": os.devnull, + "GIT_CONFIG_NOSYSTEM": "1", + "GIT_CONFIG_SYSTEM": os.devnull, + "GIT_LFS_SKIP_SMUDGE": "1", + "GIT_NO_LAZY_FETCH": "1", + "GIT_OPTIONAL_LOCKS": "0", + "GIT_SSH_COMMAND": "ssh -oBatchMode=yes", + "GIT_TERMINAL_PROMPT": "0", + "SSH_ASKPASS": os.devnull, + } + ) + return environment + + +def _git_command(repository: Path, *arguments: str) -> list[str]: + command = ["git", "-C", str(repository)] + for value in _GIT_CONFIG: + command.extend(("-c", value)) + command.extend(arguments) + return command + + +def _run_git( + repository: Path, + *arguments: str, + refusal: str, + check: bool = True, +) -> subprocess.CompletedProcess[bytes]: + try: + result = subprocess.run( + _git_command(repository, *arguments), + check=False, + capture_output=True, + env=_git_environment(), + shell=False, + ) + except OSError as exc: + raise WorktreeRefusal(refusal) from exc + if check and result.returncode != 0: + raise WorktreeRefusal(refusal) + return result + + +def _decoded_line(result: subprocess.CompletedProcess[bytes], refusal: str) -> str: + try: + value = result.stdout.decode("utf-8", errors="strict").strip() + except UnicodeDecodeError as exc: + raise WorktreeRefusal(refusal) from exc + if not value or "\n" in value or "\r" in value: + raise WorktreeRefusal(refusal) + return value + + +def _repository_paths(repository: Path) -> tuple[Path, Path]: + requested = Path(repository) + if requested.is_symlink() or not requested.is_dir(): + raise WorktreeRefusal("REPOSITORY_REFUSED") + top = Path( + _decoded_line( + _run_git(requested, "rev-parse", "--show-toplevel", refusal="REPOSITORY_REFUSED"), + "REPOSITORY_REFUSED", + ) + ).resolve() + common = Path( + _decoded_line( + _run_git(top, "rev-parse", "--path-format=absolute", "--git-common-dir", refusal="REPOSITORY_REFUSED"), + "REPOSITORY_REFUSED", + ) + ).resolve() + return top, common + + +def _validate_ref_text(reference: str) -> str: + if not isinstance(reference, str) or not reference or reference.startswith("-"): + raise WorktreeRefusal("REVISION_REF_REFUSED") + if any(ord(character) < 32 or ord(character) == 127 for character in reference): + raise WorktreeRefusal("REVISION_REF_REFUSED") + return reference + + +def resolve_commit(repository: Path | str, reference: str) -> str: + top, _ = _repository_paths(Path(repository)) + requested = _validate_ref_text(reference) + result = _run_git( + top, + "rev-parse", + "--verify", + "--end-of-options", + f"{requested}^{{commit}}", + refusal="REVISION_REF_REFUSED", + ) + commit = _decoded_line(result, "REVISION_REF_REFUSED") + if len(commit) not in (40, 64) or any(character not in "0123456789abcdef" for character in commit): + raise WorktreeRefusal("REVISION_REF_REFUSED") + return commit + + +def _validate_checkout_filters(repository: Path) -> None: + result = _run_git( + repository, + "config", + "--null", + "--get-regexp", + r"^filter\..*\.(clean|smudge|process|required)$", + refusal="REVISION_FILTER_REFUSED", + check=False, + ) + if result.returncode not in (0, 1): + raise WorktreeRefusal("REVISION_FILTER_REFUSED") + for record in result.stdout.split(b"\0"): + if not record: + continue + key = record.split(b"\n", 1)[0].decode("utf-8", errors="replace").lower() + if not key.startswith("filter.lfs."): + raise WorktreeRefusal("REVISION_FILTER_REFUSED") + + +def _tree_entries(repository: Path, commit: str) -> tuple[tuple[bytes, bytes, bytes], ...]: + result = _run_git( + repository, + "ls-tree", + "-r", + "-z", + commit, + refusal="REVISION_TREE_REFUSED", + ) + entries = [] + for record in result.stdout.split(b"\0"): + if not record: + continue + try: + metadata, path = record.split(b"\t", 1) + mode, kind, _ = metadata.split(b" ", 2) + except ValueError as exc: + raise WorktreeRefusal("REVISION_TREE_REFUSED") from exc + entries.append((mode, kind, path)) + return tuple(entries) + + +def _validate_no_submodules(repository: Path, *commits: str) -> None: + for commit in commits: + if any(mode == b"160000" or kind == b"commit" for mode, kind, _ in _tree_entries(repository, commit)): + raise WorktreeRefusal("REVISION_SUBMODULE_REFUSED") + + +def _relative_path(raw: bytes) -> Path: + try: + value = raw.decode("utf-8", errors="strict") + except UnicodeDecodeError as exc: + raise WorktreeRefusal("REVISION_PATH_ENCODING_REFUSED") from exc + relative = Path(value) + if relative.is_absolute() or not relative.parts or ".." in relative.parts: + raise WorktreeRefusal("REVISION_TREE_REFUSED") + return relative + + +def _validate_symlinks(repository: Path, worktree: Path, commit: str) -> None: + root = worktree.resolve(strict=True) + for mode, _, raw_path in _tree_entries(repository, commit): + if mode != b"120000": + continue + relative = _relative_path(raw_path) + link = root / relative + if not link.is_symlink(): + raise WorktreeRefusal("REVISION_TREE_REFUSED") + try: + target = Path(os.readlink(link)) + resolved = link.resolve(strict=False) + except (OSError, RuntimeError) as exc: + raise WorktreeRefusal("REVISION_SYMLINK_ESCAPE_REFUSED") from exc + if target.is_absolute() or not _is_within(resolved, root): + raise WorktreeRefusal("REVISION_SYMLINK_ESCAPE_REFUSED") + + +def _is_within(candidate: Path, container: Path) -> bool: + return candidate == container or container in candidate.parents + + +def _prepare_runtime_root(runtime_root: Path, repository: Path, common_git: Path) -> Path: + requested = Path(runtime_root) + if requested.exists() or requested.is_symlink(): + raise WorktreeRefusal("REVISION_RUNTIME_ROOT_REFUSED") + parent = requested.parent + if parent.is_symlink() or not parent.is_dir(): + raise WorktreeRefusal("REVISION_RUNTIME_ROOT_REFUSED") + resolved = parent.resolve(strict=True) / requested.name + if _is_within(resolved, repository) or _is_within(resolved, common_git): + raise WorktreeRefusal("REVISION_RUNTIME_ROOT_REFUSED") + try: + resolved.mkdir(mode=0o700) + except OSError as exc: + raise WorktreeRefusal("REVISION_RUNTIME_ROOT_REFUSED") from exc + return resolved + + +@dataclass +class RevisionWorktreePair: + repository: Path + runtime_root: Path + base_commit: str + head_commit: str + base_root: Path + head_root: Path + _closed: bool = False + + def close(self) -> None: + if self._closed: + return + errors = [] + for worktree in (self.head_root, self.base_root): + result = _run_git( + self.repository, + "worktree", + "remove", + "--force", + str(worktree), + refusal="REVISION_WORKTREE_CLEANUP_REFUSED", + check=False, + ) + if result.returncode != 0 and (worktree.exists() or worktree.is_symlink()): + errors.append(worktree.name) + if not errors: + try: + self.runtime_root.rmdir() + except FileNotFoundError: + pass + except OSError: + errors.append(self.runtime_root.name) + self._closed = not errors + if errors: + raise WorktreeRefusal("REVISION_WORKTREE_CLEANUP_REFUSED") + + def __enter__(self) -> "RevisionWorktreePair": + return self + + def __exit__(self, exc_type, exc, traceback) -> bool: + try: + self.close() + except WorktreeRefusal: + if exc is None: + raise + return False + + +def _add_worktree(repository: Path, destination: Path, commit: str) -> None: + _run_git( + repository, + "worktree", + "add", + "--detach", + "--no-checkout", + str(destination), + commit, + refusal="REVISION_WORKTREE_CREATE_REFUSED", + ) + _run_git( + destination, + "checkout", + "--detach", + "--force", + commit, + refusal="REVISION_WORKTREE_CHECKOUT_REFUSED", + ) + + +def create_revision_worktrees( + repository: Path | str, + base_ref: str, + head_ref: str, + runtime_root: Path | str, +) -> RevisionWorktreePair: + top, common = _repository_paths(Path(repository)) + base_commit = resolve_commit(top, base_ref) + head_commit = resolve_commit(top, head_ref) + _validate_checkout_filters(top) + _validate_no_submodules(top, base_commit, head_commit) + root = _prepare_runtime_root(Path(runtime_root), top, common) + pair = RevisionWorktreePair( + repository=top, + runtime_root=root, + base_commit=base_commit, + head_commit=head_commit, + base_root=root / "base", + head_root=root / "head", + ) + try: + _add_worktree(top, pair.base_root, base_commit) + _add_worktree(top, pair.head_root, head_commit) + _validate_symlinks(top, pair.base_root, base_commit) + _validate_symlinks(top, pair.head_root, head_commit) + except BaseException: + try: + pair.close() + except WorktreeRefusal: + pass + raise + return pair + + +@contextmanager +def revision_worktrees( + repository: Path | str, + base_ref: str, + head_ref: str, + runtime_root: Path | str, +) -> Iterator[RevisionWorktreePair]: + pair = create_revision_worktrees(repository, base_ref, head_ref, runtime_root) + try: + yield pair + finally: + pair.close() diff --git a/src/breakcheck/cli.py b/src/breakcheck/cli.py index 0a8b3c0..e6b4a11 100644 --- a/src/breakcheck/cli.py +++ b/src/breakcheck/cli.py @@ -8,11 +8,14 @@ import json import os import re +import shutil import stat import sys import tempfile from pathlib import Path +from breakcheck.adapters.python.fixtures import REFUSAL_CODES as _FIXTURE_REFUSALS + def _load_pipeline(): from breakcheck.adapters.python.files import iter_python_files as inventory from breakcheck.adapters.python.scanner import PythonUsageScanner as scanner @@ -30,14 +33,22 @@ def _load_pipeline(): normalize, compare, finding_id, render_json, render_human, exit_code, verify_report) -_HELP_PREFIX = 'target grammar: @; flags: --json, --ci, --verify, --wheelhouse, --output, --evidence, --runtime-root; an explicit local wheelhouse is required; CI coverage threshold: 80.0; exit 0, exit 3, exit 4; refusal codes: ' +_HELP_PREFIX = ( + "modes:\n" + " dependency comparison: breakcheck @ [options]\n" + " offline demonstration: breakcheck demo --output-root \n" + " revision verification: breakcheck {freeze,diff,attest} --help\n" + " machine capabilities: breakcheck --capabilities --json\n\n" + "Dependency comparison requires an explicit local wheelhouse. " + "The default coverage threshold is 80 percent. Refusal codes: " +) _MISSING_CURRENT = "CURRENT_DISTRIBUTION_MISSING" _SUPPORTED_PLATFORMS = frozenset(('linux', 'darwin')) _PLATFORM_REFUSAL_CODE = 'PLATFORM_REFUSED' _WHEELHOUSE_REQUIRED_CODE = 'WHEELHOUSE_REQUIRED' _OPERATIONAL_EXCEPTION_CODES = {'ImportError': 'PIPELINE_IMPORT_REFUSED', 'OSError': 'FILESYSTEM_REFUSED', 'UnicodeError': 'TEXT_ENCODING_REFUSED'} -_DECLARED_REFUSAL_CODES = frozenset(('NONLITERAL_ARGS', 'CURRENT_DISTRIBUTION_MISSING', 'PLATFORM_REFUSED', 'WHEELHOUSE_REQUIRED', 'MISSING_WHEEL_REFUSED', 'PIPELINE_IMPORT_REFUSED', 'FILESYSTEM_REFUSED', 'TEXT_ENCODING_REFUSED')) | frozenset(('API_ABSENT_BOTH_ENVIRONMENTS', 'CALL_SITE_PATH_REFUSED', 'CALL_SITE_SCAN_REFUSED', 'CALL_SITE_SCHEMA_REFUSED', 'CALL_SITE_SOURCE_REFUSED', 'ENVIRONMENT_ARTIFACT_SYMLINK_REFUSED', 'ENVIRONMENT_PAIR_REFUSED', 'IMPORT_ROOT_REFUSED', 'INVENTORY_ROOT_SYMLINK_REFUSED', 'OBSERVATION_ENCODING_REFUSED', 'PRESENCE_CENSUS_REFUSED', 'SOURCE_SYNTAX_REFUSED', 'TARGET_GRAMMAR_REFUSED', 'UNSUPPORTED_USAGE_SCHEMA_REFUSED', 'WHEELHOUSE_REFUSED')) +_DECLARED_REFUSAL_CODES = (frozenset(('NONLITERAL_ARGS', 'CURRENT_DISTRIBUTION_MISSING', 'PLATFORM_REFUSED', 'WHEELHOUSE_REQUIRED', 'MISSING_WHEEL_REFUSED', 'PIPELINE_IMPORT_REFUSED', 'FILESYSTEM_REFUSED', 'TEXT_ENCODING_REFUSED')) | frozenset(('API_ABSENT_BOTH_ENVIRONMENTS', 'CALL_SITE_PATH_REFUSED', 'CALL_SITE_SCAN_REFUSED', 'CALL_SITE_SCHEMA_REFUSED', 'CALL_SITE_SOURCE_REFUSED', 'ENVIRONMENT_ARTIFACT_SYMLINK_REFUSED', 'ENVIRONMENT_FINGERPRINT_REFUSED', 'ENVIRONMENT_PAIR_REFUSED', 'IMPORT_ROOT_REFUSED', 'INVENTORY_ROOT_SYMLINK_REFUSED', 'OBSERVATION_ENCODING_REFUSED', 'OUTPUT_PATH_COLLISION_REFUSED', 'OUTPUT_PATH_REFUSED', 'PRESENCE_CENSUS_REFUSED', 'SOURCE_SYNTAX_REFUSED', 'TARGET_GRAMMAR_REFUSED', 'UNSUPPORTED_USAGE_SCHEMA_REFUSED', 'WHEELHOUSE_REFUSED')) | _FIXTURE_REFUSALS) _HELP = _HELP_PREFIX + ','.join(sorted(_DECLARED_REFUSAL_CODES)) def _bounded_refusal(exc): @@ -167,6 +178,8 @@ def _replay_import_statement(tree, call, api): raise ValueError("CALL_SITE_SOURCE_REFUSED") def _call_sources(root, grouped, inventory): + from breakcheck.adapters.python.scanner import build_static_context + requested = {} for api, sites in grouped.items(): for site in sites: @@ -183,6 +196,7 @@ def _call_sources(root, grouped, inventory): text = path.read_text(encoding="utf-8") lines = text.splitlines(keepends=True) tree = ast.parse(text, filename=relative) + static_context = build_static_context(tree) matches = {item: [] for item in by_file[relative]} for node in ast.walk(tree): key = (getattr(node, "lineno", None), getattr(node, "col_offset", None)) @@ -202,9 +216,40 @@ def _call_sources(root, grouped, inventory): nodes[0], requested[(relative, line, column)], ), + "module_constants": static_context.module_constants, + "imported_names": static_context.imported_names, } return result + +def _synthesize_replay(replay_source): + """Return replay source plus deterministic argument provenance. + + Tests and embedders may inject the historical two-argument synthesizer. The + production pipeline uses the context-aware synthesizer so bounded folds, + module constants, and safe nested calls retain their provenance. + """ + + from breakcheck.adapters.python.literals import ( + synthesize_snippet, + synthesize_with_provenance, + ) + + if _synthesize is synthesize_snippet: + synthesized = synthesize_with_provenance( + replay_source["expression"], + replay_source["import_statement"], + module_constants=replay_source["module_constants"], + imported_names=replay_source["imported_names"], + ) + return synthesized.source, synthesized.provenance + return ( + _synthesize( + replay_source["expression"], replay_source["import_statement"] + ), + ("SOURCE_LITERAL",), + ) + def _canonical_call_expression(api, expression): """Bind a scanned call to its canonical distribution API identity.""" try: @@ -225,6 +270,7 @@ def _scan_inventory(root, package, inventory): imports = [] call_sites = [] unsupported = [] + candidates = [] for source_path in sorted( inventory, key=lambda item: item.relative_to(root).as_posix() ): @@ -238,12 +284,15 @@ def _scan_inventory(root, package, inventory): observed_imports = observed.get("imports") observed_calls = observed.get("call_sites") observed_unsupported = observed.get("unsupported", []) + observed_candidates = observed.get("candidates", []) if (not isinstance(observed_imports, list) or not isinstance(observed_calls, list) - or not isinstance(observed_unsupported, list)): + or not isinstance(observed_unsupported, list) + or not isinstance(observed_candidates, list)): raise ValueError("CALL_SITE_SCAN_REFUSED") imports.extend(observed_imports) call_sites.extend(observed_calls) unsupported.extend(observed_unsupported) + candidates.extend(observed_candidates) imports.sort(key=lambda row: _canonical(row)) call_sites.sort( key=lambda row: ( @@ -251,7 +300,58 @@ def _scan_inventory(root, package, inventory): ) if isinstance(row, dict) else ("", 0, 0, "") ) unsupported.sort(key=lambda row: _canonical(row)) - return {"imports": imports, "call_sites": call_sites, "unsupported": unsupported} + candidates.sort(key=lambda row: _canonical(row)) + return { + "imports": imports, + "call_sites": call_sites, + "unsupported": unsupported, + "candidates": candidates, + } + + +def _fixture_suggestions(args, repository, scan, inventory): + from breakcheck.adapters.python.fixtures import suggest_fixtures + + grouped = {} + for row in scan["call_sites"]: + grouped.setdefault(row["api"], []).append( + {"file": row["file"], "line": row["line"], "column": row["column"]} + ) + call_sources = _call_sources(repository, grouped, inventory) + candidates = [] + for row in sorted( + scan["call_sites"], + key=lambda item: (item["file"], item["line"], item["column"], item["api"]), + ): + source = call_sources[(row["file"], row["line"], row["column"])] + try: + _synthesize_replay(source) + except Exception as exc: + if str(exc) != "NONLITERAL_ARGS": + raise + candidates.append( + { + **row, + "signature": None, + "type_hints": None, + "nearby_source": source["expression"], + } + ) + digest = suggest_fixtures( + args.suggest_fixtures, + candidates, + repository_root=repository, + ) + print( + _canonical( + { + "schema_version": 1, + "fixture_suggestions": len(candidates), + "sha256": digest, + } + ) + ) + return 0 def _observation_text(data): try: @@ -283,6 +383,253 @@ def _process_observation(result): "exception_class": exception_class, "duration_ms": None}) + +def _repeat_observation(snippet, environment): + """Run an admitted snippet exactly twice and fail closed on disagreement. + + The injected legacy runner branch preserves the narrow test seam used by the + schema-1 compatibility suite. Production always uses the framed private + protocol, so ordinary stdout can never impersonate an observation. + """ + from breakcheck.adapters.python.executor import ( + run_repeated_typed_snippet_isolated, + run_snippet_isolated, + ) + + if _execute is run_snippet_isolated: + return run_repeated_typed_snippet_isolated( + snippet_source=snippet, + environment=environment, + ) + runs = [] + for _ in range(2): + observation = _process_observation( + _execute(snippet_source=snippet, environment=environment) + ) + runs.append(observation) + if _digest(runs[0]) != _digest(runs[1]): + return { + "runs": runs, + "repeatable": False, + "status": "PROTOCOL_REFUSED", + "reason_code": "NONDETERMINISTIC_OBSERVATION", + "observation": None, + } + return { + "runs": runs, + "repeatable": True, + "status": "VALUE" if runs[0]["kind"] == "value" else "EXCEPTION", + "reason_code": None, + "observation": runs[0], + } + + +def _typed_refusal(run): + if run.get("status") in {"VALUE", "EXCEPTION"} and run.get("repeatable"): + return None + reason = run.get("reason_code") + return reason if isinstance(reason, str) and reason else "PROTOCOL_REFUSED" + + +def _typed_raw_type(run): + rows = run.get("runs") + if isinstance(rows, list) and rows and isinstance(rows[0], dict): + value = rows[0].get("raw_type") + if isinstance(value, str) and value: + return value + return None + + +def _fixture_snippet(binding, replay_source): + from breakcheck.adapters.python.fixtures import render_fixture_source + + try: + call = ast.parse(replay_source["expression"], mode="eval").body + except (SyntaxError, ValueError, TypeError) as exc: + raise ValueError("CALL_SITE_SOURCE_REFUSED") from exc + if not isinstance(call, ast.Call): + raise ValueError("CALL_SITE_SOURCE_REFUSED") + callable_source = ast.unparse(call.func) + return ( + replay_source["import_statement"] + + "\n\n" + + render_fixture_source(binding, callable_source) + ) + + +def _invocation(args): + return { + "allow_empty": bool(getattr(args, "allow_empty", False)), + "ci": bool(getattr(args, "ci", False)), + "coverage_report": bool(getattr(args, "coverage_report", None)), + "fixture_file": getattr(args, "fixtures", None), + "fixture_policy": getattr(args, "fixture_policy", "forbid"), + "json": bool(getattr(args, "json", False)), + "min_coverage": float(getattr(args, "min_coverage", 80.0)), + "suggest_fixtures": bool(getattr(args, "suggest_fixtures", None)), + } + + +def _artifact_invocation(args, repository, kind): + from breakcheck.schema import canonicalize_invocation + + values = _invocation(args) + selected = { + "dependency_report": ( + "allow_empty", "ci", "coverage_report", "fixture_policy", "json", + "min_coverage", "suggest_fixtures", + ), + "coverage_report": ( + "allow_empty", "fixture_policy", "min_coverage", "suggest_fixtures", + ), + }[kind] + flags = {name: values[name] for name in selected} + fixture_path = getattr(args, "fixtures", None) + if fixture_path: + try: + relative = Path(fixture_path).resolve().relative_to(repository) + except ValueError as exc: + raise ValueError("FIXTURE_PATH_REFUSED") from exc + flags["fixture_file"] = relative.as_posix() + return canonicalize_invocation(kind, flags) + + +def _schema_two_observation(observation, provenance): + return { + "kind": observation["kind"], + "payload": copy.deepcopy(observation["payload"]), + "exception_class": observation["exception_class"], + "provenance": list(provenance), + } + + +def _schema_two_dependency_report( + legacy_report, terminal_records, args, repository +): + from breakcheck.schema import ( + artifact_digest, + make_artifact, + record_identity, + ) + + terminal_by_location = { + (row["api"], row["file"], row["line"], row["column"]): row + for row in terminal_records + } + findings = [] + witnesses = [] + for legacy in legacy_report["findings"]: + site = legacy["call_sites"][0] + terminal = terminal_by_location[ + (legacy["api"], site["file"], site["line"], site["column"]) + ] + provenance = terminal["provenance"] + replay = legacy["repro"] + projection_source = replay.get("projection") + projection = ( + None + if projection_source is None + else { + "source": projection_source, + "sha256": artifact_digest(projection_source), + } + ) + finding = { + "finding_id": "", + "candidate_id": terminal["candidate_id"], + "api": legacy["api"], + "call_sites": copy.deepcopy(legacy["call_sites"]), + "verdict": legacy["verdict"], + "old": ( + None + if legacy["old"] is None + else _schema_two_observation(legacy["old"], provenance) + ), + "new": ( + None + if legacy["new"] is None + else _schema_two_observation(legacy["new"], provenance) + ), + "reason_code": legacy["reason_code"], + "reason_detail": terminal.get("reason_detail"), + "comparison": ( + None if legacy["verdict"] == "NOT_EXERCISED" + else copy.deepcopy(legacy["comparison"]) + ), + "projection": projection, + "fixture_binding_sha256": replay.get("fixture_binding_sha256"), + "suggested_action": copy.deepcopy(legacy["suggested_action"]), + } + finding["finding_id"] = record_identity(finding, "finding_id") + findings.append(finding) + if finding["verdict"] != "NOT_EXERCISED": + old_digest = artifact_digest(finding["old"]) + new_digest = artifact_digest(finding["new"]) + witness = { + "witness_id": "", + "finding_id": finding["finding_id"], + "candidate_id": finding["candidate_id"], + "old_observation_sha256": old_digest, + "new_observation_sha256": new_digest, + "old_repeat_sha256": [old_digest, old_digest], + "new_repeat_sha256": [new_digest, new_digest], + "projection_sha256": ( + None if projection is None else projection["sha256"] + ), + "provenance": list(provenance), + "replay": { + "source": replay["code"], + "sha256": artifact_digest(replay["code"]), + }, + } + witness["witness_id"] = record_identity(witness, "witness_id") + witnesses.append(witness) + findings.sort(key=lambda row: row["finding_id"]) + witnesses.sort(key=lambda row: row["witness_id"]) + summary = { + "changed": sum(row["verdict"] == "CHANGED" for row in findings), + "changed_under_projection": sum( + row["verdict"] == "CHANGED_UNDER_PROJECTION" for row in findings + ), + "identical": sum(row["verdict"] == "IDENTICAL" for row in findings), + "identical_under_projection": sum( + row["verdict"] == "IDENTICAL_UNDER_PROJECTION" for row in findings + ), + "not_exercised": sum( + row["verdict"] == "NOT_EXERCISED" for row in findings + ), + } + payload = { + "package": legacy_report["package"], + "current_version": legacy_report["current_version"], + "new_version": legacy_report["new_version"], + "coverage": copy.deepcopy(legacy_report["coverage"]), + "findings": findings, + "witnesses": witnesses, + "summary": summary, + "invocation": _artifact_invocation(args, repository, "dependency_report"), + } + return make_artifact("dependency_report", payload) + + +def _schema_two_coverage_report( + package, current_version, new_version, terminal_records, args, repository +): + from breakcheck.adapters.python.coverage import count_terminal_records + from breakcheck.schema import make_artifact + + payload = { + "package": package, + "current_version": current_version, + "new_version": new_version, + "candidates": sorted( + copy.deepcopy(terminal_records), key=lambda row: row["candidate_id"] + ), + "counts": count_terminal_records(terminal_records), + "invocation": _artifact_invocation(args, repository, "coverage_report"), + } + return make_artifact("coverage_report", payload) + def _presence_source(apis): return ( "import importlib\n" @@ -353,16 +700,81 @@ def _contextualize_comparison(comparison, expression, old, new): detail["policy"] = "fresh_process_per_observation" return result +def _refuse_symlink_components(path): + absolute = Path(path).absolute() + current = Path(absolute.anchor) + for part in absolute.parts[1:]: + current = current / part + try: + mode = current.lstat().st_mode + except FileNotFoundError: + continue + except OSError as exc: + raise ValueError("OUTPUT_PATH_REFUSED") from exc + if stat.S_ISLNK(mode): + raise ValueError("OUTPUT_PATH_REFUSED") + + def _write(path, text): - destination = Path(path) - destination.parent.mkdir(parents=True, exist_ok=True) - temporary = destination.with_name(destination.name + ".tmp") - temporary.write_text(text, encoding="utf-8", newline="\n") - temporary.replace(destination) + destination = Path(path).absolute() + _refuse_symlink_components(destination.parent) + try: + destination.parent.mkdir(parents=True, exist_ok=True) + except OSError as exc: + raise ValueError("OUTPUT_PATH_REFUSED") from exc + _refuse_symlink_components(destination.parent) + if destination.is_symlink(): + raise ValueError("OUTPUT_PATH_REFUSED") + descriptor = None + temporary_name = None + try: + descriptor, temporary_name = tempfile.mkstemp( + prefix="." + destination.name + ".", + suffix=".tmp", + dir=destination.parent, + ) + os.fchmod(descriptor, 0o600) + with os.fdopen(descriptor, "w", encoding="utf-8", newline="\n") as stream: + descriptor = None + stream.write(text) + stream.flush() + os.fsync(stream.fileno()) + os.replace(temporary_name, destination) + temporary_name = None + except OSError as exc: + raise ValueError("OUTPUT_PATH_REFUSED") from exc + finally: + if descriptor is not None: + os.close(descriptor) + if temporary_name is not None: + try: + Path(temporary_name).unlink() + except FileNotFoundError: + pass + + +def _validate_output_paths(args): + selected = [] + for name in ("output", "evidence", "coverage_report", "suggest_fixtures"): + value = getattr(args, name, None) + if value: + selected.append((name, Path(value).absolute().resolve(strict=False))) + resolved = [path for _name, path in selected] + if len(set(resolved)) != len(resolved): + raise ValueError("OUTPUT_PATH_COLLISION_REFUSED") + return tuple(selected) def _console_payload(report, rendered, max_bytes=1000000): if len(rendered.encode("utf-8")) <= max_bytes: return rendered + if report.get("schema_version") == 2: + payload = report["payload"] + return _canonical({ + "artifact_kind": report["artifact_kind"], + "coverage": copy.deepcopy(payload.get("coverage")), + "findings": len(payload.get("findings", [])), + "summary": copy.deepcopy(payload.get("summary")), + }) return _canonical({ "coverage": copy.deepcopy(report["coverage"]), "findings": len(report["findings"]), @@ -440,17 +852,26 @@ def _public_action_sites(call_sites): return projected def _build(args): + if args.runtime_root: + return _build_with_runtime(args, Path(args.runtime_root).resolve()) + runtime_root = Path(tempfile.mkdtemp(prefix="breakcheck-runtime-")).resolve() + try: + return _build_with_runtime(args, runtime_root) + finally: + if runtime_root.is_symlink() or (runtime_root.exists() and not runtime_root.is_dir()): + runtime_root.unlink(missing_ok=True) + elif runtime_root.exists(): + shutil.rmtree(runtime_root) + + +def _build_with_runtime(args, runtime_root): + _validate_output_paths(args) if sys.platform not in _SUPPORTED_PLATFORMS: raise ValueError(_PLATFORM_REFUSAL_CODE) package, new_version = _target(args.target) wheelhouse = args.wheelhouse - if not wheelhouse: + if not wheelhouse and not getattr(args, "suggest_fixtures", None): raise ValueError(_WHEELHOUSE_REQUIRED_CODE) - try: - current_version = _metadata.version(package) - except Exception: - print(_MISSING_CURRENT, file=sys.stderr) - return 2 global _inventory, _Scanner, _synthesize, _EnvironmentBuilder global _execute, _normalize, _compare, _finding_id global _render_json, _render_human, _exit_code, _verify_report @@ -459,9 +880,6 @@ def _build(args): _exit_code, _verify_report) = _load_pipeline() import_root = _import_root(package) repository = Path.cwd().resolve() - runtime_root = Path(args.runtime_root).resolve() if args.runtime_root else Path( - tempfile.mkdtemp(prefix='breakcheck-runtime-') - ).resolve() excluded = {runtime_root} for value in (args.output, args.evidence): if value: @@ -472,6 +890,13 @@ def _build(args): call_sites = scan.get("call_sites") if isinstance(scan, dict) else None if not isinstance(call_sites, list): raise ValueError("CALL_SITE_SCAN_REFUSED") + if getattr(args, "suggest_fixtures", None): + return _fixture_suggestions(args, repository, scan, inventory) + try: + current_version = _metadata.version(package) + except Exception: + print(_MISSING_CURRENT, file=sys.stderr) + return 2 grouped = {} for row in call_sites: if not isinstance(row, dict) or set(row) != {"api", "file", "line", "column"}: @@ -479,6 +904,18 @@ def _build(args): grouped.setdefault(row["api"], []).append( {"file": row["file"], "line": row["line"], "column": row["column"]} ) + from breakcheck.adapters.python.fixtures import resolve_fixture_policy + + fixture_file = resolve_fixture_policy( + getattr(args, "fixture_policy", "forbid"), + fixture_path=getattr(args, "fixtures", None), + repository_root=repository, + inventory=call_sites, + ) + fixture_bindings = { + binding.key: binding for binding in (() if fixture_file is None else fixture_file.bindings) + } + from breakcheck.adapters.python.literals import LiteralRefusal try: pair = _EnvironmentBuilder( @@ -495,19 +932,38 @@ def _build(args): raise ValueError("ENVIRONMENT_PAIR_REFUSED") findings = [] witnesses = [] + terminal_records = [] exercised = 0 call_sources = _call_sources(repository, grouped, inventory) + from breakcheck.adapters.python.coverage import ( + count_terminal_records, + make_candidate, + terminal_record, + ) for row in scan.get("unsupported", []): if not isinstance(row, dict) or row.get("reason_code") not in { "DYNAMIC_USAGE_UNSUPPORTED", "SOURCE_SYNTAX_REFUSED" }: raise ValueError("UNSUPPORTED_USAGE_SCHEMA_REFUSED") + sites = [{"file": row.get("file"), "line": row.get("line"), + "column": row.get("column")}] findings.append(_not_exercised( str(row.get("api", "dynamic")), - [{"file": row.get("file"), "line": row.get("line"), - "column": row.get("column")}], + sites, row["reason_code"], )) + candidate = make_candidate( + api=str(row.get("api", "dynamic")), + file=str(row.get("file")), + line=int(row.get("line")), + column=int(row.get("column")), + ) + terminal_records.append(terminal_record( + candidate, + "G1_NOT_DISCOVERABLE", + reason_code=row["reason_code"], + provenance=("SOURCE_LITERAL",), + )) prepared = [] for api in sorted(grouped): internal_sites = sorted( @@ -516,21 +972,37 @@ def _build(args): for site in internal_sites: sites = [{"file": site["file"], "line": site["line"], "column": site["column"]}] + candidate = make_candidate( + api=api, + file=site["file"], + line=site["line"], + column=site["column"], + ) + binding = fixture_bindings.get( + (site["file"], site["line"], site["column"], api) + ) try: replay_source = call_sources[ (site["file"], site["line"], site["column"]) ] expression = replay_source["expression"] - snippet = _synthesize( - expression, replay_source["import_statement"] - ) - except Exception as exc: - reason = ( - "DYNAMIC_USAGE_UNSUPPORTED" - if str(exc) == "DYNAMIC_USAGE_UNSUPPORTED" - else "NONLITERAL_ARGS" - ) + if binding is None: + snippet, provenance = _synthesize_replay(replay_source) + projection = None + else: + snippet = _fixture_snippet(binding, replay_source) + provenance = ("OPERATOR_FIXTURE",) + projection = binding.projection + except LiteralRefusal as exc: + reason = "NONLITERAL_ARGS" findings.append(_not_exercised(api, sites, reason)) + terminal_records.append(terminal_record( + candidate, + "G2_NONLITERAL", + reason_code=reason, + reason_detail=getattr(exc, "reason_detail", "OTHER"), + provenance=("SOURCE_LITERAL",), + )) continue prepared.append({ "api": api, @@ -538,6 +1010,12 @@ def _build(args): "expression": expression, "snippet": snippet, "site": (site["file"], site["line"], site["column"]), + "candidate": candidate, + "provenance": provenance, + "projection": projection, + "fixture_binding_sha256": ( + None if binding is None else binding.binding_sha256 + ), }) prepared_apis = sorted({row["api"] for row in prepared}) current_presence = _presence_census( @@ -553,26 +1031,72 @@ def _build(args): findings.append(_not_exercised( api, sites, "API_ABSENT_BOTH_ENVIRONMENTS" )) + terminal_records.append(terminal_record( + row["candidate"], + "G4_IMPURE", + reason_code="API_ABSENT_BOTH_ENVIRONMENTS", + environment="both", + provenance=row["provenance"], + )) continue - old = _process_observation(_execute( - snippet_source=snippet, environment=pair["current"] - )) - new = _process_observation(_execute( - snippet_source=snippet, environment=pair["new"] - )) + old_run = _repeat_observation(snippet, pair["current"]) + new_run = _repeat_observation(snippet, pair["new"]) + old_refusal = _typed_refusal(old_run) + new_refusal = _typed_refusal(new_run) + if old_refusal is not None or new_refusal is not None: + reason = ( + old_refusal + if old_refusal is not None and old_refusal == new_refusal + else old_refusal or new_refusal or "ENVIRONMENT_PAIR_REFUSED" + ) + findings.append(_not_exercised(api, sites, reason)) + unnormalizable = ( + old_run.get("status") == "UNNORMALIZABLE" + or new_run.get("status") == "UNNORMALIZABLE" + ) + terminal_records.append(terminal_record( + row["candidate"], + "G3_UNNORMALIZABLE" if unnormalizable else "G4_IMPURE", + reason_code=reason, + raw_type=( + _typed_raw_type(old_run) or _typed_raw_type(new_run) + if unnormalizable else None + ), + environment=( + "both" if old_refusal is not None and new_refusal is not None + else "current" if old_refusal is not None else "new" + ), + provenance=row["provenance"], + )) + continue + old = old_run["observation"] + new = new_run["observation"] exercised += 1 comparison = _contextualize_comparison( _compare(old, new), expression, old, new ) verdict = comparison["verdict"] - actions = [] if verdict == "IDENTICAL" else [ + if row["projection"] is not None: + verdict = ( + "IDENTICAL_UNDER_PROJECTION" + if verdict == "IDENTICAL" + else "CHANGED_UNDER_PROJECTION" + ) + actions = [] if verdict in {"IDENTICAL", "IDENTICAL_UNDER_PROJECTION"} else [ {"kind": "pin", "argument": package + "==" + current_version}, {"kind": "adapt", "argument": _public_action_sites(sites)}, ] snippet_id = _digest({"api": api, "code": snippet, "call_sites": sites}) repro = {"snippet_id": snippet_id, "api": api, "call_sites": copy.deepcopy(sites), "code": snippet, - "args_source": "literal", "reason_code": None} + "args_source": ( + "fixture" + if "OPERATOR_FIXTURE" in row["provenance"] + else "source" + ), "reason_code": None, + "provenance": list(row["provenance"]), + "projection": row["projection"], + "fixture_binding_sha256": row["fixture_binding_sha256"]} finding = {"finding_id": "", "api": api, "call_sites": copy.deepcopy(sites), "verdict": verdict, "old": old, "new": new, "repro": repro, @@ -588,33 +1112,64 @@ def _build(args): witness["witness_id"] = _digest(witness) findings.append(finding) witnesses.append(witness) + terminal_records.append(terminal_record( + row["candidate"], + "EXERCISED", + provenance=row["provenance"], + )) findings.sort(key=lambda row: row["finding_id"]) witnesses.sort(key=lambda row: row["witness_id"]) - changed = sum(row["verdict"] == "CHANGED" for row in findings) - identical = sum(row["verdict"] == "IDENTICAL" for row in findings) + changed = sum(row["verdict"] in {"CHANGED", "CHANGED_UNDER_PROJECTION"} for row in findings) + identical = sum(row["verdict"] in {"IDENTICAL", "IDENTICAL_UNDER_PROJECTION"} for row in findings) refused = sum(row["verdict"] == "NOT_EXERCISED" for row in findings) total = len(findings) - report = {"schema_version": 1, "package": package, + legacy_report = {"schema_version": 1, "package": package, "current_version": current_version, "new_version": new_version, "coverage": {"exercised": exercised, "total": total, "percent": (100.0 * exercised / total) if total else 0.0}, "findings": findings, "witnesses": witnesses, "summary": {"changed": changed, "identical": identical, - "not_exercised": refused}} + "not_exercised": refused}, + "invocation": _invocation(args)} + report = _schema_two_dependency_report( + legacy_report, terminal_records, args, repository + ) rendered = _render_json(report) - evidence = {"report": report, "report_sha256": _digest(report), - "witnesses": witnesses, - "environment_artifacts": { - "current": _artifact_digest(pair["current"]), - "new": _artifact_digest(pair["new"]), - }} - evidence["witness_sha256"] = _digest(evidence) + from breakcheck.schema import artifact_digest, make_artifact + + environment_artifacts = [ + {"name": name, "sha256": _artifact_digest(pair[name])["sha256"]} + for name in ("current", "new") + ] + evidence = make_artifact( + "evidence", + { + "report_artifact_sha256": artifact_digest(report), + "report_payload_sha256": report["payload_sha256"], + "report_kind": report["artifact_kind"], + "witnesses": copy.deepcopy(report["payload"]["witnesses"]), + "environment_artifacts": environment_artifacts, + "invocation": copy.deepcopy(report["payload"]["invocation"]), + }, + ) if args.output: _write(args.output, rendered + "\n") if args.evidence: - _write(args.evidence, _canonical(evidence) + "\n") + _write(args.evidence, _render_json(evidence) + "\n") + if getattr(args, "coverage_report", None): + coverage = _schema_two_coverage_report( + package, current_version, new_version, terminal_records, args, repository + ) + _write(args.coverage_report, _render_json(coverage) + "\n") print(_console_payload(report, rendered) if args.json else _render_human(report)) - return _exit_code(report) if args.ci else 0 + percent = report["payload"]["coverage"]["percent"] + if exercised == 0 and not getattr(args, "allow_empty", False): + return 4 + if args.ci and percent < float(getattr(args, "min_coverage", 80.0)): + return 4 + if args.ci: + return 3 if changed else 0 + return 0 def _verify(args): try: @@ -623,6 +1178,9 @@ def _verify(args): evidence_path = args.evidence or str(Path(args.verify).with_suffix(".witnesses.json")) witness = json.loads(Path(evidence_path).read_text(encoding="utf-8")) verify_report(report, witness) + if report.get("schema_version") == 2: + print("VERIFIED") + return 0 if witness.get("witnesses") != report.get("witnesses"): raise ValueError("witness rows mismatch") artifacts = witness.get("environment_artifacts") @@ -648,12 +1206,281 @@ def _verify(args): print("VERIFIED") return 0 + +def _capabilities(): + return { + "schema_version": 2, + "python": ["3.10", "3.11", "3.12", "3.13"], + "platforms": ["linux", "macos"], + "report_schemas": [1, 2], + "features": [ + "claim_attestation", + "dependency_comparison", + "fixture_suggestions", + "revision_baselines", + "revision_comparison", + ], + "interactive": False, + "runtime_dependencies": [], + } + + +def _demo(output_root): + from breakcheck.demo import run_demo + + return run_demo(output_root, _build) + + +def _present_long_options(arguments): + return { + argument.split("=", 1)[0] + for argument in arguments + if argument.startswith("--") + } + + +def _require_mode_options(parser, arguments, *, mode, allowed): + unexpected = sorted(_present_long_options(arguments) - set(allowed)) + if unexpected: + parser.error( + mode + " mode does not accept: " + ", ".join(unexpected) + ) + + +def _revision_runtime_root(requested): + if requested: + return Path(requested).resolve(), None + temporary_parent = Path(tempfile.mkdtemp(prefix="breakcheck-revision-")) + return temporary_parent / "worktrees", temporary_parent + + +def _load_json_artifact(path): + try: + return json.loads(Path(path).read_text(encoding="utf-8")) + except (OSError, UnicodeError, json.JSONDecodeError) as exc: + raise ValueError("ARTIFACT_INPUT_REFUSED") from exc + + +def _emit_revision_result(result, args): + from breakcheck.report import render_human + from breakcheck.schema import canonical_json + + rendered_report = canonical_json(result.report) + if args.output: + _write(args.output, rendered_report + "\n") + if args.evidence: + _write(args.evidence, canonical_json(result.evidence) + "\n") + print(rendered_report if args.json else render_human(result.report)) + return result.exit_code + + +def _revision_parser(command): + parser = argparse.ArgumentParser( + prog="breakcheck " + command, + description={ + "freeze": "capture a deterministic behavioral baseline for fixture-bound symbols", + "diff": "compare fixture-bound behavior across two Git revisions", + "attest": "adjudicate a behavior-preservation claim against an independent revision comparison", + }[command], + ) + parser.add_argument( + "--fixtures", + default="breakcheck.fixtures.toml", + help="repository-relative fixture file (default: breakcheck.fixtures.toml)", + ) + parser.add_argument( + "--runtime-root", + help="absent path used for detached worktrees; a temporary path is used by default", + ) + parser.add_argument( + "--output", + required=command == "freeze", + help="write the canonical report artifact to this path", + ) + parser.add_argument( + "--evidence", help="write the matching evidence artifact to this path" + ) + parser.add_argument("--json", action="store_true", help="print canonical JSON") + if command == "freeze": + parser.add_argument( + "--revision", default="HEAD", help="Git revision to capture (default: HEAD)" + ) + parser.add_argument( + "--target", + action="append", + default=[], + help="module.path:symbol target; repeat for multiple targets", + ) + parser.add_argument( + "--allow-dirty", + action="store_true", + help="request dirty-tree capture; currently refused rather than silently omitting changes", + ) + elif command == "diff": + base = parser.add_mutually_exclusive_group(required=True) + base.add_argument("--base", help="baseline Git revision") + base.add_argument("--baseline", help="baseline artifact created by freeze") + parser.add_argument("--head", required=True, help="head Git revision") + parser.add_argument( + "--previous-report", + help="verified earlier revision report used to disclose fixture retuning", + ) + parser.add_argument( + "--target", + action="append", + default=[], + help="module.path:symbol target; repeat for multiple targets", + ) + parser.add_argument( + "--fixture-source", + choices=("base", "head", "explicit"), + default="base", + help="revision that owns fixture provenance (default: base)", + ) + parser.add_argument( + "--allow-empty", + action="store_true", + help="permit a comparison with no selected targets and record that choice", + ) + parser.add_argument( + "--min-coverage", + type=float, + default=80.0, + help="minimum exercised target percentage (default: 80)", + ) + parser.add_argument( + "--strict-separation", + action="store_true", + help="require fixtures to predate the head revision", + ) + else: + parser.add_argument("--head", required=True, help="head Git revision") + parser.add_argument( + "--claim", required=True, help="repository-relative behavior claim file" + ) + parser.add_argument( + "--previous-report", + help="verified earlier revision report used to disclose fixture retuning", + ) + parser.add_argument( + "--fixture-source", + choices=("base", "head", "explicit"), + default="base", + help="revision that owns fixture provenance (default: base)", + ) + parser.add_argument( + "--allow-empty", + action="store_true", + help="permit an empty target set and record that choice", + ) + parser.add_argument( + "--min-coverage", + type=float, + default=80.0, + help="minimum exercised target percentage (default: 80)", + ) + parser.add_argument( + "--strict", + action=argparse.BooleanOptionalAction, + default=True, + help="fail when any claim is unverifiable (default: enabled)", + ) + parser.add_argument( + "--strict-separation", + action=argparse.BooleanOptionalAction, + default=True, + help="require fixtures to predate the head revision (default: enabled)", + ) + return parser + + +def _revision_command(command, argv): + from breakcheck.revision_cli import ( + RevisionModeRefusal, + attest_revision, + diff_revisions, + freeze_revision, + ) + + parser = _revision_parser(command) + args = parser.parse_args(argv) + _validate_output_paths(args) + runtime_root, temporary_parent = _revision_runtime_root(args.runtime_root) + try: + if command == "freeze": + result = freeze_revision( + Path.cwd(), + revision=args.revision, + fixture_path=args.fixtures, + runtime_root=runtime_root, + targets=args.target, + allow_dirty=args.allow_dirty, + ) + elif command == "diff": + result = diff_revisions( + Path.cwd(), + base_revision=args.base, + baseline=( + None if args.baseline is None else _load_json_artifact(args.baseline) + ), + previous_report=( + None + if args.previous_report is None + else _load_json_artifact(args.previous_report) + ), + head_revision=args.head, + fixture_path=args.fixtures, + runtime_root=runtime_root, + targets=args.target, + fixture_source=args.fixture_source, + allow_empty=args.allow_empty, + min_coverage=args.min_coverage, + strict_separation=args.strict_separation, + ) + else: + result = attest_revision( + Path.cwd(), + head_revision=args.head, + claim_path=args.claim, + previous_report=( + None + if args.previous_report is None + else _load_json_artifact(args.previous_report) + ), + fixture_path=args.fixtures, + runtime_root=runtime_root, + fixture_source=args.fixture_source, + allow_empty=args.allow_empty, + min_coverage=args.min_coverage, + strict=args.strict, + strict_separation=args.strict_separation, + ) + return _emit_revision_result(result, args) + except RevisionModeRefusal as exc: + print("REVISION_REFUSED:" + exc.code, file=sys.stderr) + return 2 + except ValueError: + print("REVISION_REFUSED:ARTIFACT_INPUT_REFUSED", file=sys.stderr) + return 2 + finally: + if temporary_parent is not None: + try: + temporary_parent.rmdir() + except OSError: + pass + def main(argv=None): + arguments = list(sys.argv[1:] if argv is None else argv) + if arguments and arguments[0] in {"freeze", "diff", "attest"}: + return _revision_command(arguments[0], arguments[1:]) parser = argparse.ArgumentParser( prog='breakcheck', usage='breakcheck @ [options]', - description=('behavioral upgrade analysis via sandboxed replay.' + - " Isolation is best-effort and not a security sandbox."), + description=( + "deterministic behavioral comparison for Python dependency upgrades " + "and source revisions. Isolation is best-effort and is not a " + "security sandbox." + ), epilog=_HELP, formatter_class=argparse.RawDescriptionHelpFormatter, ) @@ -668,11 +1495,58 @@ def main(argv=None): parser.add_argument('--output') parser.add_argument('--evidence') parser.add_argument('--runtime-root') - args = parser.parse_args(argv) + parser.add_argument( + '--capabilities', action="store_true", help="print the machine-readable capability contract" + ) + parser.add_argument('--output-root', help="absent directory for demonstration artifacts") + parser.add_argument('--coverage-report', help="write call-site coverage diagnostics") + parser.add_argument('--fixtures', help="load operator-reviewed fixture bindings") + parser.add_argument( + '--fixture-policy', choices=("forbid", "allow", "require"), default="forbid" + ) + parser.add_argument('--suggest-fixtures', help="write fixture suggestions for unexercised calls") + parser.add_argument( + '--min-coverage', type=float, default=80.0, help="minimum exercised percentage (default: 80)" + ) + parser.add_argument( + '--allow-empty', action="store_true", help="permit an empty comparison and record that choice" + ) + args = parser.parse_args(arguments) + if args.capabilities: + if args.target: + parser.error("capabilities mode is exclusive") + _require_mode_options( + parser, + arguments, + mode="capabilities", + allowed=("--capabilities", "--json"), + ) + print(_canonical(_capabilities())) + return 0 + if args.target == "demo": + if not args.output_root: + parser.error("demo requires --output-root") + _require_mode_options( + parser, + arguments, + mode="demo", + allowed=("--output-root",), + ) + return _demo(args.output_root) if args.verify: - if args.target: parser.error("verify mode is exclusive") + if args.target: + parser.error("verify mode is exclusive") + _require_mode_options( + parser, + arguments, + mode="verify", + allowed=("--verify", "--evidence"), + ) return _verify(args) - if not args.target: parser.error("target is required") + if not args.target: + parser.error("target is required") + if args.output_root: + parser.error("--output-root is only valid in demo mode") try: return _build(args) except Exception as exc: diff --git a/src/breakcheck/core/baselines.py b/src/breakcheck/core/baselines.py new file mode 100644 index 0000000..dddc794 --- /dev/null +++ b/src/breakcheck/core/baselines.py @@ -0,0 +1,630 @@ +"""Closed, deterministic domain records for revision behavior comparisons.""" + +from __future__ import annotations + +import copy +import hashlib +import json +import math +import re +from collections.abc import Callable, Iterable, Mapping + + +__all__ = ( + "BaselineRefusal", + "compare_frozen_baseline", + "detect_fixture_revision_after_failure", + "freeze_baseline", + "validate_baseline", +) + + +_BASELINE_FIELDS = frozenset( + ( + "revision", + "tree_sha256", + "dirty", + "allow_dirty", + "environment", + "fixture", + "target_observations", + "invocation", + ) +) +_ENVIRONMENT_FIELDS = frozenset(("implementation", "python_version", "platform")) +_FIXTURE_FIELDS = frozenset( + ("sha256", "source_revision", "source", "authored_by") +) +_TARGET_FIELDS = frozenset( + ( + "symbol", + "target_sha256", + "signature_sha256", + "fixture_binding_sha256", + "provenance", + "projection", + "outcome", + ) +) +_REVISION_FINDING_FIELDS = frozenset( + ( + "symbol", + "verdict", + "reason_code", + "base_observation", + "head_observation", + "base_target_sha256", + "head_target_sha256", + "signature_sha256", + "fixture_binding_sha256", + "provenance", + "projection_scope", + ) +) +_OUTCOME_FIELDS = frozenset( + ("status", "observation", "reason_code", "repeatable") +) +_OBSERVATION_FIELDS = frozenset( + ("kind", "payload", "exception_class", "duration_ms") +) +_EXERCISED_STATUSES = frozenset(("VALUE", "EXCEPTION")) +_REFUSAL_STATUSES = frozenset( + ( + "UNNORMALIZABLE", + "NETWORK_REFUSED", + "TIMEOUT", + "OUTPUT_LIMIT_REFUSED", + "PROTOCOL_REFUSED", + ) +) +_PROVENANCE = frozenset( + ( + "SOURCE_LITERAL", + "SOURCE_FOLDED", + "SOURCE_MODULE_CONSTANT", + "SOURCE_NESTED_CALL", + "OPERATOR_FIXTURE", + "RUNTIME_CAPTURE", + ) +) +_FIXTURE_SOURCES = frozenset(("base", "head", "explicit")) +_FIXTURE_AUTHORS = frozenset(("human", "agent", "unknown")) +_HASH = re.compile(r"[0-9a-f]{64}\Z") +_REVISION = re.compile(r"(?:[0-9a-f]{40}|[0-9a-f]{64})\Z") +_SYMBOL = re.compile( + r"[A-Za-z_][A-Za-z0-9_]*(?:\.[A-Za-z_][A-Za-z0-9_]*)*:" + r"[A-Za-z_][A-Za-z0-9_]*(?:\.[A-Za-z_][A-Za-z0-9_]*)*\Z" +) +_WINDOWS_ABSOLUTE = re.compile(r"[A-Za-z]:[\\/]", re.ASCII) + + +class BaselineRefusal(ValueError): + """A baseline or revision comparison failed a closed domain check.""" + + def __init__(self, code: str): + self.code = code + super().__init__(code) + + +def _refuse(code: str) -> None: + raise BaselineRefusal(code) + + +def _plain_json(value: object, *, refusal: str) -> object: + if value is None or type(value) in (bool, int, str): + return value + if type(value) is float: + if not math.isfinite(value): + _refuse(refusal) + return value + if type(value) is list: + return [_plain_json(item, refusal=refusal) for item in value] + if isinstance(value, Mapping): + result: dict[str, object] = {} + for key, item in value.items(): + if type(key) is not str or key in result: + _refuse(refusal) + result[key] = _plain_json(item, refusal=refusal) + return result + _refuse(refusal) + + +def _canonical(value: object) -> str: + return json.dumps( + value, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=True, + allow_nan=False, + ) + + +def _digest(value: object) -> str: + return hashlib.sha256(_canonical(value).encode("utf-8")).hexdigest() + + +def _closed(value: object, fields: frozenset[str], code: str) -> dict[str, object]: + if not isinstance(value, Mapping) or set(value) != fields: + _refuse(code) + return dict(value) + + +def _revision(value: object, code: str) -> str: + if type(value) is not str or not _REVISION.fullmatch(value): + _refuse(code) + return value + + +def _sha256(value: object, code: str) -> str: + if type(value) is not str or not _HASH.fullmatch(value): + _refuse(code) + return value + + +def _symbol(value: object, code: str) -> str: + if type(value) is not str or not _SYMBOL.fullmatch(value): + _refuse(code) + return value + + +def _contains_absolute_path(value: object) -> bool: + if type(value) is str: + return value.startswith(("/", "\\\\")) or bool( + _WINDOWS_ABSOLUTE.match(value) + ) + if type(value) is list: + return any(_contains_absolute_path(item) for item in value) + if isinstance(value, Mapping): + return any( + _contains_absolute_path(key) or _contains_absolute_path(item) + for key, item in value.items() + ) + return False + + +def _invocation(value: object, code: str) -> dict[str, object]: + plain = _plain_json(value, refusal=code) + if not isinstance(plain, dict): + _refuse(code) + if _contains_absolute_path(plain): + _refuse("BASELINE_PATH_REFUSED") + return plain + + +def _environment(value: object) -> dict[str, str]: + data = _closed(value, _ENVIRONMENT_FIELDS, "BASELINE_ENVIRONMENT_REFUSED") + result: dict[str, str] = {} + for field in sorted(_ENVIRONMENT_FIELDS): + item = data[field] + if type(item) is not str or not item or len(item.encode("utf-8")) > 256: + _refuse("BASELINE_ENVIRONMENT_REFUSED") + result[field] = item + if _contains_absolute_path(result): + _refuse("BASELINE_PATH_REFUSED") + return result + + +def _fixture(value: object) -> dict[str, str]: + data = _closed(value, _FIXTURE_FIELDS, "BASELINE_FIXTURE_REFUSED") + sha256 = _sha256(data["sha256"], "BASELINE_FIXTURE_REFUSED") + source_revision = _revision( + data["source_revision"], "BASELINE_FIXTURE_REFUSED" + ) + source = data["source"] + authored_by = data["authored_by"] + if source not in _FIXTURE_SOURCES or authored_by not in _FIXTURE_AUTHORS: + _refuse("BASELINE_FIXTURE_REFUSED") + return { + "sha256": sha256, + "source_revision": source_revision, + "source": str(source), + "authored_by": str(authored_by), + } + + +def _observation(value: object) -> dict[str, object]: + data = _closed(value, _OBSERVATION_FIELDS, "BASELINE_OBSERVATION_REFUSED") + kind = data["kind"] + exception_class = data["exception_class"] + duration_ms = data["duration_ms"] + if kind not in ("value", "exception") or duration_ms is not None: + _refuse("BASELINE_OBSERVATION_REFUSED") + if kind == "value" and exception_class is not None: + _refuse("BASELINE_OBSERVATION_REFUSED") + if kind == "exception" and ( + type(exception_class) is not str or not exception_class + ): + _refuse("BASELINE_OBSERVATION_REFUSED") + payload = _plain_json(data["payload"], refusal="BASELINE_OBSERVATION_REFUSED") + return { + "kind": kind, + "payload": payload, + "exception_class": exception_class, + "duration_ms": None, + } + + +def _outcome(value: object, *, baseline: bool) -> dict[str, object]: + data = _closed(value, _OUTCOME_FIELDS, "BASELINE_OBSERVATION_REFUSED") + status = data["status"] + repeatable = data["repeatable"] + reason_code = data["reason_code"] + if status not in _EXERCISED_STATUSES | _REFUSAL_STATUSES or type(repeatable) is not bool: + _refuse("BASELINE_OBSERVATION_REFUSED") + if status in _EXERCISED_STATUSES: + if not repeatable or reason_code is not None or data["observation"] is None: + _refuse("BASELINE_OBSERVATION_REFUSED") + observation = _observation(data["observation"]) + expected_kind = "value" if status == "VALUE" else "exception" + if observation["kind"] != expected_kind: + _refuse("BASELINE_OBSERVATION_REFUSED") + else: + if baseline or data["observation"] is not None: + _refuse("BASELINE_OBSERVATION_REFUSED") + if type(reason_code) is not str or not reason_code: + _refuse("BASELINE_OBSERVATION_REFUSED") + observation = None + return { + "status": status, + "observation": observation, + "reason_code": reason_code, + "repeatable": repeatable, + } + + +def _target(value: object, *, baseline: bool) -> dict[str, object]: + data = _closed(value, _TARGET_FIELDS, "BASELINE_TARGET_REFUSED") + symbol = _symbol(data["symbol"], "BASELINE_TARGET_REFUSED") + target_sha256 = _sha256(data["target_sha256"], "BASELINE_TARGET_REFUSED") + signature_sha256 = _sha256( + data["signature_sha256"], "BASELINE_TARGET_REFUSED" + ) + fixture_binding_sha256 = _sha256( + data["fixture_binding_sha256"], "BASELINE_TARGET_REFUSED" + ) + provenance = data["provenance"] + projection = data["projection"] + if provenance not in _PROVENANCE: + _refuse("BASELINE_TARGET_REFUSED") + if projection is not None and (type(projection) is not str or not projection): + _refuse("BASELINE_TARGET_REFUSED") + if _contains_absolute_path(projection): + _refuse("BASELINE_PATH_REFUSED") + return { + "symbol": symbol, + "target_sha256": target_sha256, + "signature_sha256": signature_sha256, + "fixture_binding_sha256": fixture_binding_sha256, + "provenance": provenance, + "projection": projection, + "outcome": _outcome(data["outcome"], baseline=baseline), + } + + +def _targets(values: object, *, baseline: bool, allow_empty: bool) -> list[dict[str, object]]: + if isinstance(values, (str, bytes, Mapping)): + _refuse("BASELINE_TARGET_REFUSED") + try: + rows = [_target(value, baseline=baseline) for value in values] # type: ignore[union-attr] + except TypeError: + _refuse("BASELINE_TARGET_REFUSED") + if not rows and not allow_empty: + _refuse("VACUOUS_BASELINE_REFUSED" if baseline else "VACUOUS_REVISION_COMPARISON_REFUSED") + rows.sort(key=lambda row: str(row["symbol"])) + symbols = [row["symbol"] for row in rows] + if len(set(symbols)) != len(symbols): + _refuse("BASELINE_DUPLICATE_TARGET_REFUSED") + return rows + + +def freeze_baseline( + *, + revision: str, + tree_sha256: str, + dirty: bool, + allow_dirty: bool, + environment: Mapping[str, object], + fixture: Mapping[str, object], + target_observations: Iterable[Mapping[str, object]], + invocation: Mapping[str, object], +) -> dict[str, object]: + """Create an immutable baseline payload from already repeated observations.""" + + if type(dirty) is not bool or type(allow_dirty) is not bool: + _refuse("DIRTY_TREE_REFUSED") + if dirty and not allow_dirty: + _refuse("DIRTY_TREE_REFUSED") + payload = { + "revision": _revision(revision, "BASELINE_REVISION_REFUSED"), + "tree_sha256": _sha256(tree_sha256, "BASELINE_TREE_REFUSED"), + "dirty": dirty, + "allow_dirty": allow_dirty, + "environment": _environment(environment), + "fixture": _fixture(fixture), + "target_observations": _targets( + target_observations, baseline=True, allow_empty=False + ), + "invocation": _invocation(invocation, "BASELINE_INVOCATION_REFUSED"), + } + if bool(payload["invocation"].get("allow_dirty", allow_dirty)) != allow_dirty: + _refuse("BASELINE_INVOCATION_REFUSED") + return validate_baseline(payload) + + +def validate_baseline(value: object) -> dict[str, object]: + """Validate and detach a baseline payload from caller-owned input objects.""" + + data = _closed(value, _BASELINE_FIELDS, "BASELINE_SCHEMA_REFUSED") + dirty = data["dirty"] + allow_dirty = data["allow_dirty"] + if type(dirty) is not bool or type(allow_dirty) is not bool or (dirty and not allow_dirty): + _refuse("DIRTY_TREE_REFUSED") + result = { + "revision": _revision(data["revision"], "BASELINE_REVISION_REFUSED"), + "tree_sha256": _sha256(data["tree_sha256"], "BASELINE_TREE_REFUSED"), + "dirty": dirty, + "allow_dirty": allow_dirty, + "environment": _environment(data["environment"]), + "fixture": _fixture(data["fixture"]), + "target_observations": _targets( + data["target_observations"], baseline=True, allow_empty=False + ), + "invocation": _invocation( + data["invocation"], "BASELINE_INVOCATION_REFUSED" + ), + } + return copy.deepcopy(result) + + +def _comparison_verdict( + comparator: Callable[[object, object], object] | None, + old: object, + new: object, +) -> str: + if comparator is None: + return "IDENTICAL" if _canonical(old) == _canonical(new) else "CHANGED" + result = comparator(copy.deepcopy(old), copy.deepcopy(new)) + if type(result) is str: + verdict = result + elif isinstance(result, Mapping): + verdict = result.get("verdict") + else: + verdict = getattr(result, "verdict", None) + if verdict not in ("IDENTICAL", "CHANGED"): + _refuse("REVISION_COMPARATOR_REFUSED") + return str(verdict) + + +def _finding( + *, + symbol: str, + verdict: str, + reason_code: str | None, + base: Mapping[str, object] | None, + head: Mapping[str, object] | None, +) -> dict[str, object]: + projection = None + source = base if base is not None else head + if source is not None: + projection = source["projection"] + return { + "symbol": symbol, + "verdict": verdict, + "reason_code": reason_code, + "base_observation": ( + None if base is None else copy.deepcopy(base["outcome"]["observation"]) # type: ignore[index] + ), + "head_observation": ( + None if head is None else copy.deepcopy(head["outcome"]["observation"]) # type: ignore[index] + ), + "base_target_sha256": None if base is None else base["target_sha256"], + "head_target_sha256": None if head is None else head["target_sha256"], + "signature_sha256": None if base is None else base["signature_sha256"], + "fixture_binding_sha256": ( + None if base is None else base["fixture_binding_sha256"] + ), + "provenance": None if source is None else source["provenance"], + "projection_scope": projection, + } + + +def compare_frozen_baseline( + baseline: Mapping[str, object] | None, + *, + head_revision: str, + head_tree_sha256: str, + environment: Mapping[str, object], + fixture: Mapping[str, object], + target_observations: Iterable[Mapping[str, object]], + fixtures_predate_change: bool, + invocation: Mapping[str, object], + comparator: Callable[[object, object], object] | None = None, +) -> dict[str, object]: + """Compare a validated frozen side with repeated head observations.""" + + if baseline is None: + _refuse("NO_BASELINE_REVISION") + frozen = validate_baseline(baseline) + head_revision_value = _revision(head_revision, "HEAD_REVISION_REFUSED") + head_tree = _sha256(head_tree_sha256, "HEAD_TREE_REFUSED") + if ( + head_revision_value == frozen["revision"] + or head_tree == frozen["tree_sha256"] + ): + _refuse("IDENTICAL_REVISIONS_REFUSED") + head_environment = _environment(environment) + if head_environment != frozen["environment"]: + _refuse("BASELINE_ENVIRONMENT_MISMATCH") + head_fixture = _fixture(fixture) + if head_fixture != frozen["fixture"]: + _refuse("BASELINE_FIXTURE_MISMATCH") + if type(fixtures_predate_change) is not bool: + _refuse("BASELINE_FIXTURE_REFUSED") + head_rows = _targets(target_observations, baseline=False, allow_empty=False) + base_by_symbol = { + str(row["symbol"]): row for row in frozen["target_observations"] # type: ignore[union-attr] + } + head_by_symbol = {str(row["symbol"]): row for row in head_rows} + findings: list[dict[str, object]] = [] + for symbol in sorted(set(base_by_symbol) | set(head_by_symbol)): + base_row = base_by_symbol.get(symbol) + head_row = head_by_symbol.get(symbol) + if base_row is None: + findings.append( + _finding( + symbol=symbol, + verdict="NOT_EXERCISED", + reason_code="NO_BASELINE_REVISION", + base=None, + head=head_row, + ) + ) + continue + if head_row is None: + findings.append( + _finding( + symbol=symbol, + verdict="NOT_EXERCISED", + reason_code="SYMBOL_REMOVED", + base=base_row, + head=None, + ) + ) + continue + if base_row["signature_sha256"] != head_row["signature_sha256"]: + findings.append( + _finding( + symbol=symbol, + verdict="NOT_EXERCISED", + reason_code="FIXTURE_SIGNATURE_DRIFT", + base=base_row, + head=head_row, + ) + ) + continue + if base_row["fixture_binding_sha256"] != head_row["fixture_binding_sha256"]: + findings.append( + _finding( + symbol=symbol, + verdict="NOT_EXERCISED", + reason_code="FIXTURE_BINDING_MISMATCH", + base=base_row, + head=head_row, + ) + ) + continue + if base_row["projection"] != head_row["projection"]: + findings.append( + _finding( + symbol=symbol, + verdict="NOT_EXERCISED", + reason_code="PROJECTION_SCOPE_MISMATCH", + base=base_row, + head=head_row, + ) + ) + continue + head_outcome = head_row["outcome"] + if head_outcome["status"] not in _EXERCISED_STATUSES: # type: ignore[index] + findings.append( + _finding( + symbol=symbol, + verdict="NOT_EXERCISED", + reason_code=str(head_outcome["reason_code"]), # type: ignore[index] + base=base_row, + head=head_row, + ) + ) + continue + verdict = _comparison_verdict( + comparator, + base_row["outcome"]["observation"], # type: ignore[index] + head_outcome["observation"], # type: ignore[index] + ) + if base_row["projection"] is not None: + verdict += "_UNDER_PROJECTION" + findings.append( + _finding( + symbol=symbol, + verdict=verdict, + reason_code=None, + base=base_row, + head=head_row, + ) + ) + counts = { + "changed": sum( + row["verdict"] in ("CHANGED", "CHANGED_UNDER_PROJECTION") + for row in findings + ), + "identical": sum( + row["verdict"] in ("IDENTICAL", "IDENTICAL_UNDER_PROJECTION") + for row in findings + ), + "not_exercised": sum(row["verdict"] == "NOT_EXERCISED" for row in findings), + "total": len(findings), + } + return { + "base_revision": frozen["revision"], + "head_revision": head_revision_value, + "base_tree_sha256": frozen["tree_sha256"], + "head_tree_sha256": head_tree, + "findings": findings, + "summary": counts, + "fixtures_predate_change": fixtures_predate_change, + "invocation": _invocation(invocation, "BASELINE_INVOCATION_REFUSED"), + } + + +def detect_fixture_revision_after_failure( + previous: Mapping[str, object], current: Mapping[str, object] +) -> tuple[str, ...]: + """Return bindings whose changed result became identical after fixture edits.""" + + if not isinstance(previous, Mapping) or not isinstance(current, Mapping): + _refuse("FIXTURE_HISTORY_REFUSED") + required = frozenset(("base_revision", "fixture_sha256", "findings")) + if not required.issubset(previous) or not required.issubset(current): + _refuse("FIXTURE_HISTORY_REFUSED") + base_previous = _revision(previous["base_revision"], "FIXTURE_HISTORY_REFUSED") + base_current = _revision(current["base_revision"], "FIXTURE_HISTORY_REFUSED") + old_fixture = _sha256(previous["fixture_sha256"], "FIXTURE_HISTORY_REFUSED") + new_fixture = _sha256(current["fixture_sha256"], "FIXTURE_HISTORY_REFUSED") + if base_previous != base_current or old_fixture == new_fixture: + return () + + def verdicts(value: object) -> dict[str, str]: + if isinstance(value, (str, bytes, Mapping)): + _refuse("FIXTURE_HISTORY_REFUSED") + result: dict[str, str] = {} + try: + rows = list(value) # type: ignore[arg-type] + except TypeError: + _refuse("FIXTURE_HISTORY_REFUSED") + for row in rows: + if not isinstance(row, Mapping) or set(row) not in ( + {"symbol", "verdict"}, + _REVISION_FINDING_FIELDS, + ): + _refuse("FIXTURE_HISTORY_REFUSED") + symbol = _symbol(row["symbol"], "FIXTURE_HISTORY_REFUSED") + verdict = row["verdict"] + if verdict not in ( + "IDENTICAL", + "CHANGED", + "IDENTICAL_UNDER_PROJECTION", + "CHANGED_UNDER_PROJECTION", + "NOT_EXERCISED", + ) or symbol in result: + _refuse("FIXTURE_HISTORY_REFUSED") + result[symbol] = str(verdict) + return result + + old = verdicts(previous["findings"]) + new = verdicts(current["findings"]) + return tuple( + symbol + for symbol in sorted(set(old) & set(new)) + if old[symbol] in ("CHANGED", "CHANGED_UNDER_PROJECTION") + and new[symbol] in ("IDENTICAL", "IDENTICAL_UNDER_PROJECTION") + ) diff --git a/src/breakcheck/core/claims.py b/src/breakcheck/core/claims.py new file mode 100644 index 0000000..8450926 --- /dev/null +++ b/src/breakcheck/core/claims.py @@ -0,0 +1,483 @@ +"""Closed behavior-preservation claims and deterministic adjudication.""" + +from __future__ import annotations + +import copy +from dataclasses import dataclass +import json +import math +import re +from collections.abc import Iterable, Mapping + + +__all__ = ( + "BehaviorClaim", + "ClaimRefusal", + "adjudicate_claim", + "claim_exit_code", + "parse_claim", +) + + +_REVISION = re.compile(r"(?:[0-9a-f]{40}|[0-9a-f]{64})\Z") +_SYMBOL = re.compile( + r"[A-Za-z_][A-Za-z0-9_]*(?:\.[A-Za-z_][A-Za-z0-9_]*)*:" + r"[A-Za-z_][A-Za-z0-9_]*(?:\.[A-Za-z_][A-Za-z0-9_]*)*\Z" +) +_WINDOWS_ABSOLUTE = re.compile(r"[A-Za-z]:[\\/]", re.ASCII) +_FINDING_FIELDS = frozenset( + ("symbol", "verdict", "reason_code", "projection_scope") +) +_FINDING_VERDICTS = frozenset( + ( + "IDENTICAL", + "CHANGED", + "IDENTICAL_UNDER_PROJECTION", + "CHANGED_UNDER_PROJECTION", + "NOT_EXERCISED", + ) +) +_DISPOSITIONS = frozenset( + ( + "CLAIM_VERIFIED", + "CLAIM_REFUTED", + "CLAIM_UNVERIFIABLE", + "CLAIM_OUT_OF_SCOPE", + ) +) + + +class ClaimRefusal(ValueError): + """A claim cannot be parsed or adjudicated without guessing.""" + + def __init__(self, code: str): + self.code = code + super().__init__(code) + + +@dataclass(frozen=True) +class BehaviorClaim: + schema_version: int + claim: str + base_revision: str + targets: tuple[str, ...] + + +def _refuse(code: str) -> None: + raise ClaimRefusal(code) + + +def _revision(value: object, code: str) -> str: + if type(value) is not str or not _REVISION.fullmatch(value): + _refuse(code) + return value + + +def _symbol(value: object, code: str) -> str: + if type(value) is not str or not _SYMBOL.fullmatch(value): + _refuse(code) + return value + + +def _quoted(raw: str) -> str: + try: + value = json.loads(raw) + except (json.JSONDecodeError, TypeError, ValueError): + _refuse("CLAIM_SYNTAX_REFUSED") + if type(value) is not str or "\x00" in value: + _refuse("CLAIM_SYNTAX_REFUSED") + return value + + +def parse_claim(source: str) -> BehaviorClaim: + """Parse the intentionally small, dependency-free claim TOML subset.""" + + if type(source) is not str or len(source.encode("utf-8")) > 65_536: + _refuse("CLAIM_SYNTAX_REFUSED") + top: dict[str, object] = {} + targets: list[dict[str, object]] = [] + current: dict[str, object] = top + for raw_line in source.splitlines(): + line = raw_line.strip() + if not line or line.startswith("#"): + continue + if line == "[[target]]": + current = {} + targets.append(current) + continue + if line.startswith("[") or "=" not in line: + _refuse("CLAIM_SYNTAX_REFUSED") + name, raw_value = line.split("=", 1) + name = name.strip() + raw_value = raw_value.strip() + allowed = ( + frozenset(("schema_version", "claim", "base_revision")) + if current is top + else frozenset(("symbol",)) + ) + if name not in allowed or name in current: + _refuse("CLAIM_SCHEMA_REFUSED") + if name == "schema_version": + if raw_value != "1": + _refuse("CLAIM_SCHEMA_VERSION_REFUSED") + current[name] = 1 + else: + current[name] = _quoted(raw_value) + if set(top) != {"schema_version", "claim", "base_revision"}: + _refuse("CLAIM_SCHEMA_REFUSED") + if top["schema_version"] != 1: + _refuse("CLAIM_SCHEMA_VERSION_REFUSED") + if top["claim"] != "behavior_preserved": + _refuse("CLAIM_TYPE_REFUSED") + base_revision = _revision(top["base_revision"], "CLAIM_REVISION_REFUSED") + if not targets: + _refuse("CLAIM_VACUOUS_REFUSED") + symbols = [] + for target in targets: + if set(target) != {"symbol"}: + _refuse("CLAIM_SCHEMA_REFUSED") + symbols.append(_symbol(target["symbol"], "CLAIM_TARGET_REFUSED")) + if len(set(symbols)) != len(symbols): + _refuse("CLAIM_DUPLICATE_TARGET_REFUSED") + return BehaviorClaim( + schema_version=1, + claim="behavior_preserved", + base_revision=base_revision, + targets=tuple(sorted(symbols)), + ) + + +def _plain_json(value: object) -> object: + if value is None or type(value) in (bool, int, str): + return value + if type(value) is float: + if not math.isfinite(value): + _refuse("CLAIM_INVOCATION_REFUSED") + return value + if type(value) is list: + return [_plain_json(item) for item in value] + if isinstance(value, Mapping): + result: dict[str, object] = {} + for key, item in value.items(): + if type(key) is not str or key in result: + _refuse("CLAIM_INVOCATION_REFUSED") + result[key] = _plain_json(item) + return result + _refuse("CLAIM_INVOCATION_REFUSED") + + +def _contains_absolute_path(value: object) -> bool: + if type(value) is str: + return value.startswith(("/", "\\\\")) or bool( + _WINDOWS_ABSOLUTE.match(value) + ) + if type(value) is list: + return any(_contains_absolute_path(item) for item in value) + if isinstance(value, Mapping): + return any( + _contains_absolute_path(key) or _contains_absolute_path(item) + for key, item in value.items() + ) + return False + + +def _invocation(value: object) -> dict[str, object]: + plain = _plain_json(value) + if not isinstance(plain, dict): + _refuse("CLAIM_INVOCATION_REFUSED") + if _contains_absolute_path(plain): + _refuse("CLAIM_PATH_REFUSED") + return plain + + +def _changed_targets(values: object) -> tuple[str, ...]: + if isinstance(values, (str, bytes, Mapping)): + _refuse("CLAIM_CHANGED_TARGETS_REFUSED") + try: + symbols = [_symbol(value, "CLAIM_CHANGED_TARGETS_REFUSED") for value in values] # type: ignore[union-attr] + except TypeError: + _refuse("CLAIM_CHANGED_TARGETS_REFUSED") + if not symbols: + _refuse("CLAIM_NO_CHANGED_TARGETS") + if len(set(symbols)) != len(symbols): + _refuse("CLAIM_CHANGED_TARGETS_REFUSED") + return tuple(sorted(symbols)) + + +def _findings(values: object, changed: frozenset[str]) -> dict[str, dict[str, object]]: + if isinstance(values, (str, bytes, Mapping)): + _refuse("CLAIM_FINDING_REFUSED") + result: dict[str, dict[str, object]] = {} + try: + rows = list(values) # type: ignore[arg-type] + except TypeError: + _refuse("CLAIM_FINDING_REFUSED") + for value in rows: + if not isinstance(value, Mapping) or set(value) != _FINDING_FIELDS: + _refuse("CLAIM_FINDING_REFUSED") + symbol = _symbol(value["symbol"], "CLAIM_FINDING_REFUSED") + verdict = value["verdict"] + reason_code = value["reason_code"] + projection_scope = value["projection_scope"] + if ( + symbol in result + or symbol not in changed + or verdict not in _FINDING_VERDICTS + or (reason_code is not None and (type(reason_code) is not str or not reason_code)) + or ( + projection_scope is not None + and (type(projection_scope) is not str or not projection_scope) + ) + ): + _refuse("CLAIM_FINDING_REFUSED") + projected = verdict in ( + "IDENTICAL_UNDER_PROJECTION", + "CHANGED_UNDER_PROJECTION", + ) + if projected != (projection_scope is not None): + _refuse("CLAIM_FINDING_REFUSED") + if verdict == "NOT_EXERCISED" and reason_code is None: + _refuse("CLAIM_FINDING_REFUSED") + if verdict != "NOT_EXERCISED" and reason_code is not None: + _refuse("CLAIM_FINDING_REFUSED") + result[symbol] = { + "symbol": symbol, + "verdict": verdict, + "reason_code": reason_code, + "projection_scope": projection_scope, + } + return result + + +def _separation_reason( + *, + source: str, + fixture_revision: str, + base_revision: str, + authored_by: str, + predates: bool, + strict: bool, +) -> str | None: + if source not in ("base", "head", "explicit"): + _refuse("FIXTURE_SOURCE_REFUSED") + if authored_by not in ("human", "agent", "unknown"): + _refuse("FIXTURE_AUTHOR_REFUSED") + if type(predates) is not bool or type(strict) is not bool: + _refuse("FIXTURE_SEPARATION_REFUSED") + _revision(fixture_revision, "FIXTURE_SOURCE_REVISION_REFUSED") + if not strict: + return None + if source == "head": + return "FIXTURE_AUTHORED_AGAINST_HEAD" + if authored_by == "unknown": + return "FIXTURE_AUTHOR_UNKNOWN" + if source == "base" and fixture_revision != base_revision: + return "FIXTURE_SOURCE_REVISION_MISMATCH" + if not predates: + return "FIXTURE_POSTDATES_CHANGE" + return None + + +def _disposition( + symbol: str, + disposition: str, + reason_code: str | None, + *, + projection_scope: str | None = None, +) -> dict[str, object]: + if disposition not in _DISPOSITIONS: + _refuse("CLAIM_DISPOSITION_REFUSED") + scope = "UNDER_PROJECTION" if projection_scope is not None else ( + "FULL_OBSERVATION" if disposition in ("CLAIM_VERIFIED", "CLAIM_REFUTED") else "NONE" + ) + return { + "symbol": symbol, + "disposition": disposition, + "reason_code": reason_code, + "verification_scope": scope, + "projection_scope": projection_scope, + } + + +def adjudicate_claim( + claim: BehaviorClaim, + *, + head_revision: str, + changed_targets: Iterable[str], + findings: Iterable[Mapping[str, object]], + fixture_source: str = "base", + fixture_revision: str, + fixture_authored_by: str, + fixtures_predate_change: bool, + strict_separation: bool = True, + invocation: Mapping[str, object], +) -> dict[str, object]: + """Adjudicate an independently supplied changed-target census.""" + + if not isinstance(claim, BehaviorClaim): + _refuse("CLAIM_SCHEMA_REFUSED") + if ( + claim.schema_version != 1 + or claim.claim != "behavior_preserved" + or not claim.targets + ): + _refuse("CLAIM_SCHEMA_REFUSED") + validated_claim_targets = tuple( + sorted(_symbol(symbol, "CLAIM_SCHEMA_REFUSED") for symbol in claim.targets) + ) + if ( + validated_claim_targets != claim.targets + or len(set(validated_claim_targets)) != len(validated_claim_targets) + ): + _refuse("CLAIM_SCHEMA_REFUSED") + base_revision = _revision(claim.base_revision, "CLAIM_REVISION_REFUSED") + head = _revision(head_revision, "CLAIM_REVISION_REFUSED") + if head == base_revision: + _refuse("IDENTICAL_REVISIONS_REFUSED") + changed = _changed_targets(changed_targets) + changed_set = frozenset(changed) + by_symbol = _findings(findings, changed_set) + separation_reason = _separation_reason( + source=fixture_source, + fixture_revision=fixture_revision, + base_revision=base_revision, + authored_by=fixture_authored_by, + predates=fixtures_predate_change, + strict=strict_separation, + ) + claimed = frozenset(validated_claim_targets) + dispositions: list[dict[str, object]] = [] + for symbol in changed: + if symbol not in claimed: + dispositions.append( + _disposition( + symbol, + "CLAIM_OUT_OF_SCOPE", + "CLAIM_TARGET_OMITTED", + ) + ) + continue + finding = by_symbol.get(symbol) + if finding is None: + dispositions.append( + _disposition( + symbol, + "CLAIM_UNVERIFIABLE", + "MISSING_REVISION_FINDING", + ) + ) + continue + verdict = finding["verdict"] + projection_scope = finding["projection_scope"] + if verdict in ("CHANGED", "CHANGED_UNDER_PROJECTION"): + dispositions.append( + _disposition( + symbol, + "CLAIM_REFUTED", + None, + projection_scope=projection_scope, # type: ignore[arg-type] + ) + ) + elif verdict == "NOT_EXERCISED": + dispositions.append( + _disposition( + symbol, + "CLAIM_UNVERIFIABLE", + str(finding["reason_code"]), + ) + ) + elif separation_reason is not None: + dispositions.append( + _disposition( + symbol, + "CLAIM_UNVERIFIABLE", + separation_reason, + ) + ) + else: + dispositions.append( + _disposition( + symbol, + "CLAIM_VERIFIED", + None, + projection_scope=projection_scope, # type: ignore[arg-type] + ) + ) + for symbol in sorted(claimed - changed_set): + dispositions.append( + _disposition( + symbol, + "CLAIM_UNVERIFIABLE", + "CLAIM_TARGET_UNCHANGED_OR_UNMAPPED", + ) + ) + dispositions.sort(key=lambda row: str(row["symbol"])) + summary = { + "verified": sum(row["disposition"] == "CLAIM_VERIFIED" for row in dispositions), + "refuted": sum(row["disposition"] == "CLAIM_REFUTED" for row in dispositions), + "unverifiable": sum( + row["disposition"] == "CLAIM_UNVERIFIABLE" for row in dispositions + ), + "out_of_scope": sum( + row["disposition"] == "CLAIM_OUT_OF_SCOPE" for row in dispositions + ), + "total": len(dispositions), + } + return { + "claim": claim.claim, + "base_revision": base_revision, + "head_revision": head, + "dispositions": copy.deepcopy(dispositions), + "summary": summary, + "fixtures_predate_change": fixtures_predate_change, + "invocation": _invocation(invocation), + } + + +def claim_exit_code(report: Mapping[str, object]) -> int: + """Return the fixed public exit precedence for a claim report.""" + + if not isinstance(report, Mapping): + _refuse("CLAIM_DISPOSITION_REFUSED") + values = report.get("dispositions") + if isinstance(values, (str, bytes, Mapping)): + _refuse("CLAIM_DISPOSITION_REFUSED") + try: + rows = list(values) # type: ignore[arg-type] + except TypeError: + _refuse("CLAIM_DISPOSITION_REFUSED") + if not rows: + _refuse("CLAIM_VACUOUS_REFUSED") + dispositions = [] + for row in rows: + if not isinstance(row, Mapping) or row.get("disposition") not in _DISPOSITIONS: + _refuse("CLAIM_DISPOSITION_REFUSED") + dispositions.append(row["disposition"]) + invocation = report.get("invocation", {}) + if isinstance(invocation, Mapping): + strict = invocation.get("strict", True) + elif isinstance(invocation, list): + try: + strict = { + row["name"]: row["value"] + for row in invocation + if isinstance(row, Mapping) + }.get("strict", True) + except (KeyError, TypeError): + _refuse("CLAIM_DISPOSITION_REFUSED") + else: + _refuse("CLAIM_DISPOSITION_REFUSED") + if type(strict) is not bool: + _refuse("CLAIM_DISPOSITION_REFUSED") + if "CLAIM_OUT_OF_SCOPE" in dispositions: + return 3 + if strict and "CLAIM_UNVERIFIABLE" in dispositions: + return 2 + if "CLAIM_REFUTED" in dispositions: + return 1 + permitted = {"CLAIM_VERIFIED"} + if not strict: + permitted.add("CLAIM_UNVERIFIABLE") + if set(dispositions) <= permitted: + return 0 + _refuse("CLAIM_DISPOSITION_REFUSED") diff --git a/src/breakcheck/core/interfaces.py b/src/breakcheck/core/interfaces.py index e6f2c57..c7ef28f 100644 --- a/src/breakcheck/core/interfaces.py +++ b/src/breakcheck/core/interfaces.py @@ -15,7 +15,13 @@ ) -__all__ = ("UsageScanner", "EnvBuilder", "Executor", "EqualityRules") +__all__ = ( + "UsageScanner", + "EnvBuilder", + "Executor", + "EqualityRules", + "ArtifactVerifier", +) class UsageScanner(Protocol): @@ -46,3 +52,9 @@ class EqualityRules(Protocol): """Compare two normalized observations using ecosystem semantics.""" def compare(self, old: Observation, new: Observation) -> Comparison: ... + + +class ArtifactVerifier(Protocol): + """Verify a versioned report and its separately persisted evidence.""" + + def verify(self, report: dict, evidence: dict) -> str: ... diff --git a/src/breakcheck/core/models.py b/src/breakcheck/core/models.py index a148bd5..8dbcab5 100644 --- a/src/breakcheck/core/models.py +++ b/src/breakcheck/core/models.py @@ -1,6 +1,7 @@ from dataclasses import dataclass import copy import json +from typing import Mapping _SPECS = [{'allow_unknown': False, 'class_name': 'ReplaySnippet', 'closed_mapping_fields': {}, 'conditional_list_rules': [], 'constant_fields': {}, 'enum_fields': {'args_source': ['literal', 'refused']}, 'fields': [{'default_kind': 'required', 'name': 'snippet_id'}, {'default_kind': 'required', 'name': 'api'}, {'default_kind': 'required', 'name': 'call_sites'}, {'default_kind': 'required', 'name': 'code'}, {'default_kind': 'required', 'name': 'args_source'}, {'default_kind': 'none', 'name': 'reason_code'}], 'ordered_mapping_list_fields': {'call_sites': {'required_fields': ['file', 'line']}}, 'sort_fields': [], 'sort_list_fields': {}}, {'allow_unknown': False, 'class_name': 'Observation', 'closed_mapping_fields': {}, 'conditional_list_rules': [], 'constant_fields': {}, 'enum_fields': {'kind': ['value', 'exception', 'timeout']}, 'fields': [{'default_kind': 'required', 'name': 'kind'}, {'default_kind': 'required', 'name': 'payload'}, {'default_kind': 'none', 'name': 'exception_class'}, {'default_kind': 'none', 'name': 'duration_ms'}], 'ordered_mapping_list_fields': {}, 'sort_fields': [], 'sort_list_fields': {}}, {'allow_unknown': False, 'class_name': 'Comparison', 'closed_mapping_fields': {'detail': {'enum_fields': {'reason_code': ['EQUAL', 'KIND_MISMATCH', 'EXCEPTION_CLASS', 'VALUE_MISMATCH', 'FLOAT_MISMATCH', 'MISSING_KEY', 'LENGTH_MISMATCH']}, 'json_pointer_fields': ['path'], 'nullable_fields': ['path'], 'required_fields': ['reason_code', 'path', 'old_summary', 'new_summary', 'policy']}}, 'conditional_list_rules': [], 'constant_fields': {}, 'enum_fields': {'verdict': ['IDENTICAL', 'CHANGED']}, 'fields': [{'default_kind': 'required', 'name': 'verdict'}, {'default_kind': 'required', 'name': 'detail'}], 'ordered_mapping_list_fields': {}, 'sort_fields': [], 'sort_list_fields': {}}, {'allow_unknown': False, 'class_name': 'Finding', 'closed_mapping_fields': {}, 'conditional_list_rules': [{'cases': {'CHANGED': {'max_items': 2, 'min_items': 1}, 'IDENTICAL': {'max_items': 0, 'min_items': 0}, 'NOT_EXERCISED': {'max_items': 1, 'min_items': 1}}, 'discriminator_field': 'verdict', 'field': 'suggested_action'}], 'constant_fields': {}, 'enum_fields': {'verdict': ['IDENTICAL', 'CHANGED', 'NOT_EXERCISED']}, 'fields': [{'default_kind': 'required', 'name': 'finding_id'}, {'default_kind': 'required', 'name': 'api'}, {'default_kind': 'required', 'name': 'call_sites'}, {'default_kind': 'required', 'name': 'verdict'}, {'default_kind': 'none', 'name': 'old'}, {'default_kind': 'none', 'name': 'new'}, {'default_kind': 'required', 'name': 'repro'}, {'default_kind': 'required', 'name': 'suggested_action'}, {'default_kind': 'none', 'name': 'reason_code'}, {'default_kind': 'required', 'name': 'comparison'}], 'ordered_mapping_list_fields': {'call_sites': {'required_fields': ['file', 'line']}, 'suggested_action': {'required_fields': ['kind', 'argument']}}, 'sort_fields': [], 'sort_list_fields': {}}, {'allow_unknown': False, 'class_name': 'Report', 'closed_mapping_fields': {}, 'conditional_list_rules': [], 'constant_fields': {'schema_version': 1}, 'enum_fields': {}, 'fields': [{'default_kind': 'required', 'name': 'package'}, {'default_kind': 'required', 'name': 'current_version'}, {'default_kind': 'required', 'name': 'new_version'}, {'default_kind': 'required', 'name': 'coverage'}, {'default_kind': 'required', 'name': 'findings'}, {'default_kind': 'required', 'name': 'witnesses'}, {'default_kind': 'required', 'name': 'summary'}], 'ordered_mapping_list_fields': {}, 'sort_fields': [], 'sort_list_fields': {'findings': {'sort_fields': ['finding_id']}}}, {'allow_unknown': False, 'class_name': 'Environment', 'closed_mapping_fields': {}, 'conditional_list_rules': [], 'constant_fields': {}, 'enum_fields': {}, 'fields': [], 'ordered_mapping_list_fields': {}, 'sort_fields': [], 'sort_list_fields': {}}, {'allow_unknown': False, 'class_name': 'EnvironmentPair', 'closed_mapping_fields': {}, 'conditional_list_rules': [], 'constant_fields': {}, 'enum_fields': {}, 'fields': [], 'ordered_mapping_list_fields': {}, 'sort_fields': [], 'sort_list_fields': {}}, {'allow_unknown': False, 'class_name': 'UsageManifest', 'closed_mapping_fields': {}, 'conditional_list_rules': [], 'constant_fields': {}, 'enum_fields': {}, 'fields': [], 'ordered_mapping_list_fields': {}, 'sort_fields': [], 'sort_list_fields': {}}] @@ -196,3 +197,33 @@ def sort_key(self): def canonical_json(value): return json.dumps(_plain(value), sort_keys=True, separators=(',', ':')) + + +@dataclass(frozen=True) +class ArtifactEnvelope: + """Immutable validated view of a closed schema-2 artifact.""" + + schema_version: int + artifact_kind: str + payload: dict + payload_sha256: str + + @classmethod + def from_mapping(cls, value: Mapping[str, object]) -> "ArtifactEnvelope": + from breakcheck.schema import validate_artifact + + artifact = validate_artifact(value) + return cls( + schema_version=artifact["schema_version"], + artifact_kind=artifact["artifact_kind"], + payload=copy.deepcopy(artifact["payload"]), + payload_sha256=artifact["payload_sha256"], + ) + + def to_dict(self) -> dict: + return { + "schema_version": self.schema_version, + "artifact_kind": self.artifact_kind, + "payload": copy.deepcopy(self.payload), + "payload_sha256": self.payload_sha256, + } diff --git a/src/breakcheck/demo.py b/src/breakcheck/demo.py new file mode 100644 index 0000000..b18085f --- /dev/null +++ b/src/breakcheck/demo.py @@ -0,0 +1,147 @@ +from __future__ import annotations + +import base64 +import csv +import hashlib +import io +import json +import os +import sys +import zipfile +from pathlib import Path +from types import SimpleNamespace + + +_DISTRIBUTION = "breakcheck-demo-dependency" +_IMPORT_ROOT = "breakcheck_demo_dependency" +_CURRENT = "1.0.0" +_PROPOSED = "2.0.0" + + +def _record_digest(data: bytes) -> str: + encoded = base64.urlsafe_b64encode(hashlib.sha256(data).digest()) + return "sha256=" + encoded.rstrip(b"=").decode("ascii") + + +def _package_source(version: str) -> bytes: + increment = "0" if version == _CURRENT else "1" + return ( + f'__version__ = "{version}"\n\n' + "def behavior(value):\n" + f" return value + {increment}\n" + ).encode("utf-8") + + +def _wheel_files(version: str) -> dict[str, bytes]: + dist_info = f"breakcheck_demo_dependency-{version}.dist-info" + return { + f"{_IMPORT_ROOT}/__init__.py": _package_source(version), + f"{dist_info}/METADATA": ( + "Metadata-Version: 2.1\n" + f"Name: {_DISTRIBUTION}\n" + f"Version: {version}\n" + "Summary: Offline Breakcheck demonstration dependency\n" + ).encode("utf-8"), + f"{dist_info}/WHEEL": ( + "Wheel-Version: 1.0\n" + "Generator: breakcheck-demo\n" + "Root-Is-Purelib: true\n" + "Tag: py3-none-any\n" + ).encode("utf-8"), + f"{dist_info}/top_level.txt": f"{_IMPORT_ROOT}\n".encode("utf-8"), + } + + +def _write_wheel(wheelhouse: Path, version: str) -> Path: + wheel = wheelhouse / ( + f"breakcheck_demo_dependency-{version}-py3-none-any.whl" + ) + files = _wheel_files(version) + dist_info = f"breakcheck_demo_dependency-{version}.dist-info" + record_path = f"{dist_info}/RECORD" + rows = [ + (name, _record_digest(data), str(len(data))) + for name, data in sorted(files.items()) + ] + rows.append((record_path, "", "")) + record_stream = io.StringIO(newline="") + csv.writer(record_stream, lineterminator="\n").writerows(rows) + files[record_path] = record_stream.getvalue().encode("utf-8") + with zipfile.ZipFile(wheel, "w", compression=zipfile.ZIP_DEFLATED) as archive: + for name, data in sorted(files.items()): + info = zipfile.ZipInfo(name, date_time=(2020, 1, 1, 0, 0, 0)) + info.compress_type = zipfile.ZIP_DEFLATED + info.external_attr = 0o644 << 16 + archive.writestr(info, data) + return wheel + + +def _install_metadata_view(root: Path) -> None: + package = root / _IMPORT_ROOT + package.mkdir(parents=True) + (package / "__init__.py").write_bytes(_package_source(_CURRENT)) + dist_info = root / f"breakcheck_demo_dependency-{_CURRENT}.dist-info" + dist_info.mkdir() + (dist_info / "METADATA").write_text( + "Metadata-Version: 2.1\n" + f"Name: {_DISTRIBUTION}\n" + f"Version: {_CURRENT}\n", + encoding="utf-8", + ) + (dist_info / "top_level.txt").write_text( + _IMPORT_ROOT + "\n", encoding="utf-8" + ) + + +def run_demo(output_root: str | os.PathLike[str], build) -> int: + root = Path(output_root) + if root.exists() or root.is_symlink(): + raise ValueError("DEMO_OUTPUT_EXISTS_REFUSED") + root.mkdir(parents=True) + repository = root / "repository" + wheelhouse = root / "wheelhouse" + installed = root / "installed-current" + repository.mkdir() + wheelhouse.mkdir() + installed.mkdir() + (repository / "app.py").write_text( + "import breakcheck_demo_dependency\n\n" + "outcome = breakcheck_demo_dependency.behavior(1)\n", + encoding="utf-8", + ) + _write_wheel(wheelhouse, _CURRENT) + _write_wheel(wheelhouse, _PROPOSED) + _install_metadata_view(installed) + + report = root / "report.json" + evidence = root / "evidence.json" + runtime = root / "runtime" + args = SimpleNamespace( + target=f"{_DISTRIBUTION}@{_PROPOSED}", + wheelhouse=str(wheelhouse), + runtime_root=str(runtime), + output=str(report), + evidence=str(evidence), + json=False, + ci=False, + ) + previous_cwd = Path.cwd() + sys.path.insert(0, str(installed)) + try: + os.chdir(repository) + result = build(args) + finally: + os.chdir(previous_cwd) + if sys.path and sys.path[0] == str(installed): + sys.path.pop(0) + if result != 0 or not report.is_file() or not evidence.is_file(): + raise ValueError("DEMO_EXECUTION_REFUSED") + try: + from breakcheck.verify import verify_report + + report_payload = json.loads(report.read_text(encoding="utf-8")) + evidence_payload = json.loads(evidence.read_text(encoding="utf-8")) + verify_report(report_payload, evidence_payload) + except (OSError, UnicodeError, json.JSONDecodeError, ValueError) as exc: + raise ValueError("DEMO_VERIFICATION_REFUSED") from exc + return 0 diff --git a/src/breakcheck/report.py b/src/breakcheck/report.py index b7dadbc..9036146 100644 --- a/src/breakcheck/report.py +++ b/src/breakcheck/report.py @@ -41,9 +41,93 @@ def _ordered(report): return value def render_json(report): + if isinstance(report, dict) and report.get("schema_version") == 2: + from breakcheck.schema import canonical_json, validate_artifact + + return canonical_json(validate_artifact(report)) return json.dumps(_ordered(report), sort_keys=True, separators=(",", ":"), ensure_ascii=True) def render_human(report): + if isinstance(report, dict) and report.get("schema_version") == 2: + from breakcheck.schema import validate_artifact + + artifact = validate_artifact(report) + kind = artifact["artifact_kind"] + payload = artifact["payload"] + if kind == "claim_report": + summary = payload["summary"] + lines = [ + "claims " + + "CLAIM_VERIFIED=" + str(summary["claim_verified"]) + + " CLAIM_REFUTED=" + str(summary["claim_refuted"]) + + " CLAIM_UNVERIFIABLE=" + str(summary["claim_unverifiable"]) + + " CLAIM_OUT_OF_SCOPE=" + str(summary["claim_out_of_scope"]) + ] + for item in payload["dispositions"]: + scope = ( + " projection=" + item["projection_scope"] + if item["projection_scope"] is not None + else "" + ) + lines.append( + item["disposition_id"] + + " " + + item["disposition"] + + scope + ) + for event in payload["fixture_revision_events"]: + lines.append( + "warning " + + event["reason_code"] + + " target=" + + event["target_id"] + ) + return "\n".join(lines) + if kind == "coverage_report": + counts = payload["counts"] + return "coverage " + " ".join( + name + "=" + str(counts[name]) + for name in ( + "EXERCISED", + "G1_NOT_DISCOVERABLE", + "G2_NONLITERAL", + "G3_UNNORMALIZABLE", + "G4_IMPURE", + ) + ) + if kind not in ("dependency_report", "revision_report"): + return "artifact=" + kind + summary = payload["summary"] + summary_line = ( + "verdicts " + + "IDENTICAL=" + str(summary["identical"]) + + " CHANGED=" + str(summary["changed"]) + + " IDENTICAL_UNDER_PROJECTION=" + + str(summary["identical_under_projection"]) + + " CHANGED_UNDER_PROJECTION=" + + str(summary["changed_under_projection"]) + + " NOT_EXERCISED=" + str(summary["not_exercised"]) + ) + lines = [summary_line] + for item in payload["findings"]: + identity = item["finding_id"] + verdict = item["verdict"] + projection = item.get("projection") + scope = ( + " projection=" + projection["source"] + if projection is not None + else "" + ) + lines.append(identity + " " + verdict + scope) + if kind == "revision_report": + for event in payload["fixture_revision_events"]: + lines.append( + "warning " + + event["reason_code"] + + " target=" + + event["target_id"] + ) + return "\n".join(lines) value = _ordered(report) lines = ["coverage=" + str(value.get("coverage", {}).get("percent")), "findings="] for item in value["findings"]: @@ -51,6 +135,42 @@ def render_human(report): return "\n".join(lines) def ci_exit_code(report): + if isinstance(report, dict) and report.get("schema_version") == 2: + from breakcheck.schema import validate_artifact + + artifact = validate_artifact(report) + if artifact["artifact_kind"] == "claim_report": + from breakcheck.core.claims import claim_exit_code + + payload = artifact["payload"] + return claim_exit_code( + { + "dispositions": payload["dispositions"], + "invocation": payload["invocation"], + } + ) + if artifact["artifact_kind"] not in ("dependency_report", "revision_report"): + return _CLEAN + payload = artifact["payload"] + invocation = {row["name"]: row["value"] for row in payload["invocation"]} + if artifact["artifact_kind"] == "dependency_report": + percent = payload["coverage"]["percent"] + if percent < invocation.get("min_coverage", _THRESHOLD): + return _COVERAGE + if any( + item["verdict"] in ("CHANGED", "CHANGED_UNDER_PROJECTION") + for item in payload["findings"] + ): + return _CHANGED + if artifact["artifact_kind"] == "revision_report": + total = len(payload["findings"]) + exercised = sum( + item["verdict"] != "NOT_EXERCISED" for item in payload["findings"] + ) + percent = 0.0 if total == 0 else 100.0 * exercised / total + if percent < invocation.get("min_coverage", _THRESHOLD): + return _COVERAGE + return _CLEAN value = _ordered(report) coverage = value.get("coverage", {}) percent = coverage.get("percent", 0) if isinstance(coverage, dict) else 0 diff --git a/src/breakcheck/revision_cli.py b/src/breakcheck/revision_cli.py new file mode 100644 index 0000000..a387ed7 --- /dev/null +++ b/src/breakcheck/revision_cli.py @@ -0,0 +1,1373 @@ +"""Fail-closed orchestration for behavior comparisons across Git revisions.""" + +from __future__ import annotations + +import copy +from dataclasses import dataclass +import hashlib +import os +from pathlib import Path, PurePosixPath +import platform +import subprocess +import sys +from collections.abc import Iterable, Mapping + +from .adapters.python.equality import compare_observations +from .adapters.python.executor import run_repeated_typed_snippet_isolated +from .adapters.python.fixtures import ( + FixtureBinding, + FixtureFile, + FixtureRefusal, + load_fixture_file, + render_fixture_source, +) +from .adapters.python.symbols import ( + SymbolAnalysisRefusal, + SymbolChange, + SymbolDefinition, + compare_symbol_trees, + inventory_symbols, + tracked_tree_identity, +) +from .adapters.python.worktrees import WorktreeRefusal, revision_worktrees +from .core.baselines import BaselineRefusal, freeze_baseline +from .core.claims import ( + ClaimRefusal, + adjudicate_claim, + claim_exit_code, + parse_claim, +) +from .revision_report import make_evidence_artifact, make_revision_artifact +from .schema import ( + artifact_digest, + canonicalize_invocation, + record_identity, + validate_artifact, + verify_artifact, +) + + +_EXERCISED = frozenset(("VALUE", "EXCEPTION")) +_CHANGED_STATUSES = frozenset( + ( + "CHANGED", + "CONTEXT_CHANGED", + "FIXTURE_SIGNATURE_DRIFT", + "NO_BASELINE_REVISION", + "SYMBOL_REMOVED", + "SYMBOL_AMBIGUOUS", + ) +) + + +class RevisionModeRefusal(ValueError): + """A revision command could not produce evidence without guessing.""" + + def __init__(self, code: str): + self.code = code + super().__init__(code) + + +@dataclass(frozen=True) +class RevisionCommandResult: + report: dict[str, object] + evidence: dict[str, object] + exit_code: int + + +@dataclass(frozen=True) +class _Replay: + status: str + reason_code: str | None + observation: dict[str, object] | None + repeat_sha256: tuple[str, str] | None + provenance: tuple[str, ...] + replay_source: str + + +@dataclass(frozen=True) +class _FixtureContext: + fixture: FixtureFile + source_revision: str + authored_by: str + source: str + predates_change: bool + bindings: Mapping[str, FixtureBinding] + + +def _refuse(code: str) -> None: + raise RevisionModeRefusal(code) + + +def _translate(exc: Exception) -> RevisionModeRefusal: + code = getattr(exc, "code", None) + if type(code) is not str or not code: + code = str(exc) if str(exc) else "REVISION_MODE_REFUSED" + return RevisionModeRefusal(code) + + +def _git(repository: Path, *arguments: str) -> bytes: + environment = { + key: os.environ[key] + for key in ("PATH", "SYSTEMROOT", "TMPDIR", "TEMP", "TMP") + if key in os.environ + } + environment.update( + { + "GIT_CONFIG_GLOBAL": os.devnull, + "GIT_CONFIG_NOSYSTEM": "1", + "GIT_CONFIG_SYSTEM": os.devnull, + "GIT_OPTIONAL_LOCKS": "0", + "GIT_TERMINAL_PROMPT": "0", + } + ) + try: + result = subprocess.run( + ["git", "-C", str(repository), *arguments], + check=False, + capture_output=True, + env=environment, + shell=False, + ) + except OSError as exc: + raise RevisionModeRefusal("REPOSITORY_REFUSED") from exc + if result.returncode != 0: + _refuse("REPOSITORY_REFUSED") + return result.stdout + + +def _repository_root(repository: Path | str) -> Path: + requested = Path(repository) + if requested.is_symlink() or not requested.is_dir(): + _refuse("REPOSITORY_REFUSED") + try: + top = Path( + _git(requested, "rev-parse", "--show-toplevel") + .decode("utf-8", errors="strict") + .strip() + ).resolve(strict=True) + except (OSError, UnicodeError): + _refuse("REPOSITORY_REFUSED") + if requested.resolve(strict=True) != top: + _refuse("REPOSITORY_REFUSED") + return top + + +def _dirty(repository: Path) -> bool: + return bool( + _git(repository, "status", "--porcelain=v1", "--untracked-files=normal") + ) + + +def _relative_path(value: Path | str, code: str) -> str: + if isinstance(value, Path): + text = value.as_posix() + elif type(value) is str: + text = value + else: + _refuse(code) + if not text or "\\" in text: + _refuse(code) + relative = PurePosixPath(text) + if relative.is_absolute() or any( + part in ("", ".", "..") for part in relative.parts + ): + _refuse(code) + if relative.as_posix() != text: + _refuse(code) + return text + + +def _target_list(values: Iterable[str]) -> tuple[str, ...]: + if isinstance(values, (str, bytes, Mapping)): + _refuse("REVISION_TARGET_REFUSED") + try: + rows = tuple(sorted(values)) + except TypeError: + _refuse("REVISION_TARGET_REFUSED") + if any(type(value) is not str or not value for value in rows): + _refuse("REVISION_TARGET_REFUSED") + if len(set(rows)) != len(rows): + _refuse("REVISION_TARGET_REFUSED") + return rows + + +def _environment_schema() -> dict[str, str]: + return { + "implementation": platform.python_implementation().lower(), + "machine": platform.machine() or "unknown", + "platform": sys.platform, + "python": platform.python_version(), + } + + +def _environment_domain() -> dict[str, str]: + schema = _environment_schema() + return { + "implementation": schema["implementation"], + "python_version": schema["python"], + "platform": schema["platform"] + "-" + schema["machine"], + } + + +def _api(definition: SymbolDefinition) -> str: + return definition.module + "." + definition.symbol + + +def _inventory_rows( + definitions: Iterable[SymbolDefinition], +) -> list[dict[str, object]]: + return [ + { + "file": definition.relative_path, + "line": definition.line, + "column": definition.column, + "api": _api(definition), + } + for definition in definitions + ] + + +def _definition_map( + definitions: Iterable[SymbolDefinition], +) -> dict[str, SymbolDefinition | None]: + grouped: dict[str, list[SymbolDefinition]] = {} + for definition in definitions: + grouped.setdefault(definition.target, []).append(definition) + return { + target: values[0] if len(values) == 1 else None + for target, values in grouped.items() + } + + +def _load_fixtures( + *, + repository: Path, + pair, + fixture_path: Path | str, + fixture_source: str, + source_definitions: tuple[SymbolDefinition, ...], +) -> _FixtureContext: + relative = _relative_path(fixture_path, "FIXTURE_BINDING_PATH_REFUSED") + if fixture_source == "base": + root = pair.base_root + revision = pair.base_commit + predates = pair.base_commit != pair.head_commit + elif fixture_source == "head": + root = pair.head_root + revision = pair.head_commit + predates = False + elif fixture_source == "explicit": + root = repository + try: + revision = ( + _git(repository, "rev-parse", "--verify", "HEAD^{commit}") + .decode("ascii", errors="strict") + .strip() + ) + except UnicodeError: + _refuse("FIXTURE_SOURCE_REVISION_REFUSED") + predates = revision != pair.head_commit + else: + _refuse("FIXTURE_SOURCE_REFUSED") + fixture = load_fixture_file( + root / relative, + repository_root=root, + inventory=_inventory_rows(source_definitions), + ) + if not fixture.bindings: + _refuse("FIXTURE_VACUOUS_REFUSED") + authors = {binding.fixture_authored_by for binding in fixture.bindings} + if len(authors) != 1: + _refuse("FIXTURE_AUTHOR_MIXED_REFUSED") + by_key = { + ( + definition.relative_path, + definition.line, + definition.column, + _api(definition), + ): definition.target + for definition in source_definitions + } + bindings: dict[str, FixtureBinding] = {} + for binding in fixture.bindings: + target = by_key.get(binding.key) + if target is None or target in bindings: + _refuse("FIXTURE_AMBIGUOUS_REFUSED") + bindings[target] = binding + return _FixtureContext( + fixture=fixture, + source_revision=revision, + authored_by=next(iter(authors)), + source=fixture_source, + predates_change=predates, + bindings=bindings, + ) + + +def _fixture_schema(context: _FixtureContext) -> dict[str, str]: + return { + "authored_by": context.authored_by, + "sha256": context.fixture.canonical_sha256, + "source_revision": context.source_revision, + } + + +def _fixture_domain(context: _FixtureContext) -> dict[str, str]: + return { + "sha256": context.fixture.canonical_sha256, + "source_revision": context.source_revision, + "source": context.source, + "authored_by": context.authored_by, + } + + +def _import_roots( + root: Path, definition: SymbolDefinition +) -> tuple[tuple[str, ...], tuple[str, ...]]: + resolved_root = root.resolve(strict=True) + relative = PurePosixPath(definition.relative_path) + if not relative.parts or relative.is_absolute() or ".." in relative.parts: + _refuse("REVISION_IMPORT_ROOT_REFUSED") + if relative.parts[0] != "src": + return (str(resolved_root),), (".",) + source_root = resolved_root / "src" + try: + if source_root.is_symlink() or not source_root.is_dir(): + _refuse("REVISION_IMPORT_ROOT_REFUSED") + resolved_source = source_root.resolve(strict=True) + except OSError: + _refuse("REVISION_IMPORT_ROOT_REFUSED") + if resolved_source != source_root or resolved_source.parent != resolved_root: + _refuse("REVISION_IMPORT_ROOT_REFUSED") + return (str(resolved_source),), ("src",) + + +def _import_root_identity( + root: Path, + definitions: Iterable[SymbolDefinition], + *, + tree_sha256: str, +) -> str: + labels = set() + for definition in definitions: + _, chosen = _import_roots(root, definition) + labels.update(chosen) + return artifact_digest( + {"roots": sorted(labels), "tree_sha256": tree_sha256} + ) + + +def _verified_previous_report( + previous_report: Mapping[str, object] | None, +) -> tuple[dict[str, object] | None, str | None]: + if previous_report is None: + return None, None + try: + artifact = validate_artifact(previous_report) + if artifact["artifact_kind"] != "revision_report": + _refuse("PREVIOUS_REPORT_REFUSED") + verify_artifact(artifact) + except RevisionModeRefusal: + raise + except ValueError: + _refuse("PREVIOUS_REPORT_REFUSED") + return artifact["payload"], artifact_digest(artifact) + + +def _fixture_revision_events( + previous_payload: Mapping[str, object] | None, + *, + base_revision: str, + findings: list[dict[str, object]], +) -> list[dict[str, object]]: + if previous_payload is None: + return [] + if previous_payload["base_revision"] != base_revision: + _refuse("PREVIOUS_REPORT_BASE_MISMATCH") + prior_by_target = { + str(row["target_id"]): row for row in previous_payload["findings"] + } + events = [] + for current in findings: + prior = prior_by_target.get(str(current["target_id"])) + if prior is None: + continue + prior_fixture = prior["fixture_binding_sha256"] + current_fixture = current["fixture_binding_sha256"] + if ( + prior_fixture is None + or current_fixture is None + or prior_fixture == current_fixture + or prior["verdict"] + not in ("CHANGED", "CHANGED_UNDER_PROJECTION") + or current["verdict"] + not in ("IDENTICAL", "IDENTICAL_UNDER_PROJECTION") + ): + continue + event = { + "event_id": "", + "target_id": current["target_id"], + "prior_finding_id": prior["finding_id"], + "current_finding_id": current["finding_id"], + "prior_fixture_binding_sha256": prior_fixture, + "current_fixture_binding_sha256": current_fixture, + "prior_verdict": prior["verdict"], + "current_verdict": current["verdict"], + "reason_code": "FIXTURE_REVISED_AFTER_FAILURE", + } + event["event_id"] = record_identity(event, "event_id") + events.append(event) + events.sort(key=lambda row: str(row["event_id"])) + return events + + +def _projection(binding: FixtureBinding) -> dict[str, str] | None: + if binding.projection is None: + return None + return { + "source": binding.projection, + "sha256": artifact_digest(binding.projection), + } + + +def _snippet( + definition: SymbolDefinition, binding: FixtureBinding +) -> tuple[str, str]: + marker_suffix = hashlib.sha256( + (definition.target + "\0" + binding.binding_sha256).encode("utf-8") + ).hexdigest()[:16] + marker = "_BreakcheckImportFailed_" + marker_suffix + call_source = render_fixture_source( + binding, "_bc_target_module." + definition.symbol + ) + indented_call = "".join( + " " + line for line in call_source.splitlines(True) + ) + source = ( + "import importlib as _bc_importlib\n" + + "class " + + marker + + "(BaseException):\n pass\n" + + "try:\n" + + " _bc_target_module = _bc_importlib.import_module(" + + repr(definition.module) + + ")\n" + + "except BaseException as _bc_import_error:\n" + + " raise " + + marker + + "(type(_bc_import_error).__name__, list(_bc_import_error.args))\n" + + "else:\n" + + indented_call + ) + return source, marker + + +def _replay( + root: Path, + definition: SymbolDefinition, + binding: FixtureBinding, + *, + executor, +) -> _Replay: + source, import_marker = _snippet(definition, binding) + prefixes, _ = _import_roots(root, definition) + result = executor( + snippet_source=source, + sys_path_prefixes=prefixes, + runs=2, + ) + if not isinstance(result, Mapping): + _refuse("REVISION_EXECUTOR_REFUSED") + repeatable = result.get("repeatable") + status = result.get("status") + reason = result.get("reason_code") + observation = result.get("observation") + if type(repeatable) is not bool or type(status) is not str: + _refuse("REVISION_EXECUTOR_REFUSED") + if not repeatable: + return _Replay( + "PROTOCOL_REFUSED", + "NONDETERMINISTIC_OBSERVATION", + None, + None, + ("OPERATOR_FIXTURE",), + source, + ) + if status == "EXCEPTION" and isinstance(observation, Mapping): + if observation.get("exception_class") == import_marker: + return _Replay( + "PROTOCOL_REFUSED", + "IMPORT_FAILED", + None, + None, + ("OPERATOR_FIXTURE",), + source, + ) + if status not in _EXERCISED: + if type(reason) is not str or not reason: + _refuse("REVISION_EXECUTOR_REFUSED") + return _Replay( + status, reason, None, None, ("OPERATOR_FIXTURE",), source + ) + if not isinstance(observation, Mapping): + _refuse("REVISION_EXECUTOR_REFUSED") + try: + domain = { + "kind": observation["kind"], + "payload": copy.deepcopy(observation["payload"]), + "exception_class": observation["exception_class"], + "duration_ms": None, + } + except KeyError: + _refuse("REVISION_EXECUTOR_REFUSED") + schema_observation = { + "kind": domain["kind"], + "payload": copy.deepcopy(domain["payload"]), + "exception_class": domain["exception_class"], + "provenance": ["OPERATOR_FIXTURE"], + } + digest = artifact_digest(schema_observation) + return _Replay( + status, + None, + domain, + (digest, digest), + ("OPERATOR_FIXTURE",), + source, + ) + + +def _schema_observation(replay: _Replay) -> dict[str, object]: + if replay.observation is None: + _refuse("REVISION_EXECUTOR_REFUSED") + return { + "kind": replay.observation["kind"], + "payload": copy.deepcopy(replay.observation["payload"]), + "exception_class": replay.observation["exception_class"], + "provenance": list(replay.provenance), + } + + +def _domain_target( + definition: SymbolDefinition, + binding: FixtureBinding, + replay: _Replay, +) -> dict[str, object]: + return { + "symbol": definition.target, + "target_sha256": definition.definition_sha256, + "signature_sha256": definition.signature_sha256, + "fixture_binding_sha256": binding.binding_sha256, + "provenance": "OPERATOR_FIXTURE", + "projection": binding.projection, + "outcome": { + "status": replay.status, + "observation": copy.deepcopy(replay.observation), + "reason_code": replay.reason_code, + "repeatable": replay.status in _EXERCISED, + }, + } + + +def _baseline_target( + definition: SymbolDefinition, + binding: FixtureBinding, + replay: _Replay, +) -> dict[str, object]: + observation = _schema_observation(replay) + target = { + "target_id": "", + "module": definition.module, + "symbol": definition.symbol, + "definition_sha256": definition.definition_sha256, + "signature_sha256": definition.signature_sha256, + "observation": observation, + "repeat_sha256": list(replay.repeat_sha256 or ()), + "projection_sha256": ( + None + if binding.projection is None + else artifact_digest(binding.projection) + ), + } + target["target_id"] = record_identity(target, "target_id") + return target + + +def _result( + report: Mapping[str, object], + *, + environment_artifacts: list[dict[str, str]], + exit_code: int, +) -> RevisionCommandResult: + evidence = make_evidence_artifact( + report, + environment_artifacts=sorted( + environment_artifacts, key=lambda row: row["name"] + ), + ) + return RevisionCommandResult( + report=copy.deepcopy(dict(report)), + evidence=copy.deepcopy(evidence), + exit_code=exit_code, + ) + + +def _selection( + changes: tuple[SymbolChange, ...], targets: tuple[str, ...], allow_empty: bool +) -> tuple[str, ...]: + available = {change.target for change in changes} + if targets: + if any(target not in available for target in targets): + _refuse("REVISION_TARGET_UNMATCHED") + return targets + selected = tuple( + change.target for change in changes if change.status in _CHANGED_STATUSES + ) + if not selected and not allow_empty: + _refuse("NO_CHANGED_TARGETS") + return selected + + +def freeze_revision( + repository: Path | str, + *, + revision: str = "HEAD", + fixture_path: Path | str = "breakcheck.fixtures.toml", + runtime_root: Path | str, + targets: Iterable[str] = (), + allow_dirty: bool = False, + fixture_policy: str = "require", + executor=run_repeated_typed_snippet_isolated, +) -> RevisionCommandResult: + """Capture repeated observations for fixture-bound targets at one revision.""" + + try: + if fixture_policy != "require": + _refuse("FIXTURE_REQUIRED") + if type(allow_dirty) is not bool: + _refuse("DIRTY_TREE_REFUSED") + top = _repository_root(repository) + dirty = _dirty(top) + if dirty: + _refuse( + "DIRTY_TREE_CAPTURE_UNSUPPORTED" + if allow_dirty + else "DIRTY_TREE_REFUSED" + ) + requested_targets = _target_list(targets) + with revision_worktrees(top, revision, revision, runtime_root) as pair: + definitions = inventory_symbols(pair.base_root).definitions + by_target = _definition_map(definitions) + context = _load_fixtures( + repository=top, + pair=pair, + fixture_path=fixture_path, + fixture_source="base", + source_definitions=definitions, + ) + selected = requested_targets or tuple(sorted(context.bindings)) + if not selected: + _refuse("VACUOUS_BASELINE_REFUSED") + domain_targets = [] + artifact_targets = [] + selected_definitions = [] + for target in selected: + definition = by_target.get(target) + binding = context.bindings.get(target) + if definition is None: + _refuse("REVISION_TARGET_UNMATCHED") + if binding is None: + _refuse("FIXTURE_REQUIRED") + selected_definitions.append(definition) + replay = _replay( + pair.base_root, definition, binding, executor=executor + ) + if replay.status not in _EXERCISED: + _refuse("BASELINE_TARGET_NOT_EXERCISED") + domain_targets.append(_domain_target(definition, binding, replay)) + artifact_targets.append( + _baseline_target(definition, binding, replay) + ) + tree = tracked_tree_identity(pair.base_root) + import_root_sha256 = _import_root_identity( + pair.base_root, + selected_definitions, + tree_sha256=tree.sha256, + ) + invocation_flags: dict[str, object] = { + "allow_dirty": allow_dirty, + "fixture_file": _relative_path( + fixture_path, "FIXTURE_BINDING_PATH_REFUSED" + ), + "fixture_policy": fixture_policy, + "target": list(selected), + } + freeze_baseline( + revision=pair.base_commit, + tree_sha256=tree.sha256, + dirty=dirty, + allow_dirty=allow_dirty, + environment=_environment_domain(), + fixture=_fixture_domain(context), + target_observations=domain_targets, + invocation=copy.deepcopy(invocation_flags), + ) + artifact_targets.sort(key=lambda row: str(row["target_id"])) + report = make_revision_artifact( + "baseline", + { + "revision": pair.base_commit, + "tree_sha256": tree.sha256, + "dirty": dirty, + "allow_dirty": allow_dirty, + "environment": _environment_schema(), + "fixture": _fixture_schema(context), + "target_observations": artifact_targets, + "invocation": canonicalize_invocation( + "baseline", invocation_flags + ), + }, + ) + return _result( + report, + environment_artifacts=[ + {"name": "revision_tree", "sha256": tree.sha256}, + { + "name": "revision_import_roots", + "sha256": import_root_sha256, + }, + { + "name": "fixture", + "sha256": context.fixture.canonical_sha256, + }, + ], + exit_code=0, + ) + except ( + FixtureRefusal, + WorktreeRefusal, + SymbolAnalysisRefusal, + BaselineRefusal, + ) as exc: + raise _translate(exc) from exc + + +def _reason(base: _Replay, head: _Replay) -> str | None: + base_import = base.reason_code == "IMPORT_FAILED" + head_import = head.reason_code == "IMPORT_FAILED" + if base_import != head_import: + return "IMPORT_ASYMMETRY" + if base_import and head_import: + return "IMPORT_FAILED" + if base.status not in _EXERCISED or head.status not in _EXERCISED: + if base.reason_code == head.reason_code and base.reason_code is not None: + return base.reason_code + return "REPLAY_ASYMMETRY" + return None + + +def _finding( + *, + target: str, + base_definition: SymbolDefinition | None, + head_definition: SymbolDefinition | None, + binding: FixtureBinding | None, + base: _Replay | None, + head: _Replay | None, + reason_code: str | None, +) -> tuple[dict[str, object], dict[str, object] | None]: + definition = base_definition or head_definition + if definition is None: + _refuse("REVISION_TARGET_REFUSED") + target_id = artifact_digest({"target": target}) + projection = None if binding is None else _projection(binding) + if reason_code is not None: + verdict = "NOT_EXERCISED" + base_observation = None + head_observation = None + projection = None + else: + if ( + base is None + or head is None + or base.observation is None + or head.observation is None + ): + _refuse("REVISION_EXECUTOR_REFUSED") + comparison = compare_observations(base.observation, head.observation) + verdict = comparison["verdict"] + if binding is not None and binding.projection is not None: + verdict += "_UNDER_PROJECTION" + base_observation = _schema_observation(base) + head_observation = _schema_observation(head) + finding = { + "finding_id": "", + "target_id": target_id, + "module": definition.module, + "symbol": definition.symbol, + "verdict": verdict, + "base": base_observation, + "head": head_observation, + "reason_code": reason_code, + "projection": projection, + "fixture_binding_sha256": ( + None if binding is None else binding.binding_sha256 + ), + } + finding["finding_id"] = record_identity(finding, "finding_id") + if reason_code is not None: + return finding, None + assert base is not None and head is not None + witness = { + "witness_id": "", + "finding_id": finding["finding_id"], + "target_id": target_id, + "base_observation_sha256": artifact_digest(base_observation), + "head_observation_sha256": artifact_digest(head_observation), + "base_repeat_sha256": list(base.repeat_sha256 or ()), + "head_repeat_sha256": list(head.repeat_sha256 or ()), + "projection_sha256": ( + None if projection is None else projection["sha256"] + ), + "provenance": ["OPERATOR_FIXTURE"], + "replay": { + "source": base.replay_source, + "sha256": artifact_digest(base.replay_source), + }, + } + witness["witness_id"] = record_identity(witness, "witness_id") + return finding, witness + + +def _evaluate_pair( + *, + top: Path, + pair, + fixture_path: Path | str, + fixture_source: str, + targets: tuple[str, ...], + allow_empty: bool, + min_coverage: float, + strict_separation: bool, + executor, + previous_payload: Mapping[str, object] | None = None, + previous_report_sha256: str | None = None, +) -> tuple[ + dict[str, object], + _FixtureContext, + int, + tuple[str, ...], + dict[str, str], +]: + if type(allow_empty) is not bool or type(strict_separation) is not bool: + _refuse("REVISION_POLICY_REFUSED") + if ( + type(min_coverage) not in (int, float) + or not 0 < float(min_coverage) <= 100 + ): + _refuse("REVISION_COVERAGE_REFUSED") + if strict_separation and fixture_source == "explicit": + _refuse("FIXTURE_EXPLICIT_STRICT_REFUSED") + changes = compare_symbol_trees(pair.base_root, pair.head_root) + selected = _selection(changes, targets, allow_empty) + base_definitions = inventory_symbols(pair.base_root).definitions + head_definitions = inventory_symbols(pair.head_root).definitions + base_map = _definition_map(base_definitions) + head_map = _definition_map(head_definitions) + source_definitions = ( + base_definitions if fixture_source == "base" else head_definitions + ) + if fixture_source == "explicit": + source_definitions = base_definitions + context = _load_fixtures( + repository=top, + pair=pair, + fixture_path=fixture_path, + fixture_source=fixture_source, + source_definitions=source_definitions, + ) + if strict_separation and ( + not context.predates_change or context.authored_by == "unknown" + ): + _refuse("FIXTURE_SEPARATION_REFUSED") + change_map = {change.target: change for change in changes} + findings = [] + witnesses = [] + for target in selected: + change = change_map[target] + base_definition = base_map.get(target) + head_definition = head_map.get(target) + binding = context.bindings.get(target) + reason_code = None + base_replay = None + head_replay = None + if change.status in ( + "SYMBOL_AMBIGUOUS", + "NO_BASELINE_REVISION", + "SYMBOL_REMOVED", + "FIXTURE_SIGNATURE_DRIFT", + ): + reason_code = change.status + elif base_definition is None or head_definition is None: + reason_code = "SYMBOL_AMBIGUOUS" + elif binding is None: + reason_code = "FIXTURE_REQUIRED" + else: + base_replay = _replay( + pair.base_root, base_definition, binding, executor=executor + ) + head_replay = _replay( + pair.head_root, head_definition, binding, executor=executor + ) + reason_code = _reason(base_replay, head_replay) + finding, witness = _finding( + target=target, + base_definition=base_definition, + head_definition=head_definition, + binding=binding, + base=base_replay, + head=head_replay, + reason_code=reason_code, + ) + findings.append(finding) + if witness is not None: + witnesses.append(witness) + findings.sort(key=lambda row: str(row["finding_id"])) + witnesses.sort(key=lambda row: str(row["witness_id"])) + fixture_revision_events = _fixture_revision_events( + previous_payload, + base_revision=pair.base_commit, + findings=findings, + ) + summary = { + "changed": sum(row["verdict"] == "CHANGED" for row in findings), + "changed_under_projection": sum( + row["verdict"] == "CHANGED_UNDER_PROJECTION" for row in findings + ), + "identical": sum(row["verdict"] == "IDENTICAL" for row in findings), + "identical_under_projection": sum( + row["verdict"] == "IDENTICAL_UNDER_PROJECTION" for row in findings + ), + "not_exercised": sum( + row["verdict"] == "NOT_EXERCISED" for row in findings + ), + } + total = len(findings) + exercised = total - summary["not_exercised"] + coverage = 0.0 if total == 0 else 100.0 * exercised / total + if total == 0 and not allow_empty: + _refuse("VACUOUS_REVISION_COMPARISON_REFUSED") + if summary["changed"] or summary["changed_under_projection"]: + exit_code = 3 + elif coverage < float(min_coverage): + exit_code = 4 + else: + exit_code = 0 + base_tree = tracked_tree_identity(pair.base_root) + head_tree = tracked_tree_identity(pair.head_root) + import_root_identities = { + "base": _import_root_identity( + pair.base_root, + (base_map[target] for target in selected if target in base_map), + tree_sha256=base_tree.sha256, + ), + "head": _import_root_identity( + pair.head_root, + (head_map[target] for target in selected if target in head_map), + tree_sha256=head_tree.sha256, + ), + } + flags: dict[str, object] = { + "allow_empty": allow_empty, + "fixture_file": _relative_path( + fixture_path, "FIXTURE_BINDING_PATH_REFUSED" + ), + "fixture_policy": "require", + "fixture_source": fixture_source, + "min_coverage": float(min_coverage), + "strict_separation": strict_separation, + } + if selected: + flags["target"] = list(selected) + if previous_report_sha256 is not None: + flags["previous_report_sha256"] = previous_report_sha256 + payload = { + "base_revision": pair.base_commit, + "head_revision": pair.head_commit, + "base_tree_sha256": base_tree.sha256, + "head_tree_sha256": head_tree.sha256, + "findings": findings, + "witnesses": witnesses, + "fixture_revision_events": fixture_revision_events, + "summary": summary, + "fixture": _fixture_schema(context), + "fixtures_predate_change": context.predates_change, + "invocation": canonicalize_invocation("revision_report", flags), + } + return payload, context, exit_code, selected, import_root_identities + + +def diff_revisions( + repository: Path | str, + *, + base_revision: str | None = None, + baseline: Mapping[str, object] | None = None, + previous_report: Mapping[str, object] | None = None, + head_revision: str, + fixture_path: Path | str = "breakcheck.fixtures.toml", + runtime_root: Path | str, + targets: Iterable[str] = (), + fixture_source: str = "base", + allow_empty: bool = False, + min_coverage: float = 80.0, + strict_separation: bool = False, + executor=run_repeated_typed_snippet_isolated, +) -> RevisionCommandResult: + """Compare fixture-bound behavior across two committed revisions.""" + + try: + top = _repository_root(repository) + selected = _target_list(targets) + previous_payload, previous_report_sha256 = _verified_previous_report( + previous_report + ) + baseline_payload = None + if baseline is not None: + verified = validate_artifact(baseline) + if verified["artifact_kind"] != "baseline": + _refuse("BASELINE_ARTIFACT_REFUSED") + verify_artifact(verified) + baseline_payload = verified["payload"] + recorded_base = str(baseline_payload["revision"]) + if base_revision is not None and base_revision != recorded_base: + _refuse("BASELINE_REVISION_MISMATCH") + base_revision = recorded_base + recorded_targets = tuple( + sorted( + str(row["module"]) + ":" + str(row["symbol"]) + for row in baseline_payload["target_observations"] + ) + ) + if selected and selected != recorded_targets: + _refuse("BASELINE_TARGET_MISMATCH") + selected = recorded_targets + if base_revision is None: + _refuse("NO_BASELINE_REVISION") + with revision_worktrees( + top, base_revision, head_revision, runtime_root + ) as pair: + if pair.base_commit == pair.head_commit: + _refuse("IDENTICAL_REVISIONS_REFUSED") + if ( + previous_payload is not None + and previous_payload["base_revision"] != pair.base_commit + ): + _refuse("PREVIOUS_REPORT_BASE_MISMATCH") + payload, context, exit_code, _, import_roots = _evaluate_pair( + top=top, + pair=pair, + fixture_path=fixture_path, + fixture_source=fixture_source, + targets=selected, + allow_empty=allow_empty, + min_coverage=min_coverage, + strict_separation=strict_separation, + executor=executor, + previous_payload=previous_payload, + previous_report_sha256=previous_report_sha256, + ) + if baseline_payload is not None: + if baseline_payload["environment"] != _environment_schema(): + _refuse("BASELINE_ENVIRONMENT_MISMATCH") + if baseline_payload["tree_sha256"] != payload["base_tree_sha256"]: + _refuse("BASELINE_TREE_MISMATCH") + if baseline_payload["fixture"] != payload["fixture"]: + _refuse("BASELINE_FIXTURE_MISMATCH") + baseline_targets = { + str(row["module"]) + ":" + str(row["symbol"]): row + for row in baseline_payload["target_observations"] + } + if set(baseline_targets) != set(selected): + _refuse("BASELINE_TARGET_MISMATCH") + for finding in payload["findings"]: + target = ( + str(finding["module"]) + + ":" + + str(finding["symbol"]) + ) + baseline_target = baseline_targets.get(target) + if baseline_target is None: + _refuse("BASELINE_TARGET_MISMATCH") + if finding["base"] is not None and ( + finding["base"] != baseline_target["observation"] + ): + _refuse("BASELINE_OBSERVATION_MISMATCH") + projection = finding["projection"] + projection_sha = ( + None if projection is None else projection["sha256"] + ) + if projection_sha != baseline_target["projection_sha256"]: + _refuse("BASELINE_PROJECTION_MISMATCH") + report = make_revision_artifact("revision_report", payload) + return _result( + report, + environment_artifacts=[ + { + "name": "base_tree", + "sha256": payload["base_tree_sha256"], + }, + { + "name": "base_import_roots", + "sha256": import_roots["base"], + }, + { + "name": "fixture", + "sha256": context.fixture.canonical_sha256, + }, + { + "name": "head_tree", + "sha256": payload["head_tree_sha256"], + }, + { + "name": "head_import_roots", + "sha256": import_roots["head"], + }, + ], + exit_code=exit_code, + ) + except (FixtureRefusal, WorktreeRefusal, SymbolAnalysisRefusal) as exc: + raise _translate(exc) from exc + + +def _read_text(root: Path, relative: Path | str, code: str) -> tuple[str, str]: + name = _relative_path(relative, code) + path = root / name + if path.is_symlink() or not path.is_file(): + _refuse(code) + try: + data = path.read_bytes() + if len(data) > 65_536: + _refuse(code) + return data.decode("utf-8", errors="strict"), name + except (OSError, UnicodeError): + _refuse(code) + + +def _claim_disposition( + row: Mapping[str, object], *, strict_separation: bool +) -> dict[str, object]: + disposition = str(row["disposition"]) + reason = row["reason_code"] + if not strict_separation and disposition == "CLAIM_VERIFIED": + disposition = "CLAIM_UNVERIFIABLE" + reason = "STRICT_SEPARATION_REQUIRED" + if disposition == "CLAIM_REFUTED" and reason is None: + reason = "BEHAVIOR_CHANGED" + symbol = str(row["symbol"]) + result = { + "disposition_id": "", + "target_id": artifact_digest({"target": symbol}), + "symbol": symbol, + "disposition": disposition, + "reason_code": reason, + "projection_scope": row["projection_scope"], + } + result["disposition_id"] = record_identity(result, "disposition_id") + return result + + +def attest_revision( + repository: Path | str, + *, + head_revision: str, + claim_path: Path | str, + previous_report: Mapping[str, object] | None = None, + fixture_path: Path | str = "breakcheck.fixtures.toml", + runtime_root: Path | str, + fixture_source: str = "base", + allow_empty: bool = False, + min_coverage: float = 80.0, + strict: bool = True, + strict_separation: bool = True, + executor=run_repeated_typed_snippet_isolated, +) -> RevisionCommandResult: + """Adjudicate a closed preservation claim against the independent diff census.""" + + try: + if type(strict) is not bool: + _refuse("CLAIM_POLICY_REFUSED") + top = _repository_root(repository) + previous_payload, previous_report_sha256 = _verified_previous_report( + previous_report + ) + claim_relative = _relative_path(claim_path, "CLAIM_FILE_REFUSED") + head_commit = ( + _git( + top, + "rev-parse", + "--verify", + "--end-of-options", + f"{head_revision}^{{commit}}", + ) + .decode("ascii", errors="strict") + .strip() + ) + with revision_worktrees( + top, head_commit, head_commit, runtime_root + ) as claim_pair: + claim_text, _ = _read_text( + claim_pair.head_root, claim_relative, "CLAIM_FILE_REFUSED" + ) + claim = parse_claim(claim_text) + comparison_root = Path(runtime_root) + if comparison_root.exists() or comparison_root.is_symlink(): + _refuse("REVISION_WORKTREE_CLEANUP_REFUSED") + with revision_worktrees( + top, claim.base_revision, head_commit, comparison_root + ) as pair: + if ( + previous_payload is not None + and previous_payload["base_revision"] != pair.base_commit + ): + _refuse("PREVIOUS_REPORT_BASE_MISMATCH") + ( + payload, + context, + _, + changed_targets, + import_roots, + ) = _evaluate_pair( + top=top, + pair=pair, + fixture_path=fixture_path, + fixture_source=fixture_source, + targets=(), + allow_empty=allow_empty, + min_coverage=min_coverage, + strict_separation=strict_separation, + executor=executor, + previous_payload=previous_payload, + previous_report_sha256=previous_report_sha256, + ) + simple_findings = [ + { + "symbol": str(row["module"]) + ":" + str(row["symbol"]), + "verdict": row["verdict"], + "reason_code": row["reason_code"], + "projection_scope": ( + None + if row["projection"] is None + else row["projection"]["source"] + ), + } + for row in payload["findings"] + ] + domain = adjudicate_claim( + claim, + head_revision=pair.head_commit, + changed_targets=changed_targets, + findings=simple_findings, + fixture_source=fixture_source, + fixture_revision=context.source_revision, + fixture_authored_by=context.authored_by, + fixtures_predate_change=context.predates_change, + strict_separation=strict_separation, + invocation={ + "strict": strict, + "strict_separation": strict_separation, + }, + ) + dispositions = [ + _claim_disposition(row, strict_separation=strict_separation) + for row in domain["dispositions"] + ] + dispositions.sort(key=lambda row: str(row["disposition_id"])) + summary = { + "claim_out_of_scope": sum( + row["disposition"] == "CLAIM_OUT_OF_SCOPE" + for row in dispositions + ), + "claim_refuted": sum( + row["disposition"] == "CLAIM_REFUTED" + for row in dispositions + ), + "claim_unverifiable": sum( + row["disposition"] == "CLAIM_UNVERIFIABLE" + for row in dispositions + ), + "claim_verified": sum( + row["disposition"] == "CLAIM_VERIFIED" + for row in dispositions + ), + "total": len(dispositions), + } + flags = { + "allow_empty": allow_empty, + "claim_file": claim_relative, + "fixture_file": _relative_path( + fixture_path, "FIXTURE_BINDING_PATH_REFUSED" + ), + "fixture_source": fixture_source, + "min_coverage": float(min_coverage), + "strict": strict, + "strict_separation": strict_separation, + } + if previous_report_sha256 is not None: + flags["previous_report_sha256"] = previous_report_sha256 + claim_payload = { + "claim": claim.claim, + "base_revision": pair.base_commit, + "head_revision": pair.head_commit, + "dispositions": dispositions, + "summary": summary, + "fixture": _fixture_schema(context), + "fixtures_predate_change": context.predates_change, + "fixture_revision_events": payload[ + "fixture_revision_events" + ], + "invocation": canonicalize_invocation("claim_report", flags), + } + report = make_revision_artifact("claim_report", claim_payload) + return _result( + report, + environment_artifacts=[ + { + "name": "base_tree", + "sha256": payload["base_tree_sha256"], + }, + { + "name": "base_import_roots", + "sha256": import_roots["base"], + }, + { + "name": "fixture", + "sha256": context.fixture.canonical_sha256, + }, + { + "name": "head_tree", + "sha256": payload["head_tree_sha256"], + }, + { + "name": "head_import_roots", + "sha256": import_roots["head"], + }, + ], + exit_code=claim_exit_code(claim_payload), + ) + except ( + ClaimRefusal, + FixtureRefusal, + WorktreeRefusal, + SymbolAnalysisRefusal, + ) as exc: + raise _translate(exc) from exc + + +__all__ = ( + "RevisionCommandResult", + "RevisionModeRefusal", + "attest_revision", + "diff_revisions", + "freeze_revision", +) diff --git a/src/breakcheck/revision_report.py b/src/breakcheck/revision_report.py new file mode 100644 index 0000000..be1af1e --- /dev/null +++ b/src/breakcheck/revision_report.py @@ -0,0 +1,45 @@ +"""Builders for revision-domain schema-2 artifacts.""" + +from __future__ import annotations + +import copy +from typing import Mapping, Sequence + +from .schema import artifact_digest, make_artifact, validate_artifact + + +_REVISION_ARTIFACT_KINDS = frozenset( + ("baseline", "revision_report", "claim_report") +) + + +def make_revision_artifact(kind: str, payload: Mapping[str, object]) -> dict: + if kind not in _REVISION_ARTIFACT_KINDS: + raise ValueError("ARTIFACT_KIND_REFUSED") + return make_artifact(kind, payload) + + +def make_evidence_artifact( + report: Mapping[str, object], + *, + environment_artifacts: Sequence[Mapping[str, object]], +) -> dict: + validated = validate_artifact(report) + if validated["artifact_kind"] == "evidence": + raise ValueError("ARTIFACT_KIND_REFUSED") + payload = validated["payload"] + witnesses = payload.get("witnesses", []) + return make_artifact( + "evidence", + { + "report_artifact_sha256": artifact_digest(validated), + "report_payload_sha256": validated["payload_sha256"], + "report_kind": validated["artifact_kind"], + "witnesses": copy.deepcopy(witnesses), + "environment_artifacts": copy.deepcopy(list(environment_artifacts)), + "invocation": copy.deepcopy(payload["invocation"]), + }, + ) + + +__all__ = ("make_evidence_artifact", "make_revision_artifact") diff --git a/src/breakcheck/schema.py b/src/breakcheck/schema.py new file mode 100644 index 0000000..41716d3 --- /dev/null +++ b/src/breakcheck/schema.py @@ -0,0 +1,1068 @@ +"""Closed, deterministic schema-2 artifacts. + +Schema 1 remains owned by :mod:`breakcheck.verify`. This module defines the +major-version boundary used by all new dependency, coverage, revision, claim, +baseline, and evidence artifacts. +""" + +from __future__ import annotations + +import copy +import hashlib +import json +import math +from pathlib import PurePosixPath +import re +from typing import Mapping + + +SCHEMA_VERSION = 2 + +_ENVELOPE_FIELDS = frozenset( + ("schema_version", "artifact_kind", "payload", "payload_sha256") +) +_PAYLOAD_FIELDS = { + "dependency_report": frozenset( + ( + "package", + "current_version", + "new_version", + "coverage", + "findings", + "witnesses", + "summary", + "invocation", + ) + ), + "coverage_report": frozenset( + ( + "package", + "current_version", + "new_version", + "candidates", + "counts", + "invocation", + ) + ), + "baseline": frozenset( + ( + "revision", + "tree_sha256", + "dirty", + "allow_dirty", + "environment", + "fixture", + "target_observations", + "invocation", + ) + ), + "revision_report": frozenset( + ( + "base_revision", + "head_revision", + "base_tree_sha256", + "head_tree_sha256", + "findings", + "witnesses", + "summary", + "fixture", + "fixtures_predate_change", + "fixture_revision_events", + "invocation", + ) + ), + "claim_report": frozenset( + ( + "claim", + "base_revision", + "head_revision", + "dispositions", + "summary", + "fixture", + "fixtures_predate_change", + "fixture_revision_events", + "invocation", + ) + ), + "evidence": frozenset( + ( + "report_artifact_sha256", + "report_payload_sha256", + "report_kind", + "witnesses", + "environment_artifacts", + "invocation", + ) + ), +} + +_SUMMARY_FIELDS = frozenset( + ( + "changed", + "changed_under_projection", + "identical", + "identical_under_projection", + "not_exercised", + ) +) +_VERDICTS = frozenset( + ( + "IDENTICAL", + "CHANGED", + "IDENTICAL_UNDER_PROJECTION", + "CHANGED_UNDER_PROJECTION", + "NOT_EXERCISED", + ) +) +_EXERCISED_VERDICTS = _VERDICTS - {"NOT_EXERCISED"} +_PROJECTION_VERDICTS = frozenset( + ("IDENTICAL_UNDER_PROJECTION", "CHANGED_UNDER_PROJECTION") +) +_PROVENANCE_ORDER = ( + "SOURCE_LITERAL", + "SOURCE_FOLDED", + "SOURCE_MODULE_CONSTANT", + "SOURCE_NESTED_CALL", + "OPERATOR_FIXTURE", + "RUNTIME_CAPTURE", +) +_PROVENANCE_POSITION = { + value: position for position, value in enumerate(_PROVENANCE_ORDER) +} +_COVERAGE_BUCKETS = ( + "EXERCISED", + "G1_NOT_DISCOVERABLE", + "G2_NONLITERAL", + "G3_UNNORMALIZABLE", + "G4_IMPURE", +) +_CLAIM_DISPOSITIONS = ( + "CLAIM_VERIFIED", + "CLAIM_REFUTED", + "CLAIM_UNVERIFIABLE", + "CLAIM_OUT_OF_SCOPE", +) +_CLAIM_COUNTS = { + "CLAIM_VERIFIED": "claim_verified", + "CLAIM_REFUTED": "claim_refuted", + "CLAIM_UNVERIFIABLE": "claim_unverifiable", + "CLAIM_OUT_OF_SCOPE": "claim_out_of_scope", +} +_HEX = re.compile(r"[0-9a-f]+\Z") +_FLAG_NAME = re.compile(r"[a-z][a-z0-9_]*\Z") +_NAME = re.compile(r"[A-Za-z_][A-Za-z0-9_.:-]*\Z") +_INVOCATION_FLAGS = { + "dependency_report": frozenset( + ( + "allow_empty", + "ci", + "coverage_report", + "fixture_file", + "fixture_policy", + "json", + "min_coverage", + "suggest_fixtures", + ) + ), + "coverage_report": frozenset( + ( + "allow_empty", + "fixture_file", + "fixture_policy", + "min_coverage", + "suggest_fixtures", + ) + ), + "baseline": frozenset( + ("allow_dirty", "fixture_file", "fixture_policy", "target") + ), + "revision_report": frozenset( + ( + "allow_empty", + "fixture_file", + "fixture_policy", + "fixture_source", + "min_coverage", + "strict_separation", + "target", + "previous_report_sha256", + ) + ), + "claim_report": frozenset( + ( + "allow_empty", + "claim_file", + "fixture_file", + "fixture_source", + "min_coverage", + "strict", + "strict_separation", + "previous_report_sha256", + ) + ), +} + + +def _refuse(code: str) -> None: + raise ValueError(code) + + +def _plain_json(value, *, _depth=0, _budget=None): + if _budget is None: + _budget = [0] + _budget[0] += 1 + if _depth > 64 or _budget[0] > 100_000: + _refuse("ARTIFACT_ENCODING_REFUSED") + if value is None or type(value) in (bool, int, str): + if type(value) is str and len(value.encode("utf-8")) > 65_536: + _refuse("ARTIFACT_ENCODING_REFUSED") + return value + if type(value) is float: + if not math.isfinite(value): + _refuse("ARTIFACT_ENCODING_REFUSED") + return value + if type(value) is list: + if len(value) > 10_000: + _refuse("ARTIFACT_ENCODING_REFUSED") + return [ + _plain_json(item, _depth=_depth + 1, _budget=_budget) + for item in value + ] + if isinstance(value, Mapping): + if len(value) > 10_000: + _refuse("ARTIFACT_ENCODING_REFUSED") + result = {} + for key, item in value.items(): + if ( + type(key) is not str + or key in result + or len(key.encode("utf-8")) > 65_536 + ): + _refuse("ARTIFACT_ENCODING_REFUSED") + result[key] = _plain_json( + item, _depth=_depth + 1, _budget=_budget + ) + return result + _refuse("ARTIFACT_ENCODING_REFUSED") + + +def canonical_json(value) -> str: + return json.dumps( + _plain_json(value), + sort_keys=True, + separators=(",", ":"), + ensure_ascii=True, + allow_nan=False, + ) + + +def artifact_digest(value) -> str: + return hashlib.sha256(canonical_json(value).encode("utf-8")).hexdigest() + + +def record_identity(value: Mapping[str, object], field: str) -> str: + if not isinstance(value, Mapping) or field not in value: + _refuse("ARTIFACT_IDENTITY_REFUSED") + payload = _plain_json(copy.deepcopy(value)) + payload[field] = "" + return artifact_digest(payload) + + +def _closed(value, fields, *, code="ARTIFACT_SCHEMA_REFUSED") -> dict: + if not isinstance(value, dict) or set(value) != set(fields): + _refuse(code) + return value + + +def _text(value, *, code="ARTIFACT_SCHEMA_REFUSED", allow_empty=False) -> str: + if type(value) is not str or (not allow_empty and not value): + _refuse(code) + if len(value.encode("utf-8")) > 65_536: + _refuse(code) + return value + + +def _sha256(value, *, code="ARTIFACT_HASH_REFUSED") -> str: + if type(value) is not str or len(value) != 64 or not _HEX.fullmatch(value): + _refuse(code) + return value + + +def _revision(value) -> str: + if ( + type(value) is not str + or len(value) not in (40, 64) + or not _HEX.fullmatch(value) + ): + _refuse("ARTIFACT_REVISION_REFUSED") + return value + + +def _nonnegative_integer(value, *, code="ARTIFACT_COUNT_REFUSED") -> int: + if type(value) is not int or value < 0: + _refuse(code) + return value + + +def _relative_path(value) -> str: + _text(value, code="ARTIFACT_PATH_REFUSED") + if "\\" in value: + _refuse("ARTIFACT_PATH_REFUSED") + path = PurePosixPath(value) + if path.is_absolute() or any(part in ("", ".", "..") for part in path.parts): + _refuse("ARTIFACT_PATH_REFUSED") + if path.as_posix() != value: + _refuse("ARTIFACT_PATH_REFUSED") + return value + + +def _validate_flag_value(name, value): + if name in { + "allow_dirty", + "allow_empty", + "ci", + "coverage_report", + "json", + "strict", + "strict_separation", + "suggest_fixtures", + }: + if type(value) is not bool: + _refuse("ARTIFACT_INVOCATION_REFUSED") + elif name == "min_coverage": + if type(value) not in (int, float) or not 0 < float(value) <= 100: + _refuse("ARTIFACT_INVOCATION_REFUSED") + elif name == "fixture_policy": + if value not in ("forbid", "allow", "require"): + _refuse("ARTIFACT_INVOCATION_REFUSED") + elif name == "fixture_source": + if value not in ("base", "head", "explicit"): + _refuse("ARTIFACT_INVOCATION_REFUSED") + elif name in ("fixture_file", "claim_file"): + _relative_path(value) + elif name == "previous_report_sha256": + _sha256(value, code="ARTIFACT_INVOCATION_REFUSED") + elif name == "target": + if not isinstance(value, list) or not value: + _refuse("ARTIFACT_INVOCATION_REFUSED") + for item in value: + _text(item, code="ARTIFACT_INVOCATION_REFUSED") + else: + _refuse("ARTIFACT_INVOCATION_REFUSED") + return _plain_json(value) + + +def canonicalize_invocation(kind: str, flags: Mapping[str, object]) -> list[dict]: + allowed = _INVOCATION_FLAGS.get(kind) + if allowed is None or not isinstance(flags, Mapping): + _refuse("ARTIFACT_INVOCATION_REFUSED") + if set(flags) - allowed or any(not _FLAG_NAME.fullmatch(name) for name in flags): + _refuse("ARTIFACT_INVOCATION_REFUSED") + return [ + {"name": name, "value": _validate_flag_value(name, flags[name])} + for name in sorted(flags) + ] + + +def _validate_invocation(value, kind: str) -> list[dict]: + if not isinstance(value, list): + _refuse("ARTIFACT_INVOCATION_REFUSED") + flags = {} + observed_names = [] + for row in value: + _closed(row, ("name", "value"), code="ARTIFACT_INVOCATION_REFUSED") + name = row["name"] + if type(name) is not str or name in flags: + _refuse("ARTIFACT_INVOCATION_REFUSED") + flags[name] = row["value"] + observed_names.append(name) + if observed_names != sorted(observed_names): + _refuse("ARTIFACT_INVOCATION_REFUSED") + if canonicalize_invocation(kind, flags) != value: + _refuse("ARTIFACT_INVOCATION_REFUSED") + return value + + +def _validate_provenance(value) -> list[str]: + if not isinstance(value, list) or not value or len(set(value)) != len(value): + _refuse("ARTIFACT_PROVENANCE_REFUSED") + if any(item not in _PROVENANCE_POSITION for item in value): + _refuse("ARTIFACT_PROVENANCE_REFUSED") + if value != sorted(value, key=_PROVENANCE_POSITION.__getitem__): + _refuse("ARTIFACT_PROVENANCE_REFUSED") + return value + + +def _validate_observation(value) -> dict: + _closed(value, ("kind", "payload", "exception_class", "provenance")) + kind = value["kind"] + if kind not in ("value", "exception"): + _refuse("ARTIFACT_OBSERVATION_REFUSED") + _validate_provenance(value["provenance"]) + _plain_json(value["payload"]) + if kind == "value": + if value["exception_class"] is not None: + _refuse("ARTIFACT_OBSERVATION_REFUSED") + elif ( + type(value["exception_class"]) is not str + or not _NAME.fullmatch(value["exception_class"]) + or not isinstance(value["payload"], list) + ): + _refuse("ARTIFACT_OBSERVATION_REFUSED") + return value + + +def _validate_projection(value): + if value is None: + return None + _closed(value, ("source", "sha256")) + source = _text(value["source"], code="ARTIFACT_PROJECTION_REFUSED") + if "outcome" not in source or _sha256( + value["sha256"], code="ARTIFACT_PROJECTION_REFUSED" + ) != artifact_digest(source): + _refuse("ARTIFACT_PROJECTION_REFUSED") + return value + + +def _validate_replay(value): + _closed(value, ("source", "sha256")) + source = _text(value["source"], code="ARTIFACT_REPLAY_REFUSED") + if _sha256( + value["sha256"], code="ARTIFACT_REPLAY_REFUSED" + ) != artifact_digest(source): + _refuse("ARTIFACT_REPLAY_REFUSED") + return value + + +def _validate_call_site(value): + _closed(value, ("file", "line", "column")) + _relative_path(value["file"]) + if type(value["line"]) is not int or value["line"] < 1: + _refuse("ARTIFACT_SCHEMA_REFUSED") + if type(value["column"]) is not int or value["column"] < 0: + _refuse("ARTIFACT_SCHEMA_REFUSED") + return value + + +def _validate_comparison(value): + if value is None: + return None + _closed(value, ("verdict", "detail")) + if value["verdict"] not in ("IDENTICAL", "CHANGED"): + _refuse("ARTIFACT_SCHEMA_REFUSED") + detail = _closed( + value["detail"], + ("reason_code", "path", "old_summary", "new_summary", "policy"), + ) + for field in ("reason_code", "old_summary", "new_summary", "policy"): + _text(detail[field]) + if detail["path"] is not None and ( + type(detail["path"]) is not str + or (detail["path"] and not detail["path"].startswith("/")) + ): + _refuse("ARTIFACT_SCHEMA_REFUSED") + return value + + +def _validate_actions(value): + if not isinstance(value, list) or len(value) > 3: + _refuse("ARTIFACT_SCHEMA_REFUSED") + for row in value: + _closed(row, ("kind", "argument")) + if row["kind"] not in ("adapt", "pin", "review"): + _refuse("ARTIFACT_SCHEMA_REFUSED") + _plain_json(row["argument"]) + return value + + +def _validate_finding(value, *, revision=False): + if revision: + fields = ( + "finding_id", + "target_id", + "module", + "symbol", + "verdict", + "base", + "head", + "reason_code", + "projection", + "fixture_binding_sha256", + ) + old_name, new_name = "base", "head" + else: + fields = ( + "finding_id", + "candidate_id", + "api", + "call_sites", + "verdict", + "old", + "new", + "reason_code", + "reason_detail", + "comparison", + "projection", + "fixture_binding_sha256", + "suggested_action", + ) + old_name, new_name = "old", "new" + _closed(value, fields) + _sha256(value["finding_id"], code="ARTIFACT_IDENTITY_REFUSED") + if value["finding_id"] != record_identity(value, "finding_id"): + _refuse("ARTIFACT_IDENTITY_REFUSED") + identity_field = "target_id" if revision else "candidate_id" + _sha256(value[identity_field], code="ARTIFACT_IDENTITY_REFUSED") + if revision: + _text(value["module"]) + _text(value["symbol"]) + else: + _text(value["api"]) + if not isinstance(value["call_sites"], list) or not value["call_sites"]: + _refuse("ARTIFACT_SCHEMA_REFUSED") + for site in value["call_sites"]: + _validate_call_site(site) + _validate_comparison(value["comparison"]) + _validate_actions(value["suggested_action"]) + if value["reason_detail"] is not None: + _text(value["reason_detail"]) + verdict = value["verdict"] + if verdict not in _VERDICTS: + _refuse("ARTIFACT_SCHEMA_REFUSED") + projection = _validate_projection(value["projection"]) + if (verdict in _PROJECTION_VERDICTS) != (projection is not None): + _refuse("ARTIFACT_PROJECTION_REFUSED") + binding = value["fixture_binding_sha256"] + if binding is not None: + _sha256(binding) + if verdict == "NOT_EXERCISED": + if value[old_name] is not None or value[new_name] is not None: + _refuse("ARTIFACT_OBSERVATION_REFUSED") + _text(value["reason_code"], code="ARTIFACT_OBSERVATION_REFUSED") + else: + _validate_observation(value[old_name]) + _validate_observation(value[new_name]) + if value["reason_code"] is not None: + _refuse("ARTIFACT_OBSERVATION_REFUSED") + return value + + +def _validate_repeat_hashes(value, expected) -> None: + if ( + not isinstance(value, list) + or len(value) != 2 + or any(_sha256(item, code="ARTIFACT_REPEAT_REFUSED") != expected for item in value) + ): + _refuse("ARTIFACT_REPEAT_REFUSED") + + +def _validate_dependency_witness(value, finding): + _closed( + value, + ( + "witness_id", + "finding_id", + "candidate_id", + "old_observation_sha256", + "new_observation_sha256", + "old_repeat_sha256", + "new_repeat_sha256", + "projection_sha256", + "provenance", + "replay", + ), + ) + if value["witness_id"] != record_identity(value, "witness_id"): + _refuse("ARTIFACT_IDENTITY_REFUSED") + if ( + value["finding_id"] != finding["finding_id"] + or value["candidate_id"] != finding["candidate_id"] + ): + _refuse("ARTIFACT_IDENTITY_REFUSED") + old_digest = artifact_digest(finding["old"]) + new_digest = artifact_digest(finding["new"]) + if ( + value["old_observation_sha256"] != old_digest + or value["new_observation_sha256"] != new_digest + ): + _refuse("ARTIFACT_IDENTITY_REFUSED") + _validate_repeat_hashes(value["old_repeat_sha256"], old_digest) + _validate_repeat_hashes(value["new_repeat_sha256"], new_digest) + expected_projection = ( + None if finding["projection"] is None else finding["projection"]["sha256"] + ) + if value["projection_sha256"] != expected_projection: + _refuse("ARTIFACT_PROJECTION_REFUSED") + _validate_provenance(value["provenance"]) + _validate_replay(value["replay"]) + expected_provenance = finding["old"]["provenance"] + if ( + value["provenance"] != expected_provenance + or finding["new"]["provenance"] != expected_provenance + ): + _refuse("ARTIFACT_PROVENANCE_REFUSED") + + +def _validate_summary(value, findings): + _closed(value, _SUMMARY_FIELDS) + expected = {name: 0 for name in _SUMMARY_FIELDS} + mapping = { + "IDENTICAL": "identical", + "CHANGED": "changed", + "IDENTICAL_UNDER_PROJECTION": "identical_under_projection", + "CHANGED_UNDER_PROJECTION": "changed_under_projection", + "NOT_EXERCISED": "not_exercised", + } + for finding in findings: + expected[mapping[finding["verdict"]]] += 1 + for count in value.values(): + _nonnegative_integer(count) + if value != expected: + _refuse("ARTIFACT_COUNT_REFUSED") + + +def _validate_coverage(value, findings): + _closed(value, ("exercised", "total", "percent")) + exercised = _nonnegative_integer(value["exercised"]) + total = _nonnegative_integer(value["total"]) + if total != len(findings) or exercised != sum( + finding["verdict"] in _EXERCISED_VERDICTS for finding in findings + ): + _refuse("ARTIFACT_COUNT_REFUSED") + percent = value["percent"] + if type(percent) not in (int, float) or not math.isfinite(float(percent)): + _refuse("ARTIFACT_COUNT_REFUSED") + expected = 100.0 * exercised / total if total else 0.0 + if float(percent) != expected: + _refuse("ARTIFACT_COUNT_REFUSED") + + +def _validate_dependency(payload): + for field in ("package", "current_version", "new_version"): + _text(payload[field]) + if not isinstance(payload["findings"], list) or not isinstance( + payload["witnesses"], list + ): + _refuse("ARTIFACT_SCHEMA_REFUSED") + findings = [] + by_id = {} + for finding in payload["findings"]: + _validate_finding(finding) + if finding["finding_id"] in by_id: + _refuse("ARTIFACT_IDENTITY_REFUSED") + by_id[finding["finding_id"]] = finding + findings.append(finding) + if findings != sorted(findings, key=lambda item: item["finding_id"]): + _refuse("ARTIFACT_ORDER_REFUSED") + witnessed = set() + for witness in payload["witnesses"]: + finding = by_id.get(witness.get("finding_id") if isinstance(witness, dict) else None) + if finding is None or finding["verdict"] == "NOT_EXERCISED": + _refuse("ARTIFACT_IDENTITY_REFUSED") + _validate_dependency_witness(witness, finding) + if finding["finding_id"] in witnessed: + _refuse("ARTIFACT_IDENTITY_REFUSED") + witnessed.add(finding["finding_id"]) + if payload["witnesses"] != sorted( + payload["witnesses"], key=lambda item: item["witness_id"] + ): + _refuse("ARTIFACT_ORDER_REFUSED") + if witnessed != { + finding["finding_id"] + for finding in findings + if finding["verdict"] in _EXERCISED_VERDICTS + }: + _refuse("ARTIFACT_REPEAT_REFUSED") + _validate_coverage(payload["coverage"], findings) + _validate_summary(payload["summary"], findings) + _validate_invocation(payload["invocation"], "dependency_report") + + +def _validate_coverage_report(payload): + for field in ("package", "current_version", "new_version"): + _text(payload[field]) + candidates = payload["candidates"] + if not isinstance(candidates, list): + _refuse("ARTIFACT_SCHEMA_REFUSED") + identities = set() + for row in candidates: + required = {"candidate_id", "api", "file", "line", "column", "bucket", "provenance"} + optional = {"reason_code", "reason_detail", "raw_type", "environment"} + if not isinstance(row, dict) or not required.issubset(row) or set(row) - required - optional: + _refuse("ARTIFACT_SCHEMA_REFUSED") + _sha256(row["candidate_id"], code="ARTIFACT_IDENTITY_REFUSED") + if row["candidate_id"] in identities: + _refuse("ARTIFACT_IDENTITY_REFUSED") + identities.add(row["candidate_id"]) + _text(row["api"]) + _relative_path(row["file"]) + if type(row["line"]) is not int or row["line"] < 1 or type(row["column"]) is not int or row["column"] < 0: + _refuse("ARTIFACT_SCHEMA_REFUSED") + if row["bucket"] not in _COVERAGE_BUCKETS: + _refuse("ARTIFACT_SCHEMA_REFUSED") + _validate_provenance(row["provenance"]) + if row["bucket"] == "EXERCISED" and set(row) != required: + _refuse("ARTIFACT_SCHEMA_REFUSED") + if row["bucket"] != "EXERCISED" and not row.get("reason_code"): + _refuse("ARTIFACT_SCHEMA_REFUSED") + for name in optional & set(row): + _text(row[name]) + if candidates != sorted(candidates, key=lambda item: item["candidate_id"]): + _refuse("ARTIFACT_ORDER_REFUSED") + counts = _closed(payload["counts"], (*_COVERAGE_BUCKETS, "total")) + expected = {bucket: 0 for bucket in _COVERAGE_BUCKETS} + for row in candidates: + expected[row["bucket"]] += 1 + expected["total"] = len(candidates) + for value in counts.values(): + _nonnegative_integer(value) + if counts != expected: + _refuse("ARTIFACT_COUNT_REFUSED") + _validate_invocation(payload["invocation"], "coverage_report") + + +def _validate_environment(value): + _closed(value, ("implementation", "machine", "platform", "python")) + for item in value.values(): + _text(item) + + +def _validate_fixture(value): + _closed(value, ("authored_by", "sha256", "source_revision")) + if value["authored_by"] not in ("human", "agent", "unknown"): + _refuse("ARTIFACT_PROVENANCE_REFUSED") + _sha256(value["sha256"]) + _revision(value["source_revision"]) + + +def _validate_baseline(payload): + _revision(payload["revision"]) + _sha256(payload["tree_sha256"]) + if type(payload["dirty"]) is not bool or type(payload["allow_dirty"]) is not bool: + _refuse("ARTIFACT_SCHEMA_REFUSED") + if payload["dirty"] and not payload["allow_dirty"]: + _refuse("ARTIFACT_DIRTY_REFUSED") + _validate_environment(payload["environment"]) + _validate_fixture(payload["fixture"]) + targets = payload["target_observations"] + if not isinstance(targets, list) or not targets: + _refuse("ARTIFACT_SCHEMA_REFUSED") + identities = set() + for target in targets: + _closed( + target, + ( + "target_id", + "module", + "symbol", + "definition_sha256", + "signature_sha256", + "observation", + "repeat_sha256", + "projection_sha256", + ), + ) + if target["target_id"] != record_identity(target, "target_id") or target["target_id"] in identities: + _refuse("ARTIFACT_IDENTITY_REFUSED") + identities.add(target["target_id"]) + _text(target["module"]) + _text(target["symbol"]) + _sha256(target["definition_sha256"]) + _sha256(target["signature_sha256"]) + _validate_observation(target["observation"]) + _validate_repeat_hashes( + target["repeat_sha256"], artifact_digest(target["observation"]) + ) + if target["projection_sha256"] is not None: + _sha256(target["projection_sha256"]) + if targets != sorted(targets, key=lambda item: item["target_id"]): + _refuse("ARTIFACT_ORDER_REFUSED") + _validate_invocation(payload["invocation"], "baseline") + + +def _validate_revision_witness(value, finding): + _closed( + value, + ( + "witness_id", + "finding_id", + "target_id", + "base_observation_sha256", + "head_observation_sha256", + "base_repeat_sha256", + "head_repeat_sha256", + "projection_sha256", + "provenance", + "replay", + ), + ) + if value["witness_id"] != record_identity(value, "witness_id"): + _refuse("ARTIFACT_IDENTITY_REFUSED") + if value["finding_id"] != finding["finding_id"] or value["target_id"] != finding["target_id"]: + _refuse("ARTIFACT_IDENTITY_REFUSED") + base_digest = artifact_digest(finding["base"]) + head_digest = artifact_digest(finding["head"]) + if value["base_observation_sha256"] != base_digest or value["head_observation_sha256"] != head_digest: + _refuse("ARTIFACT_IDENTITY_REFUSED") + _validate_repeat_hashes(value["base_repeat_sha256"], base_digest) + _validate_repeat_hashes(value["head_repeat_sha256"], head_digest) + expected_projection = None if finding["projection"] is None else finding["projection"]["sha256"] + if value["projection_sha256"] != expected_projection: + _refuse("ARTIFACT_PROJECTION_REFUSED") + _validate_provenance(value["provenance"]) + _validate_replay(value["replay"]) + if value["provenance"] != finding["base"]["provenance"] or value["provenance"] != finding["head"]["provenance"]: + _refuse("ARTIFACT_PROVENANCE_REFUSED") + + +def _validate_fixture_revision_events(value, *, findings=None): + if not isinstance(value, list): + _refuse("ARTIFACT_SCHEMA_REFUSED") + current_by_id = ( + None if findings is None else {row["finding_id"]: row for row in findings} + ) + event_ids = set() + targets = set() + for row in value: + _closed( + row, + ( + "event_id", + "target_id", + "prior_finding_id", + "current_finding_id", + "prior_fixture_binding_sha256", + "current_fixture_binding_sha256", + "prior_verdict", + "current_verdict", + "reason_code", + ), + ) + if ( + row["event_id"] != record_identity(row, "event_id") + or row["event_id"] in event_ids + or row["target_id"] in targets + ): + _refuse("ARTIFACT_IDENTITY_REFUSED") + event_ids.add(row["event_id"]) + targets.add(row["target_id"]) + for name in ( + "target_id", + "prior_finding_id", + "current_finding_id", + "prior_fixture_binding_sha256", + "current_fixture_binding_sha256", + ): + _sha256(row[name], code="ARTIFACT_IDENTITY_REFUSED") + if row["prior_fixture_binding_sha256"] == row["current_fixture_binding_sha256"]: + _refuse("ARTIFACT_IDENTITY_REFUSED") + if row["prior_verdict"] not in ( + "CHANGED", + "CHANGED_UNDER_PROJECTION", + ) or row["current_verdict"] not in ( + "IDENTICAL", + "IDENTICAL_UNDER_PROJECTION", + ): + _refuse("ARTIFACT_SCHEMA_REFUSED") + if row["reason_code"] != "FIXTURE_REVISED_AFTER_FAILURE": + _refuse("ARTIFACT_SCHEMA_REFUSED") + if current_by_id is not None: + current = current_by_id.get(row["current_finding_id"]) + if ( + current is None + or current["target_id"] != row["target_id"] + or current["fixture_binding_sha256"] + != row["current_fixture_binding_sha256"] + or current["verdict"] != row["current_verdict"] + ): + _refuse("ARTIFACT_IDENTITY_REFUSED") + if value != sorted(value, key=lambda row: row["event_id"]): + _refuse("ARTIFACT_ORDER_REFUSED") + return value + + +def _validate_revision_report(payload): + for field in ("base_revision", "head_revision"): + _revision(payload[field]) + for field in ("base_tree_sha256", "head_tree_sha256"): + _sha256(payload[field]) + if type(payload["fixtures_predate_change"]) is not bool: + _refuse("ARTIFACT_SCHEMA_REFUSED") + _validate_fixture(payload["fixture"]) + findings = payload["findings"] + witnesses = payload["witnesses"] + if not isinstance(findings, list) or not isinstance(witnesses, list): + _refuse("ARTIFACT_SCHEMA_REFUSED") + by_id = {} + for finding in findings: + _validate_finding(finding, revision=True) + if finding["finding_id"] in by_id: + _refuse("ARTIFACT_IDENTITY_REFUSED") + by_id[finding["finding_id"]] = finding + if findings != sorted(findings, key=lambda item: item["finding_id"]): + _refuse("ARTIFACT_ORDER_REFUSED") + witnessed = set() + for witness in witnesses: + finding = by_id.get(witness.get("finding_id") if isinstance(witness, dict) else None) + if finding is None or finding["verdict"] == "NOT_EXERCISED": + _refuse("ARTIFACT_IDENTITY_REFUSED") + _validate_revision_witness(witness, finding) + if finding["finding_id"] in witnessed: + _refuse("ARTIFACT_IDENTITY_REFUSED") + witnessed.add(finding["finding_id"]) + if witnessed != {row["finding_id"] for row in findings if row["verdict"] in _EXERCISED_VERDICTS}: + _refuse("ARTIFACT_REPEAT_REFUSED") + if witnesses != sorted(witnesses, key=lambda item: item["witness_id"]): + _refuse("ARTIFACT_ORDER_REFUSED") + _validate_fixture_revision_events( + payload["fixture_revision_events"], findings=findings + ) + _validate_summary(payload["summary"], findings) + _validate_invocation(payload["invocation"], "revision_report") + + +def _invocation_mapping(value) -> dict: + return {row["name"]: row["value"] for row in value} + + +def _validate_claim_report(payload): + if payload["claim"] != "behavior_preserved": + _refuse("ARTIFACT_SCHEMA_REFUSED") + _revision(payload["base_revision"]) + _revision(payload["head_revision"]) + if type(payload["fixtures_predate_change"]) is not bool: + _refuse("ARTIFACT_SCHEMA_REFUSED") + _validate_fixture(payload["fixture"]) + dispositions = payload["dispositions"] + if not isinstance(dispositions, list) or not dispositions: + _refuse("ARTIFACT_SCHEMA_REFUSED") + identities = set() + counts = {name: 0 for name in _CLAIM_COUNTS.values()} + for row in dispositions: + _closed( + row, + ( + "disposition_id", + "target_id", + "symbol", + "disposition", + "reason_code", + "projection_scope", + ), + ) + if row["disposition_id"] != record_identity(row, "disposition_id") or row["disposition_id"] in identities: + _refuse("ARTIFACT_IDENTITY_REFUSED") + identities.add(row["disposition_id"]) + _sha256(row["target_id"], code="ARTIFACT_IDENTITY_REFUSED") + _text(row["symbol"]) + if row["disposition"] not in _CLAIM_DISPOSITIONS: + _refuse("ARTIFACT_SCHEMA_REFUSED") + counts[_CLAIM_COUNTS[row["disposition"]]] += 1 + if row["disposition"] == "CLAIM_VERIFIED": + if row["reason_code"] is not None: + _refuse("ARTIFACT_SCHEMA_REFUSED") + else: + _text(row["reason_code"]) + if row["projection_scope"] is not None: + _text(row["projection_scope"], code="ARTIFACT_PROJECTION_REFUSED") + if dispositions != sorted(dispositions, key=lambda item: item["disposition_id"]): + _refuse("ARTIFACT_ORDER_REFUSED") + summary = _closed(payload["summary"], (*counts, "total")) + expected = {**counts, "total": len(dispositions)} + for value in summary.values(): + _nonnegative_integer(value) + if summary != expected: + _refuse("ARTIFACT_COUNT_REFUSED") + _validate_fixture_revision_events(payload["fixture_revision_events"]) + _validate_invocation(payload["invocation"], "claim_report") + flags = _invocation_mapping(payload["invocation"]) + if flags.get("strict_separation") is True: + if ( + not payload["fixtures_predate_change"] + or payload["fixture"]["authored_by"] == "unknown" + ): + _refuse("ARTIFACT_SEPARATION_REFUSED") + + +def _validate_evidence(payload): + _sha256(payload["report_artifact_sha256"]) + _sha256(payload["report_payload_sha256"]) + report_kind = payload["report_kind"] + if report_kind not in _PAYLOAD_FIELDS or report_kind == "evidence": + _refuse("ARTIFACT_KIND_REFUSED") + if not isinstance(payload["witnesses"], list): + _refuse("ARTIFACT_SCHEMA_REFUSED") + _plain_json(payload["witnesses"]) + rows = payload["environment_artifacts"] + if not isinstance(rows, list): + _refuse("ARTIFACT_SCHEMA_REFUSED") + names = [] + for row in rows: + _closed(row, ("name", "sha256")) + if type(row["name"]) is not str or not _NAME.fullmatch(row["name"]): + _refuse("ARTIFACT_SCHEMA_REFUSED") + _sha256(row["sha256"]) + names.append(row["name"]) + if names != sorted(set(names)): + _refuse("ARTIFACT_ORDER_REFUSED") + _validate_invocation(payload["invocation"], report_kind) + + +_VALIDATORS = { + "dependency_report": _validate_dependency, + "coverage_report": _validate_coverage_report, + "baseline": _validate_baseline, + "revision_report": _validate_revision_report, + "claim_report": _validate_claim_report, + "evidence": _validate_evidence, +} + + +def validate_artifact(value): + if not isinstance(value, Mapping) or set(value) != _ENVELOPE_FIELDS: + _refuse("ARTIFACT_SCHEMA_REFUSED") + artifact = _plain_json(copy.deepcopy(value)) + if artifact["schema_version"] != SCHEMA_VERSION: + _refuse("ARTIFACT_SCHEMA_VERSION_REFUSED") + kind = artifact["artifact_kind"] + expected = _PAYLOAD_FIELDS.get(kind) + if expected is None: + _refuse("ARTIFACT_KIND_REFUSED") + payload = artifact["payload"] + if not isinstance(payload, dict) or set(payload) != expected: + _refuse("ARTIFACT_SCHEMA_REFUSED") + _sha256(artifact["payload_sha256"]) + _VALIDATORS[kind](payload) + return artifact + + +def make_artifact(kind: str, payload: Mapping[str, object]): + if kind not in _PAYLOAD_FIELDS: + _refuse("ARTIFACT_KIND_REFUSED") + plain_payload = _plain_json(copy.deepcopy(payload)) + artifact = { + "schema_version": SCHEMA_VERSION, + "artifact_kind": kind, + "payload": plain_payload, + "payload_sha256": artifact_digest(plain_payload), + } + return validate_artifact(artifact) + + +def verify_artifact(value) -> str: + artifact = validate_artifact(value) + if artifact["payload_sha256"] != artifact_digest(artifact["payload"]): + _refuse("ARTIFACT_HASH_REFUSED") + return "VERIFIED" + + +__all__ = ( + "SCHEMA_VERSION", + "artifact_digest", + "canonical_json", + "canonicalize_invocation", + "make_artifact", + "record_identity", + "validate_artifact", + "verify_artifact", +) diff --git a/src/breakcheck/verify.py b/src/breakcheck/verify.py index 296bf90..6e84e97 100644 --- a/src/breakcheck/verify.py +++ b/src/breakcheck/verify.py @@ -18,7 +18,7 @@ def _without_identity(value, field): return observed, copied -def verify_report(report, witness): +def _verify_schema_one(report, witness): report = _plain(report) if not isinstance(report, dict): raise ValueError("report malformed") @@ -73,3 +73,38 @@ def verify_report(report, witness): if witnessed_ids != exercised_ids: raise ValueError("witness coverage mismatch") return 'VERIFIED' + + +def _verify_schema_two(report, evidence): + from breakcheck.schema import artifact_digest, verify_artifact + + verify_artifact(report) + verify_artifact(evidence) + if report.get("artifact_kind") == "evidence": + raise ValueError("ARTIFACT_KIND_REFUSED") + if evidence.get("artifact_kind") != "evidence": + raise ValueError("ARTIFACT_KIND_REFUSED") + report_payload = report["payload"] + evidence_payload = evidence["payload"] + if evidence_payload["report_kind"] != report["artifact_kind"]: + raise ValueError("ARTIFACT_IDENTITY_REFUSED") + if evidence_payload["report_artifact_sha256"] != artifact_digest(report): + raise ValueError("ARTIFACT_IDENTITY_REFUSED") + if evidence_payload["report_payload_sha256"] != report["payload_sha256"]: + raise ValueError("ARTIFACT_IDENTITY_REFUSED") + if evidence_payload["invocation"] != report_payload["invocation"]: + raise ValueError("ARTIFACT_INVOCATION_REFUSED") + if evidence_payload["witnesses"] != report_payload.get("witnesses", []): + raise ValueError("ARTIFACT_IDENTITY_REFUSED") + return "VERIFIED" + + +def verify_report(report, witness): + """Verify a historical schema-1 or closed schema-2 report bundle.""" + if isinstance(report, dict): + version = report.get("schema_version") + if version == 2: + return _verify_schema_two(report, witness) + if version not in (None, 1): + raise ValueError("ARTIFACT_SCHEMA_VERSION_REFUSED") + return _verify_schema_one(report, witness) diff --git a/tests/test_agent_workflow.py b/tests/test_agent_workflow.py new file mode 100644 index 0000000..f321477 --- /dev/null +++ b/tests/test_agent_workflow.py @@ -0,0 +1,74 @@ +from __future__ import annotations + +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +SKILL = ROOT / "SKILL.md" +DISCOVERY = ROOT / "AGENTS.md" + + +def _skill_text() -> str: + return SKILL.read_text(encoding="utf-8") + + +def test_agent_skill_is_discoverable_and_concise(): + text = _skill_text() + assert text.startswith("---\n") + assert "name: verifying-python-changes-with-breakcheck" in text + assert "description: Use when" in text + assert len(text.split()) < 500 + + +def test_agent_skill_covers_both_verification_modes_and_fail_closed_results(): + text = _skill_text() + for command in ( + "breakcheck PACKAGE@NEW_VERSION", + "--suggest-fixtures", + "breakcheck diff", + "breakcheck attest", + "--strict-separation", + ): + assert command in text + for disposition in ( + "CHANGED", + "NOT_EXERCISED", + "CLAIM_REFUTED", + "CLAIM_UNVERIFIABLE", + "CLAIM_OUT_OF_SCOPE", + ): + assert disposition in text + + +def test_agent_skill_prohibits_self_grading_and_evidence_mutation(): + text = _skill_text() + for protected in ( + "report.json", + "evidence.json", + "baseline.json", + "--allow-empty", + "--min-coverage", + ): + assert protected in text + assert "Never modify a fixture after seeing `CHANGED`" in text + assert "Never modify verdict or verification logic" in text + + +def test_agent_skill_requires_prechange_baseline_and_independent_attestation(): + text = _skill_text() + ordered_steps = ( + "Run `breakcheck freeze`", + "Make the behavior-preserving code change", + "Create a claim file", + "Run `breakcheck attest`", + "Report every disposition verbatim", + ) + positions = [text.index(step) for step in ordered_steps] + assert positions == sorted(positions) + assert "Never author or modify a fixture for a target after changing" in text + + +def test_repository_discovery_file_routes_automation_to_the_skill(): + text = DISCOVERY.read_text(encoding="utf-8") + assert "[SKILL.md](SKILL.md)" in text + assert "Never weaken coverage or separation policy" in text diff --git a/tests/test_behavioral_invariants.py b/tests/test_behavioral_invariants.py new file mode 100644 index 0000000..4a03f44 --- /dev/null +++ b/tests/test_behavioral_invariants.py @@ -0,0 +1,157 @@ +from __future__ import annotations + +import ast +from pathlib import Path + +import pytest + +from breakcheck import cli +from breakcheck.adapters.python import literals +from breakcheck.adapters.python.equality import compare_observations +from breakcheck.adapters.python.normalization import normalize_outcome + + +ROOT = Path(__file__).resolve().parents[1] +ADAPTER_ROOT = ROOT / "src" / "breakcheck" / "adapters" + + +def _catches_broadly(handler: ast.ExceptHandler) -> bool: + if handler.type is None: + return True + if isinstance(handler.type, ast.Name): + return handler.type.id in {"Exception", "BaseException"} + if isinstance(handler.type, ast.Tuple): + return any( + isinstance(item, ast.Name) + and item.id in {"Exception", "BaseException"} + for item in handler.type.elts + ) + return False + + +def _terminates_explicitly(handler: ast.ExceptHandler) -> bool: + for node in ast.walk(ast.Module(body=handler.body, type_ignores=[])): + if isinstance(node, ast.Raise): + return True + if isinstance(node, ast.Return): + value = node.value + if isinstance(value, ast.Constant) and value.value is None: + continue + return True + return False + + +def test_adapter_broad_exceptions_cannot_silently_continue(): + violations = [] + for path in sorted(ADAPTER_ROOT.rglob("*.py")): + tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) + for node in ast.walk(tree): + if not isinstance(node, ast.ExceptHandler) or not _catches_broadly(node): + continue + if not _terminates_explicitly(node): + violations.append( + f"{path.relative_to(ROOT).as_posix()}:{node.lineno}" + ) + assert violations == [] + + +@pytest.mark.parametrize("code", sorted(cli._DECLARED_REFUSAL_CODES)) +def test_every_declared_refusal_is_bounded_and_machine_recognizable(code): + assert cli._bounded_refusal(ValueError(code)) == code + assert code in cli._HELP + + +def test_runtime_source_never_waits_for_interactive_input(): + violations = [] + source_root = ROOT / "src" / "breakcheck" + for path in sorted(source_root.rglob("*.py")): + tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) + for node in ast.walk(tree): + if isinstance(node, ast.Call) and isinstance(node.func, ast.Name): + if node.func.id in {"input", "breakpoint"}: + violations.append( + f"{path.relative_to(ROOT).as_posix()}:{node.lineno}" + ) + assert violations == [] + + +def test_fixture_refusal_registry_is_complete(): + from breakcheck.adapters.python import fixtures + + tree = ast.parse( + (ADAPTER_ROOT / "python" / "fixtures.py").read_text(encoding="utf-8") + ) + emitted = { + node.args[0].value + for node in ast.walk(tree) + if isinstance(node, ast.Call) + and isinstance(node.func, ast.Name) + and node.func.id == "_refuse" + and node.args + and isinstance(node.args[0], ast.Constant) + and isinstance(node.args[0].value, str) + } + assert emitted == fixtures.REFUSAL_CODES + + +def _normalized_value(payload): + return normalize_outcome( + { + "kind": "value", + "payload": payload, + "exception_class": None, + } + ) + + +@pytest.mark.parametrize( + ("old_payload", "new_payload"), + [ + (b"a", "61"), + ({1, 2}, [1, 2]), + ((1, 2), [1, 2]), + ({1, 2}, frozenset({1, 2})), + ], +) +def test_normalization_preserves_python_type_changes(old_payload, new_payload): + result = compare_observations( + _normalized_value(old_payload), + _normalized_value(new_payload), + ) + + assert result["verdict"] == "CHANGED" + assert result["detail"]["reason_code"] == "KIND_MISMATCH" + + +@pytest.mark.parametrize( + "payload", + [b"a", {1, 2}, (1, 2), frozenset({1, 2})], +) +def test_tagged_normalization_remains_identical_for_the_same_value(payload): + observation = _normalized_value(payload) + + assert compare_observations(observation, observation)["verdict"] == "IDENTICAL" + + +def test_user_mapping_cannot_collide_with_a_normalized_type_tag(): + tag_shaped_mapping = { + "$breakcheck_type": "bytes", + "$breakcheck_value": "61", + } + + result = compare_observations( + _normalized_value(b"a"), + _normalized_value(tag_shaped_mapping), + ) + + assert result["verdict"] == "CHANGED" + assert result["detail"]["reason_code"] == "KIND_MISMATCH" + + +def test_folded_values_have_a_cumulative_render_budget(): + expression = "json.dumps(['x' * 65536] * 20)" + + with pytest.raises(literals.LiteralRefusal) as caught: + literals.synthesize_snippet(expression) + + assert caught.value.reason_detail == "FOLD_REFUSED" diff --git a/tests/test_claim_attestation.py b/tests/test_claim_attestation.py new file mode 100644 index 0000000..df78b1f --- /dev/null +++ b/tests/test_claim_attestation.py @@ -0,0 +1,264 @@ +from __future__ import annotations + +import pytest + +from breakcheck.core.claims import ( + BehaviorClaim, + ClaimRefusal, + adjudicate_claim, + claim_exit_code, + parse_claim, +) + + +BASE_REVISION = "1" * 40 +HEAD_REVISION = "2" * 40 + + +def _claim_text(*symbols: str) -> str: + blocks = [ + "schema_version = 1", + 'claim = "behavior_preserved"', + f'base_revision = "{BASE_REVISION}"', + ] + for symbol in symbols: + blocks.extend(("", "[[target]]", f'symbol = "{symbol}"')) + return "\n".join(blocks) + "\n" + + +def _finding( + symbol: str, + verdict: str, + *, + reason_code: str | None = None, + projection_scope: str | None = None, +) -> dict[str, object]: + return { + "symbol": symbol, + "verdict": verdict, + "reason_code": reason_code, + "projection_scope": projection_scope, + } + + +def test_claim_parser_is_closed_sorted_and_rejects_vacuous_or_duplicate_targets() -> None: + """An open, empty, or duplicate claim could hide the actual assertion surface.""" + claim = parse_claim(_claim_text("sample.math:zeta", "sample.math:alpha")) + assert claim.schema_version == 1 + assert claim.claim == "behavior_preserved" + assert claim.base_revision == BASE_REVISION + assert claim.targets == ("sample.math:alpha", "sample.math:zeta") + + with pytest.raises(ClaimRefusal, match="^CLAIM_VACUOUS_REFUSED$"): + parse_claim(_claim_text()) + with pytest.raises(ClaimRefusal, match="^CLAIM_DUPLICATE_TARGET_REFUSED$"): + parse_claim(_claim_text("sample.math:value", "sample.math:value")) + with pytest.raises(ClaimRefusal, match="^CLAIM_SCHEMA_REFUSED$"): + parse_claim(_claim_text("sample.math:value") + 'extra = "hidden"\n') + + +def test_claim_adjudication_independently_detects_refuted_unverifiable_and_out_of_scope() -> None: + """A claim must not control the changed-symbol census or call absence success.""" + claim = parse_claim( + _claim_text( + "sample.math:changed", + "sample.math:stable", + "sample.math:unavailable", + ) + ) + report = adjudicate_claim( + claim, + head_revision=HEAD_REVISION, + changed_targets={ + "sample.math:changed", + "sample.math:stable", + "sample.math:unavailable", + "sample.math:omitted", + }, + findings=[ + _finding("sample.math:changed", "CHANGED"), + _finding("sample.math:stable", "IDENTICAL"), + _finding( + "sample.math:unavailable", + "NOT_EXERCISED", + reason_code="IMPORT_FAILED", + ), + ], + fixture_source="base", + fixture_revision=BASE_REVISION, + fixture_authored_by="human", + fixtures_predate_change=True, + strict_separation=True, + invocation={"strict_separation": True}, + ) + + assert [(row["symbol"], row["disposition"]) for row in report["dispositions"]] == [ + ("sample.math:changed", "CLAIM_REFUTED"), + ("sample.math:omitted", "CLAIM_OUT_OF_SCOPE"), + ("sample.math:stable", "CLAIM_VERIFIED"), + ("sample.math:unavailable", "CLAIM_UNVERIFIABLE"), + ] + assert report["summary"] == { + "verified": 1, + "refuted": 1, + "unverifiable": 1, + "out_of_scope": 1, + "total": 4, + } + assert claim_exit_code(report) == 3 + + +def test_claim_under_projection_states_the_scope_and_exit_precedence_is_stable() -> None: + """Projection-limited evidence must never be presented as full-observation proof.""" + claim = parse_claim(_claim_text("sample.math:value")) + verified = adjudicate_claim( + claim, + head_revision=HEAD_REVISION, + changed_targets={"sample.math:value"}, + findings=[ + _finding( + "sample.math:value", + "IDENTICAL_UNDER_PROJECTION", + projection_scope="outcome.as_dict()", + ) + ], + fixture_source="base", + fixture_revision=BASE_REVISION, + fixture_authored_by="human", + fixtures_predate_change=True, + strict_separation=True, + invocation={}, + ) + row = verified["dispositions"][0] + assert row["disposition"] == "CLAIM_VERIFIED" + assert row["verification_scope"] == "UNDER_PROJECTION" + assert row["projection_scope"] == "outcome.as_dict()" + assert claim_exit_code(verified) == 0 + + for dispositions, expected in ( + ([{"disposition": "CLAIM_REFUTED"}], 1), + ([{"disposition": "CLAIM_REFUTED"}, {"disposition": "CLAIM_UNVERIFIABLE"}], 2), + ([{"disposition": "CLAIM_UNVERIFIABLE"}, {"disposition": "CLAIM_OUT_OF_SCOPE"}], 3), + ): + assert claim_exit_code({"dispositions": dispositions}) == expected + + +@pytest.mark.parametrize( + ("source", "author", "predates", "reason"), + [ + ("head", "human", False, "FIXTURE_AUTHORED_AGAINST_HEAD"), + ("base", "unknown", True, "FIXTURE_AUTHOR_UNKNOWN"), + ("explicit", "human", False, "FIXTURE_POSTDATES_CHANGE"), + ], +) +def test_strict_separation_never_verifies_unproven_fixture_provenance( + source: str, author: str, predates: bool, reason: str +) -> None: + """Inputs chosen after a change cannot independently verify that change.""" + report = adjudicate_claim( + parse_claim(_claim_text("sample.math:value")), + head_revision=HEAD_REVISION, + changed_targets={"sample.math:value"}, + findings=[_finding("sample.math:value", "IDENTICAL")], + fixture_source=source, + fixture_revision=HEAD_REVISION if source == "head" else BASE_REVISION, + fixture_authored_by=author, + fixtures_predate_change=predates, + strict_separation=True, + invocation={}, + ) + + row = report["dispositions"][0] + assert row["disposition"] == "CLAIM_UNVERIFIABLE" + assert row["reason_code"] == reason + assert claim_exit_code(report) == 2 + + +def test_non_strict_separation_records_relaxation_without_hiding_it() -> None: + """An explicit policy relaxation must remain visible in the report evidence.""" + report = adjudicate_claim( + parse_claim(_claim_text("sample.math:value")), + head_revision=HEAD_REVISION, + changed_targets={"sample.math:value"}, + findings=[_finding("sample.math:value", "IDENTICAL")], + fixture_source="head", + fixture_revision=HEAD_REVISION, + fixture_authored_by="unknown", + fixtures_predate_change=False, + strict_separation=False, + invocation={"strict_separation": False}, + ) + + assert report["dispositions"][0]["disposition"] == "CLAIM_VERIFIED" + assert report["fixtures_predate_change"] is False + assert report["invocation"]["strict_separation"] is False + + +def test_advisory_claim_exit_records_unverifiable_without_failing() -> None: + """Only an explicit non-strict claim may treat unverifiable work as advisory.""" + dispositions = [{"disposition": "CLAIM_UNVERIFIABLE"}] + + assert claim_exit_code( + {"dispositions": dispositions, "invocation": {"strict": False}} + ) == 0 + assert claim_exit_code( + {"dispositions": dispositions, "invocation": {"strict": True}} + ) == 2 + + +@pytest.mark.parametrize( + ("mutation", "code"), + [ + ("identical", "IDENTICAL_REVISIONS_REFUSED"), + ("zero", "CLAIM_NO_CHANGED_TARGETS"), + ("unknown_finding", "CLAIM_FINDING_REFUSED"), + ("path", "CLAIM_PATH_REFUSED"), + ], +) +def test_claim_adjudication_refuses_vacuous_or_open_inputs(mutation: str, code: str) -> None: + """A vacuous or malformed claim cannot produce a successful attestation.""" + claim = parse_claim(_claim_text("sample.math:value")) + head = BASE_REVISION if mutation == "identical" else HEAD_REVISION + changed = set() if mutation == "zero" else {"sample.math:value"} + findings = [_finding("sample.math:value", "IDENTICAL")] + if mutation == "unknown_finding": + findings[0]["extra"] = True + invocation: dict[str, object] = {} + if mutation == "path": + invocation["worktree_root"] = "/private/location" + + with pytest.raises(ClaimRefusal, match=f"^{code}$"): + adjudicate_claim( + claim, + head_revision=head, + changed_targets=changed, + findings=findings, + fixture_source="base", + fixture_revision=BASE_REVISION, + fixture_authored_by="human", + fixtures_predate_change=True, + strict_separation=True, + invocation=invocation, + ) + + +def test_adjudication_revalidates_programmatically_constructed_claims() -> None: + """Bypassing the text parser must not bypass the closed claim contract.""" + forged = BehaviorClaim( + schema_version=2, + claim="different_claim", + base_revision=BASE_REVISION, + targets=("sample.math:value",), + ) + + with pytest.raises(ClaimRefusal, match="^CLAIM_SCHEMA_REFUSED$"): + adjudicate_claim( + forged, + head_revision=HEAD_REVISION, + changed_targets={"sample.math:value"}, + findings=[_finding("sample.math:value", "IDENTICAL")], + fixture_revision=BASE_REVISION, + fixture_authored_by="human", + fixtures_predate_change=True, + invocation={}, + ) diff --git a/tests/test_cli_integration.py b/tests/test_cli_integration.py new file mode 100644 index 0000000..0c34477 --- /dev/null +++ b/tests/test_cli_integration.py @@ -0,0 +1,544 @@ +from __future__ import annotations + +import copy +import json +import stat +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from breakcheck import cli +from breakcheck import revision_cli +from breakcheck.adapters.python.literals import synthesize_snippet +from breakcheck.report import finding_id +from breakcheck.verify import verify_report + + +def _fake_pipeline(source: Path, runtime: Path, executions: list[tuple[str, str]]): + class Inventory: + def __call__(self, _root, excluded_paths=()): + return [source] + + class Scanner: + def __init__(self, _package): + pass + + def scan(self, **_kwargs): + return { + "imports": [], + "call_sites": [ + {"api": "sample.api", "file": "app.py", "line": 2, "column": 0} + ], + "unsupported": [], + "candidates": [], + } + + class EnvironmentBuilder: + def __init__(self, **_kwargs): + pass + + def build(self): + return {"current": str(runtime / "current"), "new": str(runtime / "new")} + + def execute(*, snippet_source, environment): + if "_rows" in snippet_source: + return { + "returncode": 0, + "stdout": repr({"sample.api": True}).encode(), + "stderr": b"", + "timed_out": False, + } + executions.append((str(environment), snippet_source)) + return { + "returncode": 0, + "stdout": b"'stable'\n", + "stderr": b"", + "timed_out": False, + } + + def compare(old, new): + assert old == new + return { + "verdict": "IDENTICAL", + "detail": { + "reason_code": "EQUAL", + "path": None, + "old_summary": "stable", + "new_summary": "stable", + "policy": "canonical_json_strict", + }, + } + + return ( + Inventory(), + Scanner, + lambda _expression, import_statement=None: "outcome = 'stable'\n", + EnvironmentBuilder, + execute, + lambda value: copy.deepcopy(value), + compare, + finding_id, + lambda report: json.dumps(report, sort_keys=True), + lambda report: json.dumps(report, sort_keys=True), + lambda report: 0, + verify_report, + ) + + +def test_dependency_replay_runs_twice_in_each_environment(monkeypatch, tmp_path): + source = tmp_path / "app.py" + source.write_text("import sample\nsample.api()\n", encoding="utf-8") + runtime = tmp_path / "runtime" + executions: list[tuple[str, str]] = [] + monkeypatch.setattr(cli, "_load_pipeline", lambda: _fake_pipeline(source, runtime, executions)) + monkeypatch.setattr(cli._metadata, "version", lambda _package: "1.0") + monkeypatch.setattr(cli, "_import_root", lambda _package: "sample") + monkeypatch.chdir(tmp_path) + + args = SimpleNamespace( + target="sample@2.0", + wheelhouse=str(tmp_path / "wheelhouse"), + runtime_root=str(runtime), + output=None, + evidence=None, + coverage_report=None, + fixtures=None, + fixture_policy="forbid", + suggest_fixtures=None, + min_coverage=80.0, + allow_empty=False, + json=True, + ci=False, + ) + + assert cli._build(args) == 0 + assert [environment for environment, _ in executions].count(str(runtime / "current")) == 2 + assert [environment for environment, _ in executions].count(str(runtime / "new")) == 2 + + +def test_implicit_dependency_runtime_is_removed_after_the_run(monkeypatch, tmp_path): + source = tmp_path / "app.py" + source.write_text("import sample\nsample.api()\n", encoding="utf-8") + generated_runtime = tmp_path / "generated-runtime" + executions: list[tuple[str, str]] = [] + + def make_runtime(*, prefix): + assert prefix == "breakcheck-runtime-" + generated_runtime.mkdir() + return str(generated_runtime) + + monkeypatch.setattr(cli.tempfile, "mkdtemp", make_runtime) + monkeypatch.setattr( + cli, + "_load_pipeline", + lambda: _fake_pipeline(source, generated_runtime, executions), + ) + monkeypatch.setattr(cli._metadata, "version", lambda _package: "1.0") + monkeypatch.setattr(cli, "_import_root", lambda _package: "sample") + monkeypatch.chdir(tmp_path) + + result = cli._build( + SimpleNamespace( + target="sample@2.0", + wheelhouse=str(tmp_path / "wheelhouse"), + runtime_root=None, + output=None, + evidence=None, + coverage_report=None, + fixtures=None, + fixture_policy="forbid", + suggest_fixtures=None, + min_coverage=80.0, + allow_empty=False, + json=True, + ci=False, + ) + ) + + assert result == 0 + assert not generated_runtime.exists() + + +def test_implicit_dependency_runtime_is_removed_after_failure(monkeypatch, tmp_path): + generated_runtime = tmp_path / "generated-runtime" + + def make_runtime(*, prefix): + assert prefix == "breakcheck-runtime-" + generated_runtime.mkdir() + return str(generated_runtime) + + monkeypatch.setattr(cli.tempfile, "mkdtemp", make_runtime) + monkeypatch.setattr( + cli, + "_load_pipeline", + lambda: (_ for _ in ()).throw(RuntimeError("unexpected failure")), + ) + + with pytest.raises(RuntimeError, match="unexpected failure"): + cli._build( + SimpleNamespace( + target="sample@2.0", + wheelhouse=str(tmp_path / "wheelhouse"), + runtime_root=None, + output=None, + evidence=None, + coverage_report=None, + suggest_fixtures=None, + ) + ) + + assert not generated_runtime.exists() + + +def test_dependency_replay_uses_static_context_and_records_provenance( + monkeypatch, tmp_path +): + source = tmp_path / "app.py" + source.write_text("ARG = 1 + 1\nsample.api(ARG)\n", encoding="utf-8") + runtime = tmp_path / "runtime" + executions: list[tuple[str, str]] = [] + pipeline = list(_fake_pipeline(source, runtime, executions)) + pipeline[2] = synthesize_snippet + monkeypatch.setattr(cli, "_load_pipeline", lambda: tuple(pipeline)) + monkeypatch.setattr(cli._metadata, "version", lambda _package: "1.0") + monkeypatch.setattr(cli, "_import_root", lambda _package: "sample") + monkeypatch.chdir(tmp_path) + report_path = tmp_path / "report.json" + + result = cli._build( + SimpleNamespace( + target="sample@2.0", + wheelhouse=str(tmp_path / "wheelhouse"), + runtime_root=str(runtime), + output=str(report_path), + evidence=None, + coverage_report=None, + fixtures=None, + fixture_policy="forbid", + suggest_fixtures=None, + min_coverage=80.0, + allow_empty=False, + json=True, + ci=False, + ) + ) + + assert result == 0 + assert all("outcome = sample.api(2)" in snippet for _, snippet in executions) + report = json.loads(report_path.read_text(encoding="utf-8")) + finding = report["payload"]["findings"][0] + expected = ["SOURCE_FOLDED", "SOURCE_MODULE_CONSTANT"] + assert finding["old"]["provenance"] == expected + assert finding["new"]["provenance"] == expected + + +def test_unexpected_synthesis_failure_is_not_hidden_as_unexercised( + monkeypatch, tmp_path +): + source = tmp_path / "app.py" + source.write_text("import sample\nsample.api(1)\n", encoding="utf-8") + runtime = tmp_path / "runtime" + pipeline = list(_fake_pipeline(source, runtime, [])) + + def broken_synthesizer(_expression, _import_statement=None): + raise RuntimeError("boom") + + pipeline[2] = broken_synthesizer + monkeypatch.setattr(cli, "_load_pipeline", lambda: tuple(pipeline)) + monkeypatch.setattr(cli._metadata, "version", lambda _package: "1.0") + monkeypatch.setattr(cli, "_import_root", lambda _package: "sample") + monkeypatch.chdir(tmp_path) + + with pytest.raises(RuntimeError, match="boom"): + cli._build( + SimpleNamespace( + target="sample@2.0", + wheelhouse=str(tmp_path / "wheelhouse"), + runtime_root=str(runtime), + output=None, + evidence=None, + coverage_report=None, + fixtures=None, + fixture_policy="forbid", + suggest_fixtures=None, + min_coverage=80.0, + allow_empty=False, + json=True, + ci=False, + ) + ) + + +def test_artifact_write_uses_private_atomic_file_and_ignores_predictable_temp( + tmp_path, +): + destination = tmp_path / "report.json" + victim = tmp_path / "victim.txt" + victim.write_text("unchanged", encoding="utf-8") + (tmp_path / "report.json.tmp").symlink_to(victim) + + cli._write(destination, "verified\n") + + assert destination.read_text(encoding="utf-8") == "verified\n" + assert victim.read_text(encoding="utf-8") == "unchanged" + assert stat.S_IMODE(destination.stat().st_mode) == 0o600 + + +def test_artifact_write_refuses_a_symlinked_parent(tmp_path): + real_parent = tmp_path / "real" + real_parent.mkdir() + linked_parent = tmp_path / "linked" + linked_parent.symlink_to(real_parent, target_is_directory=True) + + with pytest.raises(ValueError, match="OUTPUT_PATH_REFUSED"): + cli._write(linked_parent / "report.json", "sensitive\n") + + +def test_output_artifact_destinations_must_be_distinct(tmp_path): + destination = tmp_path / "same.json" + args = SimpleNamespace( + output=str(destination), + evidence=str(destination), + coverage_report=None, + suggest_fixtures=None, + ) + + with pytest.raises(ValueError, match="OUTPUT_PATH_COLLISION_REFUSED"): + cli._validate_output_paths(args) + + destination.touch() + alias = tmp_path / "alias.json" + alias.symlink_to(destination) + args.evidence = str(alias) + with pytest.raises(ValueError, match="OUTPUT_PATH_COLLISION_REFUSED"): + cli._validate_output_paths(args) + + +def test_suggest_fixtures_needs_no_wheelhouse(monkeypatch, tmp_path): + source = tmp_path / "app.py" + source.write_text("import sample\nsample.api(object())\n", encoding="utf-8") + destination = tmp_path / "suggested.toml" + monkeypatch.chdir(tmp_path) + + result = cli.main( + ["sample@2.0", "--suggest-fixtures", str(destination)] + ) + + assert result == 0 + rendered = destination.read_text(encoding="utf-8") + assert 'api = "sample.api"' in rendered + assert 'fixture_authored_by = "unknown"' in rendered + assert "sample.api(object())" in rendered + + +def test_operator_fixture_replays_a_nonliteral_call(monkeypatch, tmp_path): + source = tmp_path / "app.py" + source.write_text("import sample\nsample.api(object())\n", encoding="utf-8") + runtime = tmp_path / "runtime" + fixture = tmp_path / "breakcheck.fixtures.toml" + fixture.write_text( + "\n".join( + ( + "schema_version = 1", + "", + "[[binding]]", + 'fixture_authored_by = "agent"', + 'file = "app.py"', + "line = 2", + "column = 0", + 'api = "sample.api"', + 'args = ["7"]', + "kwargs = {}", + "", + ) + ), + encoding="utf-8", + ) + executions: list[tuple[str, str]] = [] + pipeline = list(_fake_pipeline(source, runtime, executions)) + + def refuse_nonliteral(_expression, import_statement=None): + raise ValueError("NONLITERAL_ARGS") + + pipeline[2] = refuse_nonliteral + monkeypatch.setattr(cli, "_load_pipeline", lambda: tuple(pipeline)) + monkeypatch.setattr(cli._metadata, "version", lambda _package: "1.0") + monkeypatch.setattr(cli, "_import_root", lambda _package: "sample") + monkeypatch.chdir(tmp_path) + report_path = tmp_path / "report.json" + + result = cli._build( + SimpleNamespace( + target="sample@2.0", + wheelhouse=str(tmp_path / "wheelhouse"), + runtime_root=str(runtime), + output=str(report_path), + evidence=None, + coverage_report=None, + fixtures=str(fixture), + fixture_policy="allow", + suggest_fixtures=None, + min_coverage=80.0, + allow_empty=False, + json=True, + ci=False, + ) + ) + + assert result == 0 + assert len(executions) == 4 + assert all("outcome = sample.api(7)" in source for _, source in executions) + report = json.loads(report_path.read_text(encoding="utf-8")) + assert report["schema_version"] == 2 + finding = report["payload"]["findings"][0] + assert finding["verdict"] == "IDENTICAL" + assert finding["old"]["provenance"] == ["OPERATOR_FIXTURE"] + assert finding["new"]["provenance"] == ["OPERATOR_FIXTURE"] + + +def test_nondeterministic_replay_is_g4_without_a_witness(monkeypatch, tmp_path): + source = tmp_path / "app.py" + source.write_text("import sample\nsample.api()\n", encoding="utf-8") + runtime = tmp_path / "runtime" + executions = [] + pipeline = list(_fake_pipeline(source, runtime, executions)) + counter = {"value": 0} + + def execute(*, snippet_source, environment): + if "_rows" in snippet_source: + return { + "returncode": 0, + "stdout": repr({"sample.api": True}).encode(), + "stderr": b"", + "timed_out": False, + } + counter["value"] += 1 + return { + "returncode": 0, + "stdout": repr(counter["value"]).encode(), + "stderr": b"", + "timed_out": False, + } + + pipeline[4] = execute + monkeypatch.setattr(cli, "_load_pipeline", lambda: tuple(pipeline)) + monkeypatch.setattr(cli._metadata, "version", lambda _package: "1.0") + monkeypatch.setattr(cli, "_import_root", lambda _package: "sample") + monkeypatch.chdir(tmp_path) + report_path = tmp_path / "report.json" + coverage_path = tmp_path / "coverage.json" + + result = cli._build( + SimpleNamespace( + target="sample@2.0", + wheelhouse=str(tmp_path / "wheelhouse"), + runtime_root=str(runtime), + output=str(report_path), + evidence=None, + coverage_report=str(coverage_path), + fixtures=None, + fixture_policy="forbid", + suggest_fixtures=None, + min_coverage=80.0, + allow_empty=False, + json=True, + ci=False, + ) + ) + + assert result == 4 + report = json.loads(report_path.read_text(encoding="utf-8")) + assert report["payload"]["witnesses"] == [] + assert report["payload"]["findings"][0]["reason_code"] == "NONDETERMINISTIC_OBSERVATION" + coverage = json.loads(coverage_path.read_text(encoding="utf-8")) + assert coverage["payload"]["counts"]["G4_IMPURE"] == 1 + assert coverage["payload"]["counts"]["EXERCISED"] == 0 + + +def test_freeze_command_routes_to_revision_engine(monkeypatch, tmp_path): + observed = {} + sentinel = SimpleNamespace(report={}, evidence={}, exit_code=0) + + def freeze(repository, **kwargs): + observed["repository"] = repository + observed.update(kwargs) + assert not Path(kwargs["runtime_root"]).exists() + return sentinel + + monkeypatch.setattr(revision_cli, "freeze_revision", freeze) + monkeypatch.setattr(cli, "_emit_revision_result", lambda result, _args: result.exit_code) + monkeypatch.chdir(tmp_path) + + result = cli.main( + [ + "freeze", + "--revision", + "HEAD", + "--fixtures", + "breakcheck.fixtures.toml", + "--target", + "app.pricing:total", + "--output", + "baseline.json", + ] + ) + + assert result == 0 + assert observed["repository"] == tmp_path + assert observed["targets"] == ["app.pricing:total"] + + +def test_diff_command_loads_baseline_and_preserves_strict_options( + monkeypatch, tmp_path +): + baseline = {"schema_version": 2, "artifact_kind": "baseline"} + (tmp_path / "baseline.json").write_text(json.dumps(baseline), encoding="utf-8") + observed = {} + sentinel = SimpleNamespace(report={}, evidence={}, exit_code=3) + + def diff(repository, **kwargs): + observed["repository"] = repository + observed.update(kwargs) + return sentinel + + monkeypatch.setattr(revision_cli, "diff_revisions", diff) + monkeypatch.setattr(cli, "_emit_revision_result", lambda result, _args: result.exit_code) + monkeypatch.chdir(tmp_path) + + result = cli.main( + [ + "diff", + "--baseline", + "baseline.json", + "--head", + "HEAD", + "--strict-separation", + ] + ) + + assert result == 3 + assert observed["baseline"] == baseline + assert observed["strict_separation"] is True + + +def test_attest_command_is_strict_by_default(monkeypatch, tmp_path): + observed = {} + sentinel = SimpleNamespace(report={}, evidence={}, exit_code=0) + + def attest(repository, **kwargs): + observed["repository"] = repository + observed.update(kwargs) + return sentinel + + monkeypatch.setattr(revision_cli, "attest_revision", attest) + monkeypatch.setattr(cli, "_emit_revision_result", lambda result, _args: result.exit_code) + monkeypatch.chdir(tmp_path) + + result = cli.main( + ["attest", "--head", "HEAD", "--claim", "breakcheck.claim.toml"] + ) + + assert result == 0 + assert observed["strict"] is True + assert observed["strict_separation"] is True diff --git a/tests/test_cost_contract.py b/tests/test_cost_contract.py new file mode 100644 index 0000000..1edd45e --- /dev/null +++ b/tests/test_cost_contract.py @@ -0,0 +1,54 @@ +from __future__ import annotations + +import json +import time +from pathlib import Path + +from breakcheck import cli + + +ROOT = Path(__file__).resolve().parents[1] + + +def test_runtime_dependency_inventory_remains_empty(): + project = (ROOT / "pyproject.toml").read_text(encoding="utf-8") + compact = "".join(project.split()) + assert "dependencies=[]" in compact + + +def test_capabilities_are_one_noninteractive_machine_readable_command(capsys): + started = time.monotonic() + assert cli.main(["--capabilities", "--json"]) == 0 + elapsed = time.monotonic() - started + payload = json.loads(capsys.readouterr().out) + assert payload["schema_version"] >= 1 + assert payload["python"] == ["3.10", "3.11", "3.12", "3.13"] + assert payload["platforms"] == ["linux", "macos"] + assert "dependency_comparison" in payload["features"] + assert "fixture_suggestions" in payload["features"] + assert "revision_comparison" in payload["features"] + assert "claim_attestation" in payload["features"] + assert elapsed < 5.0 + + +def test_demo_is_one_command_and_reaches_a_report_within_five_minutes( + tmp_path, capsys +): + started = time.monotonic() + assert cli.main(["demo", "--output-root", str(tmp_path / "demo")]) == 0 + elapsed = time.monotonic() - started + output = capsys.readouterr().out + assert "CHANGED" in output + assert (tmp_path / "demo" / "report.json").is_file() + assert (tmp_path / "demo" / "evidence.json").is_file() + assert elapsed < 300.0 + + +def test_documented_ci_example_is_one_file_under_twenty_lines(): + example = ROOT / "examples" / "github-actions.yml" + lines = [ + line + for line in example.read_text(encoding="utf-8").splitlines() + if line.strip() and not line.lstrip().startswith("#") + ] + assert len(lines) < 20 diff --git a/tests/test_distribution_contract.py b/tests/test_distribution_contract.py new file mode 100644 index 0000000..b22f288 --- /dev/null +++ b/tests/test_distribution_contract.py @@ -0,0 +1,96 @@ +from __future__ import annotations + +import importlib +import os +import subprocess +import sys +from pathlib import Path + +try: + import tomllib +except ModuleNotFoundError: # pragma: no cover - exercised on Python 3.10 CI + import tomli as tomllib + + +ROOT = Path(__file__).resolve().parents[1] + + +def test_distribution_metadata_exposes_the_current_public_contract() -> None: + project = tomllib.loads((ROOT / "pyproject.toml").read_text(encoding="utf-8"))["project"] + assert project["name"] == "breakcheck" + assert project["version"] == "2.0.0" + assert project["authors"] == [{"name": "ViDale Lovett"}] + assert project["dependencies"] == [] + assert project["scripts"] == {"breakcheck": "breakcheck.cli:main"} + assert project["urls"]["Repository"] == "https://github.com/lovettsendit/breakcheck" + + package = importlib.import_module("breakcheck") + assert package.__version__ == project["version"] + + +def test_source_manifest_includes_public_operational_material() -> None: + manifest = (ROOT / "MANIFEST.in").read_text(encoding="utf-8") + for contract in ( + "include SKILL.md", + "graft docs", + "graft examples", + "graft scripts", + "recursive-include src/breakcheck *.py", + "recursive-include tests *.py", + "global-exclude ._*", + "global-exclude .DS_Store", + "prune .breakcheck", + "prune build", + "prune dist", + ): + assert contract in manifest + + +def test_github_action_example_is_small_and_uses_the_public_package() -> None: + example = (ROOT / "examples" / "github-actions.yml").read_text(encoding="utf-8") + effective_lines = [ + line + for line in example.splitlines() + if line.strip() and not line.lstrip().startswith("#") + ] + assert len(effective_lines) <= 20 + assert 'python -m pip install "breakcheck==2.0.0"' in example + assert "--ci" in example + for line in example.splitlines(): + if "uses:" in line: + revision = line.split("@", 1)[1].split()[0] + assert len(revision) == 40 + int(revision, 16) + + +def test_github_action_example_quotes_colons_in_run_scalars() -> None: + example = (ROOT / "examples" / "github-actions.yml").read_text(encoding="utf-8") + for line in example.splitlines(): + if "- run:" not in line: + continue + scalar = line.split("- run:", 1)[1].strip() + if ":" in scalar: + assert scalar.startswith(("'", '"', "|", ">")), line + + +def test_release_scanner_accepts_the_public_working_tree() -> None: + result = subprocess.run( + ["bash", str(ROOT / "scripts" / "scan_artifacts.sh"), str(ROOT)], + cwd=ROOT, + text=True, + capture_output=True, + check=False, + ) + assert result.returncode == 0, result.stdout + result.stderr + + +def test_module_and_installed_command_report_one_version(tmp_path: Path) -> None: + output = subprocess.run( + [sys.executable, "-c", "import breakcheck; print(breakcheck.__version__)"], + cwd=tmp_path, + env=os.environ | {"PYTHONPATH": str(ROOT / "src")}, + text=True, + capture_output=True, + check=True, + ) + assert output.stdout.strip() == "2.0.0" diff --git a/tests/test_fixtures_and_projections.py b/tests/test_fixtures_and_projections.py new file mode 100644 index 0000000..a41f7f9 --- /dev/null +++ b/tests/test_fixtures_and_projections.py @@ -0,0 +1,301 @@ +from __future__ import annotations + +import hashlib +import os +from pathlib import Path +import subprocess +import sys + +import pytest + +from breakcheck.adapters.python.fixtures import ( + FixtureRefusal, + coverage_delta, + deterministic, + executable, + fixture_yield, + human_minutes, + load_fixture_file, + render_fixture_source, + resolve_fixture_policy, + suggest_fixtures, + valid, +) + + +INVENTORY = [ + {"file": "src/app.py", "line": 7, "column": 4, "api": "attrs.has"}, +] + + +def _fixture_text(*, author: str = "human", extra: str = "") -> str: + return ( + 'schema_version = 1\n' + "\n" + "[[binding]]\n" + f'fixture_authored_by = "{author}"\n' + 'file = "src/app.py"\n' + "line = 7\n" + "column = 4\n" + 'api = "attrs.has"\n' + 'args = ["Point(1, 2)"]\n' + 'kwargs = { strict = "True" }\n' + 'setup = "class Point:\\n pass"\n' + 'projection = "(outcome, type(outcome).__name__)"\n' + + extra + ) + + +def _write_fixture(root: Path, text: str) -> Path: + destination = root / "breakcheck.fixtures.toml" + destination.write_text(text, encoding="utf-8") + return destination + + +def test_valid_fixture_is_closed_matched_and_hash_stable(tmp_path: Path) -> None: + path = _write_fixture(tmp_path, _fixture_text()) + + first = load_fixture_file(path, repository_root=tmp_path, inventory=INVENTORY) + second = load_fixture_file(path, repository_root=tmp_path, inventory=INVENTORY) + + assert first.file_sha256 == hashlib.sha256(path.read_bytes()).hexdigest() + assert first.canonical_sha256 == second.canonical_sha256 + assert len(first.bindings) == 1 + binding = first.bindings[0] + assert binding.fixture_authored_by == "human" + assert binding.key == ("src/app.py", 7, 4, "attrs.has") + assert binding.args == ("Point(1, 2)",) + assert binding.kwargs == (("strict", "True"),) + assert binding.setup == "class Point:\n pass" + assert binding.projection == "(outcome, type(outcome).__name__)" + assert binding.binding_sha256 == second.bindings[0].binding_sha256 + + +def test_fixture_rendering_preserves_reviewed_source_without_executing_it( + tmp_path: Path, +) -> None: + path = _write_fixture( + tmp_path, + _fixture_text().replace( + 'setup = "class Point:\\n pass"', + 'setup = "raise RuntimeError(\\\"must not run while loading\\\")"', + ), + ) + fixture = load_fixture_file(path, repository_root=tmp_path, inventory=INVENTORY) + + rendered = render_fixture_source(fixture.bindings[0], "attrs.has") + + assert rendered == ( + 'raise RuntimeError("must not run while loading")\n' + "outcome = attrs.has(Point(1, 2), strict=True)\n" + "outcome = (outcome, type(outcome).__name__)\n" + ) + + +@pytest.mark.parametrize("author", ["", "model", "Human", "operator"]) +def test_fixture_author_is_required_and_closed(tmp_path: Path, author: str) -> None: + text = _fixture_text(author=author) if author else _fixture_text().replace( + 'fixture_authored_by = "human"\n', "", 1 + ) + path = _write_fixture(tmp_path, text) + + with pytest.raises(FixtureRefusal, match="FIXTURE_AUTHOR_REFUSED"): + load_fixture_file(path, repository_root=tmp_path, inventory=INVENTORY) + + +@pytest.mark.parametrize( + ("mutation", "code"), + [ + ("schema_version = 2", "FIXTURE_SCHEMA_VERSION_REFUSED"), + ('unknown = "value"', "FIXTURE_SCHEMA_REFUSED"), + ('api = "attrs.has"\napi = "attrs.has"', "FIXTURE_DUPLICATE_FIELD_REFUSED"), + ('args = ["unterminated]', "FIXTURE_SYNTAX_REFUSED"), + ], +) +def test_closed_schema_refuses_unknown_duplicate_and_malformed_values( + tmp_path: Path, mutation: str, code: str +) -> None: + text = _fixture_text() + if mutation.startswith("schema_version"): + text = text.replace("schema_version = 1", mutation) + elif mutation.startswith("unknown"): + text = text.replace("[[binding]]", mutation + "\n\n[[binding]]") + elif mutation.startswith("api ="): + text = text.replace('api = "attrs.has"', mutation) + else: + text = text.replace('args = ["Point(1, 2)"]', mutation) + path = _write_fixture(tmp_path, text) + + with pytest.raises(FixtureRefusal, match=code): + load_fixture_file(path, repository_root=tmp_path, inventory=INVENTORY) + + +def test_fixture_file_must_be_confined_regular_and_nonsymlink(tmp_path: Path) -> None: + repository = tmp_path / "repository" + repository.mkdir() + outside = _write_fixture(tmp_path, _fixture_text()) + + with pytest.raises(FixtureRefusal, match="FIXTURE_PATH_REFUSED"): + load_fixture_file(outside, repository_root=repository, inventory=INVENTORY) + + link = repository / "breakcheck.fixtures.toml" + link.symlink_to(outside) + with pytest.raises(FixtureRefusal, match="FIXTURE_SYMLINK_REFUSED"): + load_fixture_file(link, repository_root=repository, inventory=INVENTORY) + + +@pytest.mark.parametrize( + ("inventory", "mutation", "code"), + [ + (INVENTORY, "duplicate", "FIXTURE_DUPLICATE_BINDING_REFUSED"), + (INVENTORY, "line", "FIXTURE_STALE_REFUSED"), + ([], "none", "FIXTURE_UNMATCHED_REFUSED"), + (INVENTORY + INVENTORY, "none", "FIXTURE_AMBIGUOUS_REFUSED"), + ], +) +def test_inventory_matching_refuses_duplicate_stale_unmatched_and_ambiguous( + tmp_path: Path, inventory: list[dict[str, object]], mutation: str, code: str +) -> None: + text = _fixture_text() + if mutation == "duplicate": + text += text[text.index("[[binding]]") :] + elif mutation == "line": + text = text.replace("line = 7", "line = 8") + path = _write_fixture(tmp_path, text) + + with pytest.raises(FixtureRefusal, match=code): + load_fixture_file(path, repository_root=tmp_path, inventory=inventory) + + +def test_source_and_projection_caps_fail_closed(tmp_path: Path) -> None: + too_many_args = ", ".join('"1"' for _ in range(65)) + path = _write_fixture( + tmp_path, + _fixture_text().replace('args = ["Point(1, 2)"]', f"args = [{too_many_args}]"), + ) + with pytest.raises(FixtureRefusal, match="FIXTURE_ARGUMENT_CAP_REFUSED"): + load_fixture_file(path, repository_root=tmp_path, inventory=INVENTORY) + + path.write_text( + _fixture_text().replace( + 'projection = "(outcome, type(outcome).__name__)"', + 'projection = "42"', + ), + encoding="utf-8", + ) + with pytest.raises(FixtureRefusal, match="FIXTURE_PROJECTION_REFUSED"): + load_fixture_file(path, repository_root=tmp_path, inventory=INVENTORY) + + path.write_text( + _fixture_text().replace('args = ["Point(1, 2)"]', 'args = ["("]'), + encoding="utf-8", + ) + with pytest.raises(FixtureRefusal, match="FIXTURE_EXPRESSION_REFUSED"): + load_fixture_file(path, repository_root=tmp_path, inventory=INVENTORY) + + +def test_fixture_policy_defaults_forbid_and_allow_require_are_explicit(tmp_path: Path) -> None: + path = _write_fixture(tmp_path, _fixture_text()) + + with pytest.raises(FixtureRefusal, match="FIXTURE_POLICY_FORBID"): + resolve_fixture_policy( + fixture_path=path, repository_root=tmp_path, inventory=INVENTORY + ) + allowed = resolve_fixture_policy( + "allow", fixture_path=path, repository_root=tmp_path, inventory=INVENTORY + ) + assert len(allowed.bindings) == 1 + empty_root = tmp_path / "empty" + empty_root.mkdir() + absent = resolve_fixture_policy( + "allow", fixture_path=None, repository_root=empty_root, inventory=INVENTORY + ) + assert absent is None + with pytest.raises(FixtureRefusal, match="FIXTURE_REQUIRED"): + resolve_fixture_policy( + "require", fixture_path=None, repository_root=empty_root, inventory=INVENTORY + ) + with pytest.raises(FixtureRefusal, match="FIXTURE_POLICY_REFUSED"): + resolve_fixture_policy( + "sometimes", fixture_path=path, repository_root=tmp_path, inventory=INVENTORY + ) + + +def test_suggestions_are_deterministic_contextual_warn_and_never_overwrite( + tmp_path: Path, +) -> None: + destination = tmp_path / "breakcheck.fixtures.toml" + candidates = [ + { + "file": "src/b.py", + "line": 9, + "column": 2, + "api": "attrs.asdict", + "signature": "attrs.asdict(inst, *, recurse=True)", + "type_hints": "inst: object -> dict[str, object]", + "nearby_source": "result = attrs.asdict(value)", + }, + { + "file": "src/a.py", + "line": 3, + "column": 1, + "api": "attrs.has", + }, + ] + + first = suggest_fixtures( + destination, candidates, repository_root=tmp_path + ) + text = destination.read_text(encoding="utf-8") + assert first == hashlib.sha256(destination.read_bytes()).hexdigest() + assert "CONFIDENTIALITY WARNING" in text + assert text.index('file = "src/a.py"') < text.index('file = "src/b.py"') + assert "# signature: attrs.asdict(inst, *, recurse=True)" in text + assert "# type_hints: inst: object -> dict[str, object]" in text + assert "# nearby_source: result = attrs.asdict(value)" in text + assert text.count('fixture_authored_by = "unknown"') == 2 + + with pytest.raises(FixtureRefusal, match="FIXTURE_SUGGESTION_EXISTS"): + suggest_fixtures(destination, candidates, repository_root=tmp_path) + assert destination.read_text(encoding="utf-8") == text + + second_destination = tmp_path / "second.toml" + suggest_fixtures(second_destination, list(reversed(candidates)), repository_root=tmp_path) + assert second_destination.read_text(encoding="utf-8") == text + + +def test_metrics_require_observed_inputs_and_do_not_invent_values() -> None: + assert fixture_yield(3, 4) == 75.0 + assert valid(2, 3) == pytest.approx(66.66666666666667) + assert executable(1, 2) == 50.0 + assert deterministic(1, 1) == 100.0 + assert coverage_delta(40, 80) == 2.0 + assert human_minutes(12.25) == 12.25 + + with pytest.raises(TypeError): + fixture_yield() # type: ignore[call-arg] + with pytest.raises(FixtureRefusal, match="FIXTURE_METRIC_REFUSED"): + valid(4, 3) + with pytest.raises(FixtureRefusal, match="FIXTURE_METRIC_REFUSED"): + human_minutes(-1) + with pytest.raises(FixtureRefusal, match="FIXTURE_METRIC_REFUSED"): + coverage_delta(0, 1) + + +def test_fixture_module_imports_with_stdlib_only() -> None: + source_root = Path(__file__).parents[1] / "src" + code = ( + "import sys; " + f"sys.path.insert(0, {str(source_root)!r}); " + "import breakcheck.adapters.python.fixtures as fixtures; " + "print(fixtures.__name__)" + ) + result = subprocess.run( + [sys.executable, "-S", "-c", code], + check=False, + capture_output=True, + text=True, + env={"PATH": os.environ.get("PATH", "")}, + ) + assert result.returncode == 0, result.stderr + assert result.stdout.strip() == "breakcheck.adapters.python.fixtures" diff --git a/tests/test_github_community.py b/tests/test_github_community.py new file mode 100644 index 0000000..e697d38 --- /dev/null +++ b/tests/test_github_community.py @@ -0,0 +1,144 @@ +from __future__ import annotations + +from pathlib import Path + + +ROOT = Path(__file__).parents[1] +ISSUE_TEMPLATE = ROOT / ".github" / "ISSUE_TEMPLATE" + +REQUIRED_FORMS = { + "bug.yml": { + "version_breakcheck", + "python_version", + "operating_system", + "mode", + "sanitized_command", + "reproduction_steps", + "observed_behavior", + "expected_behavior", + "privacy_confirmation", + }, + "compatibility-case.yml": { + "version_breakcheck", + "python_version", + "operating_system", + "package_name", + "current_version", + "proposed_version", + "sanitized_command", + "reproduction_steps", + "observed_behavior", + "expected_behavior", + "privacy_confirmation", + }, + "revision-claim-case.yml": { + "version_breakcheck", + "python_version", + "operating_system", + "mode", + "base_revision", + "head_revision", + "target_symbols", + "fixture_source", + "sanitized_command", + "reproduction_steps", + "observed_behavior", + "expected_behavior", + "privacy_confirmation", + }, +} + +PR_REQUIREMENTS = ( + "Linked issue", + "Smallest change", + "Regression test evidence", + "Preserved counterexample", + "Privacy/security", + "Docs/changelog", + "Breaking/release assessment", +) + +PRIVACY_WARNING_TERMS = ( + "private paths", + "private source", + "literals", + "outputs", + "credentials", + ".breakcheck/", +) + + +def _read(relative: str) -> str: + return (ROOT / relative).read_text(encoding="utf-8") + + +def test_public_forms_require_supported_version_and_reproduction_details(): + for filename, required_ids in REQUIRED_FORMS.items(): + text = _read(f".github/ISSUE_TEMPLATE/{filename}") + assert "description:" in text + assert "title:" in text + assert "labels:" in text + assert "body:" in text + for field_id in required_ids: + assert f"id: {field_id}" in text, (filename, field_id) + assert text.count("required: true") >= len(required_ids) + + +def test_evidence_forms_warn_about_private_material_before_posting(): + for filename in ("bug.yml", "compatibility-case.yml", "revision-claim-case.yml"): + text = _read(f".github/ISSUE_TEMPLATE/{filename}").lower() + for term in PRIVACY_WARNING_TERMS: + assert term in text, (filename, term) + assert "remove" in text + assert "before posting" in text + + +def test_bug_form_supports_every_product_mode_without_forcing_dependency_fields(): + text = _read(".github/ISSUE_TEMPLATE/bug.yml") + for mode in ( + "Dependency comparison", + "Revision diff", + "Baseline freeze", + "Claim attestation", + "Demo or capabilities", + "Other", + ): + assert f"- {mode}" in text + for dependency_only_field in ("package_name", "current_version", "proposed_version"): + assert f"id: {dependency_only_field}" not in text + + +def test_revision_claim_form_covers_separation_and_claim_outcomes(): + text = _read(".github/ISSUE_TEMPLATE/revision-claim-case.yml") + for operation in ("Revision diff", "Baseline freeze", "Claim attestation"): + assert f"- {operation}" in text + for outcome in ( + "CLAIM_VERIFIED", + "CLAIM_REFUTED", + "CLAIM_UNVERIFIABLE", + "CLAIM_OUT_OF_SCOPE", + ): + assert outcome in text + assert "base revision" in text.lower() + assert "fixture" in text.lower() + + +def test_feature_form_is_structured_and_private_safe(): + text = _read(".github/ISSUE_TEMPLATE/feature.yml").lower() + for field_id in ("problem", "proposed_solution", "alternatives"): + assert f"id: {field_id}" in text + for term in PRIVACY_WARNING_TERMS: + assert term in text, term + + +def test_issue_config_disables_blanks_and_links_security_and_readme(): + text = _read(".github/ISSUE_TEMPLATE/config.yml") + assert "blank_issues_enabled: false" in text + assert "https://github.com/lovettsendit/breakcheck/security/advisories/new" in text + assert "https://github.com/lovettsendit/breakcheck#readme" in text + + +def test_pull_request_template_requires_release_quality_evidence(): + text = _read(".github/pull_request_template.md") + for requirement in PR_REQUIREMENTS: + assert requirement.lower() in text.lower(), requirement diff --git a/tests/test_product_smoke.py b/tests/test_product_smoke.py index ef78ab7..59d7fe0 100644 --- a/tests/test_product_smoke.py +++ b/tests/test_product_smoke.py @@ -1,5 +1,7 @@ import importlib +import pytest + def test_package_imports(): assert importlib.import_module('breakcheck') is not None @@ -12,3 +14,19 @@ def test_cli_help_exits_cleanly(): except SystemExit as exc: result = int(exc.code or 0) assert int(result or 0) == 0 + + +@pytest.mark.parametrize( + "arguments", + [ + ["--capabilities", "--json", "--wheelhouse", "/unused"], + ["demo", "--output-root", "/unused", "--allow-empty"], + ["--verify", "/unused/report.json", "--ci"], + ["sample@2.0", "--wheelhouse", "/unused", "--output-root", "/unused"], + ], +) +def test_cli_modes_refuse_irrelevant_options(arguments): + main = getattr(importlib.import_module("breakcheck.cli"), "main") + with pytest.raises(SystemExit) as refused: + main(arguments) + assert refused.value.code == 2 diff --git a/tests/test_production_hardening.py b/tests/test_production_hardening.py index 607c26c..9adf123 100644 --- a/tests/test_production_hardening.py +++ b/tests/test_production_hardening.py @@ -198,9 +198,11 @@ def render_json(report): evidence=None, json=True, ci=False, + allow_empty=True, ) assert cli._build(args) == 0 - return captured["report"] + observed = captured["report"] + return observed.get("payload", observed) def test_present_api_remains_exercised(monkeypatch, tmp_path): @@ -310,46 +312,16 @@ def test_build_backend_policy_does_not_exclude_newer_compatible_tools(): assert requirements == ["setuptools>=77", "wheel>=0.43"] -def test_public_release_roots_exclude_private_identity_and_internal_provenance(): +def test_public_release_roots_pass_the_generic_artifact_scan(): project_root = Path(__file__).resolve().parents[1] - private_term_fingerprints = { - 3: {"8b850164b5a2e503567a6779b9a31a4f9bf202fb3f5d1327c789cdc948ebb794"}, - 6: { - "147e78b1e6476d75bf35a0b85796f18369fb025928d6963a8eeeefb6c8330e64", - "6a487b4fd1d59b9a6d73b542978c4d03778842385b603085f2d4880476509bf3", - "b6f4bf975b93ad705f194752a5119dfeb93d1c8a20cf3c3aeff372fcf94d4e71", - }, - 7: {"3e44fb009899c0f900c1e74cd803b171d70a5d799d2cc933898d78e8d5fc17ca"}, - 9: {"6592078a7321c458cd4586c3945ef79b7fd3b13947eba1faa78568364918178d"}, - 11: { - "06ae6b52ed986ed5349967715ae18b67f83e52e67984ab7e0b2b5404a7ef8cf2", - "a3626979818075a734058bdf0847410e2bf597ce85eac770ae2dd6285fe1bc62", - }, - } - paths = sorted( - path - for path in project_root.rglob("*") - if path.is_file() - and path.resolve() != Path(__file__).resolve() - and not path.name.startswith("._") - and not {".benchmarks", ".pytest_cache", "__pycache__"}.intersection(path.parts) - and path.suffix.lower() in {".in", ".md", ".py", ".toml", ".yml"} + result = subprocess.run( + ["sh", str(project_root / "scripts" / "scan_artifacts.sh"), str(project_root)], + cwd=project_root, + text=True, + capture_output=True, + check=False, ) - matches = [] - for path in paths: - text = path.read_text(encoding="utf-8").lower() - if re.search(r"[\w.+-]+@[\w.-]+\.[a-z]{2,}", text): - matches.append(path.relative_to(project_root).as_posix()) - continue - for length, forbidden in private_term_fingerprints.items(): - if any( - hashlib.sha256(text[index : index + length].encode("utf-8")).hexdigest() - in forbidden - for index in range(max(0, len(text) - length + 1)) - ): - matches.append(path.relative_to(project_root).as_posix()) - break - assert matches == [] + assert result.returncode == 0, result.stdout + result.stderr def test_public_documentation_explains_practical_and_ai_assisted_use(): diff --git a/tests/test_project_metadata_and_demo.py b/tests/test_project_metadata_and_demo.py new file mode 100644 index 0000000..879022e --- /dev/null +++ b/tests/test_project_metadata_and_demo.py @@ -0,0 +1,238 @@ +from __future__ import annotations + +import base64 +import csv +import hashlib +import io +import json +import os +from pathlib import Path +import shutil +import shlex +import subprocess +import sys +import zipfile + +import pytest + +from breakcheck.demo import run_demo + +try: + import tomllib +except ModuleNotFoundError: # Python 3.10 + import tomli as tomllib + + +ROOT = Path(__file__).resolve().parents[1] +DEMO = ROOT / "examples" / "run_demo.sh" +APP = ROOT / "examples" / "packaging-change" / "app.py" + + +def _wheel_record(files: dict[str, bytes]) -> bytes: + rows = [] + for name, content in sorted(files.items()): + digest = base64.urlsafe_b64encode(hashlib.sha256(content).digest()).rstrip(b"=").decode() + rows.append((name, f"sha256={digest}", str(len(content)))) + rows.append(("packaging-PLACEHOLDER.dist-info/RECORD", "", "")) + output = io.StringIO() + writer = csv.writer(output) + for row in rows: + writer.writerow(row) + return output.getvalue().encode() + + +def _write_packaging_wheel(wheelhouse: Path, version: str, utils_source: str) -> None: + dist_info = f"packaging-{version}.dist-info" + files = { + "packaging/__init__.py": f"__version__ = {version!r}\n".encode(), + "packaging/utils.py": utils_source.encode(), + f"{dist_info}/METADATA": ( + "Metadata-Version: 2.1\n" + "Name: packaging\n" + f"Version: {version}\n" + ).encode(), + f"{dist_info}/WHEEL": ( + "Wheel-Version: 1.0\n" + "Generator: breakcheck-test-suite\n" + "Root-Is-Purelib: true\n" + "Tag: py3-none-any\n" + ).encode(), + } + record = _wheel_record(files).replace( + b"packaging-PLACEHOLDER.dist-info/RECORD", + f"{dist_info}/RECORD".encode(), + ) + files[f"{dist_info}/RECORD"] = record + wheel = wheelhouse / f"packaging-{version}-py3-none-any.whl" + with zipfile.ZipFile(wheel, "w", compression=zipfile.ZIP_DEFLATED) as archive: + for name, content in files.items(): + archive.writestr(name, content) + + +def _offline_wheelhouse(tmp_path: Path) -> Path: + wheelhouse = tmp_path / "wheelhouse" + wheelhouse.mkdir() + _write_packaging_wheel( + wheelhouse, + "21.3", + "def canonicalize_version(version, strip_trailing_zero=True):\n" + " return version.rstrip('.0') if strip_trailing_zero else version\n", + ) + _write_packaging_wheel( + wheelhouse, + "22.0", + "def canonicalize_version(version):\n" + " return version.rstrip('.0')\n", + ) + return wheelhouse + + +def _line_value(output: str, name: str) -> Path: + prefix = f"{name}=" + for line in output.splitlines(): + if line.startswith(prefix): + return Path(line.removeprefix(prefix)) + raise AssertionError(f"missing {prefix!r} in demo output:\n{output}") + + +def _python_without_build_backend(tmp_path: Path, executable: Path | None = None) -> Path: + wrapper = tmp_path / "python-without-build-backend" + selected = Path(sys.executable) if executable is None else executable + wrapper.write_text( + "#!/bin/sh\n" + "if [ \"${1:-}\" = \"-m\" ] && [ \"${2:-}\" = \"pip\" ] " + "&& [ \"${3:-}\" = \"wheel\" ]; then\n" + " printf '%s\\n' 'BUILD_BACKEND_MUST_NOT_RUN' >&2\n" + " exit 97\n" + "fi\n" + "if [ \"${1:-}\" = \"-m\" ] && [ \"${2:-}\" = \"pip\" ] " + "&& [ \"${3:-}\" = \"install\" ]; then\n" + " has_target=0\n" + " for argument in \"$@\"; do\n" + " [ \"$argument\" = \"--target\" ] && has_target=1\n" + " done\n" + " if [ \"$has_target\" -ne 1 ]; then\n" + " printf '%s\\n' 'CALLER_ENVIRONMENT_MUST_NOT_BE_MODIFIED' >&2\n" + " exit 98\n" + " fi\n" + "fi\n" + f"exec {shlex.quote(str(selected))} \"$@\"\n", + encoding="utf-8", + ) + wrapper.chmod(0o755) + return wrapper + + +def test_current_project_metadata_and_public_artifacts_are_declared(): + project = tomllib.loads((ROOT / "pyproject.toml").read_text()) + assert project["project"]["version"] == "2.0.0" + assert project["project"]["authors"] == [{"name": "ViDale Lovett"}] + assert project["project"]["urls"] == { + "Homepage": "https://github.com/lovettsendit/breakcheck", + "Repository": "https://github.com/lovettsendit/breakcheck", + "Issues": "https://github.com/lovettsendit/breakcheck/issues", + "Changelog": "https://github.com/lovettsendit/breakcheck/blob/main/CHANGELOG.md", + } + assert "Copyright (c) 2026 ViDale Lovett and contributors" in (ROOT / "LICENSE").read_text() + assert "## 2.0.0 - 2026-08-26" in (ROOT / "CHANGELOG.md").read_text() + manifest = (ROOT / "MANIFEST.in").read_text() + assert "include SKILL.md" in manifest + assert "graft examples" in manifest + assert "graft docs" in manifest + readme = (ROOT / "README.md").read_text() + assert "Dependabot tells you a new version exists." in readme + assert "python -m pip install breakcheck" in readme + assert "docs/assets/breakcheck-social-preview.png" in readme + + +def test_demo_uses_installed_breakcheck_without_a_build_backend(tmp_path: Path): + wheelhouse = _offline_wheelhouse(tmp_path) + python = _python_without_build_backend(tmp_path) + environment = os.environ | { + "BREAKCHECK_DEMO_WHEELHOUSE": str(wheelhouse), + "BREAKCHECK_DEMO_KEEP": "1", + "PYTHON": str(python), + } + result = subprocess.run( + ["sh", str(DEMO)], + cwd=ROOT, + env=environment, + text=True, + capture_output=True, + check=False, + timeout=120, + ) + assert result.returncode == 0, result.stdout + result.stderr + assert "BUILD_BACKEND_MUST_NOT_RUN" not in result.stderr + assert "CALLER_ENVIRONMENT_MUST_NOT_BE_MODIFIED" not in result.stderr + assert "BREAKCHECK_EXIT=3" in result.stdout + assert "VERIFIED" in result.stdout + assert "DEMO_VERDICT=PASS" in result.stdout + assert APP.read_text() == ( + "from packaging.utils import canonicalize_version\n\n" + "canonicalize_version('1.0.0', strip_trailing_zero=False)\n" + ) + root = _line_value(result.stdout, "DEMO_ROOT") + report_path = _line_value(result.stdout, "REPORT_PATH") + evidence_path = _line_value(result.stdout, "EVIDENCE_PATH") + try: + assert root.is_dir() + assert report_path.is_file() + assert evidence_path.is_file() + report = json.loads(report_path.read_text()) + assert report["schema_version"] == 2 + assert report["payload"]["summary"]["changed"] == 1 + assert len(report["payload"]["findings"]) == 1 + finding = report["payload"]["findings"][0] + assert finding["verdict"] == "CHANGED" + assert finding["old"]["kind"] == "value" + assert finding["old"]["payload"] == "1.0.0" + assert finding["new"]["kind"] == "exception" + assert finding["new"]["exception_class"] == "TypeError" + assert "unexpected keyword argument" in finding["new"]["payload"][0] + finally: + shutil.rmtree(root, ignore_errors=True) + + +def test_demo_source_checkout_fallback_does_not_require_a_build_backend(tmp_path: Path): + wheelhouse = _offline_wheelhouse(tmp_path) + environment_root = tmp_path / "clean-python" + subprocess.run( + [sys.executable, "-m", "venv", str(environment_root)], + check=True, + capture_output=True, + text=True, + ) + executable = "Scripts/python.exe" if os.name == "nt" else "bin/python" + clean_python = environment_root / executable + python = _python_without_build_backend(tmp_path, clean_python) + result = subprocess.run( + ["sh", str(DEMO)], + cwd=ROOT, + env=os.environ + | { + "BREAKCHECK_DEMO_WHEELHOUSE": str(wheelhouse), + "PYTHON": str(python), + "PYTHONPATH": "", + }, + text=True, + capture_output=True, + check=False, + timeout=120, + ) + assert result.returncode == 0, result.stdout + result.stderr + assert "BUILD_BACKEND_MUST_NOT_RUN" not in result.stderr + assert "CALLER_ENVIRONMENT_MUST_NOT_BE_MODIFIED" not in result.stderr + assert "DEMO_VERDICT=PASS" in result.stdout + + +def test_builtin_demo_refuses_an_unverified_artifact_bundle(tmp_path: Path) -> None: + """The one-command demo must verify its output before reporting success.""" + + def write_unverified_bundle(arguments) -> int: + Path(arguments.output).write_text('{"schema_version": 2}', encoding="utf-8") + Path(arguments.evidence).write_text("{}", encoding="utf-8") + return 0 + + with pytest.raises(ValueError, match="^DEMO_VERIFICATION_REFUSED$"): + run_demo(tmp_path / "demo", write_unverified_bundle) diff --git a/tests/test_release_automation.py b/tests/test_release_automation.py new file mode 100644 index 0000000..ff3484c --- /dev/null +++ b/tests/test_release_automation.py @@ -0,0 +1,317 @@ +from __future__ import annotations + +import subprocess +import tarfile +import warnings +import zipfile +from io import BytesIO +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +CI = ROOT / ".github" / "workflows" / "ci.yml" +RELEASE = ROOT / ".github" / "workflows" / "release.yml" +SCANNER = ROOT / "scripts" / "scan_artifacts.sh" +PUBLISH_ACTION = "pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33" +UPLOAD_ACTION = "actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a" +DOWNLOAD_ACTION = "actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c" + + +def _read(path: Path) -> str: + return path.read_text(encoding="utf-8") + + +def test_ci_keeps_full_eight_job_matrix_and_runs_one_acceptance_job() -> None: + ci = _read(CI) + assert "branches: [main]" in ci + assert "pull_request:" in ci + assert 'os: ["ubuntu-latest", "macos-latest"]' in ci + assert 'python-version: ["3.10", "3.11", "3.12", "3.13"]' in ci + assert ci.count("runs-on: ${{ matrix.os }}") == 1 + assert "acceptance:" in ci + assert "needs: test" in ci + assert "breakcheck demo --output-root" in ci + assert "breakcheck --capabilities --json" in ci + assert "matrix.os == 'ubuntu-latest'" not in ci + assert "matrix.python-version == '3.13'" not in ci + for action in ("actions/checkout@", "actions/setup-python@"): + line = next(line for line in ci.splitlines() if action in line) + assert len(line.split("@", 1)[1].split()[0]) == 40 + + +def test_release_workflow_fails_closed_before_trusted_publishing() -> None: + release = _read(RELEASE) + assert "types: [published]" in release + assert "permissions: {}" in release + assert "validate:" in release + assert "publish:" in release + assert "needs: validate" in release + assert "name: pypi" in release + assert "url: https://pypi.org/p/breakcheck" in release + assert "contents: read" in release + assert "id-token: write" in release + assert "attestations: true" in release + assert PUBLISH_ACTION in release + assert UPLOAD_ACTION in release + assert DOWNLOAD_ACTION in release + assert "packages-dir: dist/" in release + assert "print-hash: true" in release + assert "actions/checkout@" in release + assert "secrets." not in release + assignment_marker = "token" + ":" + id_token_permission = "id-token" + ":" + assert assignment_marker not in release.replace(id_token_permission, "") + assert "draft" in release and "prerelease" in release + assert "breakcheck-2.0.0-py3-none-any.whl" in release + assert "breakcheck-2.0.0.tar.gz" in release + assert "sha256sum" in release + assert "test -z" not in release + assert release.count("id-token: write") == 1 + validate_block, publish_block = release.split(" publish:", 1) + assert "id-token: write" not in validate_block + assert "python -m build --outdir dist ." in validate_block + assert "SOURCE_DATE_EPOCH" in validate_block + assert "python -m twine check --strict" in validate_block + assert "scripts/scan_artifacts.sh" in validate_block + assert "gh api" not in validate_block + assert "github.event.release.id" not in validate_block + assert "releases/assets" not in validate_block + assert "id-token: write" in publish_block + assert "pip install" not in publish_block + assert "python -m build" not in publish_block + assert "skip-existing: false" in publish_block + assert "verify-metadata: true" in publish_block + assert "outputs.wheel-sha256" not in release + assert "outputs.sdist-sha256" not in release + assert "wheel_sha256:" in release + assert "sdist_sha256:" in release + for line in release.splitlines(): + if "uses:" in line: + assert "@" in line + assert len(line.split("@", 1)[1].split()[0]) == 40 + + +def test_release_provenance_is_the_exact_tag_not_uploaded_release_assets() -> None: + release = _read(RELEASE) + validate_block, publish_block = release.split(" publish:", 1) + + assert "ref: ${{ github.event.release.tag_name }}" in validate_block + assert "persist-credentials: false" in validate_block + assert "git describe --exact-match --tags HEAD" in validate_block + assert "git status --porcelain" in validate_block + assert validate_block.index("git describe --exact-match --tags HEAD") < validate_block.index( + "python -m build --outdir dist ." + ) + assert "release_json" not in release + assert "RELEASE_ID" not in release + assert "GH_TOKEN" not in release + assert "releases/assets" not in release + assert "python -m build --outdir dist ." not in publish_block + assert "actions/upload-artifact@" in validate_block + assert "actions/download-artifact@" in publish_block + + +def test_artifact_scanner_rejects_personal_and_macos_payloads(tmp_path: Path) -> None: + clean = tmp_path / "clean" + clean.mkdir() + (clean / "public.txt").write_text("public release artifact\n", encoding="utf-8") + clean_result = subprocess.run(["bash", str(SCANNER), str(clean)], text=True, capture_output=True) + assert clean_result.returncode == 0, clean_result.stderr + + dirty = tmp_path / "dirty" + dirty.mkdir() + (dirty / "._metadata").write_text("AppleDouble", encoding="utf-8") + personal_email = "person" + "@" + "example" + ".com" + (dirty / "metadata.json").write_text('{"email":"' + personal_email + '"}', encoding="utf-8") + dirty_result = subprocess.run(["bash", str(SCANNER), str(dirty)], text=True, capture_output=True) + assert dirty_result.returncode != 0 + assert "FORBIDDEN" in dirty_result.stdout + + wheel = tmp_path / "dirty.whl" + local_path = "/" + "Users/example/work/project" + with zipfile.ZipFile(wheel, "w") as archive: + archive.writestr("breakcheck-2.0.0.dist-info/build-path.txt", local_path) + wheel_result = subprocess.run(["bash", str(SCANNER), str(wheel)], text=True, capture_output=True) + assert wheel_result.returncode != 0 + + sdist = tmp_path / "dirty.tar.gz" + payload = tmp_path / "payload.txt" + credential_name = "api" + "_key" + payload.write_text(credential_name + "=not-a-release-secret", encoding="utf-8") + with tarfile.open(sdist, "w:gz") as archive: + archive.add(payload, arcname="breakcheck-2.0.0/.git/config") + sdist_result = subprocess.run(["bash", str(SCANNER), str(sdist)], text=True, capture_output=True) + assert sdist_result.returncode != 0 + + +def test_artifact_scanner_allows_clean_checkout_and_id_token(tmp_path: Path) -> None: + checkout_result = subprocess.run(["bash", str(SCANNER), str(ROOT)], text=True, capture_output=True) + assert checkout_result.returncode == 0, checkout_result.stdout + checkout_result.stderr + + artifact = tmp_path / "workflow.yml" + artifact.write_text("permissions:\n id-token: write\n", encoding="utf-8") + result = subprocess.run(["bash", str(SCANNER), str(artifact.parent)], text=True, capture_output=True) + assert result.returncode == 0, result.stdout + result.stderr + + +def test_artifact_scanner_refuses_local_path_without_self_triggering( + tmp_path: Path, +) -> None: + local_root = "/" + "Volumes/external-work/project" + artifact = tmp_path / "local-path.txt" + artifact.write_text(local_root + "\n", encoding="utf-8") + + result = subprocess.run( + ["bash", str(SCANNER), str(tmp_path)], + text=True, + capture_output=True, + ) + assert result.returncode != 0 + assert "local filesystem path" in result.stdout + + scanner_only = tmp_path / "scanner-only" + scanner_only.mkdir() + (scanner_only / SCANNER.name).write_bytes(SCANNER.read_bytes()) + result = subprocess.run( + ["bash", str(SCANNER), str(scanner_only)], + text=True, + capture_output=True, + ) + assert result.returncode == 0, result.stdout + result.stderr + + +def test_artifact_scanner_refuses_real_email_local_path_and_credential(tmp_path: Path) -> None: + email = "release" + "@" + "example" + ".com" + local_root = "/" + "home/example/work/project" + credential_name = "api" + "_key" + artifact = tmp_path / "private.txt" + artifact.write_text( + "\n".join((email, local_root, credential_name + "=not-a-secret")), + encoding="utf-8", + ) + result = subprocess.run(["bash", str(SCANNER), str(artifact.parent)], text=True, capture_output=True) + assert result.returncode != 0 + assert "FORBIDDEN" in result.stdout + + +def test_artifact_scanner_checks_git_patch_content_without_author_headers(tmp_path: Path) -> None: + repository = tmp_path / "repository" + repository.mkdir() + subprocess.run(["git", "init", "-q", str(repository)], check=True) + subprocess.run(["git", "-C", str(repository), "config", "user.name", "Release Test"], check=True) + subprocess.run( + ["git", "-C", str(repository), "config", "user.email", "release" + "@" + "example" + ".com"], + check=True, + ) + tracked = repository / "tracked.txt" + tracked.write_text("safe content\n", encoding="utf-8") + subprocess.run(["git", "-C", str(repository), "add", "tracked.txt"], check=True) + subprocess.run(["git", "-C", str(repository), "commit", "-qm", "safe"], check=True) + historical_value = "token" + ": historical-secret" + tracked.write_text(historical_value + "\n", encoding="utf-8") + subprocess.run(["git", "-C", str(repository), "commit", "-am", "add secret"], check=True) + tracked.write_text("safe again\n", encoding="utf-8") + subprocess.run(["git", "-C", str(repository), "commit", "-am", "remove secret"], check=True) + + result = subprocess.run(["bash", str(SCANNER), str(repository)], text=True, capture_output=True) + assert result.returncode != 0 + assert "git history" in result.stdout + + +def test_artifact_scanner_checks_commit_email_metadata(tmp_path: Path) -> None: + repository = tmp_path / "repository" + repository.mkdir() + subprocess.run(["git", "init", "-q", str(repository)], check=True) + subprocess.run(["git", "-C", str(repository), "config", "user.name", "Release Test"], check=True) + personal_email = "release" + "@" + "example" + ".com" + subprocess.run( + ["git", "-C", str(repository), "config", "user.email", personal_email], + check=True, + ) + (repository / "tracked.txt").write_text("safe content\n", encoding="utf-8") + subprocess.run(["git", "-C", str(repository), "add", "tracked.txt"], check=True) + subprocess.run(["git", "-C", str(repository), "commit", "-qm", "safe"], check=True) + + result = subprocess.run( + ["bash", str(SCANNER), str(repository)], + text=True, + capture_output=True, + ) + assert result.returncode != 0 + assert "personal email in git metadata" in result.stdout + + +def test_artifact_scanner_allows_github_noreply_merge_metadata(tmp_path: Path) -> None: + repository = tmp_path / "repository" + repository.mkdir() + subprocess.run(["git", "init", "-q", str(repository)], check=True) + subprocess.run(["git", "-C", str(repository), "config", "user.name", "GitHub"], check=True) + subprocess.run( + ["git", "-C", str(repository), "config", "user.email", "noreply@github.com"], + check=True, + ) + (repository / "tracked.txt").write_text("safe content\n", encoding="utf-8") + subprocess.run(["git", "-C", str(repository), "add", "tracked.txt"], check=True) + subprocess.run( + [ + "git", + "-C", + str(repository), + "commit", + "-qm", + "Merge pull request", + "--author", + "Contributor <12345+contributor@users.noreply.github.com>", + ], + check=True, + ) + + result = subprocess.run( + ["bash", str(SCANNER), str(repository)], + text=True, + capture_output=True, + ) + assert result.returncode == 0, result.stdout + result.stderr + + +def test_artifact_scanner_refuses_archive_traversal_and_links(tmp_path: Path) -> None: + traversal = tmp_path / "traversal.whl" + with zipfile.ZipFile(traversal, "w") as archive: + archive.writestr("../outside.txt", "public") + result = subprocess.run( + ["bash", str(SCANNER), str(traversal)], + text=True, + capture_output=True, + ) + assert result.returncode != 0 + assert "unsafe archive path" in result.stdout + + linked = tmp_path / "linked.tar.gz" + with tarfile.open(linked, "w:gz") as archive: + info = tarfile.TarInfo("breakcheck-2.0.0/link") + info.type = tarfile.SYMTYPE + info.linkname = "../../outside" + archive.addfile(info, BytesIO()) + result = subprocess.run( + ["bash", str(SCANNER), str(linked)], + text=True, + capture_output=True, + ) + assert result.returncode != 0 + assert "archive link or special file" in result.stdout + + +def test_artifact_scanner_refuses_duplicate_archive_members(tmp_path: Path) -> None: + duplicate = tmp_path / "duplicate.whl" + with warnings.catch_warnings(): + warnings.simplefilter("ignore", UserWarning) + with zipfile.ZipFile(duplicate, "w") as archive: + archive.writestr("breakcheck/module.py", "first") + archive.writestr("breakcheck/module.py", "second") + result = subprocess.run( + ["bash", str(SCANNER), str(duplicate)], + text=True, + capture_output=True, + ) + assert result.returncode != 0 + assert "duplicate archive path" in result.stdout diff --git a/tests/test_replay_protocol_and_coverage.py b/tests/test_replay_protocol_and_coverage.py new file mode 100644 index 0000000..d8cf121 --- /dev/null +++ b/tests/test_replay_protocol_and_coverage.py @@ -0,0 +1,229 @@ +from __future__ import annotations + +import json +import subprocess +from types import SimpleNamespace + +from breakcheck.adapters.python import executor, protocol +from breakcheck.adapters.python.equality import compare_observations + + +def test_protocol_frame_is_closed_and_tamper_evident(): + packet = { + "protocol_version": 1, + "status": "VALUE", + "payload": {"answer": 42}, + "exception_class": None, + "reason_code": None, + "raw_type": None, + } + encoded = protocol.encode_packet(packet) + assert protocol.decode_packet(encoded) == packet + + tampered = bytearray(encoded) + tampered[-1] ^= 1 + try: + protocol.decode_packet(bytes(tampered)) + except ValueError as exc: + assert str(exc) == "PROTOCOL_REFUSED" + else: # pragma: no cover - the assertion above is the required path + raise AssertionError("tampered protocol frame was accepted") + + open_packet = dict(packet, unexpected=True) + try: + protocol.encode_packet(open_packet) + except ValueError as exc: + assert str(exc) == "PROTOCOL_REFUSED" + else: # pragma: no cover - the assertion above is the required path + raise AssertionError("open protocol packet was accepted") + + +def test_package_stdout_cannot_impersonate_typed_protocol(): + forged = json.dumps( + { + "protocol_version": 1, + "status": "VALUE", + "payload": "forged", + "exception_class": None, + "reason_code": None, + "raw_type": None, + }, + sort_keys=True, + ) + result = executor.run_typed_snippet_isolated( + snippet_source=f"print({forged!r})\noutcome = {{'actual': 7}}\n" + ) + + assert result["status"] == "VALUE" + assert result["observation"] == { + "kind": "value", + "payload": {"actual": 7}, + "exception_class": None, + "duration_ms": None, + } + assert b'"payload": "forged"' in result["stdout"] + + +def test_rich_object_is_refused_without_repr_fallback(): + result = executor.run_typed_snippet_isolated( + snippet_source="outcome = object()\n" + ) + + assert result["status"] == "UNNORMALIZABLE" + assert result["observation"] is None + assert result["reason_code"] == "UNSTABLE_OBSERVATION_REFUSED" + assert result["raw_type"] == "object" + assert "object at 0x" not in result["stdout"].decode("utf-8", "replace") + + +def test_network_timeout_and_output_limit_are_typed_refusals(): + network = executor.run_typed_snippet_isolated( + snippet_source="import socket\noutcome = socket.socket()\n" + ) + assert network["status"] == "NETWORK_REFUSED" + assert network["observation"] is None + assert network["reason_code"] == "NETWORK_ACCESS_REFUSED" + + caught_network = executor.run_typed_snippet_isolated( + snippet_source=( + "import socket\n" + "try:\n" + " socket.socket()\n" + "except BaseException:\n" + " pass\n" + "outcome = 'attempt was caught'\n" + ) + ) + assert caught_network["status"] == "NETWORK_REFUSED" + assert caught_network["observation"] is None + + timeout = executor.run_typed_snippet_isolated( + snippet_source="import time\ntime.sleep(5)\noutcome = 1\n", + timeout_seconds=0.1, + ) + assert timeout["status"] == "TIMEOUT" + assert timeout["observation"] is None + assert timeout["reason_code"] == "EXECUTION_TIMEOUT" + + limited = executor.run_typed_snippet_isolated( + snippet_source="print('x' * 100000)\noutcome = 1\n", + max_output_bytes=256, + ) + assert limited["status"] == "OUTPUT_LIMIT_REFUSED" + assert limited["observation"] is None + assert limited["reason_code"] == "OUTPUT_LIMIT_REFUSED" + assert len(limited["stdout"]) == 256 + + +def test_malformed_or_missing_child_packet_fails_closed(): + def malformed_runner(*_args, **_kwargs): + return SimpleNamespace( + returncode=0, + stdout=b"ordinary output", + stderr=b"", + protocol=b"not-a-frame", + ) + + result = executor.run_typed_snippet_isolated( + snippet_source="outcome = 1\n", runner=malformed_runner + ) + assert result["status"] == "PROTOCOL_REFUSED" + assert result["observation"] is None + assert result["reason_code"] == "PROTOCOL_REFUSED" + + +def test_deterministic_ordinary_exceptions_remain_comparable(): + repeated = executor.run_repeated_typed_snippet_isolated( + snippet_source="raise ValueError('stable failure')\n" + ) + + assert repeated["repeatable"] is True + assert repeated["status"] == "EXCEPTION" + assert repeated["reason_code"] is None + assert repeated["observation"] == { + "kind": "exception", + "payload": ["stable failure"], + "exception_class": "ValueError", + "duration_ms": None, + } + assert len(repeated["runs"]) == 2 + + +def test_same_environment_mismatch_is_explicit_and_has_no_observation(): + repeated = executor.run_repeated_typed_snippet_isolated( + snippet_source="import time\noutcome = time.time_ns()\n" + ) + + assert repeated["repeatable"] is False + assert repeated["status"] == "PROTOCOL_REFUSED" + assert repeated["reason_code"] == "NONDETERMINISTIC_OBSERVATION" + assert repeated["observation"] is None + assert len(repeated["runs"]) == 2 + + +def test_sys_path_prefixes_are_ephemeral_and_all_roots_are_scrubbed(tmp_path): + base_root = tmp_path / "base-private-root" + head_root = tmp_path / "head-private-root" + base_root.mkdir() + head_root.mkdir() + + result = executor.run_typed_snippet_isolated( + snippet_source=( + "import sys\n" + "print(sys.path[0], sys.path[1], file=sys.stderr)\n" + "outcome = [sys.path[0], sys.path[1]]\n" + ), + sys_path_prefixes=(base_root, head_root), + ) + + assert result["status"] == "VALUE" + assert result["observation"]["payload"] == [ + "", + "", + ] + combined = result["stdout"] + result["stderr"] + assert str(base_root).encode() not in combined + assert str(head_root).encode() not in combined + assert b"" in combined + assert b"" in combined + + +def test_protocol_packet_bytes_are_bounded(): + result = executor.run_typed_snippet_isolated( + snippet_source="outcome = 'x' * 10000\n", + max_protocol_bytes=256, + ) + assert result["status"] == "PROTOCOL_REFUSED" + assert result["observation"] is None + assert result["reason_code"] == "PROTOCOL_SIZE_REFUSED" + + +def test_legacy_runner_contract_is_unchanged(): + result = executor.run_snippet_isolated(snippet_source="print('legacy')") + assert result["returncode"] == 0 + assert result["stdout"] == b"legacy\n" + assert set(result) == { + "returncode", + "stdout", + "stderr", + "timed_out", + "output_limited", + "elapsed_ms", + } + + +def test_isolated_protocol_preserves_return_type_changes_end_to_end(): + cases = [ + ("outcome = b'a'\n", "outcome = '61'\n"), + ("outcome = {1, 2}\n", "outcome = [1, 2]\n"), + ("outcome = (1, 2)\n", "outcome = [1, 2]\n"), + ] + + for old_source, new_source in cases: + old = executor.run_typed_snippet_isolated(snippet_source=old_source) + new = executor.run_typed_snippet_isolated(snippet_source=new_source) + + assert old["status"] == new["status"] == "VALUE" + comparison = compare_observations(old["observation"], new["observation"]) + assert comparison["verdict"] == "CHANGED" + assert comparison["detail"]["reason_code"] == "KIND_MISMATCH" diff --git a/tests/test_revision_cli.py b/tests/test_revision_cli.py new file mode 100644 index 0000000..7460934 --- /dev/null +++ b/tests/test_revision_cli.py @@ -0,0 +1,666 @@ +from __future__ import annotations + +import copy +import json +from pathlib import Path + +import pytest + +from breakcheck.revision_cli import ( + RevisionModeRefusal, + attest_revision, + diff_revisions, +) +from breakcheck.revision_report import make_revision_artifact +from breakcheck.revision_cli import freeze_revision +from breakcheck.report import ci_exit_code +from breakcheck.schema import artifact_digest, verify_artifact + +from test_revision_executor import _commit, _git, _write, revision_repository + + +def _src_revision_repository(tmp_path: Path) -> tuple[Path, str, str]: + repository = tmp_path / "src-repository" + repository.mkdir() + _git(repository, "init", "-q") + _write( + repository, + "src/sample.py", + "def compute_total(value):\n return value + 1\n", + ) + _write( + repository, + "breakcheck.fixtures.toml", + "\n".join( + ( + "schema_version = 1", + "", + "[[binding]]", + 'file = "src/sample.py"', + "line = 1", + "column = 0", + 'api = "sample.compute_total"', + 'args = ["2"]', + "kwargs = {}", + 'fixture_authored_by = "human"', + "", + ) + ), + ) + base = _commit( + repository, + "src base", + ("src/sample.py", "breakcheck.fixtures.toml"), + ) + _write( + repository, + "src/sample.py", + "def compute_total(value):\n return value + 2\n", + ) + head = _commit(repository, "src head", ("src/sample.py",)) + return repository, base, head + + +def test_src_layout_freeze_diff_and_attest_use_confined_import_roots( + tmp_path: Path, +) -> None: + repository, base, head = _src_revision_repository(tmp_path) + + frozen = freeze_revision( + repository, + revision=base, + runtime_root=tmp_path / "freeze-runtime", + ) + assert frozen.report["payload"]["target_observations"][0]["observation"][ + "payload" + ] == 3 + assert { + row["name"] + for row in frozen.evidence["payload"]["environment_artifacts"] + } >= {"revision_import_roots", "revision_tree"} + + compared = diff_revisions( + repository, + base_revision=base, + head_revision=head, + runtime_root=tmp_path / "diff-runtime", + ) + assert compared.exit_code == 3 + assert compared.report["payload"]["findings"][0]["verdict"] == "CHANGED" + assert { + row["name"] + for row in compared.evidence["payload"]["environment_artifacts"] + } >= {"base_import_roots", "head_import_roots"} + + _write( + repository, + "claim.toml", + "\n".join( + ( + "schema_version = 1", + 'claim = "behavior_preserved"', + f'base_revision = "{base}"', + "", + "[[target]]", + 'symbol = "sample:compute_total"', + "", + ) + ), + ) + claim_head = _commit(repository, "src claim", ("claim.toml",)) + attested = attest_revision( + repository, + head_revision=claim_head, + claim_path="claim.toml", + runtime_root=tmp_path / "attest-runtime", + ) + assert attested.exit_code == 1 + assert attested.report["payload"]["dispositions"][0][ + "disposition" + ] == "CLAIM_REFUTED" + assert { + row["name"] + for row in attested.evidence["payload"]["environment_artifacts"] + } >= {"base_import_roots", "head_import_roots"} + + for result in (frozen, compared, attested): + encoded = json.dumps(result.evidence, sort_keys=True) + assert str(tmp_path) not in encoded + assert verify_artifact(result.report) == "VERIFIED" + assert verify_artifact(result.evidence) == "VERIFIED" + + +def test_diff_catches_a_real_behavior_change_and_emits_a_verified_witness( + tmp_path: Path, +) -> None: + repository, base, head = revision_repository(tmp_path) + + result = diff_revisions( + repository, + base_revision=base, + head_revision=head, + fixture_path="breakcheck.fixtures.toml", + runtime_root=tmp_path / "runtime", + ) + + assert result.exit_code == 3 + assert ci_exit_code(result.report) == result.exit_code + assert verify_artifact(result.report) == "VERIFIED" + assert verify_artifact(result.evidence) == "VERIFIED" + payload = result.report["payload"] + assert payload["summary"] == { + "changed": 1, + "changed_under_projection": 0, + "identical": 0, + "identical_under_projection": 0, + "not_exercised": 0, + } + finding = payload["findings"][0] + assert finding["symbol"] == "compute_total" + assert finding["verdict"] == "CHANGED" + assert finding["base"]["payload"] == 3 + assert finding["head"]["payload"] == 4 + assert len(payload["witnesses"]) == 1 + replay = payload["witnesses"][0]["replay"] + assert "importlib.import_module('sample')" in replay["source"] + assert replay["sha256"] == artifact_digest(replay["source"]) + assert str(tmp_path) not in replay["source"] + assert not (tmp_path / "runtime").exists() + + +def test_diff_reports_import_asymmetry_without_a_witness(tmp_path: Path) -> None: + repository, base, _ = revision_repository(tmp_path) + _write( + repository, + "sample.py", + "import dependency_that_is_not_installed\n\ndef compute_total(value):\n return value + 1\n", + ) + head = _commit(repository, "import failure", ("sample.py",)) + + result = diff_revisions( + repository, + base_revision=base, + head_revision=head, + fixture_path="breakcheck.fixtures.toml", + runtime_root=tmp_path / "runtime", + targets=("sample:compute_total",), + ) + + assert result.exit_code == 4 + assert ci_exit_code(result.report) == result.exit_code + finding = result.report["payload"]["findings"][0] + assert finding["verdict"] == "NOT_EXERCISED" + assert finding["reason_code"] == "IMPORT_ASYMMETRY" + assert finding["base"] is None + assert finding["head"] is None + assert result.report["payload"]["witnesses"] == [] + + +def test_default_target_selection_ignores_unchanged_symbols_but_explicit_targets_do_not( + tmp_path: Path, +) -> None: + repository, base, _ = revision_repository(tmp_path) + _write( + repository, + "sample.py", + "def compute_total(value):\n return value + 1\n", + ) + _write(repository, "stable.py", "def untouched(value):\n return value\n") + _write( + repository, + "breakcheck.fixtures.toml", + "\n".join( + ( + "schema_version = 1", + "", + "[[binding]]", + 'file = "sample.py"', + "line = 1", + "column = 0", + 'api = "sample.compute_total"', + 'args = ["2"]', + "kwargs = {}", + 'fixture_authored_by = "human"', + "", + "[[binding]]", + 'file = "stable.py"', + "line = 1", + "column = 0", + 'api = "stable.untouched"', + 'args = ["2"]', + "kwargs = {}", + 'fixture_authored_by = "human"', + "", + ) + ), + ) + base = _commit( + repository, + "fixture coverage", + ("sample.py", "stable.py", "breakcheck.fixtures.toml"), + ) + _write( + repository, + "sample.py", + "def compute_total(value):\n return value + 2\n", + ) + head = _commit(repository, "one changed", ("sample.py",)) + + default = diff_revisions( + repository, + base_revision=base, + head_revision=head, + fixture_path="breakcheck.fixtures.toml", + runtime_root=tmp_path / "default-runtime", + ) + explicit = diff_revisions( + repository, + base_revision=base, + head_revision=head, + fixture_path="breakcheck.fixtures.toml", + runtime_root=tmp_path / "explicit-runtime", + targets=("stable:untouched",), + ) + + assert [row["symbol"] for row in default.report["payload"]["findings"]] == [ + "compute_total" + ] + assert explicit.report["payload"]["findings"][0]["verdict"] == "IDENTICAL" + assert explicit.exit_code == 0 + + +def test_attest_adjudicates_an_independent_claim_and_detects_omitted_changes( + tmp_path: Path, +) -> None: + repository, base, _ = revision_repository(tmp_path) + _write( + repository, + "sample.py", + "def compute_total(value):\n added = value + 1\n return added\n\ndef omitted(value):\n return value * 2\n", + ) + _write( + repository, + "breakcheck.fixtures.toml", + "\n".join( + ( + "schema_version = 1", + "", + "[[binding]]", + 'file = "sample.py"', + "line = 1", + "column = 0", + 'api = "sample.compute_total"', + 'args = ["2"]', + "kwargs = {}", + 'fixture_authored_by = "human"', + "", + ) + ), + ) + _write( + repository, + "claim.toml", + "\n".join( + ( + "schema_version = 1", + 'claim = "behavior_preserved"', + f'base_revision = "{base}"', + "", + "[[target]]", + 'symbol = "sample:compute_total"', + "", + ) + ), + ) + head = _commit( + repository, + "refactor and extra change", + ("sample.py", "claim.toml"), + ) + + result = attest_revision( + repository, + head_revision=head, + claim_path="claim.toml", + fixture_path="breakcheck.fixtures.toml", + runtime_root=tmp_path / "runtime", + ) + + assert result.exit_code == 3 + assert verify_artifact(result.report) == "VERIFIED" + assert verify_artifact(result.evidence) == "VERIFIED" + dispositions = { + row["symbol"]: row["disposition"] + for row in result.report["payload"]["dispositions"] + } + assert dispositions["sample:compute_total"] == "CLAIM_VERIFIED" + assert dispositions["sample:omitted"] == "CLAIM_OUT_OF_SCOPE" + + +def test_diff_rejects_an_existing_runtime_root_before_touching_it(tmp_path: Path) -> None: + repository, base, head = revision_repository(tmp_path) + runtime = tmp_path / "runtime" + runtime.mkdir() + sentinel = runtime / "keep.txt" + sentinel.write_text("keep", encoding="utf-8") + + with pytest.raises(ValueError, match="^REVISION_RUNTIME_ROOT_REFUSED$"): + diff_revisions( + repository, + base_revision=base, + head_revision=head, + fixture_path="breakcheck.fixtures.toml", + runtime_root=runtime, + ) + + assert sentinel.read_text(encoding="utf-8") == "keep" + + +def test_diff_from_verified_baseline_rechecks_environment_fixture_and_base_observation( + tmp_path: Path, +) -> None: + repository, base, head = revision_repository(tmp_path) + baseline = freeze_revision( + repository, + revision=base, + fixture_path="breakcheck.fixtures.toml", + runtime_root=tmp_path / "freeze-runtime", + ).report + + result = diff_revisions( + repository, + baseline=baseline, + head_revision=head, + fixture_path="breakcheck.fixtures.toml", + runtime_root=tmp_path / "diff-runtime", + ) + assert result.exit_code == 3 + + payload = copy.deepcopy(baseline["payload"]) + payload["environment"]["platform"] = "different-platform" + incompatible = make_revision_artifact("baseline", payload) + with pytest.raises(ValueError, match="^BASELINE_ENVIRONMENT_MISMATCH$"): + diff_revisions( + repository, + baseline=incompatible, + head_revision=head, + fixture_path="breakcheck.fixtures.toml", + runtime_root=tmp_path / "incompatible-runtime", + ) + + +def test_projection_verdict_and_non_strict_claim_scope_remain_explicit( + tmp_path: Path, +) -> None: + repository, base, _ = revision_repository(tmp_path) + _write( + repository, + "sample.py", + "def compute_total(value):\n return value + 1\n", + ) + _write( + repository, + "breakcheck.fixtures.toml", + "\n".join( + ( + "schema_version = 1", + "", + "[[binding]]", + 'file = "sample.py"', + "line = 1", + "column = 0", + 'api = "sample.compute_total"', + 'args = ["2"]', + "kwargs = {}", + 'projection = "outcome % 2"', + 'fixture_authored_by = "human"', + "", + ) + ), + ) + base = _commit( + repository, "projection", ("sample.py", "breakcheck.fixtures.toml") + ) + _write( + repository, + "sample.py", + "def compute_total(value):\n changed_structure = value + 3\n return changed_structure\n", + ) + _write( + repository, + "claim.toml", + "\n".join( + ( + "schema_version = 1", + 'claim = "behavior_preserved"', + f'base_revision = "{base}"', + "", + "[[target]]", + 'symbol = "sample:compute_total"', + "", + ) + ), + ) + head = _commit(repository, "projected refactor", ("sample.py", "claim.toml")) + + compared = diff_revisions( + repository, + base_revision=base, + head_revision=head, + fixture_path="breakcheck.fixtures.toml", + runtime_root=tmp_path / "diff-runtime", + ) + finding = compared.report["payload"]["findings"][0] + assert finding["verdict"] == "IDENTICAL_UNDER_PROJECTION" + assert finding["projection"]["source"] == "outcome % 2" + + attested = attest_revision( + repository, + head_revision=head, + claim_path="claim.toml", + fixture_path="breakcheck.fixtures.toml", + runtime_root=tmp_path / "attest-runtime", + strict_separation=False, + ) + disposition = attested.report["payload"]["dispositions"][0] + assert disposition["disposition"] == "CLAIM_UNVERIFIABLE" + assert disposition["reason_code"] == "STRICT_SEPARATION_REQUIRED" + assert attested.exit_code == 2 + + +def test_attest_strict_policy_controls_only_unverifiable_exit_status( + tmp_path: Path, +) -> None: + repository, base, _ = revision_repository(tmp_path) + _write( + repository, + "sample.py", + "import dependency_that_is_not_installed\n\ndef compute_total(value):\n return value + 1\n", + ) + _write( + repository, + "claim.toml", + "\n".join( + ( + "schema_version = 1", + 'claim = "behavior_preserved"', + f'base_revision = "{base}"', + "", + "[[target]]", + 'symbol = "sample:compute_total"', + "", + ) + ), + ) + head = _commit(repository, "unverifiable import", ("sample.py", "claim.toml")) + + strict_result = attest_revision( + repository, + head_revision=head, + claim_path="claim.toml", + fixture_path="breakcheck.fixtures.toml", + runtime_root=tmp_path / "strict-runtime", + strict=True, + ) + advisory_result = attest_revision( + repository, + head_revision=head, + claim_path="claim.toml", + fixture_path="breakcheck.fixtures.toml", + runtime_root=tmp_path / "advisory-runtime", + strict=False, + ) + + for result in (strict_result, advisory_result): + disposition = result.report["payload"]["dispositions"][0] + assert disposition["disposition"] == "CLAIM_UNVERIFIABLE" + assert disposition["reason_code"] == "IMPORT_ASYMMETRY" + assert result.report["payload"]["summary"]["claim_verified"] == 0 + assert strict_result.exit_code == 2 + assert advisory_result.exit_code == 0 + assert ci_exit_code(strict_result.report) == strict_result.exit_code + assert ci_exit_code(advisory_result.report) == advisory_result.exit_code + + +def test_diff_surfaces_fixture_revision_after_failure_without_changing_exit( + tmp_path: Path, +) -> None: + repository, base, head = revision_repository(tmp_path) + prior = diff_revisions( + repository, + base_revision=base, + head_revision=head, + fixture_path="breakcheck.fixtures.toml", + fixture_source="head", + runtime_root=tmp_path / "prior-runtime", + ) + assert prior.exit_code == 3 + + fixture = (repository / "breakcheck.fixtures.toml").read_text( + encoding="utf-8" + ) + _write( + repository, + "breakcheck.fixtures.toml", + fixture.replace( + 'fixture_authored_by = "human"', + 'projection = "outcome * 0"\nfixture_authored_by = "human"', + ), + ) + tuned_head = _commit( + repository, + "fixture revision", + ("breakcheck.fixtures.toml",), + ) + + current = diff_revisions( + repository, + base_revision=base, + head_revision=tuned_head, + fixture_path="breakcheck.fixtures.toml", + fixture_source="head", + previous_report=prior.report, + runtime_root=tmp_path / "current-runtime", + ) + + assert current.exit_code == 0 + payload = current.report["payload"] + assert payload["findings"][0]["verdict"] == "IDENTICAL_UNDER_PROJECTION" + assert len(payload["fixture_revision_events"]) == 1 + event = payload["fixture_revision_events"][0] + assert event["target_id"] == payload["findings"][0]["target_id"] + assert event["prior_verdict"] == "CHANGED" + assert event["current_verdict"] == "IDENTICAL_UNDER_PROJECTION" + assert event["reason_code"] == "FIXTURE_REVISED_AFTER_FAILURE" + assert ( + event["prior_fixture_binding_sha256"] + != event["current_fixture_binding_sha256"] + ) + assert verify_artifact(current.report) == "VERIFIED" + + mismatched_payload = copy.deepcopy(prior.report["payload"]) + mismatched_payload["base_revision"] = head + mismatched = make_revision_artifact( + "revision_report", mismatched_payload + ) + mismatch_runtime = tmp_path / "mismatch-runtime" + with pytest.raises( + ValueError, match="^PREVIOUS_REPORT_BASE_MISMATCH$" + ): + diff_revisions( + repository, + base_revision=base, + head_revision=tuned_head, + fixture_path="breakcheck.fixtures.toml", + fixture_source="head", + previous_report=mismatched, + runtime_root=mismatch_runtime, + ) + assert not mismatch_runtime.exists() + + _write( + repository, + "claim.toml", + "\n".join( + ( + "schema_version = 1", + 'claim = "behavior_preserved"', + f'base_revision = "{base}"', + "", + "[[target]]", + 'symbol = "sample:compute_total"', + "", + ) + ), + ) + claim_head = _commit(repository, "claim", ("claim.toml",)) + attested = attest_revision( + repository, + head_revision=claim_head, + claim_path="claim.toml", + fixture_path="breakcheck.fixtures.toml", + fixture_source="head", + previous_report=prior.report, + runtime_root=tmp_path / "attest-runtime", + strict_separation=False, + ) + assert ( + attested.report["payload"]["fixture_revision_events"] + == payload["fixture_revision_events"] + ) + assert verify_artifact(attested.report) == "VERIFIED" + + +@pytest.mark.parametrize("fixture_state", ("untracked", "edited")) +def test_strict_separation_refuses_explicit_checkout_fixtures( + tmp_path: Path, fixture_state: str +) -> None: + repository, base, head = revision_repository(tmp_path) + fixture_path = "breakcheck.fixtures.toml" + if fixture_state == "untracked": + fixture_path = "untracked-fixtures.toml" + _write( + repository, + fixture_path, + (repository / "breakcheck.fixtures.toml").read_text( + encoding="utf-8" + ), + ) + else: + with (repository / fixture_path).open("a", encoding="utf-8") as stream: + stream.write("\n# edited after the recorded revision\n") + + runtime = tmp_path / (fixture_state + "-runtime") + with pytest.raises( + RevisionModeRefusal, match="^FIXTURE_EXPLICIT_STRICT_REFUSED$" + ): + diff_revisions( + repository, + base_revision=base, + head_revision=head, + fixture_path=fixture_path, + fixture_source="explicit", + runtime_root=runtime, + strict_separation=True, + ) + assert not runtime.exists() diff --git a/tests/test_revision_diff.py b/tests/test_revision_diff.py new file mode 100644 index 0000000..4f60160 --- /dev/null +++ b/tests/test_revision_diff.py @@ -0,0 +1,307 @@ +from __future__ import annotations + +import copy + +import pytest + +from breakcheck.core.baselines import ( + BaselineRefusal, + compare_frozen_baseline, + detect_fixture_revision_after_failure, + freeze_baseline, +) + + +BASE_REVISION = "1" * 40 +HEAD_REVISION = "2" * 40 +BASE_TREE = "3" * 64 +HEAD_TREE = "4" * 64 +FIXTURE_SHA256 = "5" * 64 +BINDING_SHA256 = "6" * 64 +TARGET_SHA256 = "7" * 64 +HEAD_TARGET_SHA256 = "8" * 64 +SIGNATURE_SHA256 = "9" * 64 + + +def _environment() -> dict[str, str]: + return { + "implementation": "CPython", + "python_version": "3.12.5", + "platform": "linux-x86_64", + } + + +def _fixture(sha256: str = FIXTURE_SHA256) -> dict[str, str]: + return { + "sha256": sha256, + "source_revision": BASE_REVISION, + "source": "base", + "authored_by": "human", + } + + +def _outcome(payload: object = 7) -> dict[str, object]: + return { + "status": "VALUE", + "observation": { + "kind": "value", + "payload": payload, + "exception_class": None, + "duration_ms": None, + }, + "reason_code": None, + "repeatable": True, + } + + +def _target( + *, + symbol: str = "sample.math:value", + payload: object = 7, + target_sha256: str = TARGET_SHA256, + signature_sha256: str = SIGNATURE_SHA256, + projection: str | None = None, +) -> dict[str, object]: + return { + "symbol": symbol, + "target_sha256": target_sha256, + "signature_sha256": signature_sha256, + "fixture_binding_sha256": BINDING_SHA256, + "provenance": "OPERATOR_FIXTURE", + "projection": projection, + "outcome": _outcome(payload), + } + + +def _baseline(*targets: dict[str, object]) -> dict[str, object]: + return freeze_baseline( + revision=BASE_REVISION, + tree_sha256=BASE_TREE, + dirty=False, + allow_dirty=False, + environment=_environment(), + fixture=_fixture(), + target_observations=list(targets or (_target(),)), + invocation={"fixture_source": "base"}, + ) + + +def test_revision_diff_reports_identical_changed_added_and_removed_without_guessing() -> None: + """A wrong target census must not disappear additions, removals, or changes.""" + baseline = _baseline( + _target(symbol="sample.math:changed", payload=1), + _target(symbol="sample.math:identical", payload=2), + _target(symbol="sample.math:removed", payload=3), + ) + head = [ + _target( + symbol="sample.math:changed", + payload=4, + target_sha256=HEAD_TARGET_SHA256, + ), + _target( + symbol="sample.math:identical", + payload=2, + target_sha256=HEAD_TARGET_SHA256, + ), + _target( + symbol="sample.math:added", + payload=5, + target_sha256=HEAD_TARGET_SHA256, + ), + ] + + report = compare_frozen_baseline( + baseline, + head_revision=HEAD_REVISION, + head_tree_sha256=HEAD_TREE, + environment=_environment(), + fixture=_fixture(), + target_observations=head, + fixtures_predate_change=True, + invocation={"fixture_source": "base"}, + ) + + assert [(row["symbol"], row["verdict"], row["reason_code"]) for row in report["findings"]] == [ + ("sample.math:added", "NOT_EXERCISED", "NO_BASELINE_REVISION"), + ("sample.math:changed", "CHANGED", None), + ("sample.math:identical", "IDENTICAL", None), + ("sample.math:removed", "NOT_EXERCISED", "SYMBOL_REMOVED"), + ] + assert report["summary"] == { + "changed": 1, + "identical": 1, + "not_exercised": 2, + "total": 4, + } + assert report["base_revision"] == BASE_REVISION + assert report["head_revision"] == HEAD_REVISION + + +def test_signature_drift_projection_scope_and_unrepeatable_head_are_explicit() -> None: + """A changed signature, projection, or unstable run must not become IDENTICAL.""" + projected = _target( + symbol="sample.math:projected", + projection="outcome.value", + ) + baseline = _baseline( + projected, + _target(symbol="sample.math:signature"), + _target(symbol="sample.math:unstable"), + ) + projected_head = copy.deepcopy(projected) + projected_head["target_sha256"] = HEAD_TARGET_SHA256 + signature_head = _target( + symbol="sample.math:signature", + target_sha256=HEAD_TARGET_SHA256, + signature_sha256="a" * 64, + ) + unstable_head = _target( + symbol="sample.math:unstable", target_sha256=HEAD_TARGET_SHA256 + ) + unstable_head["outcome"] = { + "status": "PROTOCOL_REFUSED", + "observation": None, + "reason_code": "NONDETERMINISTIC_OBSERVATION", + "repeatable": False, + } + + report = compare_frozen_baseline( + baseline, + head_revision=HEAD_REVISION, + head_tree_sha256=HEAD_TREE, + environment=_environment(), + fixture=_fixture(), + target_observations=[projected_head, signature_head, unstable_head], + fixtures_predate_change=True, + invocation={}, + ) + findings = {row["symbol"]: row for row in report["findings"]} + + assert findings["sample.math:projected"]["verdict"] == "IDENTICAL_UNDER_PROJECTION" + assert findings["sample.math:projected"]["projection_scope"] == "outcome.value" + assert findings["sample.math:signature"]["verdict"] == "NOT_EXERCISED" + assert findings["sample.math:signature"]["reason_code"] == "FIXTURE_SIGNATURE_DRIFT" + assert findings["sample.math:unstable"]["verdict"] == "NOT_EXERCISED" + assert findings["sample.math:unstable"]["reason_code"] == "NONDETERMINISTIC_OBSERVATION" + + +@pytest.mark.parametrize( + ("mutation", "code"), + [ + ("missing", "NO_BASELINE_REVISION"), + ("revision", "IDENTICAL_REVISIONS_REFUSED"), + ("tree", "IDENTICAL_REVISIONS_REFUSED"), + ("environment", "BASELINE_ENVIRONMENT_MISMATCH"), + ("fixture", "BASELINE_FIXTURE_MISMATCH"), + ("zero", "VACUOUS_REVISION_COMPARISON_REFUSED"), + ], +) +def test_revision_diff_refuses_missing_vacuous_or_incomparable_inputs( + mutation: str, code: str +) -> None: + """No comparison may succeed without two distinct, comparable sides.""" + baseline = None if mutation == "missing" else _baseline() + head_revision = BASE_REVISION if mutation == "revision" else HEAD_REVISION + head_tree = BASE_TREE if mutation == "tree" else HEAD_TREE + environment = _environment() + fixture = _fixture() + targets = [_target(target_sha256=HEAD_TARGET_SHA256)] + if mutation == "environment": + environment["platform"] = "different" + elif mutation == "fixture": + fixture["sha256"] = "a" * 64 + elif mutation == "zero": + baseline = _baseline(_target(symbol="sample.math:only")) + targets = [] + + with pytest.raises(BaselineRefusal, match=f"^{code}$"): + compare_frozen_baseline( + baseline, + head_revision=head_revision, + head_tree_sha256=head_tree, + environment=environment, + fixture=fixture, + target_observations=targets, + fixtures_predate_change=True, + invocation={}, + ) + + +def test_custom_comparator_is_used_instead_of_a_second_equality_engine() -> None: + """Integration must be able to delegate equality to the shared comparison rules.""" + calls: list[tuple[object, object]] = [] + + def comparator(old: object, new: object) -> str: + calls.append((old, new)) + return "IDENTICAL" + + report = compare_frozen_baseline( + _baseline(), + head_revision=HEAD_REVISION, + head_tree_sha256=HEAD_TREE, + environment=_environment(), + fixture=_fixture(), + target_observations=[_target(payload=999, target_sha256=HEAD_TARGET_SHA256)], + fixtures_predate_change=True, + invocation={}, + comparator=comparator, + ) + + assert len(calls) == 1 + assert report["findings"][0]["verdict"] == "IDENTICAL" + + +def test_fixture_revision_after_changed_result_is_disclosed_deterministically() -> None: + """Changing a fixture after a failure must remain visible to reviewers.""" + previous = { + "base_revision": BASE_REVISION, + "fixture_sha256": FIXTURE_SHA256, + "findings": [ + {"symbol": "sample.math:value", "verdict": "CHANGED"}, + {"symbol": "sample.math:stable", "verdict": "IDENTICAL"}, + ], + } + current = { + "base_revision": BASE_REVISION, + "fixture_sha256": "a" * 64, + "findings": [ + {"symbol": "sample.math:value", "verdict": "IDENTICAL"}, + {"symbol": "sample.math:stable", "verdict": "IDENTICAL"}, + ], + } + + assert detect_fixture_revision_after_failure(previous, current) == ( + "sample.math:value", + ) + assert detect_fixture_revision_after_failure(previous, previous) == () + + +def test_fixture_revision_disclosure_accepts_full_revision_findings() -> None: + """The disclosure helper must consume the real revision finding shape.""" + changed = compare_frozen_baseline( + _baseline(), + head_revision=HEAD_REVISION, + head_tree_sha256=HEAD_TREE, + environment=_environment(), + fixture=_fixture(), + target_observations=[_target(payload=8, target_sha256=HEAD_TARGET_SHA256)], + fixtures_predate_change=True, + invocation={}, + ) + identical = copy.deepcopy(changed) + identical["findings"][0]["verdict"] = "IDENTICAL" + previous = { + "base_revision": BASE_REVISION, + "fixture_sha256": FIXTURE_SHA256, + "findings": changed["findings"], + } + current = { + "base_revision": BASE_REVISION, + "fixture_sha256": "a" * 64, + "findings": identical["findings"], + } + + assert detect_fixture_revision_after_failure(previous, current) == ( + "sample.math:value", + ) diff --git a/tests/test_revision_executor.py b/tests/test_revision_executor.py new file mode 100644 index 0000000..3f2f932 --- /dev/null +++ b/tests/test_revision_executor.py @@ -0,0 +1,260 @@ +from __future__ import annotations + +import copy +import os +from pathlib import Path +import subprocess + +import pytest + +from breakcheck.adapters.python.symbols import SymbolDefinition +from breakcheck.revision_cli import ( + RevisionModeRefusal, + _import_roots, + freeze_revision, +) +from breakcheck.schema import verify_artifact + + +def _git(repository: Path, *arguments: str, check: bool = True) -> subprocess.CompletedProcess[str]: + environment = { + "PATH": os.environ.get("PATH", ""), + "GIT_CONFIG_GLOBAL": os.devnull, + "GIT_CONFIG_SYSTEM": os.devnull, + "GIT_TERMINAL_PROMPT": "0", + } + return subprocess.run( + ["git", "-C", str(repository), *arguments], + check=check, + capture_output=True, + text=True, + env=environment, + ) + + +def _write(repository: Path, relative: str, content: str) -> None: + path = repository / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + + +def _commit(repository: Path, message: str, paths: tuple[str, ...]) -> str: + _git(repository, "add", "--", *paths) + _git( + repository, + "-c", + "user.name=Release Test", + "-c", + "user.email=release-test@invalid", + "commit", + "-q", + "-m", + message, + ) + return _git(repository, "rev-parse", "HEAD").stdout.strip() + + +def revision_repository(tmp_path: Path) -> tuple[Path, str, str]: + repository = tmp_path / "repository" + repository.mkdir() + _git(repository, "init", "-q") + _write(repository, "sample.py", "def compute_total(value):\n return value + 1\n") + _write( + repository, + "breakcheck.fixtures.toml", + "\n".join( + ( + "schema_version = 1", + "", + "[[binding]]", + 'file = "sample.py"', + "line = 1", + "column = 0", + 'api = "sample.compute_total"', + 'args = ["2"]', + "kwargs = {}", + 'fixture_authored_by = "human"', + "", + ) + ), + ) + base = _commit( + repository, + "base", + ("sample.py", "breakcheck.fixtures.toml"), + ) + _write(repository, "sample.py", "def compute_total(value):\n return value + 2\n") + head = _commit(repository, "head", ("sample.py",)) + return repository, base, head + + +def test_freeze_uses_a_detached_tree_repeats_exactly_twice_and_cleans_up( + tmp_path: Path, +) -> None: + repository, base, _ = revision_repository(tmp_path) + original_head = _git(repository, "rev-parse", "HEAD").stdout.strip() + original_status = _git(repository, "status", "--porcelain=v1").stdout + original_worktrees = _git(repository, "worktree", "list", "--porcelain").stdout + + result = freeze_revision( + repository, + revision=base, + fixture_path="breakcheck.fixtures.toml", + runtime_root=tmp_path / "runtime", + ) + + assert result.exit_code == 0 + assert result.report["artifact_kind"] == "baseline" + assert verify_artifact(result.report) == "VERIFIED" + assert verify_artifact(result.evidence) == "VERIFIED" + target = result.report["payload"]["target_observations"][0] + assert target["module"] == "sample" + assert target["symbol"] == "compute_total" + assert target["observation"]["payload"] == 3 + assert len(target["repeat_sha256"]) == 2 + assert target["repeat_sha256"][0] == target["repeat_sha256"][1] + assert not (tmp_path / "runtime").exists() + assert _git(repository, "rev-parse", "HEAD").stdout.strip() == original_head + assert _git(repository, "status", "--porcelain=v1").stdout == original_status + assert _git(repository, "worktree", "list", "--porcelain").stdout == original_worktrees + + +def test_freeze_refuses_dirty_checkout_even_when_relaxation_is_requested( + tmp_path: Path, +) -> None: + repository, base, _ = revision_repository(tmp_path) + _write(repository, "untracked.txt", "not part of the committed revision\n") + + with pytest.raises(RevisionModeRefusal, match="^DIRTY_TREE_REFUSED$"): + freeze_revision( + repository, + revision=base, + fixture_path="breakcheck.fixtures.toml", + runtime_root=tmp_path / "refused-runtime", + ) + + with pytest.raises( + RevisionModeRefusal, match="^DIRTY_TREE_CAPTURE_UNSUPPORTED$" + ): + freeze_revision( + repository, + revision=base, + fixture_path="breakcheck.fixtures.toml", + runtime_root=tmp_path / "allowed-runtime", + allow_dirty=True, + ) + + assert not (tmp_path / "allowed-runtime").exists() + + +def test_freeze_refuses_an_unrepeatable_or_unexercised_baseline( + tmp_path: Path, +) -> None: + repository, base, _ = revision_repository(tmp_path) + _write(repository, "sample.py", "def compute_total(value):\n return object()\n") + refused = _commit(repository, "rich result", ("sample.py",)) + + with pytest.raises( + RevisionModeRefusal, match="^BASELINE_TARGET_NOT_EXERCISED$" + ): + freeze_revision( + repository, + revision=refused, + fixture_path="breakcheck.fixtures.toml", + runtime_root=tmp_path / "runtime", + ) + + +def test_freeze_result_is_detached_from_caller_mutation(tmp_path: Path) -> None: + repository, base, _ = revision_repository(tmp_path) + result = freeze_revision( + repository, + revision=base, + fixture_path="breakcheck.fixtures.toml", + runtime_root=tmp_path / "runtime", + ) + report = copy.deepcopy(result.report) + report["payload"]["target_observations"][0]["observation"]["payload"] = 99 + assert result.report["payload"]["target_observations"][0]["observation"]["payload"] == 3 + + +def test_revision_executor_contract_requests_exactly_two_runs_and_an_explicit_tree( + tmp_path: Path, +) -> None: + repository, base, _ = revision_repository(tmp_path) + calls: list[dict[str, object]] = [] + + def executor(**kwargs: object) -> dict[str, object]: + calls.append(dict(kwargs)) + return { + "runs": [{}, {}], + "repeatable": True, + "status": "VALUE", + "reason_code": None, + "observation": { + "kind": "value", + "payload": 3, + "exception_class": None, + "duration_ms": None, + }, + } + + freeze_revision( + repository, + revision=base, + fixture_path="breakcheck.fixtures.toml", + runtime_root=tmp_path / "runtime", + executor=executor, + ) + + assert len(calls) == 1 + assert calls[0]["runs"] == 2 + prefixes = calls[0]["sys_path_prefixes"] + assert isinstance(prefixes, tuple) and len(prefixes) == 1 + assert Path(prefixes[0]).name == "base" + + +def test_revision_cleanup_runs_when_execution_is_interrupted(tmp_path: Path) -> None: + repository, base, _ = revision_repository(tmp_path) + original_worktrees = _git(repository, "worktree", "list", "--porcelain").stdout + + def interrupted(**kwargs: object) -> dict[str, object]: + raise KeyboardInterrupt + + with pytest.raises(KeyboardInterrupt): + freeze_revision( + repository, + revision=base, + fixture_path="breakcheck.fixtures.toml", + runtime_root=tmp_path / "runtime", + executor=interrupted, + ) + + assert not (tmp_path / "runtime").exists() + assert _git(repository, "worktree", "list", "--porcelain").stdout == original_worktrees + + +def test_src_import_root_refuses_a_symlink_escape(tmp_path: Path) -> None: + root = tmp_path / "tree" + outside = tmp_path / "outside" + root.mkdir() + outside.mkdir() + (root / "src").symlink_to(outside, target_is_directory=True) + definition = SymbolDefinition( + target="sample:compute_total", + module="sample", + symbol="compute_total", + kind="function", + relative_path="src/sample.py", + line=1, + column=0, + signature_sha256="0" * 64, + behavior_sha256="1" * 64, + context_sha256="2" * 64, + definition_sha256="3" * 64, + ) + + with pytest.raises( + RevisionModeRefusal, match="^REVISION_IMPORT_ROOT_REFUSED$" + ): + _import_roots(root, definition) diff --git a/tests/test_revision_freeze.py b/tests/test_revision_freeze.py new file mode 100644 index 0000000..339f3d6 --- /dev/null +++ b/tests/test_revision_freeze.py @@ -0,0 +1,203 @@ +from __future__ import annotations + +import copy + +import pytest + +from breakcheck.core.baselines import BaselineRefusal, freeze_baseline, validate_baseline + + +BASE_REVISION = "1" * 40 +TREE_SHA256 = "2" * 64 +FIXTURE_SHA256 = "3" * 64 +BINDING_SHA256 = "4" * 64 +TARGET_SHA256 = "5" * 64 +SIGNATURE_SHA256 = "6" * 64 + + +def _environment() -> dict[str, str]: + return { + "implementation": "CPython", + "python_version": "3.12.5", + "platform": "darwin-arm64", + } + + +def _fixture() -> dict[str, str]: + return { + "sha256": FIXTURE_SHA256, + "source_revision": BASE_REVISION, + "source": "base", + "authored_by": "human", + } + + +def _target(symbol: str = "sample.math:value") -> dict[str, object]: + return { + "symbol": symbol, + "target_sha256": TARGET_SHA256, + "signature_sha256": SIGNATURE_SHA256, + "fixture_binding_sha256": BINDING_SHA256, + "provenance": "OPERATOR_FIXTURE", + "projection": None, + "outcome": { + "status": "VALUE", + "observation": { + "kind": "value", + "payload": 7, + "exception_class": None, + "duration_ms": None, + }, + "reason_code": None, + "repeatable": True, + }, + } + + +def test_freeze_binds_revision_environment_fixture_target_and_invocation() -> None: + """Removing any identity field would make a baseline mutable or ambiguous.""" + baseline = freeze_baseline( + revision=BASE_REVISION, + tree_sha256=TREE_SHA256, + dirty=False, + allow_dirty=False, + environment=_environment(), + fixture=_fixture(), + target_observations=[_target()], + invocation={"allow_dirty": False, "fixture_source": "base"}, + ) + + assert baseline == { + "revision": BASE_REVISION, + "tree_sha256": TREE_SHA256, + "dirty": False, + "allow_dirty": False, + "environment": _environment(), + "fixture": _fixture(), + "target_observations": [_target()], + "invocation": {"allow_dirty": False, "fixture_source": "base"}, + } + assert validate_baseline(copy.deepcopy(baseline)) == baseline + + +def test_dirty_tree_is_refused_by_default_and_explicitly_recorded_when_allowed() -> None: + """A dirty checkout must never be silently represented as a committed revision.""" + with pytest.raises(BaselineRefusal, match="^DIRTY_TREE_REFUSED$"): + freeze_baseline( + revision=BASE_REVISION, + tree_sha256=TREE_SHA256, + dirty=True, + allow_dirty=False, + environment=_environment(), + fixture=_fixture(), + target_observations=[_target()], + invocation={"allow_dirty": False}, + ) + + baseline = freeze_baseline( + revision=BASE_REVISION, + tree_sha256=TREE_SHA256, + dirty=True, + allow_dirty=True, + environment=_environment(), + fixture=_fixture(), + target_observations=[_target()], + invocation={"allow_dirty": True}, + ) + assert baseline["dirty"] is True + assert baseline["allow_dirty"] is True + assert baseline["invocation"]["allow_dirty"] is True + + +@pytest.mark.parametrize( + ("mutation", "code"), + [ + ("environment", "BASELINE_ENVIRONMENT_REFUSED"), + ("fixture", "BASELINE_FIXTURE_REFUSED"), + ("target_hash", "BASELINE_TARGET_REFUSED"), + ("signature_hash", "BASELINE_TARGET_REFUSED"), + ("binding_hash", "BASELINE_TARGET_REFUSED"), + ("repeatability", "BASELINE_OBSERVATION_REFUSED"), + ("status", "BASELINE_OBSERVATION_REFUSED"), + ("duplicate", "BASELINE_DUPLICATE_TARGET_REFUSED"), + ("empty", "VACUOUS_BASELINE_REFUSED"), + ("path", "BASELINE_PATH_REFUSED"), + ], +) +def test_freeze_fails_closed_when_required_evidence_is_missing_or_unsafe( + mutation: str, code: str +) -> None: + """Malformed evidence must refuse instead of producing a partial baseline.""" + environment = _environment() + fixture = _fixture() + targets = [_target()] + if mutation == "environment": + environment.pop("platform") + elif mutation == "fixture": + fixture["unknown"] = "value" + elif mutation == "target_hash": + targets[0]["target_sha256"] = "bad" + elif mutation == "signature_hash": + targets[0]["signature_sha256"] = "bad" + elif mutation == "binding_hash": + targets[0]["fixture_binding_sha256"] = "bad" + elif mutation == "repeatability": + targets[0]["outcome"]["repeatable"] = False # type: ignore[index] + elif mutation == "status": + targets[0]["outcome"]["status"] = "NETWORK_REFUSED" # type: ignore[index] + elif mutation == "duplicate": + targets.append(_target()) + elif mutation == "empty": + targets = [] + + invocation: dict[str, object] = {"allow_dirty": False} + if mutation == "path": + invocation["runtime_root"] = "/private/location" + + with pytest.raises(BaselineRefusal, match=f"^{code}$"): + freeze_baseline( + revision=BASE_REVISION, + tree_sha256=TREE_SHA256, + dirty=False, + allow_dirty=False, + environment=environment, + fixture=fixture, + target_observations=targets, + invocation=invocation, + ) + + +def test_freeze_is_deterministic_and_does_not_mutate_inputs() -> None: + """Input ordering and caller mutation must not alter persisted baseline bytes.""" + first_target = _target("sample.math:zeta") + second_target = _target("sample.math:alpha") + targets = [first_target, second_target] + original = copy.deepcopy(targets) + + first = freeze_baseline( + revision=BASE_REVISION, + tree_sha256=TREE_SHA256, + dirty=False, + allow_dirty=False, + environment=_environment(), + fixture=_fixture(), + target_observations=targets, + invocation={"fixture_source": "base", "allow_dirty": False}, + ) + second = freeze_baseline( + revision=BASE_REVISION, + tree_sha256=TREE_SHA256, + dirty=False, + allow_dirty=False, + environment=_environment(), + fixture=_fixture(), + target_observations=list(reversed(targets)), + invocation={"allow_dirty": False, "fixture_source": "base"}, + ) + + assert first == second + assert [row["symbol"] for row in first["target_observations"]] == [ + "sample.math:alpha", + "sample.math:zeta", + ] + assert targets == original diff --git a/tests/test_revision_symbols.py b/tests/test_revision_symbols.py new file mode 100644 index 0000000..03c2302 --- /dev/null +++ b/tests/test_revision_symbols.py @@ -0,0 +1,308 @@ +from __future__ import annotations + +import os +from pathlib import Path +import subprocess + +import pytest + +from breakcheck.adapters.python.symbols import ( + SymbolAnalysisRefusal, + compare_symbol_trees, + inventory_symbols, + tracked_tree_identity, +) + + +def _write(root: Path, relative: str, source: str) -> None: + path = root / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(source, encoding="utf-8") + + +def test_location_only_source_changes_do_not_change_symbol_fingerprints(tmp_path: Path) -> None: + """Blank lines and source locations must not create behavioral targets.""" + base = tmp_path / "base" + head = tmp_path / "head" + base.mkdir() + head.mkdir() + _write( + base, + "pricing.py", + "def total(value: int) -> int:\n return value + 1\n\n" + "class Calculator:\n def apply(self, value):\n return value * 2\n", + ) + _write( + head, + "pricing.py", + "\n\n\ndef total(value: int) -> int:\n\n return value + 1\n\n\n" + "class Calculator:\n\n def apply(self, value):\n return value * 2\n", + ) + + changes = {change.target: change for change in compare_symbol_trees(base, head)} + + assert changes["pricing:total"].status == "UNCHANGED" + assert changes["pricing:Calculator.apply"].status == "UNCHANGED" + + +def test_changed_sibling_does_not_mark_an_unchanged_function_as_context_changed( + tmp_path: Path, +) -> None: + """A changed function must not make every sibling an out-of-scope target.""" + base = tmp_path / "base" + head = tmp_path / "head" + base.mkdir() + head.mkdir() + _write( + base, + "sample.py", + "def compute_total(value):\n return value + 1\n\n" + "def auxiliary(value):\n return value * 2\n", + ) + _write( + head, + "sample.py", + "def compute_total(value):\n result = value + 1\n return result\n\n" + "def auxiliary(value):\n return value * 2\n", + ) + + changes = {change.target: change for change in compare_symbol_trees(base, head)} + + assert changes["sample:compute_total"].status == "CHANGED" + assert changes["sample:auxiliary"].status == "UNCHANGED" + + +def test_symbol_changes_classify_body_signature_addition_and_removal(tmp_path: Path) -> None: + """Each structural transition must retain its distinct fail-closed disposition.""" + base = tmp_path / "base" + head = tmp_path / "head" + base.mkdir() + head.mkdir() + _write( + base, + "service.py", + "def changed(value):\n return value + 1\n\n" + "def signature(value):\n return value\n\n" + "def removed():\n return 'old'\n", + ) + _write( + head, + "service.py", + "def changed(value):\n return value + 2\n\n" + "def signature(value, scale=1):\n return value\n\n" + "def added():\n return 'new'\n", + ) + + changes = {change.target: change for change in compare_symbol_trees(base, head)} + + assert changes["service:changed"].status == "CHANGED" + assert changes["service:signature"].status == "FIXTURE_SIGNATURE_DRIFT" + assert changes["service:removed"].status == "SYMBOL_REMOVED" + assert changes["service:added"].status == "NO_BASELINE_REVISION" + + +def test_module_and_class_context_changes_are_not_silently_ignored(tmp_path: Path) -> None: + """Globals, imports, bases, and class attributes can change unchanged method bodies.""" + base = tmp_path / "base" + head = tmp_path / "head" + base.mkdir() + head.mkdir() + _write( + base, + "context.py", + "OFFSET = 1\n" + "def total(value):\n return value + OFFSET\n\n" + "class Base:\n pass\n\n" + "class Calculator(Base):\n scale = 2\n def apply(self, value):\n return value * self.scale\n", + ) + _write( + head, + "context.py", + "OFFSET = 2\n" + "def total(value):\n return value + OFFSET\n\n" + "class Base:\n pass\n\n" + "class Calculator(Base):\n scale = 3\n def apply(self, value):\n return value * self.scale\n", + ) + + changes = {change.target: change for change in compare_symbol_trees(base, head)} + + assert changes["context:total"].status == "CONTEXT_CHANGED" + assert changes["context:Calculator.apply"].status == "CONTEXT_CHANGED" + + +def test_inventory_includes_only_top_level_functions_and_direct_methods(tmp_path: Path) -> None: + """Nested functions and nested-class methods are outside the declared symbol surface.""" + root = tmp_path / "root" + root.mkdir() + _write( + root, + "workers.py", + "async def fetch():\n return 1\n\n" + "def outer():\n def nested():\n return 2\n return nested()\n\n" + "class Direct:\n async def run(self):\n return 3\n" + " class Nested:\n def hidden(self):\n return 4\n", + ) + + definitions = {item.target: item for item in inventory_symbols(root).definitions} + + assert set(definitions) == {"workers:fetch", "workers:outer", "workers:Direct.run"} + assert definitions["workers:fetch"].kind == "async_function" + assert definitions["workers:Direct.run"].kind == "async_method" + + +def test_duplicate_targets_and_module_path_collisions_are_ambiguous(tmp_path: Path) -> None: + """Python's last-definition-wins behavior must not choose a comparison target silently.""" + base = tmp_path / "base" + head = tmp_path / "head" + for root in (base, head): + root.mkdir() + _write(root, "duplicate.py", "def value():\n return 1\ndef value():\n return 2\n") + _write(root, "pkg.py", "def run():\n return 1\n") + _write(root, "pkg/__init__.py", "def run():\n return 1\n") + + changes = {change.target: change for change in compare_symbol_trees(base, head)} + + assert changes["duplicate:value"].status == "SYMBOL_AMBIGUOUS" + assert changes["pkg:run"].status == "SYMBOL_AMBIGUOUS" + + +def test_ambiguity_takes_precedence_over_added_or_removed_dispositions(tmp_path: Path) -> None: + """A duplicated one-sided target must not be represented as a single new or removed symbol.""" + empty = tmp_path / "empty" + duplicate = tmp_path / "duplicate" + empty.mkdir() + duplicate.mkdir() + _write(duplicate, "module.py", "def value():\n return 1\ndef value():\n return 2\n") + + added = compare_symbol_trees(empty, duplicate) + removed = compare_symbol_trees(duplicate, empty) + + assert added[0].status == "SYMBOL_AMBIGUOUS" + assert removed[0].status == "SYMBOL_AMBIGUOUS" + + +def test_syntax_failure_refuses_the_whole_inventory(tmp_path: Path) -> None: + """A partial inventory could make a claim look narrower than the changed tree.""" + root = tmp_path / "root" + root.mkdir() + _write(root, "broken.py", "def broken(:\n") + + with pytest.raises(SymbolAnalysisRefusal, match="^SYMBOL_SOURCE_SYNTAX_REFUSED$"): + inventory_symbols(root) + + +def test_missing_or_symlinked_symbol_root_is_refused_with_a_stable_code(tmp_path: Path) -> None: + """Root resolution errors must not leak host paths or escape the selected tree.""" + real = tmp_path / "real" + real.mkdir() + symlink = tmp_path / "linked" + symlink.symlink_to(real, target_is_directory=True) + + for root in (tmp_path / "missing", symlink): + with pytest.raises(SymbolAnalysisRefusal, match="^SYMBOL_ROOT_REFUSED$") as caught: + inventory_symbols(root) + assert str(tmp_path) not in str(caught.value) + + +def _git(repository: Path, *args: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + ["git", "-C", str(repository), *args], + check=True, + capture_output=True, + text=True, + env={ + "PATH": os.environ.get("PATH", ""), + "GIT_CONFIG_GLOBAL": os.devnull, + "GIT_CONFIG_SYSTEM": os.devnull, + "GIT_TERMINAL_PROMPT": "0", + }, + ) + + +def _commit_all(repository: Path, message: str) -> None: + _git(repository, "add", "--", "module.py", "data.txt") + _git( + repository, + "-c", + "user.name=Release Test", + "-c", + "user.email=release-test@invalid", + "commit", + "-m", + message, + ) + + +def test_tracked_tree_identity_binds_every_tracked_regular_file_and_ignores_untracked( + tmp_path: Path, +) -> None: + """The tree identity must cover tracked content without leaking an absolute path.""" + repository = tmp_path / "repository" + repository.mkdir() + _git(repository, "init", "-q") + _write(repository, "module.py", "def value():\n return 1\n") + _write(repository, "data.txt", "first\n") + _commit_all(repository, "base") + + first = tracked_tree_identity(repository) + (repository / "untracked.txt").write_text("ignored\n", encoding="utf-8") + second = tracked_tree_identity(repository) + (repository / "module.py").chmod(0o755) + executable = tracked_tree_identity(repository) + (repository / "data.txt").write_text("second\n", encoding="utf-8") + third = tracked_tree_identity(repository) + + assert first.sha256 == second.sha256 + assert first.sha256 != executable.sha256 + assert first.sha256 != third.sha256 + assert first.regular_files == 2 + assert [entry.path for entry in first.entries] == ["data.txt", "module.py"] + assert all(not Path(entry.path).is_absolute() for entry in first.entries) + assert all(str(tmp_path) not in repr(entry) for entry in first.entries) + + +def test_tracked_tree_identity_binds_internal_symlinks_and_refuses_missing_files( + tmp_path: Path, +) -> None: + """Non-regular tracked entries and missing checkout content cannot disappear from identity.""" + repository = tmp_path / "repository" + repository.mkdir() + _git(repository, "init", "-q") + _write(repository, "module.py", "def value():\n return 1\n") + _write(repository, "data.txt", "payload\n") + (repository / "alias.txt").symlink_to("data.txt") + _git(repository, "add", "--", "module.py", "data.txt", "alias.txt") + _git( + repository, + "-c", + "user.name=Release Test", + "-c", + "user.email=release-test@invalid", + "commit", + "-m", + "symlink", + ) + + identity = tracked_tree_identity(repository) + assert identity.regular_files == 2 + assert identity.symlinks == 1 + assert next(entry for entry in identity.entries if entry.path == "alias.txt").kind == "symlink" + + (repository / "data.txt").unlink() + with pytest.raises(SymbolAnalysisRefusal, match="^TRACKED_TREE_REFUSED$"): + tracked_tree_identity(repository) + + +def test_tracked_tree_identity_refuses_gitlinks(tmp_path: Path) -> None: + """A submodule commit cannot be represented as a complete local file-tree hash.""" + repository = tmp_path / "repository" + repository.mkdir() + _git(repository, "init", "-q") + _write(repository, "module.py", "def value():\n return 1\n") + _write(repository, "data.txt", "payload\n") + _commit_all(repository, "base") + commit = _git(repository, "rev-parse", "HEAD").stdout.strip() + _git(repository, "update-index", "--add", "--cacheinfo", f"160000,{commit},vendor/sub") + + with pytest.raises(SymbolAnalysisRefusal, match="^TRACKED_TREE_SUBMODULE_REFUSED$"): + tracked_tree_identity(repository) diff --git a/tests/test_revision_verification.py b/tests/test_revision_verification.py new file mode 100644 index 0000000..66e7d90 --- /dev/null +++ b/tests/test_revision_verification.py @@ -0,0 +1,275 @@ +from __future__ import annotations + +import copy + +import pytest + +from breakcheck.report import ci_exit_code, render_human +from breakcheck.revision_report import make_evidence_artifact, make_revision_artifact +from breakcheck.schema import ( + artifact_digest, + canonicalize_invocation, + record_identity, + verify_artifact, +) +from breakcheck.verify import verify_report + + +def _environment(): + return { + "implementation": "cpython", + "machine": "x86_64", + "platform": "linux", + "python": "3.12.0", + } + + +def _fixture(): + return { + "authored_by": "human", + "sha256": "f" * 64, + "source_revision": "a" * 40, + } + + +def _observation(): + return { + "kind": "value", + "payload": 3, + "exception_class": None, + "provenance": ["OPERATOR_FIXTURE"], + } + + +def _baseline_payload(): + observation = _observation() + target = { + "target_id": "", + "module": "sample.pricing", + "symbol": "compute_total", + "definition_sha256": "d" * 64, + "signature_sha256": "e" * 64, + "observation": observation, + "repeat_sha256": [artifact_digest(observation), artifact_digest(observation)], + "projection_sha256": None, + } + target["target_id"] = record_identity(target, "target_id") + return { + "revision": "a" * 40, + "tree_sha256": "b" * 64, + "dirty": False, + "allow_dirty": False, + "environment": _environment(), + "fixture": _fixture(), + "target_observations": [target], + "invocation": canonicalize_invocation( + "baseline", {"allow_dirty": False, "fixture_policy": "require"} + ), + } + + +def _revision_payload(): + base = _observation() + head = _observation() + finding = { + "finding_id": "", + "target_id": "1" * 64, + "module": "sample.pricing", + "symbol": "compute_total", + "verdict": "IDENTICAL", + "base": base, + "head": head, + "reason_code": None, + "projection": None, + "fixture_binding_sha256": "2" * 64, + } + finding["finding_id"] = record_identity(finding, "finding_id") + witness = { + "witness_id": "", + "finding_id": finding["finding_id"], + "target_id": finding["target_id"], + "base_observation_sha256": artifact_digest(base), + "head_observation_sha256": artifact_digest(head), + "base_repeat_sha256": [artifact_digest(base), artifact_digest(base)], + "head_repeat_sha256": [artifact_digest(head), artifact_digest(head)], + "projection_sha256": None, + "provenance": ["OPERATOR_FIXTURE"], + "replay": { + "source": "import sample.pricing\n\noutcome = sample.pricing.compute_total(1)\n", + "sha256": artifact_digest( + "import sample.pricing\n\noutcome = sample.pricing.compute_total(1)\n" + ), + }, + } + witness["witness_id"] = record_identity(witness, "witness_id") + return { + "base_revision": "a" * 40, + "head_revision": "b" * 40, + "base_tree_sha256": "c" * 64, + "head_tree_sha256": "d" * 64, + "findings": [finding], + "witnesses": [witness], + "summary": { + "changed": 0, + "changed_under_projection": 0, + "identical": 1, + "identical_under_projection": 0, + "not_exercised": 0, + }, + "fixture": _fixture(), + "fixtures_predate_change": True, + "fixture_revision_events": [], + "invocation": canonicalize_invocation( + "revision_report", + { + "fixture_policy": "require", + "fixture_source": "base", + "strict_separation": True, + }, + ), + } + + +def _claim_payload(): + disposition = { + "disposition_id": "", + "target_id": "1" * 64, + "symbol": "sample.pricing:compute_total", + "disposition": "CLAIM_VERIFIED", + "reason_code": None, + "projection_scope": None, + } + disposition["disposition_id"] = record_identity( + disposition, "disposition_id" + ) + return { + "claim": "behavior_preserved", + "base_revision": "a" * 40, + "head_revision": "b" * 40, + "dispositions": [disposition], + "summary": { + "claim_out_of_scope": 0, + "claim_refuted": 0, + "claim_unverifiable": 0, + "claim_verified": 1, + "total": 1, + }, + "fixture": _fixture(), + "fixtures_predate_change": True, + "fixture_revision_events": [], + "invocation": canonicalize_invocation( + "claim_report", + { + "fixture_source": "base", + "strict": True, + "strict_separation": True, + }, + ), + } + + +@pytest.mark.parametrize( + "kind,payload", + [ + ("baseline", _baseline_payload()), + ("revision_report", _revision_payload()), + ("claim_report", _claim_payload()), + ], +) +def test_revision_artifact_kinds_are_closed_self_verifying_and_bundle_bound(kind, payload): + report = make_revision_artifact(kind, payload) + evidence = make_evidence_artifact( + report, + environment_artifacts=[{"name": "revision", "sha256": "9" * 64}], + ) + assert verify_artifact(report) == "VERIFIED" + assert verify_report(report, evidence) == "VERIFIED" + + +def test_claim_summary_count_mismatch_and_out_of_scope_omission_are_refused(): + payload = _claim_payload() + payload["summary"]["claim_verified"] = 0 + with pytest.raises(ValueError, match="ARTIFACT_COUNT_REFUSED"): + make_revision_artifact("claim_report", payload) + + +def test_claim_human_output_states_projection_scope_explicitly(): + payload = _claim_payload() + payload["dispositions"][0]["projection_scope"] = "outcome.total" + payload["dispositions"][0]["disposition_id"] = record_identity( + payload["dispositions"][0], "disposition_id" + ) + rendered = render_human(make_revision_artifact("claim_report", payload)) + assert "CLAIM_UNVERIFIABLE=0" in rendered.splitlines()[0] + assert "projection=outcome.total" in rendered + + +def test_schema_two_claim_exit_policy_honors_the_recorded_strict_flag(): + payload = _claim_payload() + payload["dispositions"][0]["disposition"] = "CLAIM_UNVERIFIABLE" + payload["dispositions"][0]["reason_code"] = "IMPORT_FAILED" + payload["dispositions"][0]["disposition_id"] = record_identity( + payload["dispositions"][0], "disposition_id" + ) + payload["summary"] = { + "claim_out_of_scope": 0, + "claim_refuted": 0, + "claim_unverifiable": 1, + "claim_verified": 0, + "total": 1, + } + + strict = make_revision_artifact("claim_report", payload) + assert ci_exit_code(strict) == 2 + + payload["invocation"] = canonicalize_invocation( + "claim_report", + { + "fixture_source": "base", + "strict": False, + "strict_separation": True, + }, + ) + advisory = make_revision_artifact("claim_report", payload) + assert ci_exit_code(advisory) == 0 + + +def test_strict_claim_cannot_verify_unknown_or_post_change_fixture_authorship(): + payload = _claim_payload() + payload["fixtures_predate_change"] = False + with pytest.raises(ValueError, match="ARTIFACT_SEPARATION_REFUSED"): + make_revision_artifact("claim_report", payload) + payload = _claim_payload() + payload["fixture"]["authored_by"] = "unknown" + with pytest.raises(ValueError, match="ARTIFACT_SEPARATION_REFUSED"): + make_revision_artifact("claim_report", payload) + + +def test_evidence_cannot_be_reused_for_another_report(): + report = make_revision_artifact("revision_report", _revision_payload()) + evidence = make_evidence_artifact(report, environment_artifacts=[]) + other = copy.deepcopy(report) + other["payload"]["head_tree_sha256"] = "e" * 64 + other["payload_sha256"] = "0" * 64 + with pytest.raises(ValueError, match="ARTIFACT_HASH_REFUSED"): + verify_report(other, evidence) + + +def test_unknown_artifact_kind_version_and_environment_row_are_refused(): + payload = _baseline_payload() + with pytest.raises(ValueError, match="ARTIFACT_KIND_REFUSED"): + make_revision_artifact("other", payload) + artifact = make_revision_artifact("baseline", payload) + artifact["schema_version"] = 3 + with pytest.raises(ValueError, match="ARTIFACT_SCHEMA_VERSION_REFUSED"): + verify_artifact(artifact) + with pytest.raises(ValueError, match="ARTIFACT_SCHEMA_VERSION_REFUSED"): + verify_report(artifact, {}) + clean = make_revision_artifact("baseline", _baseline_payload()) + with pytest.raises(ValueError, match="ARTIFACT_SCHEMA_REFUSED"): + make_evidence_artifact( + clean, + environment_artifacts=[ + {"name": "revision", "sha256": "9" * 64, "path": "/tmp/private"} + ], + ) diff --git a/tests/test_revision_worktrees.py b/tests/test_revision_worktrees.py new file mode 100644 index 0000000..874e8f8 --- /dev/null +++ b/tests/test_revision_worktrees.py @@ -0,0 +1,256 @@ +from __future__ import annotations + +import hashlib +import os +from pathlib import Path +import subprocess + +import pytest + +import breakcheck.adapters.python.worktrees as worktree_module +from breakcheck.adapters.python.worktrees import WorktreeRefusal, revision_worktrees + + +def _git(repository: Path, *args: str, check: bool = True) -> subprocess.CompletedProcess[str]: + environment = { + "PATH": os.environ.get("PATH", ""), + "GIT_CONFIG_GLOBAL": os.devnull, + "GIT_CONFIG_SYSTEM": os.devnull, + "GIT_TERMINAL_PROMPT": "0", + } + return subprocess.run( + ["git", "-C", str(repository), *args], + check=check, + capture_output=True, + text=True, + env=environment, + ) + + +def _commit(repository: Path, message: str, content: str) -> str: + (repository / "module.py").write_text(content, encoding="utf-8") + _git(repository, "add", "--", "module.py") + _git( + repository, + "-c", + "user.name=Release Test", + "-c", + "user.email=release-test@invalid", + "commit", + "-m", + message, + ) + return _git(repository, "rev-parse", "HEAD").stdout.strip() + + +def _repository(tmp_path: Path) -> tuple[Path, str, str]: + repository = tmp_path / "repository" + repository.mkdir() + _git(repository, "init", "-q") + base = _commit(repository, "base", "def value():\n return 1\n") + head = _commit(repository, "head", "def value():\n return 2\n") + return repository, base, head + + +def _index_sha256(repository: Path) -> str: + index = Path(_git(repository, "rev-parse", "--git-path", "index").stdout.strip()) + if not index.is_absolute(): + index = repository / index + return hashlib.sha256(index.read_bytes()).hexdigest() + + +def test_revision_worktrees_are_detached_owned_and_leave_checkout_unchanged(tmp_path: Path) -> None: + """A revision comparison must not move HEAD, alter the index, or leave worktrees.""" + repository, base, head = _repository(tmp_path) + runtime_root = tmp_path / "runtime" + original_head = _git(repository, "rev-parse", "HEAD").stdout.strip() + original_index = _index_sha256(repository) + original_worktrees = _git(repository, "worktree", "list", "--porcelain").stdout + + with revision_worktrees(repository, base, head, runtime_root) as pair: + assert pair.base_commit == base + assert pair.head_commit == head + assert pair.base_root.parent == runtime_root + assert pair.head_root.parent == runtime_root + assert _git(pair.base_root, "rev-parse", "HEAD").stdout.strip() == base + assert _git(pair.head_root, "rev-parse", "HEAD").stdout.strip() == head + assert _git(pair.base_root, "symbolic-ref", "-q", "HEAD", check=False).returncode != 0 + assert _git(pair.head_root, "symbolic-ref", "-q", "HEAD", check=False).returncode != 0 + + assert not runtime_root.exists() + assert _git(repository, "rev-parse", "HEAD").stdout.strip() == original_head + assert _index_sha256(repository) == original_index + assert _git(repository, "worktree", "list", "--porcelain").stdout == original_worktrees + + +@pytest.mark.parametrize("reference", ["", "--help", "HEAD\n--help", "HEAD\x00suffix"]) +def test_revision_ref_metacharacters_are_refused_without_creating_runtime( + tmp_path: Path, reference: str +) -> None: + """Option-like or control-bearing refs must never reach Git as arguments.""" + repository, base, _ = _repository(tmp_path) + runtime_root = tmp_path / "runtime" + + with pytest.raises(WorktreeRefusal, match="^REVISION_REF_REFUSED$"): + with revision_worktrees(repository, reference, base, runtime_root): + pass + + assert not runtime_root.exists() + + +def test_runtime_root_must_be_absent_and_outside_repository_and_git_data(tmp_path: Path) -> None: + """A caller must not make checkout cleanup capable of deleting existing data.""" + repository, base, head = _repository(tmp_path) + existing = tmp_path / "existing" + existing.mkdir() + sentinel = existing / "keep.txt" + sentinel.write_text("keep", encoding="utf-8") + + for runtime_root in (existing, repository / "runtime", repository / ".git" / "runtime"): + with pytest.raises(WorktreeRefusal, match="^REVISION_RUNTIME_ROOT_REFUSED$"): + with revision_worktrees(repository, base, head, runtime_root): + pass + + assert sentinel.read_text(encoding="utf-8") == "keep" + assert not (repository / "runtime").exists() + assert not (repository / ".git" / "runtime").exists() + + +def test_cleanup_runs_for_base_exception_and_is_idempotent(tmp_path: Path) -> None: + """Catchable termination must not leave owned worktrees registered or on disk.""" + repository, base, head = _repository(tmp_path) + runtime_root = tmp_path / "runtime" + original_worktrees = _git(repository, "worktree", "list", "--porcelain").stdout + + with pytest.raises(KeyboardInterrupt): + with revision_worktrees(repository, base, head, runtime_root) as pair: + raise KeyboardInterrupt + + pair.close() + assert not runtime_root.exists() + assert _git(repository, "worktree", "list", "--porcelain").stdout == original_worktrees + + +def test_checkout_hooks_and_lfs_smudge_are_disabled(tmp_path: Path) -> None: + """Repository-controlled checkout helpers must not execute during materialization.""" + repository, base, _ = _repository(tmp_path) + marker = tmp_path / "executed" + hooks = repository / "hooks" + hooks.mkdir() + post_checkout = hooks / "post-checkout" + post_checkout.write_text(f"#!/bin/sh\ntouch '{marker}'\n", encoding="utf-8") + post_checkout.chmod(0o755) + _git(repository, "config", "core.hooksPath", str(hooks)) + (repository / ".gitattributes").write_text("payload.txt filter=lfs\n", encoding="utf-8") + (repository / "payload.txt").write_text("payload\n", encoding="utf-8") + _git(repository, "config", "filter.lfs.clean", "cat") + _git(repository, "config", "filter.lfs.smudge", f"touch '{marker}'; cat") + _git(repository, "config", "filter.lfs.required", "true") + _git(repository, "add", "--", ".gitattributes", "payload.txt") + _git( + repository, + "-c", + "user.name=Release Test", + "-c", + "user.email=release-test@invalid", + "commit", + "-m", + "filtered", + ) + head = _git(repository, "rev-parse", "HEAD").stdout.strip() + + with revision_worktrees(repository, base, head, tmp_path / "runtime") as pair: + assert (pair.head_root / "payload.txt").read_text(encoding="utf-8") == "payload\n" + + assert not marker.exists() + + +def test_non_lfs_checkout_filter_is_refused_before_materialization(tmp_path: Path) -> None: + """An unknown smudge/process filter could execute arbitrary repository configuration.""" + repository, base, head = _repository(tmp_path) + _git(repository, "config", "filter.unsafe.smudge", "cat") + runtime_root = tmp_path / "runtime" + + with pytest.raises(WorktreeRefusal, match="^REVISION_FILTER_REFUSED$"): + with revision_worktrees(repository, base, head, runtime_root): + pass + + assert not runtime_root.exists() + + +def test_gitlink_is_refused_before_materialization(tmp_path: Path) -> None: + """Submodule gitlinks must never trigger nested repository behavior.""" + repository, base, _ = _repository(tmp_path) + _git(repository, "update-index", "--add", "--cacheinfo", f"160000,{base},vendor/sub") + _git( + repository, + "-c", + "user.name=Release Test", + "-c", + "user.email=release-test@invalid", + "commit", + "-m", + "gitlink", + ) + head = _git(repository, "rev-parse", "HEAD").stdout.strip() + runtime_root = tmp_path / "runtime" + + with pytest.raises(WorktreeRefusal, match="^REVISION_SUBMODULE_REFUSED$"): + with revision_worktrees(repository, base, head, runtime_root): + pass + + assert not runtime_root.exists() + + +def test_symlink_escape_is_refused_and_owned_worktrees_are_cleaned(tmp_path: Path) -> None: + """A tracked symlink may not make revision execution escape its owned tree.""" + repository, base, _ = _repository(tmp_path) + (repository / "escape").symlink_to("../outside") + _git(repository, "add", "--", "escape") + _git( + repository, + "-c", + "user.name=Release Test", + "-c", + "user.email=release-test@invalid", + "commit", + "-m", + "escape", + ) + head = _git(repository, "rev-parse", "HEAD").stdout.strip() + runtime_root = tmp_path / "runtime" + original_worktrees = _git(repository, "worktree", "list", "--porcelain").stdout + + with pytest.raises(WorktreeRefusal, match="^REVISION_SYMLINK_ESCAPE_REFUSED$"): + with revision_worktrees(repository, base, head, runtime_root): + pass + + assert not runtime_root.exists() + assert _git(repository, "worktree", "list", "--porcelain").stdout == original_worktrees + + +def test_partial_creation_failure_does_not_delete_foreign_runtime_content( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Failure cleanup may remove owned worktrees but never a later foreign sibling.""" + repository, base, head = _repository(tmp_path) + runtime_root = tmp_path / "runtime" + real_add = worktree_module._add_worktree + calls = 0 + + def fail_second_add(repository_path: Path, destination: Path, commit: str) -> None: + nonlocal calls + calls += 1 + if calls == 2: + (runtime_root / "foreign.txt").write_text("keep", encoding="utf-8") + raise WorktreeRefusal("REVISION_WORKTREE_CREATE_REFUSED") + real_add(repository_path, destination, commit) + + monkeypatch.setattr(worktree_module, "_add_worktree", fail_second_add) + + with pytest.raises(WorktreeRefusal, match="^REVISION_WORKTREE_CREATE_REFUSED$"): + with revision_worktrees(repository, base, head, runtime_root): + pass + + assert (runtime_root / "foreign.txt").read_text(encoding="utf-8") == "keep" + assert not (runtime_root / "base").exists() diff --git a/tests/test_schema_compatibility.py b/tests/test_schema_compatibility.py new file mode 100644 index 0000000..46c9205 --- /dev/null +++ b/tests/test_schema_compatibility.py @@ -0,0 +1,356 @@ +from __future__ import annotations + +import copy +import hashlib +import json + +import pytest + +from breakcheck.core.models import ArtifactEnvelope +from breakcheck.report import render_human +from breakcheck.schema import ( + artifact_digest, + canonicalize_invocation, + make_artifact, + record_identity, + validate_artifact, + verify_artifact, +) +from breakcheck.verify import verify_report + + +def _sha(value: object) -> str: + encoded = json.dumps( + value, sort_keys=True, separators=(",", ":"), ensure_ascii=True + ).encode("utf-8") + return hashlib.sha256(encoded).hexdigest() + + +def _observation(value: object = False, *, provenance: str = "SOURCE_LITERAL"): + return { + "kind": "value", + "payload": value, + "exception_class": None, + "provenance": [provenance], + } + + +def _dependency_payload(*, projected: bool = False): + old = _observation() + new = _observation() + projection = None + verdict = "IDENTICAL" + if projected: + projection = { + "source": "outcome.value", + "sha256": _sha("outcome.value"), + } + verdict = "IDENTICAL_UNDER_PROJECTION" + finding = { + "finding_id": "", + "candidate_id": "a" * 64, + "api": "sample.has", + "call_sites": [{"file": "app.py", "line": 3, "column": 4}], + "verdict": verdict, + "old": old, + "new": new, + "reason_code": None, + "reason_detail": None, + "comparison": { + "verdict": "IDENTICAL", + "detail": { + "reason_code": "EQUAL", + "path": None, + "old_summary": "value:bool:false", + "new_summary": "value:bool:false", + "policy": "canonical_json_strict", + }, + }, + "projection": projection, + "fixture_binding_sha256": None, + "suggested_action": [], + } + finding["finding_id"] = record_identity(finding, "finding_id") + witness = { + "witness_id": "", + "finding_id": finding["finding_id"], + "candidate_id": finding["candidate_id"], + "old_observation_sha256": _sha(old), + "new_observation_sha256": _sha(new), + "old_repeat_sha256": [_sha(old), _sha(old)], + "new_repeat_sha256": [_sha(new), _sha(new)], + "projection_sha256": None if projection is None else projection["sha256"], + "provenance": ["SOURCE_LITERAL"], + "replay": { + "source": "import sample\n\noutcome = sample.has(1)\n", + "sha256": _sha("import sample\n\noutcome = sample.has(1)\n"), + }, + } + witness["witness_id"] = record_identity(witness, "witness_id") + summary = { + "changed": 0, + "changed_under_projection": 0, + "identical": 0 if projected else 1, + "identical_under_projection": 1 if projected else 0, + "not_exercised": 0, + } + return { + "package": "sample", + "current_version": "1.0", + "new_version": "2.0", + "coverage": {"exercised": 1, "total": 1, "percent": 100.0}, + "findings": [finding], + "witnesses": [witness], + "summary": summary, + "invocation": canonicalize_invocation( + "dependency_report", + { + "allow_empty": False, + "fixture_policy": "forbid", + "min_coverage": 80.0, + }, + ), + } + + +def _evidence_for(report): + payload = report["payload"] + return make_artifact( + "evidence", + { + "report_artifact_sha256": artifact_digest(report), + "report_payload_sha256": report["payload_sha256"], + "report_kind": report["artifact_kind"], + "witnesses": copy.deepcopy(payload["witnesses"]), + "environment_artifacts": [ + {"name": "current", "sha256": "c" * 64}, + {"name": "new", "sha256": "d" * 64}, + ], + "invocation": copy.deepcopy(payload["invocation"]), + }, + ) + + +def test_schema_two_dependency_artifact_is_closed_and_self_verifying(): + artifact = make_artifact("dependency_report", _dependency_payload()) + assert artifact["schema_version"] == 2 + assert artifact["artifact_kind"] == "dependency_report" + assert verify_artifact(artifact) == "VERIFIED" + + +def test_artifact_envelope_model_validates_and_does_not_alias_mutable_input(): + artifact = make_artifact("dependency_report", _dependency_payload()) + model = ArtifactEnvelope.from_mapping(artifact) + artifact["payload"]["package"] = "mutated" + assert model.to_dict()["payload"]["package"] == "sample" + assert model.to_dict()["payload_sha256"] == model.payload_sha256 + + +def test_schema_two_dependency_bundle_verifies_report_and_evidence_identity(): + report = make_artifact("dependency_report", _dependency_payload()) + evidence = _evidence_for(report) + assert verify_report(report, evidence) == "VERIFIED" + + +def test_schema_two_rejects_unknown_envelope_payload_and_nested_fields(): + artifact = make_artifact("dependency_report", _dependency_payload()) + unknown_envelope = copy.deepcopy(artifact) + unknown_envelope["extra"] = True + unknown_payload = copy.deepcopy(artifact) + unknown_payload["payload"]["extra"] = True + unknown_nested = copy.deepcopy(artifact) + unknown_nested["payload"]["coverage"]["extra"] = True + for mutated in (unknown_envelope, unknown_payload, unknown_nested): + with pytest.raises(ValueError, match="ARTIFACT_SCHEMA_REFUSED"): + validate_artifact(mutated) + + +@pytest.mark.parametrize( + "mutation,code", + [ + ("count", "ARTIFACT_COUNT_REFUSED"), + ("finding_identity", "ARTIFACT_IDENTITY_REFUSED"), + ("missing_provenance", "ARTIFACT_PROVENANCE_REFUSED"), + ("repeat_mismatch", "ARTIFACT_REPEAT_REFUSED"), + ("projection_scope", "ARTIFACT_PROJECTION_REFUSED"), + ], +) +def test_schema_two_rejects_semantically_invalid_dependency_artifacts(mutation, code): + payload = _dependency_payload(projected=mutation == "projection_scope") + if mutation == "count": + payload["summary"]["identical"] = 0 + elif mutation == "finding_identity": + payload["findings"][0]["finding_id"] = "0" * 64 + elif mutation == "missing_provenance": + payload["findings"][0]["old"]["provenance"] = [] + payload["findings"][0]["finding_id"] = record_identity( + payload["findings"][0], "finding_id" + ) + payload["witnesses"][0]["finding_id"] = payload["findings"][0]["finding_id"] + payload["witnesses"][0]["witness_id"] = record_identity( + payload["witnesses"][0], "witness_id" + ) + elif mutation == "repeat_mismatch": + payload["witnesses"][0]["old_repeat_sha256"][1] = "0" * 64 + payload["witnesses"][0]["witness_id"] = record_identity( + payload["witnesses"][0], "witness_id" + ) + else: + payload["findings"][0]["verdict"] = "IDENTICAL" + payload["findings"][0]["finding_id"] = record_identity( + payload["findings"][0], "finding_id" + ) + with pytest.raises(ValueError, match=code): + make_artifact("dependency_report", payload) + + +def test_schema_two_rejects_tampering_and_non_json_values(): + artifact = make_artifact("dependency_report", _dependency_payload()) + artifact["payload"]["package"] = "changed" + with pytest.raises(ValueError, match="ARTIFACT_HASH_REFUSED"): + verify_artifact(artifact) + payload = _dependency_payload() + payload["package"] = object() + with pytest.raises(ValueError, match="ARTIFACT_ENCODING_REFUSED"): + make_artifact("dependency_report", payload) + payload = _dependency_payload() + nested = None + for _ in range(70): + nested = [nested] + payload["findings"][0]["old"]["payload"] = nested + with pytest.raises(ValueError, match="ARTIFACT_ENCODING_REFUSED"): + make_artifact("dependency_report", payload) + + +def test_coverage_artifact_enforces_one_terminal_bucket_per_candidate_and_counts(): + candidate = { + "candidate_id": "a" * 64, + "api": "sample.has", + "file": "app.py", + "line": 3, + "column": 4, + "bucket": "EXERCISED", + "provenance": ["SOURCE_LITERAL"], + } + payload = { + "package": "sample", + "current_version": "1.0", + "new_version": "2.0", + "candidates": [candidate], + "counts": { + "EXERCISED": 1, + "G1_NOT_DISCOVERABLE": 0, + "G2_NONLITERAL": 0, + "G3_UNNORMALIZABLE": 0, + "G4_IMPURE": 0, + "total": 1, + }, + "invocation": canonicalize_invocation( + "coverage_report", + { + "allow_empty": False, + "fixture_policy": "forbid", + "min_coverage": 80.0, + }, + ), + } + assert verify_artifact(make_artifact("coverage_report", payload)) == "VERIFIED" + payload["counts"]["EXERCISED"] = 0 + with pytest.raises(ValueError, match="ARTIFACT_COUNT_REFUSED"): + make_artifact("coverage_report", payload) + + +def test_invocation_flags_are_sorted_closed_and_reject_unsanitized_paths(): + observed = canonicalize_invocation( + "dependency_report", + {"min_coverage": 80.0, "allow_empty": False, "fixture_policy": "forbid"}, + ) + assert [row["name"] for row in observed] == [ + "allow_empty", + "fixture_policy", + "min_coverage", + ] + with pytest.raises(ValueError, match="ARTIFACT_INVOCATION_REFUSED"): + canonicalize_invocation("dependency_report", {"unknown": True}) + with pytest.raises(ValueError, match="ARTIFACT_PATH_REFUSED"): + canonicalize_invocation( + "dependency_report", {"fixture_file": "/private/location/fixtures.toml"} + ) + + +def test_human_output_puts_not_exercised_count_on_summary_line_and_names_projection(): + artifact = make_artifact("dependency_report", _dependency_payload(projected=True)) + rendered = render_human(artifact) + assert "NOT_EXERCISED=0" in rendered.splitlines()[0] + assert "IDENTICAL_UNDER_PROJECTION" in rendered + assert "projection=outcome.value" in rendered + + +def test_historical_schema_one_bundle_still_verifies_with_unchanged_contract(): + observation = { + "kind": "value", + "payload": False, + "exception_class": None, + "duration_ms": None, + } + finding = { + "finding_id": "", + "api": "sample.has", + "call_sites": [{"file": "app.py", "line": 3, "column": 4}], + "verdict": "IDENTICAL", + "old": observation, + "new": observation, + "repro": { + "snippet_id": "a" * 64, + "api": "sample.has", + "call_sites": [{"file": "app.py", "line": 3, "column": 4}], + "code": "import sample\noutcome = sample.has(1)\n", + "args_source": "literal", + "reason_code": None, + }, + "suggested_action": [], + "reason_code": None, + "comparison": { + "verdict": "IDENTICAL", + "detail": { + "reason_code": "EQUAL", + "path": None, + "old_summary": "value:bool:false", + "new_summary": "value:bool:false", + "policy": "canonical_json_strict", + }, + }, + } + finding["finding_id"] = _sha( + {key: value for key, value in finding.items() if key != "finding_id"} + ) + witness = { + "witness_id": "", + "finding_id": finding["finding_id"], + "snippet_id": "a" * 64, + "api": "sample.has", + "code": "import sample\noutcome = sample.has(1)\n", + "current_version": "1.0", + "new_version": "2.0", + "old_observation_sha256": _sha(observation), + "new_observation_sha256": _sha(observation), + } + witness["witness_id"] = _sha(witness) + report = { + "schema_version": 1, + "package": "sample", + "current_version": "1.0", + "new_version": "2.0", + "coverage": {"exercised": 1, "total": 1, "percent": 100.0}, + "findings": [finding], + "witnesses": [witness], + "summary": {"changed": 0, "identical": 1, "not_exercised": 0}, + } + evidence = { + "report": copy.deepcopy(report), + "report_sha256": _sha(report), + "witnesses": copy.deepcopy(report["witnesses"]), + "environment_artifacts": {}, + } + evidence["witness_sha256"] = _sha(evidence) + assert verify_report(report, evidence) == "VERIFIED" diff --git a/tests/test_static_resolution.py b/tests/test_static_resolution.py new file mode 100644 index 0000000..ea7e669 --- /dev/null +++ b/tests/test_static_resolution.py @@ -0,0 +1,487 @@ +from __future__ import annotations + +import ast +import subprocess +import sys + +import pytest + +from breakcheck.adapters.python import coverage, literals, scanner + + +def _refusal(expression: str) -> literals.LiteralRefusal: + with pytest.raises(literals.LiteralRefusal) as caught: + literals.synthesize_snippet(expression) + return caught.value + + +def test_literal_refusal_remains_backward_compatible_and_is_diagnostic(): + refusal = _refusal("json.dumps(cfg)") + + assert isinstance(refusal, ValueError) + assert str(refusal) == "NONLITERAL_ARGS" + assert refusal.family == "NONLITERAL_ARGS" + assert refusal.reason_detail == "LOCAL_NAME" + assert refusal.as_dict() == { + "reason_code": "NONLITERAL_ARGS", + "reason_detail": "LOCAL_NAME", + } + + +@pytest.mark.parametrize( + ("expression", "detail"), + [ + ("json.dumps({'x': f'{url}'})", "FOLDABLE_EXPR"), + ("json.dumps(obj.value)", "ATTRIBUTE_ACCESS"), + ("json.dumps([x for x in values])", "COMPREHENSION"), + ("json.dumps(*values)", "STARRED"), + ("json.dumps(**values)", "STARRED"), + ("json.dumps(lambda: None)", "OTHER"), + ], +) +def test_nonliteral_taxonomy_is_stable(expression: str, detail: str): + assert _refusal(expression).reason_detail == detail + + +def test_eight_call_golden_preserves_lift_split_and_adds_diagnostics(): + cases = [ + ("json.dumps({'x': 1})", "LIFTED"), + ("json.dumps(cfg)", "LOCAL_NAME"), + ("json.dumps({'x': 1 + 1})", "LIFTED"), + ("json.dumps({'x': f'{url}'})", "FOLDABLE_EXPR"), + ("json.dumps({'x': 1}, indent=cfg)", "LOCAL_NAME"), + ("json.dumps(json.loads('{}'))", "LIFTED"), + ("json.loads('{}')", "LIFTED"), + ("cfg.keys()", "LIFTED"), + ] + + observed = [] + for expression, _expected in cases: + try: + literals.synthesize_snippet(expression) + except literals.LiteralRefusal as exc: + observed.append(exc.reason_detail) + else: + observed.append("LIFTED") + + assert observed == [expected for _expression, expected in cases] + + +def _run_snippet(snippet: str) -> object: + completed = subprocess.run( + [sys.executable, "-I", "-c", snippet], + check=True, + capture_output=True, + text=True, + ) + return ast.literal_eval(completed.stdout.strip()) + + +@pytest.mark.parametrize( + ("expression", "expected"), + [ + ("json.dumps(1 + 2)", "3"), + ("json.dumps(8 - 3)", "5"), + ("json.dumps(3 * 4)", "12"), + ("json.dumps(7 / 2)", "3.5"), + ("json.dumps(7 // 2)", "3"), + ("json.dumps(7 % 3)", "1"), + ("json.dumps(2 ** 8)", "256"), + ("json.dumps(f'value={1 + 1}')", '"value=2"'), + ], +) +def test_constant_folding_is_replayed_as_a_bounded_literal(expression, expected): + synthesized = literals.synthesize_with_provenance(expression) + + assert _run_snippet(synthesized.source) == expected + assert synthesized.provenance == ("SOURCE_FOLDED",) + + +@pytest.mark.parametrize( + "expression", + [ + "json.dumps(2 ** 65)", + "json.dumps(2.0 ** 2)", + "json.dumps([0] * 10001)", + "json.dumps('x' * 65537)", + "json.dumps(1 / 0)", + "json.dumps(1 + 'x')", + "json.dumps(1e308 * 1e308)", + "json.dumps(f'{1!r}')", + "json.dumps(f'{1:02d}')", + ], +) +def test_constant_folding_guards_refuse_without_leaking_exceptions(expression): + assert _refusal(expression).reason_detail == "FOLD_REFUSED" + + +def test_constant_folding_has_one_shared_256_node_budget_per_call_site(): + expression = "json.dumps([" + ",".join("1" for _ in range(257)) + "])" + + assert _refusal(expression).reason_detail == "FOLD_REFUSED" + + +def test_module_constant_table_resolves_only_a_single_stable_module_binding(): + source = "LIMIT = 1 + 1\nimport json\njson.dumps(LIMIT)\n" + context = scanner.build_static_context(source) + call = ast.parse(source).body[-1].value + synthesized = literals.synthesize_with_provenance( + ast.unparse(call), + module_constants=context.module_constants, + imported_names=context.imported_names, + ) + + assert context.module_constants["LIMIT"].value == 2 + assert _run_snippet(synthesized.source) == "2" + assert synthesized.provenance == ( + "SOURCE_FOLDED", + "SOURCE_MODULE_CONSTANT", + ) + assert scanner.PythonUsageScanner.build_static_context(source) == context + + +@pytest.mark.parametrize( + ("source", "constant_name"), + [ + ( + "VALUES = [1]\nVALUES.append(2)\nimport json\njson.dumps(VALUES)\n", + "VALUES", + ), + ( + ( + "OPTIONS = {'mode': 'safe'}\n" + "OPTIONS['mode'] = 'unsafe'\n" + "import json\njson.dumps(OPTIONS)\n" + ), + "OPTIONS", + ), + ( + "MEMBERS = {1}\nMEMBERS.add(2)\nimport json\njson.dumps(MEMBERS)\n", + "MEMBERS", + ), + ( + ( + "NESTED = ([1],)\n" + "ALIAS = NESTED[0]\n" + "ALIAS.append(2)\n" + "import json\njson.dumps(NESTED)\n" + ), + "NESTED", + ), + ( + "ALIASED = [1]\nCOPY = ALIASED\nimport json\njson.dumps(ALIASED)\n", + "ALIASED", + ), + ], +) +def test_mutable_module_bindings_are_refused_instead_of_replayed_from_stale_ast( + source, constant_name +): + context = scanner.build_static_context(source) + call = ast.parse(source).body[-1].value + + assert constant_name not in context.module_constants + with pytest.raises(literals.LiteralRefusal) as caught: + literals.synthesize_with_provenance( + ast.unparse(call), + module_constants=context.module_constants, + imported_names=context.imported_names, + ) + assert caught.value.reason_detail == "LOCAL_NAME" + + +def test_deeply_immutable_module_tuple_is_still_replayed_with_provenance(): + source = ( + "COORDINATES = ('origin', (1, 2), None, True, 1.5, b'x')\n" + "import json\njson.dumps(COORDINATES)\n" + ) + context = scanner.build_static_context(source) + call = ast.parse(source).body[-1].value + synthesized = literals.synthesize_with_provenance( + ast.unparse(call), + module_constants=context.module_constants, + imported_names=context.imported_names, + ) + + assert context.module_constants["COORDINATES"].value == ( + "origin", + (1, 2), + None, + True, + 1.5, + b"x", + ) + assert "SOURCE_MODULE_CONSTANT" in synthesized.provenance + + +def test_nonfinite_float_is_not_admitted_as_an_immutable_module_constant(): + context = scanner.build_static_context("LIMIT = 1e309\n") + + assert "LIMIT" not in context.module_constants + + +def test_mutable_literals_remain_supported_when_passed_directly_to_a_call(): + synthesized = literals.synthesize_with_provenance( + "json.dumps([1, {'mode': 'safe'}])" + ) + + assert _run_snippet(synthesized.source) == '[1, {"mode": "safe"}]' + assert synthesized.provenance == ("SOURCE_LITERAL",) + + +@pytest.mark.parametrize( + "source", + [ + "LIMIT = 1\ndef f():\n LIMIT = 2\n", + "LIMIT = 1\ndef f(LIMIT):\n return LIMIT\n", + "LIMIT = 1\nfor LIMIT in []:\n pass\n", + "LIMIT = 1\nwith open('x') as LIMIT:\n pass\n", + "LIMIT = 1\ntry:\n pass\nexcept Exception as LIMIT:\n pass\n", + "LIMIT = 1\nvalues = [LIMIT for LIMIT in []]\n", + "LIMIT = 1\n(LIMIT := 2)\n", + "LIMIT = 1\nLIMIT += 1\n", + "LIMIT = 1\ndel LIMIT\n", + "LIMIT = 1\ndef f():\n global LIMIT\n", + ], +) +def test_module_constant_is_refused_for_every_shadow_or_rebind_form(source): + assert "LIMIT" not in scanner.build_static_context(source).module_constants + + +def test_globals_call_refuses_the_whole_module_constant_table(): + context = scanner.build_static_context("A = 1\nB = 2\nglobals()\n") + + assert context.module_constants == {} + + +def test_multi_target_nested_and_unpack_assignments_are_not_module_constants(): + source = "A = B = 1\n(C, D) = (2, 3)\nif True:\n E = 4\nF = 5; G = 6\n" + + assert set(scanner.build_static_context(source).module_constants) == {"F"} + + +@pytest.mark.parametrize( + ("source", "expression"), + [ + ("from .config import TIMEOUT\n", "json.dumps(TIMEOUT)"), + ("import config\n", "json.dumps(config.TIMEOUT)"), + ], +) +def test_cross_module_constants_have_a_distinct_fail_closed_detail(source, expression): + context = scanner.build_static_context(source) + + with pytest.raises(literals.LiteralRefusal) as caught: + literals.synthesize_with_provenance( + expression, + module_constants=context.module_constants, + imported_names=context.imported_names, + ) + + assert caught.value.reason_detail == "MODULE_CONSTANT_CROSS_MODULE" + + +def test_nested_call_is_emitted_not_evaluated_at_scan_time_with_sorted_imports(): + synthesized = literals.synthesize_with_provenance( + "json.dumps(json.loads('{}'), default=base64.b64encode(b'x'))" + ) + + assert synthesized.source.splitlines()[:2] == ["import base64", "import json"] + assert "json.loads('{}')" in synthesized.source + assert synthesized.provenance == ( + "SOURCE_LITERAL", + "SOURCE_NESTED_CALL", + ) + + +def test_nested_call_replays_a_real_composition(): + synthesized = literals.synthesize_with_provenance( + "json.dumps(json.loads('{\"x\": 1}'))" + ) + + assert _run_snippet(synthesized.source) == '{"x": 1}' + + +@pytest.mark.parametrize( + ("expression", "required_import"), + [ + ("json.dumps(base64.b64encode(b'x'))", "import base64"), + ("json.dumps(binascii.hexlify(b'x'))", "import binascii"), + ("json.dumps(decimal.Decimal('1'))", "import decimal"), + ("json.dumps(fractions.Fraction(1, 2))", "import fractions"), + ("json.dumps(hashlib.sha256(b'x'))", "import hashlib"), + ("json.dumps(math.floor(1.5))", "import math"), + ("json.dumps(re.escape('x'))", "import re"), + ("json.dumps(struct.pack('B', 1))", "import struct"), + ("json.dumps(textwrap.dedent('x'))", "import textwrap"), + ("json.dumps(urllib.parse.quote('x'))", "import urllib.parse"), + ], +) +def test_nested_call_stdlib_allowlist_is_exact_and_deterministic( + expression, required_import +): + source = literals.synthesize_snippet(expression) + + assert required_import in source.splitlines() + + +@pytest.mark.parametrize( + ("expression", "detail"), + [ + ("json.dumps(os.getcwd())", "NESTED_CALL"), + ("json.dumps(open('x'))", "NESTED_CALL"), + ( + "json.dumps(json.loads(json.dumps(json.loads(json.dumps('{}')))))", + "NESTED_CALL_DEPTH_EXCEEDED", + ), + ("json.dumps(json.loads(cfg))", "LOCAL_NAME"), + ("json.dumps(json.loads(*values))", "STARRED"), + ], +) +def test_nested_call_allowlist_depth_and_arguments_fail_closed(expression, detail): + assert _refusal(expression).reason_detail == detail + + +def test_nested_target_package_alias_is_allowed_without_an_extra_import(): + synthesized = literals.synthesize_with_provenance( + "tools.changed(tools.normalize(1 + 1))", + "import samplepkg.tools as tools", + ) + + assert synthesized.source.startswith("import samplepkg.tools as tools\n\n") + assert "tools.normalize(2)" in synthesized.source + assert synthesized.provenance == ( + "SOURCE_FOLDED", + "SOURCE_NESTED_CALL", + ) + + +def test_static_expansion_is_deterministic_across_repeated_runs(): + context = scanner.build_static_context("VALUE = 1 + 1\n") + observed = [ + literals.synthesize_with_provenance( + "json.dumps(json.loads(json.dumps(VALUE)))", + module_constants=context.module_constants, + imported_names=context.imported_names, + ) + for _ in range(2) + ] + + assert observed[0] == observed[1] + + +def test_provenance_scaffold_is_closed_ordered_and_immutable(): + lifted = literals.lift_with_provenance(ast.parse("{'x': [1, 2]}", mode="eval").body) + + assert lifted.value == {"x": [1, 2]} + assert lifted.provenance == ("SOURCE_LITERAL",) + assert literals.order_provenance( + ["SOURCE_NESTED_CALL", "SOURCE_LITERAL", "SOURCE_FOLDED", "SOURCE_LITERAL"] + ) == ("SOURCE_LITERAL", "SOURCE_FOLDED", "SOURCE_NESTED_CALL") + with pytest.raises(ValueError, match="PROVENANCE_REFUSED"): + literals.order_provenance(["UNKNOWN"]) + + +def test_candidate_identity_is_canonical_stable_and_location_attributable(): + first = coverage.make_candidate( + api="json.dumps", file="src/app.py", line=7, column=4 + ) + second = coverage.make_candidate( + column=4, line=7, file="src/app.py", api="json.dumps" + ) + + assert first == second + assert len(first["candidate_id"]) == 64 + assert set(first) == {"candidate_id", "api", "file", "line", "column"} + assert coverage.make_candidate( + api="json.dumps", file="src/app.py", line=8, column=4 + )["candidate_id"] != first["candidate_id"] + + +def test_terminal_records_are_closed_sorted_and_one_per_candidate(): + first = coverage.make_candidate( + api="json.dumps", file="b.py", line=2, column=0 + ) + second = coverage.make_candidate( + api="json.loads", file="a.py", line=1, column=0 + ) + records = coverage.finalize_terminal_records( + [ + coverage.terminal_record( + first, + "G2_NONLITERAL", + reason_code="NONLITERAL_ARGS", + reason_detail="LOCAL_NAME", + provenance=("SOURCE_LITERAL",), + ), + coverage.terminal_record( + second, + "EXERCISED", + provenance=("SOURCE_LITERAL",), + ), + ] + ) + + assert [row["candidate_id"] for row in records] == sorted( + [first["candidate_id"], second["candidate_id"]] + ) + assert coverage.count_terminal_records(records) == { + "EXERCISED": 1, + "G1_NOT_DISCOVERABLE": 0, + "G2_NONLITERAL": 1, + "G3_UNNORMALIZABLE": 0, + "G4_IMPURE": 0, + "total": 2, + } + with pytest.raises(ValueError, match="CANDIDATE_TERMINAL_DUPLICATE"): + coverage.finalize_terminal_records([records[0], records[0]]) + + +def test_scanner_exposes_candidates_separately_without_changing_legacy_calls(): + observed = scanner.PythonUsageScanner("json").scan( + source=( + "import json\n" + "json.dumps({'x': 1})\n" + "getattr(json, 'loads')('{}')\n" + ), + path="app.py", + package="json", + ) + + assert observed["call_sites"] == [ + {"api": "json.dumps", "line": 2, "file": "app.py", "column": 0} + ] + assert len(observed["candidates"]) == 2 + assert len({row["candidate_id"] for row in observed["candidates"]}) == 2 + assert [row["reason_code"] for row in observed["candidates"]] == [ + None, + "DYNAMIC_USAGE_UNSUPPORTED", + ] + + +@pytest.mark.parametrize( + "source", + [ + "import attrs\ndef check(attrs):\n return attrs.has(1)\n", + ( + "import attrs\ndef check(manager):\n" + " with manager as attrs:\n return attrs.has(1)\n" + ), + ( + "import attrs\ndef check():\n try:\n raise RuntimeError\n" + " except RuntimeError as attrs:\n return attrs.has(1)\n" + ), + "import attrs\ndef check():\n attrs += 1\n return attrs.has(1)\n", + "import attrs\ndef check():\n del attrs\n return attrs.has(1)\n", + ], +) +def test_scanner_refuses_calls_when_the_import_alias_is_shadowed(source): + observed = scanner.PythonUsageScanner("attrs").scan( + source=source, + path="app.py", + package="attrs", + ) + + assert observed["call_sites"] == [] + assert [row["reason_code"] for row in observed["unsupported"]] == [ + "DYNAMIC_USAGE_UNSUPPORTED" + ]