From d88cb7ee5539a7e186ab13d92471b504de658b55 Mon Sep 17 00:00:00 2001 From: ViDale Lovett <229783427+lovettsendit@users.noreply.github.com> Date: Wed, 26 Aug 2026 21:15:57 +0200 Subject: [PATCH] Release Breakcheck 2.0.1 --- .github/ISSUE_TEMPLATE/bug.yml | 2 +- .github/ISSUE_TEMPLATE/compatibility-case.yml | 2 +- .../ISSUE_TEMPLATE/revision-claim-case.yml | 2 +- .github/workflows/release.yml | 48 +-- CHANGELOG.md | 13 + README.md | 16 +- SKILL.md | 24 +- examples/github-actions.yml | 2 +- examples/run_demo.sh | 16 +- pyproject.toml | 2 +- src/breakcheck/__init__.py | 2 +- src/breakcheck/adapters/python/envs.py | 21 +- src/breakcheck/adapters/python/executor.py | 9 +- src/breakcheck/adapters/python/fixtures.py | 316 ++++++++++++++---- src/breakcheck/adapters/python/protocol.py | 11 +- src/breakcheck/cli.py | 125 ++++++- src/breakcheck/demo.py | 2 +- src/breakcheck/revision_cli.py | 9 +- tests/test_agent_workflow.py | 18 + tests/test_cli_integration.py | 179 ++++++++++ tests/test_cost_contract.py | 1 + tests/test_distribution_contract.py | 6 +- tests/test_fixtures_and_projections.py | 106 ++++++ tests/test_production_hardening.py | 27 ++ tests/test_project_metadata_and_demo.py | 81 ++++- tests/test_release_automation.py | 4 +- tests/test_release_contract.py | 53 +++ tests/test_replay_protocol_and_coverage.py | 69 +++- tests/test_revision_cli.py | 11 + 29 files changed, 1028 insertions(+), 149 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug.yml b/.github/ISSUE_TEMPLATE/bug.yml index 887144d..0494506 100644 --- a/.github/ISSUE_TEMPLATE/bug.yml +++ b/.github/ISSUE_TEMPLATE/bug.yml @@ -13,7 +13,7 @@ body: attributes: label: Breakcheck version description: Which Breakcheck version produced this result? - placeholder: 2.0.0 + placeholder: 2.0.1 validations: required: true - type: input diff --git a/.github/ISSUE_TEMPLATE/compatibility-case.yml b/.github/ISSUE_TEMPLATE/compatibility-case.yml index ac4dbdc..6e11834 100644 --- a/.github/ISSUE_TEMPLATE/compatibility-case.yml +++ b/.github/ISSUE_TEMPLATE/compatibility-case.yml @@ -13,7 +13,7 @@ body: attributes: label: Breakcheck version description: Which Breakcheck version produced this result? - placeholder: 2.0.0 + placeholder: 2.0.1 validations: required: true - type: input diff --git a/.github/ISSUE_TEMPLATE/revision-claim-case.yml b/.github/ISSUE_TEMPLATE/revision-claim-case.yml index 8a7fb17..639ba5a 100644 --- a/.github/ISSUE_TEMPLATE/revision-claim-case.yml +++ b/.github/ISSUE_TEMPLATE/revision-claim-case.yml @@ -13,7 +13,7 @@ body: attributes: label: Breakcheck version description: Which Breakcheck version produced this result? - placeholder: 2.0.0 + placeholder: 2.0.1 validations: required: true - type: input diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a7caf64..19e4d17 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -41,7 +41,7 @@ jobs: echo "refusing draft or prerelease" exit 1 fi - if [ "$RELEASE_TAG" != "v2.0.0" ]; then + if [ "$RELEASE_TAG" != "v2.0.1" ]; then echo "release tag does not match project version" exit 1 fi @@ -51,8 +51,8 @@ jobs: import tomllib project = tomllib.loads(pathlib.Path("pyproject.toml").read_text(encoding="utf-8"))["project"] - if project["name"] != "breakcheck" or project["version"] != "2.0.0": - raise SystemExit("project metadata does not match v2.0.0") + if project["name"] != "breakcheck" or project["version"] != "2.0.1": + raise SystemExit("project metadata does not match v2.0.1") if project.get("dependencies") != []: raise SystemExit("release unexpectedly has runtime dependencies") PY @@ -72,47 +72,47 @@ jobs: from pathlib import Path expected = { - "breakcheck-2.0.0-py3-none-any.whl", - "breakcheck-2.0.0.tar.gz", + "breakcheck-2.0.1-py3-none-any.whl", + "breakcheck-2.0.1.tar.gz", } actual = {path.name for path in Path("dist").iterdir() if path.is_file()} if actual != expected: raise SystemExit(f"unexpected distribution inventory: {sorted(actual)}") PY - echo "wheel_sha256=$(sha256sum dist/breakcheck-2.0.0-py3-none-any.whl | awk '{print $1}')" >> "$GITHUB_OUTPUT" - echo "sdist_sha256=$(sha256sum dist/breakcheck-2.0.0.tar.gz | awk '{print $1}')" >> "$GITHUB_OUTPUT" + echo "wheel_sha256=$(sha256sum dist/breakcheck-2.0.1-py3-none-any.whl | awk '{print $1}')" >> "$GITHUB_OUTPUT" + echo "sdist_sha256=$(sha256sum dist/breakcheck-2.0.1.tar.gz | awk '{print $1}')" >> "$GITHUB_OUTPUT" - name: Validate metadata, inventory, privacy, and installed CLI run: | set -euo pipefail python -m twine check --strict dist/* - bash scripts/scan_artifacts.sh dist/breakcheck-2.0.0-py3-none-any.whl - bash scripts/scan_artifacts.sh dist/breakcheck-2.0.0.tar.gz + bash scripts/scan_artifacts.sh dist/breakcheck-2.0.1-py3-none-any.whl + bash scripts/scan_artifacts.sh dist/breakcheck-2.0.1.tar.gz python - <<'PY' import email import tarfile import zipfile - wheel = "dist/breakcheck-2.0.0-py3-none-any.whl" - sdist = "dist/breakcheck-2.0.0.tar.gz" + wheel = "dist/breakcheck-2.0.1-py3-none-any.whl" + sdist = "dist/breakcheck-2.0.1.tar.gz" with zipfile.ZipFile(wheel) as archive: metadata_name = next(name for name in archive.namelist() if name.endswith(".dist-info/METADATA")) metadata = email.message_from_bytes(archive.read(metadata_name)) - if metadata["Name"] != "breakcheck" or metadata["Version"] != "2.0.0": + if metadata["Name"] != "breakcheck" or metadata["Version"] != "2.0.1": raise SystemExit("wheel identity mismatch") with tarfile.open(sdist, "r:gz") as archive: names = set(archive.getnames()) required = { - "breakcheck-2.0.0/README.md", - "breakcheck-2.0.0/SECURITY.md", - "breakcheck-2.0.0/SKILL.md", - "breakcheck-2.0.0/examples/github-actions.yml", - "breakcheck-2.0.0/scripts/scan_artifacts.sh", + "breakcheck-2.0.1/README.md", + "breakcheck-2.0.1/SECURITY.md", + "breakcheck-2.0.1/SKILL.md", + "breakcheck-2.0.1/examples/github-actions.yml", + "breakcheck-2.0.1/scripts/scan_artifacts.sh", } if not required <= names: raise SystemExit("source distribution inventory incomplete") PY python -m venv "$RUNNER_TEMP/breakcheck-release-smoke" - "$RUNNER_TEMP/breakcheck-release-smoke/bin/python" -m pip install --no-deps dist/breakcheck-2.0.0-py3-none-any.whl + "$RUNNER_TEMP/breakcheck-release-smoke/bin/python" -m pip install --no-deps dist/breakcheck-2.0.1-py3-none-any.whl ( cd "$RUNNER_TEMP" "$RUNNER_TEMP/breakcheck-release-smoke/bin/breakcheck" --help @@ -122,10 +122,10 @@ jobs: - name: Transfer only validated distributions uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: breakcheck-2.0.0-distributions + name: breakcheck-2.0.1-distributions path: | - dist/breakcheck-2.0.0-py3-none-any.whl - dist/breakcheck-2.0.0.tar.gz + dist/breakcheck-2.0.1-py3-none-any.whl + dist/breakcheck-2.0.1.tar.gz if-no-files-found: error include-hidden-files: false retention-days: 1 @@ -144,7 +144,7 @@ jobs: - name: Receive validated distributions uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - name: breakcheck-2.0.0-distributions + name: breakcheck-2.0.1-distributions path: dist digest-mismatch: error - name: Recheck transferred artifact identity @@ -154,8 +154,8 @@ jobs: run: | set -euo pipefail test "$(find dist -maxdepth 1 -type f | wc -l | tr -d ' ')" = 2 - test "$(sha256sum dist/breakcheck-2.0.0-py3-none-any.whl | awk '{print $1}')" = "$EXPECTED_WHEEL_SHA256" - test "$(sha256sum dist/breakcheck-2.0.0.tar.gz | awk '{print $1}')" = "$EXPECTED_SDIST_SHA256" + test "$(sha256sum dist/breakcheck-2.0.1-py3-none-any.whl | awk '{print $1}')" = "$EXPECTED_WHEEL_SHA256" + test "$(sha256sum dist/breakcheck-2.0.1.tar.gz | awk '{print $1}')" = "$EXPECTED_SDIST_SHA256" - name: Publish with Trusted Publishing and PEP 740 attestations uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 with: diff --git a/CHANGELOG.md b/CHANGELOG.md index f993215..07139fd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,19 @@ All notable changes to Breakcheck are documented here. +## 2.0.1 - 2026-08-26 + +- Resolved relative demo output roots before entering the generated repository. +- Reported expected demo refusals as bounded CLI errors without Python tracebacks. +- Accepted documented multiline and literal TOML fixture strings and reported syntax-error line numbers. +- Rejected invalid coverage thresholds before replay with `MIN_COVERAGE_REFUSED`. +- Added replay-backed fixture suggestions for deterministic rich results while leaving projection choice under explicit review. +- Reported exact line and column drift for stale fixture bindings without changing refusal codes. +- Required the shell demonstration to prove the expected `packaging` 21.3-to-22.0 observations before reporting success. +- Allowed only inert INET/INET6 non-raw socket allocation while refusing local socket pairs, bind, connect, name-resolution, and other socket operations. +- Resolved common distribution/import-name differences for PyYAML, Beautiful Soup, Pillow, and python-dateutil. +- Bounded offline installation and duplicate-wheel failures with actionable refusal codes. + ## 2.0.0 - 2026-08-26 - Added PyPI-ready project metadata and trusted release automation. diff --git a/README.md b/README.md index 489b2ee..734a9fe 100644 --- a/README.md +++ b/README.md @@ -71,6 +71,8 @@ python -m pip download --only-binary=:all: --dest wheelhouse 'attrs==24.2.0' Download exact versions in separate commands. Asking `pip download` to resolve two versions of the same distribution in one command can produce a dependency-resolution error. +Do not use `--no-deps`: the wheelhouse must contain the complete transitive dependency closure needed by both versions. If offline installation cannot resolve that closure, Breakcheck refuses with `ENVIRONMENT_INSTALL_REFUSED` and identifies the requirement without exposing pip output or local paths. + Run the comparison: ```console @@ -116,7 +118,7 @@ Every discovered candidate reaches exactly one terminal bucket: ### Generate fixture suggestions -When source arguments cannot be resolved statically, generate a reviewable skeleton without creating replay environments: +For a fast static pass, run `--suggest-fixtures` without `--wheelhouse`. This scans for unresolved G2 arguments and generates reviewable skeletons without creating replay environments: ```console breakcheck attrs@24.2.0 \ @@ -125,6 +127,16 @@ breakcheck attrs@24.2.0 \ The generated file identifies each unresolved call by repository-relative file, line, column, API, and nearby source. A human or coding tool fills in concrete expressions and changes `fixture_authored_by` from `unknown` to `human` or `agent`. +To also find deterministic rich results that reach `G3_UNNORMALIZABLE`, supply the explicit wheelhouse used for comparison: + +```console +breakcheck attrs@24.2.0 \ + --wheelhouse wheelhouse \ + --suggest-fixtures breakcheck.fixtures.toml +``` + +With `--wheelhouse`, Breakcheck performs isolated replay in both dependency environments. A repeatable rich result adds a skeleton marked `G3_UNNORMALIZABLE` with `projection = ""`. Breakcheck does not invent a projection: an agent or human must fill in a stable expression that references `outcome`, then present the fixture diff for human review. Impure or nondeterministic calls remain excluded from replay-backed suggestions. + Example: ```toml @@ -411,7 +423,7 @@ To build release artifacts: ```console python -m pip install 'build>=1.2,<2' python -m build -python -m pip install dist/breakcheck-2.0.0-py3-none-any.whl +python -m pip install dist/breakcheck-2.0.1-py3-none-any.whl breakcheck --capabilities --json ``` diff --git a/SKILL.md b/SKILL.md index c58823c..89a88af 100644 --- a/SKILL.md +++ b/SKILL.md @@ -7,11 +7,13 @@ description: Use when a Python dependency version changes or when a code change ## Purpose -Use Breakcheck as the deterministic measurement step after proposing a dependency upgrade or a behavior-preserving code change. A coding agent may propose inputs and interpret results; Breakcheck owns replay, comparison, refusal, and evidence. +Use Breakcheck after proposing a dependency upgrade or behavior-preserving change. A coding tool may propose inputs; Breakcheck owns replay, comparison, refusal, and evidence. ## Dependency upgrades -Run from the repository root after preparing exact trusted wheels in a local wheelhouse: +Run from the repository root with trusted wheels: + +The wheelhouse must include the complete transitive dependency closure for both compared versions. Do not use `pip download --no-deps`. ```console breakcheck PACKAGE@NEW_VERSION \ @@ -22,13 +24,21 @@ breakcheck PACKAGE@NEW_VERSION \ --json --ci ``` -If coverage is limited by unresolved arguments, generate proposals with: +For fast G2 proposals, run without `--wheelhouse`: ```console breakcheck PACKAGE@NEW_VERSION --suggest-fixtures breakcheck.fixtures.toml ``` -Fill only fixtures that can be justified from repository context, mark `fixture_authored_by = "agent"`, and present the fixture diff for human review before replay. +To also suggest fixtures for repeatable `G3_UNNORMALIZABLE` rich results, use isolated replay against the exact wheelhouse: + +```console +breakcheck PACKAGE@NEW_VERSION \ + --wheelhouse wheelhouse \ + --suggest-fixtures breakcheck.fixtures.toml +``` + +Replay-backed rich-result skeletons contain `projection = ""`. Breakcheck does not supply the projection. Fill it only with a stable expression referencing `outcome`, mark `fixture_authored_by = "agent"`, and present the fixture diff for human review. Impure and nondeterministic calls remain excluded. ## Behavior-preserving code changes @@ -40,7 +50,7 @@ Fixtures must be authored, reviewed, and committed against the base revision bef 4. Run `breakcheck attest` against the changed revision. 5. Report every disposition verbatim to the human, including all unverifiable and out-of-scope counts. -An explicit comparison between committed revisions is also available: +Compare committed revisions: ```console breakcheck diff \ @@ -51,7 +61,7 @@ breakcheck diff \ --evidence .breakcheck/revision-evidence.json ``` -For an explicit preservation claim, create a reviewed `breakcheck.claim.toml` and run: +Attest a reviewed `breakcheck.claim.toml`: ```console breakcheck attest \ @@ -78,4 +88,4 @@ breakcheck attest \ - Never modify verdict or verification logic as part of the change being verified. - Inspect and sanitize artifacts before sending them to an external service; they may contain source locations, replay source, arguments, setup, projections, and observed values. -Report the exact Breakcheck command, exit code, verdict counts, and artifact paths alongside the ordinary project test results. +Report the command, exit code, verdict counts, artifact paths, and project tests. diff --git a/examples/github-actions.yml b/examples/github-actions.yml index 020c369..a24c342 100644 --- a/examples/github-actions.yml +++ b/examples/github-actions.yml @@ -10,7 +10,7 @@ jobs: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 with: {python-version: "3.13"} - - run: python -m pip install "breakcheck==2.0.0" "$DEPENDENCY==$CURRENT_VERSION" + - run: python -m pip install "breakcheck==2.0.1" "$DEPENDENCY==$CURRENT_VERSION" - run: mkdir wheelhouse - run: 'python -m pip download --only-binary=:all: -d wheelhouse "$DEPENDENCY==$CURRENT_VERSION"' - run: 'python -m pip download --only-binary=:all: -d wheelhouse "$DEPENDENCY==$PROPOSED_VERSION"' diff --git a/examples/run_demo.sh b/examples/run_demo.sh index 1921dbb..c0e882f 100644 --- a/examples/run_demo.sh +++ b/examples/run_demo.sh @@ -83,7 +83,7 @@ if [ -z "${BREAKCHECK_DEMO_WHEELHOUSE:-}" ]; then fi TOOL_PYTHON="$PYTHON" -if "$PYTHON" -c 'import breakcheck; raise SystemExit(0 if breakcheck.__version__ == "2.0.0" else 1)' 2>/dev/null; then +if "$PYTHON" -c 'import breakcheck; raise SystemExit(0 if breakcheck.__version__ == "2.0.1" else 1)' 2>/dev/null; then BREAKCHECK_IMPORT_ROOT=$("$PYTHON" -c 'from pathlib import Path; import breakcheck; print(Path(breakcheck.__file__).resolve().parent.parent)') else BREAKCHECK_IMPORT_ROOT="$CHECKOUT/src" @@ -133,10 +133,22 @@ if report.get("schema_version") != 2 or report.get("artifact_kind") != "dependen payload = report.get("payload", {}) findings = payload.get("findings") summary = payload.get("summary") +if payload.get("current_version") != "21.3" or payload.get("new_version") != "22.0": + raise SystemExit("BREAKCHECK_DEMO_REFUSED: expected packaging 21.3 to 22.0") if not isinstance(findings, list) or len(findings) != 1: raise SystemExit("BREAKCHECK_DEMO_REFUSED: expected exactly one finding") -if findings[0].get("verdict") != "CHANGED": +finding = findings[0] +if finding.get("verdict") != "CHANGED": raise SystemExit("BREAKCHECK_DEMO_REFUSED: expected CHANGED finding") +old = finding.get("old") +new = finding.get("new") +if not isinstance(old, dict) or old.get("kind") != "exception" or old.get("exception_class") != "TypeError": + raise SystemExit("BREAKCHECK_DEMO_REFUSED: unexpected packaging 21.3 observation") +old_payload = old.get("payload") +if not isinstance(old_payload, list) or not old_payload or "unexpected keyword argument" not in str(old_payload[0]): + raise SystemExit("BREAKCHECK_DEMO_REFUSED: missing changed-behavior detail") +if not isinstance(new, dict) or new.get("kind") != "value" or new.get("payload") != "1.0.0": + raise SystemExit("BREAKCHECK_DEMO_REFUSED: unexpected packaging 22.0 observation") if not isinstance(summary, dict) or summary.get("changed") != 1: raise SystemExit("BREAKCHECK_DEMO_REFUSED: expected summary.changed == 1") PY diff --git a/pyproject.toml b/pyproject.toml index 09a540e..93ccc9c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "breakcheck" -version = "2.0.0" +version = "2.0.1" description = "Deterministic behavioral comparison for Python dependency and code changes." requires-python = ">=3.10,<3.14" dependencies = [] diff --git a/src/breakcheck/__init__.py b/src/breakcheck/__init__.py index 8fc7dc1..a30aba0 100644 --- a/src/breakcheck/__init__.py +++ b/src/breakcheck/__init__.py @@ -1,5 +1,5 @@ 'Public command surface.' -__version__ = "2.0.0" +__version__ = "2.0.1" from .cli import main diff --git a/src/breakcheck/adapters/python/envs.py b/src/breakcheck/adapters/python/envs.py index c78776f..35019b0 100644 --- a/src/breakcheck/adapters/python/envs.py +++ b/src/breakcheck/adapters/python/envs.py @@ -16,6 +16,15 @@ "new_version", "package", "version", "wheelhouse") +class EnvironmentRefusal(RuntimeError): + """A bounded environment-construction failure safe for CLI reporting.""" + + def __init__(self, code, *, detail=None): + self.code = code + self.detail = None if detail is None else dict(detail) + super().__init__(code) + + def _normalized_distribution(value): return re.sub(r"[-_.]+", "-", str(value)).lower() @@ -74,12 +83,20 @@ def _install(wheelhouse, package, version, allow_network, runner=None, environme else: argv.append(requirement) execute = runner or subprocess.run + refusal_detail = { + "requirement": requirement, + "wheelhouse_requirement": "complete_dependency_closure", + } try: result = execute(argv, check=False, capture_output=True, text=True, shell=False) except Exception as exc: - raise RuntimeError('INSTALL_FAILED') from exc + raise EnvironmentRefusal( + "ENVIRONMENT_INSTALL_REFUSED", detail=refusal_detail + ) from exc if getattr(result, "returncode", 1) != 0: - raise RuntimeError('INSTALL_FAILED') + raise EnvironmentRefusal( + "ENVIRONMENT_INSTALL_REFUSED", detail=refusal_detail + ) return result diff --git a/src/breakcheck/adapters/python/executor.py b/src/breakcheck/adapters/python/executor.py index 3b34219..676ce56 100644 --- a/src/breakcheck/adapters/python/executor.py +++ b/src/breakcheck/adapters/python/executor.py @@ -28,9 +28,14 @@ _INHERITED_ENVIRONMENT_KEYS = {"PATH", "SYSTEMROOT", "TMPDIR", "TEMP", "TMP"} _NETWORK_GUARD = ( "import sys as _guard_sys\n" + "import socket as _guard_socket\n" "def _guard_audit(event, args):\n" - " if event.startswith('socket.'):\n" - " raise RuntimeError('NETWORK_ACCESS_REFUSED')\n" + " if not event.startswith('socket.'):\n" + " return\n" + " if event == 'socket.__new__' and len(args) >= 4:\n" + " if args[1] in (_guard_socket.AF_INET, _guard_socket.AF_INET6) and args[2] in (_guard_socket.SOCK_STREAM, _guard_socket.SOCK_DGRAM) and args[3] == 0:\n" + " return\n" + " raise RuntimeError('NETWORK_ACCESS_REFUSED')\n" "_guard_sys.addaudithook(_guard_audit)\n" ) diff --git a/src/breakcheck/adapters/python/fixtures.py b/src/breakcheck/adapters/python/fixtures.py index 0352188..cfcec84 100644 --- a/src/breakcheck/adapters/python/fixtures.py +++ b/src/breakcheck/adapters/python/fixtures.py @@ -87,9 +87,22 @@ class FixtureRefusal(ValueError): """A stable fail-closed fixture refusal.""" - def __init__(self, code: str): + def __init__( + self, + code: str, + *, + line: int | None = None, + detail: Mapping[str, object] | None = None, + ): self.code = code - super().__init__(code) + self.line = line + self.detail = ( + {"line": line} + if detail is None and line is not None + else None if detail is None else dict(detail) + ) + message = code if line is None else f"{code}:line={line}" + super().__init__(message) @dataclass(frozen=True) @@ -118,10 +131,15 @@ class FixtureFile: canonical_sha256: str -def _refuse(code: str) -> None: +def _refuse( + code: str, + *, + line: int | None = None, + detail: Mapping[str, object] | None = None, +) -> None: if code not in REFUSAL_CODES: raise RuntimeError("FIXTURE_REFUSAL_UNDECLARED") - raise FixtureRefusal(code) + raise FixtureRefusal(code, line=line, detail=detail) def _canonical(value: object) -> bytes: @@ -135,21 +153,25 @@ def _digest(value: object) -> str: def _strip_comment(line: str) -> str: - quoted = False + quote: str | None = None escaped = False for index, character in enumerate(line): if escaped: escaped = False continue - if quoted and character == "\\": + if quote == '"' and character == "\\": escaped = True continue - if character == '"': - quoted = not quoted + if quote is not None: + if character == quote: + quote = None continue - if character == "#" and not quoted: + if character in ('"', "'"): + quote = character + continue + if character == "#": return line[:index] - if quoted or escaped: + if quote is not None or escaped: _refuse("FIXTURE_SYNTAX_REFUSED") return line @@ -157,20 +179,22 @@ def _strip_comment(line: str) -> str: def _split_unquoted(value: str, delimiter: str) -> list[str]: pieces: list[str] = [] start = 0 - quoted = False + quote: str | None = None escaped = False nesting = 0 for index, character in enumerate(value): if escaped: escaped = False continue - if quoted and character == "\\": + if quote == '"' and character == "\\": escaped = True continue - if character == '"': - quoted = not quoted + if quote is not None: + if character == quote: + quote = None continue - if quoted: + if character in ('"', "'"): + quote = character continue if character in "[{(": nesting += 1 @@ -181,22 +205,83 @@ def _split_unquoted(value: str, delimiter: str) -> list[str]: elif character == delimiter and nesting == 0: pieces.append(value[start:index]) start = index + 1 - if quoted or escaped or nesting != 0: + if quote is not None or escaped or nesting != 0: _refuse("FIXTURE_SYNTAX_REFUSED") pieces.append(value[start:]) return pieces -def _parse_string(value: str) -> str: - if not value.startswith('"') or not value.endswith('"'): - _refuse("FIXTURE_SYNTAX_REFUSED") - try: - parsed = json.loads(value) - except (json.JSONDecodeError, UnicodeError): - _refuse("FIXTURE_SYNTAX_REFUSED") - if not isinstance(parsed, str): +def _decode_basic_string(value: str, *, multiline: bool) -> str: + decoded: list[str] = [] + index = 0 + escapes = { + "b": "\b", + "t": "\t", + "n": "\n", + "f": "\f", + "r": "\r", + '"': '"', + "\\": "\\", + } + while index < len(value): + character = value[index] + if character != "\\": + if character == '"' and not multiline: + _refuse("FIXTURE_SYNTAX_REFUSED") + if ord(character) < 0x20 and character not in ("\t", "\n"): + _refuse("FIXTURE_SYNTAX_REFUSED") + if character == "\n" and not multiline: + _refuse("FIXTURE_SYNTAX_REFUSED") + decoded.append(character) + index += 1 + continue + index += 1 + if index >= len(value): + _refuse("FIXTURE_SYNTAX_REFUSED") + escaped = value[index] + if multiline and escaped == "\n": + index += 1 + while index < len(value) and value[index] in (" ", "\t", "\n"): + index += 1 + continue + if escaped in escapes: + decoded.append(escapes[escaped]) + index += 1 + continue + if escaped in ("u", "U"): + width = 4 if escaped == "u" else 8 + digits = value[index + 1 : index + 1 + width] + if len(digits) != width or not re.fullmatch(r"[0-9A-Fa-f]+", digits): + _refuse("FIXTURE_SYNTAX_REFUSED") + codepoint = int(digits, 16) + if codepoint > 0x10FFFF or 0xD800 <= codepoint <= 0xDFFF: + _refuse("FIXTURE_SYNTAX_REFUSED") + decoded.append(chr(codepoint)) + index += width + 1 + continue _refuse("FIXTURE_SYNTAX_REFUSED") - return parsed + return "".join(decoded) + + +def _parse_string(value: str) -> str: + if value.startswith('"""') and value.endswith('"""') and len(value) >= 6: + content = value[3:-3] + if content.startswith("\n"): + content = content[1:] + return _decode_basic_string(content, multiline=True) + if value.startswith("'''") and value.endswith("'''") and len(value) >= 6: + content = value[3:-3] + return content[1:] if content.startswith("\n") else content + if value.startswith('"') and value.endswith('"') and len(value) >= 2: + return _decode_basic_string(value[1:-1], multiline=False) + if value.startswith("'") and value.endswith("'") and len(value) >= 2: + content = value[1:-1] + if "'" in content or "\n" in content or "\r" in content: + _refuse("FIXTURE_SYNTAX_REFUSED") + if any(ord(character) < 0x20 and character != "\t" for character in content): + _refuse("FIXTURE_SYNTAX_REFUSED") + return content + _refuse("FIXTURE_SYNTAX_REFUSED") def _parse_string_array(value: str) -> list[str]: @@ -246,37 +331,95 @@ def _parse_value(field: str, raw: str) -> object: return _parse_string(raw) +def _triple_string_end(value: str, delimiter: str) -> int | None: + position = 3 + while True: + found = value.find(delimiter, position) + if found < 0: + return None + if delimiter == "'''": + return found + backslashes = 0 + cursor = found - 1 + while cursor >= 0 and value[cursor] == "\\": + backslashes += 1 + cursor -= 1 + if backslashes % 2 == 0: + return found + position = found + 1 + + +def _collect_multiline_string( + lines: Sequence[str], start: int, raw_value: str +) -> tuple[str, int]: + delimiter = raw_value[:3] + combined = raw_value + end = start + while True: + closing = _triple_string_end(combined, delimiter) + if closing is not None: + trailing = combined[closing + 3 :].strip() + if trailing and not trailing.startswith("#"): + _refuse("FIXTURE_SYNTAX_REFUSED") + return combined[: closing + 3], end + end += 1 + if end >= len(lines): + _refuse("FIXTURE_SYNTAX_REFUSED") + combined += "\n" + lines[end] + + def _parse_document(text: str) -> tuple[dict[str, object], list[dict[str, object]]]: top: dict[str, object] = {} bindings: list[dict[str, object]] = [] current: dict[str, object] = top - for raw_line in text.splitlines(): - line = _strip_comment(raw_line).strip() - if not line: - continue - if line == "[[binding]]": - if len(bindings) >= _MAX_BINDINGS: - _refuse("FIXTURE_BINDING_CAP_REFUSED") - current = {} - bindings.append(current) - continue - if line.startswith("["): - _refuse("FIXTURE_SCHEMA_REFUSED") - if "=" not in line: - _refuse("FIXTURE_SYNTAX_REFUSED") - key, raw_value = line.split("=", 1) - key = key.strip() - raw_value = raw_value.strip() - allowed = ( - _TOP_LEVEL_FIELDS - if current is top - else _BINDING_REQUIRED_FIELDS | _BINDING_OPTIONAL_FIELDS - ) - if key not in allowed: - _refuse("FIXTURE_SCHEMA_REFUSED") - if key in current: - _refuse("FIXTURE_DUPLICATE_FIELD_REFUSED") - current[key] = _parse_value(key, raw_value) + lines = text.splitlines() + index = 0 + while index < len(lines): + line_number = index + 1 + try: + raw_line = lines[index] + probe = raw_line.strip() + triple_assignment = False + if "=" in probe: + _, probe_value = probe.split("=", 1) + triple_assignment = probe_value.strip().startswith(('"""', "'''")) + line = probe if triple_assignment else _strip_comment(raw_line).strip() + if not line: + index += 1 + continue + if line == "[[binding]]": + if len(bindings) >= _MAX_BINDINGS: + _refuse("FIXTURE_BINDING_CAP_REFUSED") + current = {} + bindings.append(current) + index += 1 + continue + if line.startswith("["): + _refuse("FIXTURE_SCHEMA_REFUSED") + if "=" not in line: + _refuse("FIXTURE_SYNTAX_REFUSED") + key, raw_value = line.split("=", 1) + key = key.strip() + raw_value = raw_value.strip() + if raw_value.startswith(('"""', "'''")): + raw_value, index = _collect_multiline_string( + lines, index, raw_value + ) + allowed = ( + _TOP_LEVEL_FIELDS + if current is top + else _BINDING_REQUIRED_FIELDS | _BINDING_OPTIONAL_FIELDS + ) + if key not in allowed: + _refuse("FIXTURE_SCHEMA_REFUSED") + if key in current: + _refuse("FIXTURE_DUPLICATE_FIELD_REFUSED") + current[key] = _parse_value(key, raw_value) + index += 1 + except FixtureRefusal as exc: + if exc.code == "FIXTURE_SYNTAX_REFUSED" and exc.line is None: + raise FixtureRefusal(exc.code, line=line_number) from None + raise if set(top) != _TOP_LEVEL_FIELDS: _refuse("FIXTURE_SCHEMA_REFUSED") return top, bindings @@ -406,9 +549,12 @@ def _binding_payload(row: Mapping[str, object]) -> dict[str, object]: def _inventory_keys( inventory: Iterable[Mapping[str, object]], -) -> tuple[dict[tuple[str, int, int, str], int], set[tuple[str, str]]]: +) -> tuple[ + dict[tuple[str, int, int, str], int], + dict[tuple[str, str], list[tuple[str, int, int, str]]], +]: counts: dict[tuple[str, int, int, str], int] = {} - nearby: set[tuple[str, str]] = set() + nearby: dict[tuple[str, str], list[tuple[str, int, int, str]]] = {} try: rows = list(inventory) except TypeError: @@ -434,7 +580,9 @@ def _inventory_keys( _refuse("FIXTURE_INVENTORY_REFUSED") key = (file_name, line, column, api) counts[key] = counts.get(key, 0) + 1 - nearby.add((file_name, api)) + nearby.setdefault((file_name, api), []).append(key) + for values in nearby.values(): + values.sort(key=lambda item: (item[1], item[2], item[0], item[3])) return counts, nearby @@ -477,8 +625,34 @@ def load_fixture_file( if matches > 1: _refuse("FIXTURE_AMBIGUOUS_REFUSED") if matches == 0: - if (key[0], key[3]) in nearby: - _refuse("FIXTURE_STALE_REFUSED") + candidates = nearby.get((key[0], key[3]), []) + if candidates: + binding = { + "file": key[0], "line": key[1], "column": key[2], "api": key[3] + } + inventory_candidates = [] + for candidate_key in candidates: + mismatched_fields = [] + if candidate_key[1] != key[1]: + mismatched_fields.append("line") + if candidate_key[2] != key[2]: + mismatched_fields.append("column") + inventory_candidates.append( + { + "file": candidate_key[0], + "line": candidate_key[1], + "column": candidate_key[2], + "api": candidate_key[3], + "mismatched_fields": mismatched_fields, + } + ) + _refuse( + "FIXTURE_STALE_REFUSED", + detail={ + "binding": binding, + "inventory_candidates": inventory_candidates, + }, + ) _refuse("FIXTURE_UNMATCHED_REFUSED") binding_sha256 = _digest(payload) payloads.append(payload) @@ -604,12 +778,24 @@ def suggest_fixtures( if key in seen: _refuse("FIXTURE_AMBIGUOUS_REFUSED") seen.add(key) + projection_required = candidate.get("projection_required", False) + if type(projection_required) is not bool: + _refuse("FIXTURE_INVENTORY_REFUSED") + coverage_bucket = _context(candidate.get("coverage_bucket")) + reason_code = _context(candidate.get("reason_code")) + raw_type = _context(candidate.get("raw_type")) + if projection_required and coverage_bucket != "G3_UNNORMALIZABLE": + _refuse("FIXTURE_INVENTORY_REFUSED") normalized.append( { "key": key, "signature": _context(candidate.get("signature")), "type_hints": _context(candidate.get("type_hints")), "nearby_source": _context(candidate.get("nearby_source")), + "coverage_bucket": coverage_bucket, + "reason_code": reason_code, + "raw_type": raw_type, + "projection_required": projection_required, } ) normalized.sort(key=lambda item: item["key"]) @@ -621,7 +807,14 @@ def suggest_fixtures( for item in normalized: file_name, line, column, api = item["key"] lines.append("") - for label in ("signature", "type_hints", "nearby_source"): + for label in ( + "signature", + "type_hints", + "nearby_source", + "coverage_bucket", + "reason_code", + "raw_type", + ): if item[label] is not None: lines.append("# " + label + ": " + str(item[label])) lines.extend( @@ -636,6 +829,13 @@ def suggest_fixtures( "kwargs = {}", ] ) + if item["projection_required"]: + lines.extend( + [ + "# Projection must reference outcome and reduce it to stable, normalizable data.", + 'projection = ""', + ] + ) data = ("\n".join(lines) + "\n").encode("utf-8") flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL if hasattr(os, "O_NOFOLLOW"): diff --git a/src/breakcheck/adapters/python/protocol.py b/src/breakcheck/adapters/python/protocol.py index 7f7ee88..86d1335 100644 --- a/src/breakcheck/adapters/python/protocol.py +++ b/src/breakcheck/adapters/python/protocol.py @@ -233,6 +233,7 @@ def child_source( import json as _bc_json import math as _bc_math import os as _bc_os +import socket as _bc_socket import struct as _bc_struct import sys as _bc_sys @@ -256,9 +257,13 @@ def __init__(self, raw_type): self.raw_type = raw_type def _bc_audit(event, args): - if event.startswith("socket."): - _bc_network_attempted[0] = True - raise _BreakcheckNetworkRefused() + if not event.startswith("socket."): + return + if event == "socket.__new__" and len(args) >= 4: + if args[1] in (_bc_socket.AF_INET, _bc_socket.AF_INET6) and args[2] in (_bc_socket.SOCK_STREAM, _bc_socket.SOCK_DGRAM) and args[3] == 0: + return + _bc_network_attempted[0] = True + raise _BreakcheckNetworkRefused() _bc_network_attempted = [False] _bc_sys.addaudithook(_bc_audit) diff --git a/src/breakcheck/cli.py b/src/breakcheck/cli.py index e6b4a11..a6fc910 100644 --- a/src/breakcheck/cli.py +++ b/src/breakcheck/cli.py @@ -43,12 +43,26 @@ def _load_pipeline(): "The default coverage threshold is 80 percent. Refusal codes: " ) _MISSING_CURRENT = "CURRENT_DISTRIBUTION_MISSING" +_MIN_COVERAGE_REFUSAL_CODE = "MIN_COVERAGE_REFUSED" +_DEMO_REFUSAL_CODES = frozenset( + ( + "DEMO_EXECUTION_REFUSED", + "DEMO_OUTPUT_EXISTS_REFUSED", + "DEMO_VERIFICATION_REFUSED", + ) +) +_IMPORT_ROOT_OVERRIDES = { + "beautifulsoup4": "bs4", + "pillow": "PIL", + "pyyaml": "yaml", + "python-dateutil": "dateutil", +} _SUPPORTED_PLATFORMS = frozenset(('linux', 'darwin')) _PLATFORM_REFUSAL_CODE = 'PLATFORM_REFUSED' _WHEELHOUSE_REQUIRED_CODE = 'WHEELHOUSE_REQUIRED' _OPERATIONAL_EXCEPTION_CODES = {'ImportError': 'PIPELINE_IMPORT_REFUSED', 'OSError': 'FILESYSTEM_REFUSED', 'UnicodeError': 'TEXT_ENCODING_REFUSED'} -_DECLARED_REFUSAL_CODES = (frozenset(('NONLITERAL_ARGS', 'CURRENT_DISTRIBUTION_MISSING', 'PLATFORM_REFUSED', 'WHEELHOUSE_REQUIRED', 'MISSING_WHEEL_REFUSED', 'PIPELINE_IMPORT_REFUSED', 'FILESYSTEM_REFUSED', 'TEXT_ENCODING_REFUSED')) | frozenset(('API_ABSENT_BOTH_ENVIRONMENTS', 'CALL_SITE_PATH_REFUSED', 'CALL_SITE_SCAN_REFUSED', 'CALL_SITE_SCHEMA_REFUSED', 'CALL_SITE_SOURCE_REFUSED', 'ENVIRONMENT_ARTIFACT_SYMLINK_REFUSED', 'ENVIRONMENT_FINGERPRINT_REFUSED', 'ENVIRONMENT_PAIR_REFUSED', 'IMPORT_ROOT_REFUSED', 'INVENTORY_ROOT_SYMLINK_REFUSED', 'OBSERVATION_ENCODING_REFUSED', 'OUTPUT_PATH_COLLISION_REFUSED', 'OUTPUT_PATH_REFUSED', 'PRESENCE_CENSUS_REFUSED', 'SOURCE_SYNTAX_REFUSED', 'TARGET_GRAMMAR_REFUSED', 'UNSUPPORTED_USAGE_SCHEMA_REFUSED', 'WHEELHOUSE_REFUSED')) | _FIXTURE_REFUSALS) +_DECLARED_REFUSAL_CODES = (frozenset(('NONLITERAL_ARGS', 'CURRENT_DISTRIBUTION_MISSING', 'PLATFORM_REFUSED', 'WHEELHOUSE_REQUIRED', 'MISSING_WHEEL_REFUSED', 'AMBIGUOUS_WHEEL_REFUSED', 'PIPELINE_IMPORT_REFUSED', 'FILESYSTEM_REFUSED', 'TEXT_ENCODING_REFUSED', 'ENVIRONMENT_INSTALL_REFUSED', _MIN_COVERAGE_REFUSAL_CODE)) | frozenset(('API_ABSENT_BOTH_ENVIRONMENTS', 'CALL_SITE_PATH_REFUSED', 'CALL_SITE_SCAN_REFUSED', 'CALL_SITE_SCHEMA_REFUSED', 'CALL_SITE_SOURCE_REFUSED', 'ENVIRONMENT_ARTIFACT_SYMLINK_REFUSED', 'ENVIRONMENT_FINGERPRINT_REFUSED', 'ENVIRONMENT_PAIR_REFUSED', 'IMPORT_ROOT_REFUSED', 'INVENTORY_ROOT_SYMLINK_REFUSED', 'OBSERVATION_ENCODING_REFUSED', 'OUTPUT_PATH_COLLISION_REFUSED', 'OUTPUT_PATH_REFUSED', 'PRESENCE_CENSUS_REFUSED', 'SOURCE_SYNTAX_REFUSED', 'TARGET_GRAMMAR_REFUSED', 'UNSUPPORTED_USAGE_SCHEMA_REFUSED', 'WHEELHOUSE_REFUSED')) | _FIXTURE_REFUSALS | _DEMO_REFUSAL_CODES) _HELP = _HELP_PREFIX + ','.join(sorted(_DECLARED_REFUSAL_CODES)) def _bounded_refusal(exc): @@ -65,6 +79,12 @@ def _bounded_refusal(exc): def _canonical(value): return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=True) + +def _print_refusal_detail(exc): + detail = getattr(exc, "detail", None) + if isinstance(detail, dict): + print("REFUSAL_DETAIL:" + _canonical(detail), file=sys.stderr) + def _digest(value): return hashlib.sha256(_canonical(value).encode("utf-8")).hexdigest() @@ -72,6 +92,16 @@ def _bounded_workers(value): if value < 1: raise ValueError("workers must be positive") return min(value, 8) + +def _coverage_threshold(value): + try: + threshold = float(value) + except (TypeError, ValueError) as exc: + raise argparse.ArgumentTypeError(_MIN_COVERAGE_REFUSAL_CODE) from exc + if not 0 < threshold <= 100: + raise argparse.ArgumentTypeError(_MIN_COVERAGE_REFUSAL_CODE) + return threshold + def _target(value): if not isinstance(value, str) or value.count("@") != 1: raise ValueError("TARGET_GRAMMAR_REFUSED") @@ -82,6 +112,10 @@ def _target(value): return package, version def _import_root(package): + distribution_key = re.sub(r"[-_.]+", "-", str(package)).lower() + override = _IMPORT_ROOT_OVERRIDES.get(distribution_key) + if override is not None: + return override try: distribution = _metadata.distribution(package) declared = distribution.read_text("top_level.txt") or "" @@ -206,16 +240,25 @@ def _call_sources(root, grouped, inventory): nodes = matches[(line, column)] if not nodes: raise ValueError("CALL_SITE_SOURCE_REFUSED") - source = _node_source(lines, nodes[0]) + api = requested[(relative, line, column)] + selected = [] + for node in nodes: + try: + import_statement = _replay_import_statement(tree, node, api) + except ValueError as exc: + if str(exc) != "CALL_SITE_SOURCE_REFUSED": + raise + continue + selected.append((node, import_statement)) + if len(selected) != 1: + raise ValueError("CALL_SITE_SOURCE_REFUSED") + node, import_statement = selected[0] + source = _node_source(lines, node) if not isinstance(source, str) or not source: raise ValueError("CALL_SITE_SOURCE_REFUSED") result[(relative, line, column)] = { "expression": source, - "import_statement": _replay_import_statement( - tree, - nodes[0], - requested[(relative, line, column)], - ), + "import_statement": import_statement, "module_constants": static_context.module_constants, "imported_names": static_context.imported_names, } @@ -309,9 +352,7 @@ def _scan_inventory(root, package, inventory): } -def _fixture_suggestions(args, repository, scan, inventory): - from breakcheck.adapters.python.fixtures import suggest_fixtures - +def _fixture_suggestion_candidates(repository, scan, inventory): grouped = {} for row in scan["call_sites"]: grouped.setdefault(row["api"], []).append( @@ -337,6 +378,12 @@ def _fixture_suggestions(args, repository, scan, inventory): "nearby_source": source["expression"], } ) + return candidates + + +def _write_fixture_suggestions(args, repository, candidates): + from breakcheck.adapters.python.fixtures import suggest_fixtures + digest = suggest_fixtures( args.suggest_fixtures, candidates, @@ -353,6 +400,7 @@ def _fixture_suggestions(args, repository, scan, inventory): ) return 0 + def _observation_text(data): try: return bytes(data or b"").decode("utf-8", "strict") @@ -890,8 +938,13 @@ def _build_with_runtime(args, runtime_root): call_sites = scan.get("call_sites") if isinstance(scan, dict) else None if not isinstance(call_sites, list): raise ValueError("CALL_SITE_SCAN_REFUSED") - if getattr(args, "suggest_fixtures", None): - return _fixture_suggestions(args, repository, scan, inventory) + suggesting_fixtures = bool(getattr(args, "suggest_fixtures", None)) + suggestion_candidates = ( + _fixture_suggestion_candidates(repository, scan, inventory) + if suggesting_fixtures else [] + ) + if suggesting_fixtures and not wheelhouse: + return _write_fixture_suggestions(args, repository, suggestion_candidates) try: current_version = _metadata.version(package) except Exception: @@ -927,6 +980,8 @@ def _build_with_runtime(args, runtime_root): code = _bounded_refusal(exc) if code is None: raise + if isinstance(getattr(exc, "detail", None), dict): + raise raise RuntimeError(code) from None if not isinstance(pair, dict) or set(pair) != {"current", "new"}: raise ValueError("ENVIRONMENT_PAIR_REFUSED") @@ -1054,6 +1109,32 @@ def _build_with_runtime(args, runtime_root): old_run.get("status") == "UNNORMALIZABLE" or new_run.get("status") == "UNNORMALIZABLE" ) + if ( + suggesting_fixtures + and unnormalizable + and old_run.get("repeatable") is True + and new_run.get("repeatable") is True + ): + old_type = _typed_raw_type(old_run) + new_type = _typed_raw_type(new_run) + raw_type = old_type if old_type == new_type else _canonical( + {"current": old_type, "new": new_type} + ) + suggestion_candidates.append( + { + "api": api, + "file": row["site"][0], + "line": row["site"][1], + "column": row["site"][2], + "signature": None, + "type_hints": None, + "nearby_source": expression, + "coverage_bucket": "G3_UNNORMALIZABLE", + "reason_code": reason, + "raw_type": raw_type, + "projection_required": True, + } + ) terminal_records.append(terminal_record( row["candidate"], "G3_UNNORMALIZABLE" if unnormalizable else "G4_IMPURE", @@ -1117,6 +1198,8 @@ def _build_with_runtime(args, runtime_root): "EXERCISED", provenance=row["provenance"], )) + if suggesting_fixtures: + return _write_fixture_suggestions(args, repository, suggestion_candidates) findings.sort(key=lambda row: row["finding_id"]) witnesses.sort(key=lambda row: row["witness_id"]) changed = sum(row["verdict"] in {"CHANGED", "CHANGED_UNDER_PROJECTION"} for row in findings) @@ -1217,6 +1300,7 @@ def _capabilities(): "claim_attestation", "dependency_comparison", "fixture_suggestions", + "projection_suggestions", "revision_baselines", "revision_comparison", ], @@ -1344,7 +1428,7 @@ def _revision_parser(command): ) parser.add_argument( "--min-coverage", - type=float, + type=_coverage_threshold, default=80.0, help="minimum exercised target percentage (default: 80)", ) @@ -1375,7 +1459,7 @@ def _revision_parser(command): ) parser.add_argument( "--min-coverage", - type=float, + type=_coverage_threshold, default=80.0, help="minimum exercised target percentage (default: 80)", ) @@ -1458,6 +1542,7 @@ def _revision_command(command, argv): return _emit_revision_result(result, args) except RevisionModeRefusal as exc: print("REVISION_REFUSED:" + exc.code, file=sys.stderr) + _print_refusal_detail(exc) return 2 except ValueError: print("REVISION_REFUSED:ARTIFACT_INPUT_REFUSED", file=sys.stderr) @@ -1506,7 +1591,7 @@ def main(argv=None): ) parser.add_argument('--suggest-fixtures', help="write fixture suggestions for unexercised calls") parser.add_argument( - '--min-coverage', type=float, default=80.0, help="minimum exercised percentage (default: 80)" + '--min-coverage', type=_coverage_threshold, default=80.0, help="minimum exercised percentage (default: 80)" ) parser.add_argument( '--allow-empty', action="store_true", help="permit an empty comparison and record that choice" @@ -1532,7 +1617,14 @@ def main(argv=None): mode="demo", allowed=("--output-root",), ) - return _demo(args.output_root) + try: + return _demo(args.output_root) + except ValueError as exc: + code = str(exc) + if code not in _DEMO_REFUSAL_CODES: + raise + print("DEMO_REFUSED:" + code, file=sys.stderr) + return 2 if args.verify: if args.target: parser.error("verify mode is exclusive") @@ -1554,6 +1646,7 @@ def main(argv=None): if code is None: raise print("BUILD_REFUSED:" + code, file=sys.stderr) + _print_refusal_detail(exc) return 2 if __name__ == "__main__": diff --git a/src/breakcheck/demo.py b/src/breakcheck/demo.py index b18085f..55c01d3 100644 --- a/src/breakcheck/demo.py +++ b/src/breakcheck/demo.py @@ -94,7 +94,7 @@ def _install_metadata_view(root: Path) -> None: def run_demo(output_root: str | os.PathLike[str], build) -> int: - root = Path(output_root) + root = Path(output_root).resolve() if root.exists() or root.is_symlink(): raise ValueError("DEMO_OUTPUT_EXISTS_REFUSED") root.mkdir(parents=True) diff --git a/src/breakcheck/revision_cli.py b/src/breakcheck/revision_cli.py index a387ed7..7d68428 100644 --- a/src/breakcheck/revision_cli.py +++ b/src/breakcheck/revision_cli.py @@ -63,8 +63,9 @@ class RevisionModeRefusal(ValueError): """A revision command could not produce evidence without guessing.""" - def __init__(self, code: str): + def __init__(self, code: str, *, detail: Mapping[str, object] | None = None): self.code = code + self.detail = None if detail is None else dict(detail) super().__init__(code) @@ -103,7 +104,11 @@ def _translate(exc: Exception) -> RevisionModeRefusal: code = getattr(exc, "code", None) if type(code) is not str or not code: code = str(exc) if str(exc) else "REVISION_MODE_REFUSED" - return RevisionModeRefusal(code) + detail = getattr(exc, "detail", None) + return RevisionModeRefusal( + code, + detail=detail if isinstance(detail, Mapping) else None, + ) def _git(repository: Path, *arguments: str) -> bytes: diff --git a/tests/test_agent_workflow.py b/tests/test_agent_workflow.py index f321477..62aaece 100644 --- a/tests/test_agent_workflow.py +++ b/tests/test_agent_workflow.py @@ -6,6 +6,7 @@ ROOT = Path(__file__).resolve().parents[1] SKILL = ROOT / "SKILL.md" DISCOVERY = ROOT / "AGENTS.md" +README = ROOT / "README.md" def _skill_text() -> str: @@ -72,3 +73,20 @@ def test_repository_discovery_file_routes_automation_to_the_skill(): text = DISCOVERY.read_text(encoding="utf-8") assert "[SKILL.md](SKILL.md)" in text assert "Never weaken coverage or separation policy" in text + + +def test_fixture_suggestion_guidance_distinguishes_scan_and_replay_modes(): + readme = README.read_text(encoding="utf-8") + skill = _skill_text() + for text in (readme, skill): + lowered = text.lower() + assert "without `--wheelhouse`" in text + assert "G2" in text + assert "G3_UNNORMALIZABLE" in text + assert 'projection = ""' in text + assert "outcome" in text + assert "impure" in lowered + assert "nondeterministic" in lowered + assert "isolated replay" in readme + assert "does not invent a projection" in readme + assert "human review" in readme diff --git a/tests/test_cli_integration.py b/tests/test_cli_integration.py index 0c34477..bc23af2 100644 --- a/tests/test_cli_integration.py +++ b/tests/test_cli_integration.py @@ -10,6 +10,8 @@ from breakcheck import cli from breakcheck import revision_cli +from breakcheck.adapters.python import envs +from breakcheck.adapters.python.fixtures import FixtureRefusal from breakcheck.adapters.python.literals import synthesize_snippet from breakcheck.report import finding_id from breakcheck.verify import verify_report @@ -332,6 +334,183 @@ def test_suggest_fixtures_needs_no_wheelhouse(monkeypatch, tmp_path): assert "sample.api(object())" in rendered +def test_collocated_chained_call_selects_the_scanned_dependency_api( + monkeypatch, tmp_path +): + source = tmp_path / "app.py" + source.write_text( + "import click\n\n" + "def decorate(function):\n" + " return click.option('--flag', callback=function)(function)\n", + encoding="utf-8", + ) + scan = { + "call_sites": [ + {"api": "click.option", "file": "app.py", "line": 4, "column": 11} + ] + } + monkeypatch.setattr(cli, "_synthesize", synthesize_snippet, raising=False) + + candidates = cli._fixture_suggestion_candidates( + tmp_path, scan, {source.resolve()} + ) + + assert len(candidates) == 1 + assert candidates[0]["api"] == "click.option" + assert candidates[0]["nearby_source"] == "click.option('--flag', callback=function)" + + +def test_dependency_refusal_prints_canonical_fixture_detail(monkeypatch, capsys): + detail = { + "binding": {"file": "app.py", "line": 8, "column": 9, "api": "sample.api"}, + "inventory_candidates": [ + { + "file": "app.py", + "line": 7, + "column": 0, + "api": "sample.api", + "mismatched_fields": ["line", "column"], + } + ], + } + monkeypatch.setattr( + cli, + "_build", + lambda _args: (_ for _ in ()).throw( + FixtureRefusal("FIXTURE_STALE_REFUSED", detail=detail) + ), + ) + + result = cli.main(["sample@2.0", "--wheelhouse", "wheelhouse"]) + + assert result == 2 + assert capsys.readouterr().err.splitlines() == [ + "BUILD_REFUSED:FIXTURE_STALE_REFUSED", + "REFUSAL_DETAIL:" + json.dumps(detail, sort_keys=True, separators=(",", ":")), + ] + + +def test_environment_install_detail_survives_the_build_boundary( + monkeypatch, tmp_path, capsys +): + source = tmp_path / "app.py" + source.write_text("import sample\nsample.api()\n", encoding="utf-8") + runtime = tmp_path / "runtime" + pipeline = list(_fake_pipeline(source, runtime, [])) + + class RefusingEnvironmentBuilder: + def __init__(self, **_kwargs): + pass + + def build(self): + raise envs.EnvironmentRefusal( + "ENVIRONMENT_INSTALL_REFUSED", + detail={ + "requirement": "sample==1.0", + "wheelhouse_requirement": "complete_dependency_closure", + }, + ) + + pipeline[3] = RefusingEnvironmentBuilder + monkeypatch.setattr(cli, "_load_pipeline", lambda: tuple(pipeline)) + monkeypatch.setattr(cli._metadata, "version", lambda _package: "1.0") + monkeypatch.setattr(cli, "_import_root", lambda _package: "sample") + monkeypatch.chdir(tmp_path) + + result = cli.main( + ["sample@2.0", "--wheelhouse", str(tmp_path / "wheelhouse")] + ) + + assert result == 2 + assert capsys.readouterr().err.splitlines() == [ + "BUILD_REFUSED:ENVIRONMENT_INSTALL_REFUSED", + 'REFUSAL_DETAIL:{"requirement":"sample==1.0",' + '"wheelhouse_requirement":"complete_dependency_closure"}', + ] + + +def test_replay_backed_suggestions_include_repeatable_rich_results( + monkeypatch, tmp_path, capsys +): + source = tmp_path / "app.py" + source.write_text("import sample\nsample.api()\n", encoding="utf-8") + runtime = tmp_path / "runtime" + destination = tmp_path / "suggested.toml" + executions: list[tuple[str, str]] = [] + monkeypatch.setattr(cli, "_load_pipeline", lambda: _fake_pipeline(source, runtime, executions)) + monkeypatch.setattr(cli._metadata, "version", lambda _package: "1.0") + monkeypatch.setattr(cli, "_import_root", lambda _package: "sample") + monkeypatch.setattr( + cli, + "_repeat_observation", + lambda _snippet, _environment: { + "runs": [{"raw_type": "SampleResult"}, {"raw_type": "SampleResult"}], + "repeatable": True, + "status": "UNNORMALIZABLE", + "reason_code": "UNSTABLE_OBSERVATION_REFUSED", + "observation": None, + }, + ) + monkeypatch.chdir(tmp_path) + + result = cli.main( + [ + "sample@2.0", + "--wheelhouse", + str(tmp_path / "wheelhouse"), + "--suggest-fixtures", + str(destination), + ] + ) + + assert result == 0 + payload = json.loads(capsys.readouterr().out) + assert payload["fixture_suggestions"] == 1 + rendered = destination.read_text(encoding="utf-8") + assert "# coverage_bucket: G3_UNNORMALIZABLE" in rendered + assert "# raw_type: SampleResult" in rendered + assert 'projection = ""' in rendered + + +def test_replay_backed_suggestions_exclude_impure_or_nondeterministic_calls( + monkeypatch, tmp_path, capsys +): + source = tmp_path / "app.py" + source.write_text("import sample\nsample.api()\n", encoding="utf-8") + runtime = tmp_path / "runtime" + destination = tmp_path / "suggested.toml" + executions: list[tuple[str, str]] = [] + monkeypatch.setattr(cli, "_load_pipeline", lambda: _fake_pipeline(source, runtime, executions)) + monkeypatch.setattr(cli._metadata, "version", lambda _package: "1.0") + monkeypatch.setattr(cli, "_import_root", lambda _package: "sample") + monkeypatch.setattr( + cli, + "_repeat_observation", + lambda _snippet, _environment: { + "runs": [], + "repeatable": False, + "status": "PROTOCOL_REFUSED", + "reason_code": "NONDETERMINISTIC_OBSERVATION", + "observation": None, + }, + ) + monkeypatch.chdir(tmp_path) + + result = cli.main( + [ + "sample@2.0", + "--wheelhouse", + str(tmp_path / "wheelhouse"), + "--suggest-fixtures", + str(destination), + ] + ) + + assert result == 0 + assert json.loads(capsys.readouterr().out)["fixture_suggestions"] == 0 + assert "[[binding]]" not in destination.read_text(encoding="utf-8") + + def test_operator_fixture_replays_a_nonliteral_call(monkeypatch, tmp_path): source = tmp_path / "app.py" source.write_text("import sample\nsample.api(object())\n", encoding="utf-8") diff --git a/tests/test_cost_contract.py b/tests/test_cost_contract.py index 1edd45e..d50cc5b 100644 --- a/tests/test_cost_contract.py +++ b/tests/test_cost_contract.py @@ -26,6 +26,7 @@ def test_capabilities_are_one_noninteractive_machine_readable_command(capsys): assert payload["platforms"] == ["linux", "macos"] assert "dependency_comparison" in payload["features"] assert "fixture_suggestions" in payload["features"] + assert "projection_suggestions" in payload["features"] assert "revision_comparison" in payload["features"] assert "claim_attestation" in payload["features"] assert elapsed < 5.0 diff --git a/tests/test_distribution_contract.py b/tests/test_distribution_contract.py index b22f288..cdfb043 100644 --- a/tests/test_distribution_contract.py +++ b/tests/test_distribution_contract.py @@ -18,7 +18,7 @@ def test_distribution_metadata_exposes_the_current_public_contract() -> None: project = tomllib.loads((ROOT / "pyproject.toml").read_text(encoding="utf-8"))["project"] assert project["name"] == "breakcheck" - assert project["version"] == "2.0.0" + assert project["version"] == "2.0.1" assert project["authors"] == [{"name": "ViDale Lovett"}] assert project["dependencies"] == [] assert project["scripts"] == {"breakcheck": "breakcheck.cli:main"} @@ -54,7 +54,7 @@ def test_github_action_example_is_small_and_uses_the_public_package() -> None: if line.strip() and not line.lstrip().startswith("#") ] assert len(effective_lines) <= 20 - assert 'python -m pip install "breakcheck==2.0.0"' in example + assert 'python -m pip install "breakcheck==2.0.1"' in example assert "--ci" in example for line in example.splitlines(): if "uses:" in line: @@ -93,4 +93,4 @@ def test_module_and_installed_command_report_one_version(tmp_path: Path) -> None capture_output=True, check=True, ) - assert output.stdout.strip() == "2.0.0" + assert output.stdout.strip() == "2.0.1" diff --git a/tests/test_fixtures_and_projections.py b/tests/test_fixtures_and_projections.py index a41f7f9..de76b55 100644 --- a/tests/test_fixtures_and_projections.py +++ b/tests/test_fixtures_and_projections.py @@ -71,6 +71,47 @@ def test_valid_fixture_is_closed_matched_and_hash_stable(tmp_path: Path) -> None assert binding.binding_sha256 == second.bindings[0].binding_sha256 +def test_documented_multiline_setup_is_valid_toml(tmp_path: Path) -> None: + text = _fixture_text().replace( + 'setup = "class Point:\\n pass"', + 'setup = """\nclass Point:\n pass\n"""', + ) + path = _write_fixture(tmp_path, text) + + fixture = load_fixture_file(path, repository_root=tmp_path, inventory=INVENTORY) + + assert fixture.bindings[0].setup == "class Point:\n pass\n" + + +def test_toml_literal_strings_are_accepted_in_fixture_values(tmp_path: Path) -> None: + text = _fixture_text().replace( + 'args = ["Point(1, 2)"]', "args = ['Point(1, 2)']" + ).replace( + 'kwargs = { strict = "True" }', "kwargs = { strict = 'True' }" + ) + path = _write_fixture(tmp_path, text) + + fixture = load_fixture_file(path, repository_root=tmp_path, inventory=INVENTORY) + + binding = fixture.bindings[0] + assert binding.args == ("Point(1, 2)",) + assert binding.kwargs == (("strict", "True"),) + + +def test_fixture_syntax_refusal_identifies_the_source_line(tmp_path: Path) -> None: + path = _write_fixture( + tmp_path, + _fixture_text().replace( + 'args = ["Point(1, 2)"]', 'args = ["unterminated]' + ), + ) + + with pytest.raises( + FixtureRefusal, match=r"^FIXTURE_SYNTAX_REFUSED:line=9$" + ): + load_fixture_file(path, repository_root=tmp_path, inventory=INVENTORY) + + def test_fixture_rendering_preserves_reviewed_source_without_executing_it( tmp_path: Path, ) -> None: @@ -167,6 +208,39 @@ def test_inventory_matching_refuses_duplicate_stale_unmatched_and_ambiguous( load_fixture_file(path, repository_root=tmp_path, inventory=inventory) +def test_stale_fixture_reports_the_exact_drift_without_changing_its_code( + tmp_path: Path, +) -> None: + path = _write_fixture( + tmp_path, + _fixture_text().replace("line = 7", "line = 8").replace("column = 4", "column = 9"), + ) + + with pytest.raises(FixtureRefusal) as captured: + load_fixture_file(path, repository_root=tmp_path, inventory=INVENTORY) + + refusal = captured.value + assert refusal.code == "FIXTURE_STALE_REFUSED" + assert str(refusal) == "FIXTURE_STALE_REFUSED" + assert refusal.detail == { + "binding": { + "api": "attrs.has", + "column": 9, + "file": "src/app.py", + "line": 8, + }, + "inventory_candidates": [ + { + "api": "attrs.has", + "column": 4, + "file": "src/app.py", + "line": 7, + "mismatched_fields": ["line", "column"], + } + ], + } + + def test_source_and_projection_caps_fail_closed(tmp_path: Path) -> None: too_many_args = ", ".join('"1"' for _ in range(65)) path = _write_fixture( @@ -264,6 +338,38 @@ def test_suggestions_are_deterministic_contextual_warn_and_never_overwrite( assert second_destination.read_text(encoding="utf-8") == text +def test_rich_result_suggestion_requests_a_projection_without_inventing_one( + tmp_path: Path, +) -> None: + destination = tmp_path / "breakcheck.fixtures.toml" + + suggest_fixtures( + destination, + [ + { + "file": "src/app.py", + "line": 11, + "column": 4, + "api": "attrs.define", + "nearby_source": "attrs.define(slots=True)", + "coverage_bucket": "G3_UNNORMALIZABLE", + "reason_code": "UNSTABLE_OBSERVATION_REFUSED", + "raw_type": "type", + "projection_required": True, + } + ], + repository_root=tmp_path, + ) + + rendered = destination.read_text(encoding="utf-8") + assert "# coverage_bucket: G3_UNNORMALIZABLE" in rendered + assert "# reason_code: UNSTABLE_OBSERVATION_REFUSED" in rendered + assert "# raw_type: type" in rendered + assert "# Projection must reference outcome" in rendered + assert 'projection = ""' in rendered + assert "type(outcome).__name__" not in rendered + + def test_metrics_require_observed_inputs_and_do_not_invent_values() -> None: assert fixture_yield(3, 4) == 75.0 assert valid(2, 3) == pytest.approx(66.66666666666667) diff --git a/tests/test_production_hardening.py b/tests/test_production_hardening.py index 9adf123..a5daadc 100644 --- a/tests/test_production_hardening.py +++ b/tests/test_production_hardening.py @@ -533,6 +533,33 @@ def test_symlinked_wheel_refuses(tmp_path): envs._local_wheel(wheelhouse, "sample", "1.0") +def test_failed_offline_install_explains_that_the_wheelhouse_needs_dependencies( + tmp_path, +): + wheelhouse = tmp_path / "wheelhouse" + wheelhouse.mkdir() + (wheelhouse / "sample-1.0-py3-none-any.whl").write_bytes(b"placeholder") + + def failed_install(_argv, **_kwargs): + return SimpleNamespace(returncode=1, stdout="", stderr="No matching distribution") + + with pytest.raises(envs.EnvironmentRefusal) as captured: + envs._install( + wheelhouse, + "sample", + "1.0", + False, + runner=failed_install, + environment=tmp_path, + ) + + assert captured.value.code == "ENVIRONMENT_INSTALL_REFUSED" + assert captured.value.detail == { + "requirement": "sample==1.0", + "wheelhouse_requirement": "complete_dependency_closure", + } + + def test_build_venv_rolls_back_both_environments_on_second_install_failure( monkeypatch, tmp_path ): diff --git a/tests/test_project_metadata_and_demo.py b/tests/test_project_metadata_and_demo.py index 879022e..1b46f59 100644 --- a/tests/test_project_metadata_and_demo.py +++ b/tests/test_project_metadata_and_demo.py @@ -15,6 +15,7 @@ import pytest +from breakcheck import cli from breakcheck.demo import run_demo try: @@ -75,14 +76,14 @@ def _offline_wheelhouse(tmp_path: Path) -> Path: _write_packaging_wheel( wheelhouse, "21.3", - "def canonicalize_version(version, strip_trailing_zero=True):\n" - " return version.rstrip('.0') if strip_trailing_zero else version\n", + "def canonicalize_version(version):\n" + " return version.rstrip('.0')\n", ) _write_packaging_wheel( wheelhouse, "22.0", - "def canonicalize_version(version):\n" - " return version.rstrip('.0')\n", + "def canonicalize_version(version, strip_trailing_zero=True):\n" + " return version.rstrip('.0') if strip_trailing_zero else version\n", ) return wheelhouse @@ -95,6 +96,22 @@ def _line_value(output: str, name: str) -> Path: raise AssertionError(f"missing {prefix!r} in demo output:\n{output}") +def _assert_expected_demo_report(report: dict[str, object]) -> None: + assert report["schema_version"] == 2 + payload = report["payload"] + assert payload["current_version"] == "21.3" + assert payload["new_version"] == "22.0" + assert payload["summary"]["changed"] == 1 + assert len(payload["findings"]) == 1 + finding = payload["findings"][0] + assert finding["verdict"] == "CHANGED" + assert finding["old"]["kind"] == "exception" + assert finding["old"]["exception_class"] == "TypeError" + assert "unexpected keyword argument" in finding["old"]["payload"][0] + assert finding["new"]["kind"] == "value" + assert finding["new"]["payload"] == "1.0.0" + + def _python_without_build_backend(tmp_path: Path, executable: Path | None = None) -> Path: wrapper = tmp_path / "python-without-build-backend" selected = Path(sys.executable) if executable is None else executable @@ -125,7 +142,7 @@ def _python_without_build_backend(tmp_path: Path, executable: Path | None = None def test_current_project_metadata_and_public_artifacts_are_declared(): project = tomllib.loads((ROOT / "pyproject.toml").read_text()) - assert project["project"]["version"] == "2.0.0" + assert project["project"]["version"] == "2.0.1" assert project["project"]["authors"] == [{"name": "ViDale Lovett"}] assert project["project"]["urls"] == { "Homepage": "https://github.com/lovettsendit/breakcheck", @@ -134,7 +151,7 @@ def test_current_project_metadata_and_public_artifacts_are_declared(): "Changelog": "https://github.com/lovettsendit/breakcheck/blob/main/CHANGELOG.md", } assert "Copyright (c) 2026 ViDale Lovett and contributors" in (ROOT / "LICENSE").read_text() - assert "## 2.0.0 - 2026-08-26" in (ROOT / "CHANGELOG.md").read_text() + assert "## 2.0.1 - 2026-08-26" in (ROOT / "CHANGELOG.md").read_text() manifest = (ROOT / "MANIFEST.in").read_text() assert "include SKILL.md" in manifest assert "graft examples" in manifest @@ -179,17 +196,7 @@ def test_demo_uses_installed_breakcheck_without_a_build_backend(tmp_path: Path): assert root.is_dir() assert report_path.is_file() assert evidence_path.is_file() - report = json.loads(report_path.read_text()) - assert report["schema_version"] == 2 - assert report["payload"]["summary"]["changed"] == 1 - assert len(report["payload"]["findings"]) == 1 - finding = report["payload"]["findings"][0] - assert finding["verdict"] == "CHANGED" - assert finding["old"]["kind"] == "value" - assert finding["old"]["payload"] == "1.0.0" - assert finding["new"]["kind"] == "exception" - assert finding["new"]["exception_class"] == "TypeError" - assert "unexpected keyword argument" in finding["new"]["payload"][0] + _assert_expected_demo_report(json.loads(report_path.read_text())) finally: shutil.rmtree(root, ignore_errors=True) @@ -212,6 +219,7 @@ def test_demo_source_checkout_fallback_does_not_require_a_build_backend(tmp_path env=os.environ | { "BREAKCHECK_DEMO_WHEELHOUSE": str(wheelhouse), + "BREAKCHECK_DEMO_KEEP": "1", "PYTHON": str(python), "PYTHONPATH": "", }, @@ -224,6 +232,14 @@ def test_demo_source_checkout_fallback_does_not_require_a_build_backend(tmp_path assert "BUILD_BACKEND_MUST_NOT_RUN" not in result.stderr assert "CALLER_ENVIRONMENT_MUST_NOT_BE_MODIFIED" not in result.stderr assert "DEMO_VERDICT=PASS" in result.stdout + root = _line_value(result.stdout, "DEMO_ROOT") + report_path = _line_value(result.stdout, "REPORT_PATH") + try: + assert root.is_dir() + assert report_path.is_file() + _assert_expected_demo_report(json.loads(report_path.read_text())) + finally: + shutil.rmtree(root, ignore_errors=True) def test_builtin_demo_refuses_an_unverified_artifact_bundle(tmp_path: Path) -> None: @@ -236,3 +252,34 @@ def write_unverified_bundle(arguments) -> int: with pytest.raises(ValueError, match="^DEMO_VERIFICATION_REFUSED$"): run_demo(tmp_path / "demo", write_unverified_bundle) + + +def test_builtin_demo_accepts_a_relative_output_root( + tmp_path: Path, monkeypatch, capsys +) -> None: + """Changing into the demo repository must not invalidate output paths.""" + + monkeypatch.chdir(tmp_path) + + assert cli.main(["demo", "--output-root", ".breakcheck/demo"]) == 0 + + captured = capsys.readouterr() + assert "CHANGED" in captured.out + assert "Traceback" not in captured.err + assert (tmp_path / ".breakcheck" / "demo" / "report.json").is_file() + assert (tmp_path / ".breakcheck" / "demo" / "evidence.json").is_file() + + +def test_builtin_demo_reports_a_bounded_refusal_without_a_traceback( + tmp_path: Path, capsys +) -> None: + """Expected demo refusals are CLI results, not unhandled exceptions.""" + + output_root = tmp_path / "existing" + output_root.mkdir() + + assert cli.main(["demo", "--output-root", str(output_root)]) == 2 + + captured = capsys.readouterr() + assert captured.err.strip() == "DEMO_REFUSED:DEMO_OUTPUT_EXISTS_REFUSED" + assert "Traceback" not in captured.err diff --git a/tests/test_release_automation.py b/tests/test_release_automation.py index ff3484c..402993a 100644 --- a/tests/test_release_automation.py +++ b/tests/test_release_automation.py @@ -62,8 +62,8 @@ def test_release_workflow_fails_closed_before_trusted_publishing() -> None: id_token_permission = "id-token" + ":" assert assignment_marker not in release.replace(id_token_permission, "") assert "draft" in release and "prerelease" in release - assert "breakcheck-2.0.0-py3-none-any.whl" in release - assert "breakcheck-2.0.0.tar.gz" in release + assert "breakcheck-2.0.1-py3-none-any.whl" in release + assert "breakcheck-2.0.1.tar.gz" in release assert "sha256sum" in release assert "test -z" not in release assert release.count("id-token: write") == 1 diff --git a/tests/test_release_contract.py b/tests/test_release_contract.py index f6d54d3..45df4e1 100644 --- a/tests/test_release_contract.py +++ b/tests/test_release_contract.py @@ -70,6 +70,59 @@ def test_platform_refusal(monkeypatch, tmp_path, capsys): assert result == 2 assert 'BUILD_REFUSED:' + PLATFORM_REFUSAL in capsys.readouterr().err + +@pytest.mark.parametrize("value", ("0", "-1", "100.1", "nan")) +def test_min_coverage_is_rejected_before_execution_without_a_traceback( + value, capsys +): + module = importlib.import_module(CLI_MODULE) + + with pytest.raises(SystemExit) as exc: + module.main(["coverage-probe@1.0", "--min-coverage", value]) + + assert int(exc.value.code or 0) == 2 + error = capsys.readouterr().err + assert "MIN_COVERAGE_REFUSED" in error + assert "Traceback" not in error + + +@pytest.mark.parametrize( + ("distribution", "import_root"), + [ + ("PyYAML", "yaml"), + ("beautifulsoup4", "bs4"), + ("Pillow", "PIL"), + ("python-dateutil", "dateutil"), + ], +) +def test_known_distribution_names_resolve_to_their_public_import_roots( + monkeypatch, distribution, import_root +): + module = importlib.import_module(CLI_MODULE) + monkeypatch.setattr( + module._metadata, + "distribution", + lambda _name: (_ for _ in ()).throw(module._metadata.PackageNotFoundError()), + ) + + assert module._import_root(distribution) == import_root + + +def test_duplicate_exact_wheels_are_a_bounded_cli_refusal(monkeypatch, capsys): + module = importlib.import_module(CLI_MODULE) + monkeypatch.setattr( + module, + "_build", + lambda _args: (_ for _ in ()).throw( + RuntimeError("AMBIGUOUS_WHEEL_REFUSED:sample==1.0") + ), + ) + + assert module.main(["sample@2.0", "--wheelhouse", "wheelhouse"]) == 2 + error = capsys.readouterr().err + assert error.strip() == "BUILD_REFUSED:AMBIGUOUS_WHEEL_REFUSED" + assert "Traceback" not in error + def test_report_tamper(): verify = getattr(importlib.import_module(VERIFIER_MODULE), VERIFIER_FUNCTION) with pytest.raises(ValueError): diff --git a/tests/test_replay_protocol_and_coverage.py b/tests/test_replay_protocol_and_coverage.py index d8cf121..a1b7eff 100644 --- a/tests/test_replay_protocol_and_coverage.py +++ b/tests/test_replay_protocol_and_coverage.py @@ -77,8 +77,65 @@ def test_rich_object_is_refused_without_repr_fallback(): def test_network_timeout_and_output_limit_are_typed_refusals(): + legacy_allocation = executor.run_snippet_isolated( + snippet_source=( + "import socket\n" + "candidate = socket.socket()\n" + "candidate.close()\n" + "print(repr(True))\n" + ) + ) + assert legacy_allocation["returncode"] == 0 + assert legacy_allocation["stdout"].strip() == b"True" + + allocation = executor.run_typed_snippet_isolated( + snippet_source=( + "import socket\n" + "candidate = socket.socket()\n" + "outcome = candidate.family == socket.AF_INET\n" + "candidate.close()\n" + ) + ) + assert allocation["status"] == "VALUE" + assert allocation["observation"]["payload"] is True + + local_pair = executor.run_typed_snippet_isolated( + snippet_source=( + "import socket\n" + "left, right = socket.socketpair()\n" + "left.send(b'x')\n" + "outcome = right.recv(1)\n" + ) + ) + assert local_pair["status"] == "NETWORK_REFUSED" + assert local_pair["reason_code"] == "NETWORK_ACCESS_REFUSED" + + raw_socket = executor.run_typed_snippet_isolated( + snippet_source=( + "import socket\n" + "outcome = socket.socket(socket.AF_INET, socket.SOCK_RAW, socket.IPPROTO_ICMP)\n" + ) + ) + assert raw_socket["status"] == "NETWORK_REFUSED" + assert raw_socket["reason_code"] == "NETWORK_ACCESS_REFUSED" + + public_bind = executor.run_typed_snippet_isolated( + snippet_source=( + "import socket\n" + "candidate = socket.socket()\n" + "candidate.bind(('0.0.0.0', 0))\n" + "outcome = 'unreachable'\n" + ) + ) + assert public_bind["status"] == "NETWORK_REFUSED" + network = executor.run_typed_snippet_isolated( - snippet_source="import socket\noutcome = socket.socket()\n" + snippet_source=( + "import socket\n" + "candidate = socket.socket()\n" + "candidate.connect(('127.0.0.1', 9))\n" + "outcome = 'unreachable'\n" + ) ) assert network["status"] == "NETWORK_REFUSED" assert network["observation"] is None @@ -87,16 +144,24 @@ def test_network_timeout_and_output_limit_are_typed_refusals(): caught_network = executor.run_typed_snippet_isolated( snippet_source=( "import socket\n" + "candidate = socket.socket()\n" "try:\n" - " socket.socket()\n" + " candidate.connect(('127.0.0.1', 9))\n" "except BaseException:\n" " pass\n" + "candidate.close()\n" "outcome = 'attempt was caught'\n" ) ) assert caught_network["status"] == "NETWORK_REFUSED" assert caught_network["observation"] is None + name_resolution = executor.run_typed_snippet_isolated( + snippet_source="import socket\noutcome = socket.getaddrinfo('example.com', 443)\n" + ) + assert name_resolution["status"] == "NETWORK_REFUSED" + assert name_resolution["reason_code"] == "NETWORK_ACCESS_REFUSED" + timeout = executor.run_typed_snippet_isolated( snippet_source="import time\ntime.sleep(5)\noutcome = 1\n", timeout_seconds=0.1, diff --git a/tests/test_revision_cli.py b/tests/test_revision_cli.py index 7460934..a25373f 100644 --- a/tests/test_revision_cli.py +++ b/tests/test_revision_cli.py @@ -6,11 +6,13 @@ import pytest +from breakcheck import revision_cli from breakcheck.revision_cli import ( RevisionModeRefusal, attest_revision, diff_revisions, ) +from breakcheck.adapters.python.fixtures import FixtureRefusal from breakcheck.revision_report import make_revision_artifact from breakcheck.revision_cli import freeze_revision from breakcheck.report import ci_exit_code @@ -664,3 +666,12 @@ def test_strict_separation_refuses_explicit_checkout_fixtures( strict_separation=True, ) assert not runtime.exists() +def test_fixture_refusal_translation_preserves_structured_detail() -> None: + detail = {"binding": {"file": "app.py", "line": 8}} + + translated = revision_cli._translate( + FixtureRefusal("FIXTURE_STALE_REFUSED", detail=detail) + ) + + assert translated.code == "FIXTURE_STALE_REFUSED" + assert translated.detail == detail