Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/ISSUE_TEMPLATE/bug.yml
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ body:
attributes:
label: Breakcheck version
description: Which Breakcheck version produced this result?
placeholder: 2.0.0
placeholder: 2.0.1
validations:
required: true
- type: input
Expand Down
2 changes: 1 addition & 1 deletion .github/ISSUE_TEMPLATE/compatibility-case.yml
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ body:
attributes:
label: Breakcheck version
description: Which Breakcheck version produced this result?
placeholder: 2.0.0
placeholder: 2.0.1
validations:
required: true
- type: input
Expand Down
2 changes: 1 addition & 1 deletion .github/ISSUE_TEMPLATE/revision-claim-case.yml
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ body:
attributes:
label: Breakcheck version
description: Which Breakcheck version produced this result?
placeholder: 2.0.0
placeholder: 2.0.1
validations:
required: true
- type: input
Expand Down
48 changes: 24 additions & 24 deletions .github/workflows/release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ jobs:
echo "refusing draft or prerelease"
exit 1
fi
if [ "$RELEASE_TAG" != "v2.0.0" ]; then
if [ "$RELEASE_TAG" != "v2.0.1" ]; then
echo "release tag does not match project version"
exit 1
fi
Expand All @@ -51,8 +51,8 @@ jobs:
import tomllib

project = tomllib.loads(pathlib.Path("pyproject.toml").read_text(encoding="utf-8"))["project"]
if project["name"] != "breakcheck" or project["version"] != "2.0.0":
raise SystemExit("project metadata does not match v2.0.0")
if project["name"] != "breakcheck" or project["version"] != "2.0.1":
raise SystemExit("project metadata does not match v2.0.1")
if project.get("dependencies") != []:
raise SystemExit("release unexpectedly has runtime dependencies")
PY
Expand All @@ -72,47 +72,47 @@ jobs:
from pathlib import Path

expected = {
"breakcheck-2.0.0-py3-none-any.whl",
"breakcheck-2.0.0.tar.gz",
"breakcheck-2.0.1-py3-none-any.whl",
"breakcheck-2.0.1.tar.gz",
}
actual = {path.name for path in Path("dist").iterdir() if path.is_file()}
if actual != expected:
raise SystemExit(f"unexpected distribution inventory: {sorted(actual)}")
PY
echo "wheel_sha256=$(sha256sum dist/breakcheck-2.0.0-py3-none-any.whl | awk '{print $1}')" >> "$GITHUB_OUTPUT"
echo "sdist_sha256=$(sha256sum dist/breakcheck-2.0.0.tar.gz | awk '{print $1}')" >> "$GITHUB_OUTPUT"
echo "wheel_sha256=$(sha256sum dist/breakcheck-2.0.1-py3-none-any.whl | awk '{print $1}')" >> "$GITHUB_OUTPUT"
echo "sdist_sha256=$(sha256sum dist/breakcheck-2.0.1.tar.gz | awk '{print $1}')" >> "$GITHUB_OUTPUT"
- name: Validate metadata, inventory, privacy, and installed CLI
run: |
set -euo pipefail
python -m twine check --strict dist/*
bash scripts/scan_artifacts.sh dist/breakcheck-2.0.0-py3-none-any.whl
bash scripts/scan_artifacts.sh dist/breakcheck-2.0.0.tar.gz
bash scripts/scan_artifacts.sh dist/breakcheck-2.0.1-py3-none-any.whl
bash scripts/scan_artifacts.sh dist/breakcheck-2.0.1.tar.gz
python - <<'PY'
import email
import tarfile
import zipfile

wheel = "dist/breakcheck-2.0.0-py3-none-any.whl"
sdist = "dist/breakcheck-2.0.0.tar.gz"
wheel = "dist/breakcheck-2.0.1-py3-none-any.whl"
sdist = "dist/breakcheck-2.0.1.tar.gz"
with zipfile.ZipFile(wheel) as archive:
metadata_name = next(name for name in archive.namelist() if name.endswith(".dist-info/METADATA"))
metadata = email.message_from_bytes(archive.read(metadata_name))
if metadata["Name"] != "breakcheck" or metadata["Version"] != "2.0.0":
if metadata["Name"] != "breakcheck" or metadata["Version"] != "2.0.1":
raise SystemExit("wheel identity mismatch")
with tarfile.open(sdist, "r:gz") as archive:
names = set(archive.getnames())
required = {
"breakcheck-2.0.0/README.md",
"breakcheck-2.0.0/SECURITY.md",
"breakcheck-2.0.0/SKILL.md",
"breakcheck-2.0.0/examples/github-actions.yml",
"breakcheck-2.0.0/scripts/scan_artifacts.sh",
"breakcheck-2.0.1/README.md",
"breakcheck-2.0.1/SECURITY.md",
"breakcheck-2.0.1/SKILL.md",
"breakcheck-2.0.1/examples/github-actions.yml",
"breakcheck-2.0.1/scripts/scan_artifacts.sh",
}
if not required <= names:
raise SystemExit("source distribution inventory incomplete")
PY
python -m venv "$RUNNER_TEMP/breakcheck-release-smoke"
"$RUNNER_TEMP/breakcheck-release-smoke/bin/python" -m pip install --no-deps dist/breakcheck-2.0.0-py3-none-any.whl
"$RUNNER_TEMP/breakcheck-release-smoke/bin/python" -m pip install --no-deps dist/breakcheck-2.0.1-py3-none-any.whl
(
cd "$RUNNER_TEMP"
"$RUNNER_TEMP/breakcheck-release-smoke/bin/breakcheck" --help
Expand All @@ -122,10 +122,10 @@ jobs:
- name: Transfer only validated distributions
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: breakcheck-2.0.0-distributions
name: breakcheck-2.0.1-distributions
path: |
dist/breakcheck-2.0.0-py3-none-any.whl
dist/breakcheck-2.0.0.tar.gz
dist/breakcheck-2.0.1-py3-none-any.whl
dist/breakcheck-2.0.1.tar.gz
if-no-files-found: error
include-hidden-files: false
retention-days: 1
Expand All @@ -144,7 +144,7 @@ jobs:
- name: Receive validated distributions
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: breakcheck-2.0.0-distributions
name: breakcheck-2.0.1-distributions
path: dist
digest-mismatch: error
- name: Recheck transferred artifact identity
Expand All @@ -154,8 +154,8 @@ jobs:
run: |
set -euo pipefail
test "$(find dist -maxdepth 1 -type f | wc -l | tr -d ' ')" = 2
test "$(sha256sum dist/breakcheck-2.0.0-py3-none-any.whl | awk '{print $1}')" = "$EXPECTED_WHEEL_SHA256"
test "$(sha256sum dist/breakcheck-2.0.0.tar.gz | awk '{print $1}')" = "$EXPECTED_SDIST_SHA256"
test "$(sha256sum dist/breakcheck-2.0.1-py3-none-any.whl | awk '{print $1}')" = "$EXPECTED_WHEEL_SHA256"
test "$(sha256sum dist/breakcheck-2.0.1.tar.gz | awk '{print $1}')" = "$EXPECTED_SDIST_SHA256"
- name: Publish with Trusted Publishing and PEP 740 attestations
uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33
with:
Expand Down
13 changes: 13 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,19 @@

All notable changes to Breakcheck are documented here.

## 2.0.1 - 2026-08-26

- Resolved relative demo output roots before entering the generated repository.
- Reported expected demo refusals as bounded CLI errors without Python tracebacks.
- Accepted documented multiline and literal TOML fixture strings and reported syntax-error line numbers.
- Rejected invalid coverage thresholds before replay with `MIN_COVERAGE_REFUSED`.
- Added replay-backed fixture suggestions for deterministic rich results while leaving projection choice under explicit review.
- Reported exact line and column drift for stale fixture bindings without changing refusal codes.
- Required the shell demonstration to prove the expected `packaging` 21.3-to-22.0 observations before reporting success.
- Allowed only inert INET/INET6 non-raw socket allocation while refusing local socket pairs, bind, connect, name-resolution, and other socket operations.
- Resolved common distribution/import-name differences for PyYAML, Beautiful Soup, Pillow, and python-dateutil.
- Bounded offline installation and duplicate-wheel failures with actionable refusal codes.

## 2.0.0 - 2026-08-26

- Added PyPI-ready project metadata and trusted release automation.
Expand Down
16 changes: 14 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,8 @@ python -m pip download --only-binary=:all: --dest wheelhouse 'attrs==24.2.0'

Download exact versions in separate commands. Asking `pip download` to resolve two versions of the same distribution in one command can produce a dependency-resolution error.

Do not use `--no-deps`: the wheelhouse must contain the complete transitive dependency closure needed by both versions. If offline installation cannot resolve that closure, Breakcheck refuses with `ENVIRONMENT_INSTALL_REFUSED` and identifies the requirement without exposing pip output or local paths.

Run the comparison:

```console
Expand Down Expand Up @@ -116,7 +118,7 @@ Every discovered candidate reaches exactly one terminal bucket:

### Generate fixture suggestions

When source arguments cannot be resolved statically, generate a reviewable skeleton without creating replay environments:
For a fast static pass, run `--suggest-fixtures` without `--wheelhouse`. This scans for unresolved G2 arguments and generates reviewable skeletons without creating replay environments:

```console
breakcheck attrs@24.2.0 \
Expand All @@ -125,6 +127,16 @@ breakcheck attrs@24.2.0 \

The generated file identifies each unresolved call by repository-relative file, line, column, API, and nearby source. A human or coding tool fills in concrete expressions and changes `fixture_authored_by` from `unknown` to `human` or `agent`.

To also find deterministic rich results that reach `G3_UNNORMALIZABLE`, supply the explicit wheelhouse used for comparison:

```console
breakcheck attrs@24.2.0 \
--wheelhouse wheelhouse \
--suggest-fixtures breakcheck.fixtures.toml
```

With `--wheelhouse`, Breakcheck performs isolated replay in both dependency environments. A repeatable rich result adds a skeleton marked `G3_UNNORMALIZABLE` with `projection = ""`. Breakcheck does not invent a projection: an agent or human must fill in a stable expression that references `outcome`, then present the fixture diff for human review. Impure or nondeterministic calls remain excluded from replay-backed suggestions.

Example:

```toml
Expand Down Expand Up @@ -411,7 +423,7 @@ To build release artifacts:
```console
python -m pip install 'build>=1.2,<2'
python -m build
python -m pip install dist/breakcheck-2.0.0-py3-none-any.whl
python -m pip install dist/breakcheck-2.0.1-py3-none-any.whl
breakcheck --capabilities --json
```

Expand Down
24 changes: 17 additions & 7 deletions SKILL.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,13 @@ description: Use when a Python dependency version changes or when a code change

## Purpose

Use Breakcheck as the deterministic measurement step after proposing a dependency upgrade or a behavior-preserving code change. A coding agent may propose inputs and interpret results; Breakcheck owns replay, comparison, refusal, and evidence.
Use Breakcheck after proposing a dependency upgrade or behavior-preserving change. A coding tool may propose inputs; Breakcheck owns replay, comparison, refusal, and evidence.

## Dependency upgrades

Run from the repository root after preparing exact trusted wheels in a local wheelhouse:
Run from the repository root with trusted wheels:

The wheelhouse must include the complete transitive dependency closure for both compared versions. Do not use `pip download --no-deps`.

```console
breakcheck PACKAGE@NEW_VERSION \
Expand All @@ -22,13 +24,21 @@ breakcheck PACKAGE@NEW_VERSION \
--json --ci
```

If coverage is limited by unresolved arguments, generate proposals with:
For fast G2 proposals, run without `--wheelhouse`:

```console
breakcheck PACKAGE@NEW_VERSION --suggest-fixtures breakcheck.fixtures.toml
```

Fill only fixtures that can be justified from repository context, mark `fixture_authored_by = "agent"`, and present the fixture diff for human review before replay.
To also suggest fixtures for repeatable `G3_UNNORMALIZABLE` rich results, use isolated replay against the exact wheelhouse:

```console
breakcheck PACKAGE@NEW_VERSION \
--wheelhouse wheelhouse \
--suggest-fixtures breakcheck.fixtures.toml
```

Replay-backed rich-result skeletons contain `projection = ""`. Breakcheck does not supply the projection. Fill it only with a stable expression referencing `outcome`, mark `fixture_authored_by = "agent"`, and present the fixture diff for human review. Impure and nondeterministic calls remain excluded.

## Behavior-preserving code changes

Expand All @@ -40,7 +50,7 @@ Fixtures must be authored, reviewed, and committed against the base revision bef
4. Run `breakcheck attest` against the changed revision.
5. Report every disposition verbatim to the human, including all unverifiable and out-of-scope counts.

An explicit comparison between committed revisions is also available:
Compare committed revisions:

```console
breakcheck diff \
Expand All @@ -51,7 +61,7 @@ breakcheck diff \
--evidence .breakcheck/revision-evidence.json
```

For an explicit preservation claim, create a reviewed `breakcheck.claim.toml` and run:
Attest a reviewed `breakcheck.claim.toml`:

```console
breakcheck attest \
Expand All @@ -78,4 +88,4 @@ breakcheck attest \
- Never modify verdict or verification logic as part of the change being verified.
- Inspect and sanitize artifacts before sending them to an external service; they may contain source locations, replay source, arguments, setup, projections, and observed values.

Report the exact Breakcheck command, exit code, verdict counts, and artifact paths alongside the ordinary project test results.
Report the command, exit code, verdict counts, artifact paths, and project tests.
2 changes: 1 addition & 1 deletion examples/github-actions.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ jobs:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405
with: {python-version: "3.13"}
- run: python -m pip install "breakcheck==2.0.0" "$DEPENDENCY==$CURRENT_VERSION"
- run: python -m pip install "breakcheck==2.0.1" "$DEPENDENCY==$CURRENT_VERSION"
- run: mkdir wheelhouse
- run: 'python -m pip download --only-binary=:all: -d wheelhouse "$DEPENDENCY==$CURRENT_VERSION"'
- run: 'python -m pip download --only-binary=:all: -d wheelhouse "$DEPENDENCY==$PROPOSED_VERSION"'
Expand Down
16 changes: 14 additions & 2 deletions examples/run_demo.sh
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@ if [ -z "${BREAKCHECK_DEMO_WHEELHOUSE:-}" ]; then
fi

TOOL_PYTHON="$PYTHON"
if "$PYTHON" -c 'import breakcheck; raise SystemExit(0 if breakcheck.__version__ == "2.0.0" else 1)' 2>/dev/null; then
if "$PYTHON" -c 'import breakcheck; raise SystemExit(0 if breakcheck.__version__ == "2.0.1" else 1)' 2>/dev/null; then
BREAKCHECK_IMPORT_ROOT=$("$PYTHON" -c 'from pathlib import Path; import breakcheck; print(Path(breakcheck.__file__).resolve().parent.parent)')
else
BREAKCHECK_IMPORT_ROOT="$CHECKOUT/src"
Expand Down Expand Up @@ -133,10 +133,22 @@ if report.get("schema_version") != 2 or report.get("artifact_kind") != "dependen
payload = report.get("payload", {})
findings = payload.get("findings")
summary = payload.get("summary")
if payload.get("current_version") != "21.3" or payload.get("new_version") != "22.0":
raise SystemExit("BREAKCHECK_DEMO_REFUSED: expected packaging 21.3 to 22.0")
if not isinstance(findings, list) or len(findings) != 1:
raise SystemExit("BREAKCHECK_DEMO_REFUSED: expected exactly one finding")
if findings[0].get("verdict") != "CHANGED":
finding = findings[0]
if finding.get("verdict") != "CHANGED":
raise SystemExit("BREAKCHECK_DEMO_REFUSED: expected CHANGED finding")
old = finding.get("old")
new = finding.get("new")
if not isinstance(old, dict) or old.get("kind") != "exception" or old.get("exception_class") != "TypeError":
raise SystemExit("BREAKCHECK_DEMO_REFUSED: unexpected packaging 21.3 observation")
old_payload = old.get("payload")
if not isinstance(old_payload, list) or not old_payload or "unexpected keyword argument" not in str(old_payload[0]):
raise SystemExit("BREAKCHECK_DEMO_REFUSED: missing changed-behavior detail")
if not isinstance(new, dict) or new.get("kind") != "value" or new.get("payload") != "1.0.0":
raise SystemExit("BREAKCHECK_DEMO_REFUSED: unexpected packaging 22.0 observation")
if not isinstance(summary, dict) or summary.get("changed") != 1:
raise SystemExit("BREAKCHECK_DEMO_REFUSED: expected summary.changed == 1")
PY
Expand Down
2 changes: 1 addition & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"

[project]
name = "breakcheck"
version = "2.0.0"
version = "2.0.1"
description = "Deterministic behavioral comparison for Python dependency and code changes."
requires-python = ">=3.10,<3.14"
dependencies = []
Expand Down
2 changes: 1 addition & 1 deletion src/breakcheck/__init__.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
'Public command surface.'
__version__ = "2.0.0"
__version__ = "2.0.1"

from .cli import main

Expand Down
21 changes: 19 additions & 2 deletions src/breakcheck/adapters/python/envs.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,15 @@
"new_version", "package", "version", "wheelhouse")


class EnvironmentRefusal(RuntimeError):
"""A bounded environment-construction failure safe for CLI reporting."""

def __init__(self, code, *, detail=None):
self.code = code
self.detail = None if detail is None else dict(detail)
super().__init__(code)


def _normalized_distribution(value):
return re.sub(r"[-_.]+", "-", str(value)).lower()

Expand Down Expand Up @@ -74,12 +83,20 @@ def _install(wheelhouse, package, version, allow_network, runner=None, environme
else:
argv.append(requirement)
execute = runner or subprocess.run
refusal_detail = {
"requirement": requirement,
"wheelhouse_requirement": "complete_dependency_closure",
}
try:
result = execute(argv, check=False, capture_output=True, text=True, shell=False)
except Exception as exc:
raise RuntimeError('INSTALL_FAILED') from exc
raise EnvironmentRefusal(
"ENVIRONMENT_INSTALL_REFUSED", detail=refusal_detail
) from exc
if getattr(result, "returncode", 1) != 0:
raise RuntimeError('INSTALL_FAILED')
raise EnvironmentRefusal(
"ENVIRONMENT_INSTALL_REFUSED", detail=refusal_detail
)
return result


Expand Down
9 changes: 7 additions & 2 deletions src/breakcheck/adapters/python/executor.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,9 +28,14 @@
_INHERITED_ENVIRONMENT_KEYS = {"PATH", "SYSTEMROOT", "TMPDIR", "TEMP", "TMP"}
_NETWORK_GUARD = (
"import sys as _guard_sys\n"
"import socket as _guard_socket\n"
"def _guard_audit(event, args):\n"
" if event.startswith('socket.'):\n"
" raise RuntimeError('NETWORK_ACCESS_REFUSED')\n"
" if not event.startswith('socket.'):\n"
" return\n"
" if event == 'socket.__new__' and len(args) >= 4:\n"
" if args[1] in (_guard_socket.AF_INET, _guard_socket.AF_INET6) and args[2] in (_guard_socket.SOCK_STREAM, _guard_socket.SOCK_DGRAM) and args[3] == 0:\n"
" return\n"
" raise RuntimeError('NETWORK_ACCESS_REFUSED')\n"
"_guard_sys.addaudithook(_guard_audit)\n"
)

Expand Down
Loading