diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 08e2a20..d195e5b 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -7,3 +7,7 @@ /action.yml @SollanSystems /.github/workflows/ @SollanSystems /.github/CODEOWNERS @SollanSystems +# The anchor carries the head a gate corroborates, so an actor who can edit it +# re-points the anchor at a head they had attested — a gate-defining path in +# exactly the sense /loop/ and /action.yml are (ADR 0002 amendment, 2026-07-29). +loop-anchor.json @SollanSystems diff --git a/.github/workflows/attest.yml b/.github/workflows/attest.yml index 7b3de20..5661646 100644 --- a/.github/workflows/attest.yml +++ b/.github/workflows/attest.yml @@ -107,3 +107,85 @@ jobs: fi echo "chain head: $OBSERVED_HEAD" >> "$GITHUB_STEP_SUMMARY" echo "attestation: $ATTESTATION" >> "$GITHUB_STEP_SUMMARY" + + - name: resolve the attestation we just minted + id: resolve + # HONEST SCOPE: this is a WITHIN-RUN exercise. The anchor is written from a head + # minted seconds earlier in this same job, so it proves the resolve path works + # against the real gh and the real index — it does NOT prove cross-run detection, + # because this workflow seeds an ephemeral RUNNER_TEMP workspace and there is no + # persistent store to anchor across runs. Do not read this green as a cross-run + # proof. A true cross-run dogfood needs a persistent store and is out of scope. + # + # It runs THE SHIPPED script — the same entry point action.yml's `anchor` input + # calls. No verify-and-classify logic is reimplemented here: if it were, the only + # thing getting real-gh mileage would be a duplicate no adopter ever runs. + env: + GH_TOKEN: ${{ github.token }} + HEAD: ${{ steps.gate.outputs.chain-head }} + run: | + python -B - "$HEAD" "${RUNNER_TEMP}/loop-anchor.json" <<'PY' + import json + import sys + + json.dump({"schema": "loop-engineer/anchor@1", "chain_head": sys.argv[1]}, + open(sys.argv[2], "w")) + PY + # The attestation index is eventually consistent; bound the wait rather than + # assuming it, and fail loud when the budget is exhausted. The retry wraps the + # SCRIPT, so `anchor_attestation_unavailable` — its honest answer while the + # index catches up — is what is being retried. + for attempt in 1 2 3 4 5 6; do + if python -B scripts/action_anchor_resolve.py \ + --anchor "${RUNNER_TEMP}/loop-anchor.json" \ + --repo "$GITHUB_REPOSITORY" \ + --signer-workflow SollanSystems/loop-engineer/.github/workflows/attest.yml \ + --runner-temp "$RUNNER_TEMP" \ + --github-output "$GITHUB_OUTPUT"; then + break + fi + if [ "$attempt" = "6" ]; then + echo "::error::action_anchor_resolve.py never corroborated the attestation we just minted" + exit 1 + fi + sleep 5 + done + + - name: assert the resolved attestation proves D1 landed + env: + OUTCOME: ${{ steps.resolve.outputs.anchor-outcome }} + SUBJECT: ${{ steps.resolve.outputs.subject-path }} + PREDICATE: ${{ steps.resolve.outputs.predicate-path }} + WS: ${{ steps.seed.outputs.workspace }} + run: | + if [ "$OUTCOME" != "corroborated" ]; then + echo "::error::anchor-outcome was '$OUTCOME', expected 'corroborated'" + exit 1 + fi + python -B - "$SUBJECT" "$PREDICATE" <<'PY' + import hashlib + import json + import pathlib + import sys + + subject = pathlib.Path(sys.argv[1]) + predicate = json.loads(pathlib.Path(sys.argv[2]).read_text(encoding="utf-8")) + raw = subject.read_bytes() + # D1's byte form, regenerated by loop.verdict.subject_bytes from the carried + # head alone: 64 bytes, lowercase hex, no trailing newline. + assert subject.name == "loop-chain-head", subject.name + assert len(raw) == 64, len(raw) + assert not raw.endswith(b"\n") + head = raw.decode("ascii") + assert head == predicate["chain"]["head"], (head, predicate["chain"]["head"]) + # THE crispest proof that D1 landed. In all three attestations this repo minted + # before this slice, subject[0].digest.sha256 EQUALLED predicate.chain.head. The + # subject digest is now the hash of a FILE CONTAINING the head, so it must differ. + subject_digest = hashlib.sha256(raw).hexdigest() + assert subject_digest != head, "subject digest still equals the head: D1 did not land" + print(f"subject sha256 {subject_digest} != chain head {head}") + PY + # The first live exercise of the agreement path, on the same extraction an + # adopter gets. The signer-trust policy already passed inside the script — + # `corroborated` is unreachable without it. + python -B -m loop verdict --compare "$PREDICATE" "$WS" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c0de5cf..dd6944d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -189,6 +189,99 @@ jobs: echo "::error::the always-run anchor step recorded '$GATE_HEAD', not the observed head" exit 1 fi + + - name: Ancestry survives a grown store where head equality cannot + env: + ANCHOR_WS: ${{ steps.seed.outputs.workspace }} + ANCHOR_HEAD: ${{ steps.seed.outputs.head }} + # F3, proven live and within a single run: appending events MOVES the head, so + # feeding run N's head to --expect-chain-head at run N+1 fails by construction. + # The inverted pair below is the only ancestry coverage this repo can honestly + # claim — its attest workspace is ephemeral, so a true cross-run dogfood needs a + # persistent store and is out of scope. + run: | + python -B - "$ANCHOR_WS" <<'PY' + import json + import pathlib + import sys + + sys.path.insert(0, ".") + from loop.events import SQLiteEventStore + + ws = pathlib.Path(sys.argv[1]) + store = SQLiteEventStore(ws / ".loop" / "events.db") + for iteration_id in (2, 3): + store.append("ci-anchor-probe", "iteration_appended", + {"iteration_id": iteration_id, "outcome": "task_passed"}, + actor="ci_anchor_probe") + state_path = ws / ".loop" / "state.json" + state = json.loads(state_path.read_text(encoding="utf-8")) + state["iteration_id"] = 3 + state_path.write_text(json.dumps(state, indent=2) + "\n", encoding="utf-8") + PY + # The anchored head is still an ANCESTOR: exit 0. + python -B -m loop doctor --expect-chain-ancestor "$ANCHOR_HEAD" "$ANCHOR_WS" + # The same digest as an exact head EXPECTATION now fails: exit 1, with the + # equality code and NOT the ancestry code. Inverted so an unexpected pass fails. + set +e + out="$(python -B -m loop doctor --expect-chain-head "$ANCHOR_HEAD" "$ANCHOR_WS" 2>&1)" + status=$? + set -e + if [ "$status" -ne 1 ]; then + echo "::error::expect-chain-head exited $status on a grown store; expected 1" + echo "$out" + exit 1 + fi + case "$out" in + *chain_anchor_not_ancestor*) + echo "::error::the grown store reported chain_anchor_not_ancestor; the two codes must not collapse" + echo "$out"; exit 1 ;; + esac + case "$out" in + *chain_anchor_mismatch*) ;; + *) echo "::error::doctor did not report chain_anchor_mismatch"; echo "$out"; exit 1 ;; + esac + + gates-fallback: + name: gates (structural-fallback leg) + runs-on: ubuntu-latest + # Every other job installs jsonschema, so the pyyaml-only leg — the one the anchor@1 + # and verdict@1 structural hand-checks actually run in — had NO CI cover at all. That + # was measured: loosening a fullmatch to a match in loop/anchor.py kills 4 tests in + # this leg but only 2 with jsonschema installed, because the schema layer masks the + # rest, so a drift between _structural_violation and anchor.schema.json would pass CI. + # Deliberately a SEPARATE job rather than a matrix leg on `gates`: a matrix would + # rename `gates`' check contexts, and this repo's branch ruleset pins its required + # contexts by name. + steps: + - uses: actions/checkout@v7 + + - uses: actions/setup-python@v7 + with: + python-version: "3.12" + + - name: Install gate dependencies (no jsonschema) + # pyyaml is required even here: scripts/validate_frontmatter.py imports yaml + # unconditionally, so a truly bare environment fails collection. + run: python -m pip install --upgrade pip pyyaml pytest + + - name: Assert the leg really ran without jsonschema + # Without this the job silently becomes a duplicate of `gates` the moment + # something adds jsonschema to the install line above. + run: | + python -B - <<'PY' + import importlib.util + import sys + + if importlib.util.find_spec("jsonschema") is not None: + print("::error::jsonschema is installed: this leg is not the fallback leg") + sys.exit(1) + print("jsonschema absent: structural-fallback leg confirmed") + PY + + - name: Test suite (structural fallback) + run: python -B -m pytest -q -p no:cacheprovider scripts + recipe-openhands: name: recipe (openhands) runs-on: ubuntu-latest diff --git a/CHANGELOG.md b/CHANGELOG.md index ff3e67d..81d6170 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,9 +28,11 @@ Statement, and never reads an environment variable — `scripts/test_verdict_pur makes each boundary mechanical. The composite action gains an opt-in `attest` input (default false): it writes -the predicate to the runner temp dir and hands it to `actions/attest` with -`subject-name: loop-chain-head` / `subject-digest: sha256:`, -exposing `attestation-url`/`attestation-id` outputs; a legible permission +the predicate to the runner temp dir and hands it to `actions/attest` as a +`subject-path` — a file whose entire content is the chain head, exactly 64 +lowercase hex bytes with no trailing newline, produced by the single definition +`loop verdict --emit-subject` — exposing `attestation-url`/`attestation-id` +outputs; a legible permission precheck replaces the raw OIDC 403, and an empty chain head skips with a warning rather than shipping a malformed subject. `.github/workflows/attest.yml` mints a real attestation on every push to main over a workspace seeded through @@ -43,8 +45,47 @@ just a signed weakened gate. An agent with ordinary merge rights can loosen `loop/**`/`schemas/**`/`action.yml`/the workflow and then mint a perfectly genuine attestation for the result — the control is code-owner review on those paths, which is in force only once the repository ruleset requires it — and an -unattested chain rewrite is detected at best one run late. Verification (`--compare`, anchor auto-resolution, signer-trust policy) -is slice 4b and does not ship here. +unattested chain rewrite is detected at best one run late. + +**A verdict you can check (slice 4b of tamper-evident provenance).** Verification +ships alongside emission, so this release describes one coherent state rather +than half a mechanism. + +`loop verdict --compare ` compares an attested predicate +against the local projection over four facets — `run_id`, `chain.head`, the whole +`terminal` object, and the verified-evidence digest set — exiting 0 on agreement, +1 on disagreement and 2 on refusal. It accepts a **bare** predicate only: an +in-toto Statement or a `gh --format json` envelope is refused by name with the +documented jq path to unwrap. `signature_checked` is the literal `false` on every +path and there is no flag to flip it — authenticity is `gh attestation verify`'s +job, it runs first, and neither check implies the other. `doctor` and `tool` are +deliberately not compared: both are environment-coupled, so comparing them would +make an honest environment difference read as tampering. + +`loop doctor --expect-chain-ancestor ` (or `--anchor `, resolving +the digest from a tracked `loop-engineer/anchor@1` file) asks the answerable +cross-run question — *was this digest ever my head?* — because +`--expect-chain-head` is exact current-head equality and fails by construction +once a store grows. Ancestry is established by **replay**, recomputing every hash, +never by trusting the stored `event_hash` column: a tamperer who can rewrite the +store can also insert a row bearing the anchored digest. `loop/attestation.py` +adds a pure signer-trust policy over already-verified certificate claims that +**refuses** when a claim it needs is absent, and `scripts/action_anchor_resolve.py` +is the single `gh` call site, fail-closed on anything it cannot confidently +classify. All of it is normative in `reference/repo-os-contract.md` §24. + +**Behavioral change:** the attested subject is now a head-bearing file, so the +three attestations minted before this release carry a different subject form. +They remain valid records of what they were. + +What this does not buy, beyond the limits above: anchor trust is **exactly +ordinary write access** to the anchor file — an actor who can edit it re-points it +at a head they had attested. An attestation can corroborate a carried head but can +never discover one, because GitHub exposes no endpoint that lists attestations +without a subject digest. Attestations are deletable and no retention window is +documented, so a missing one is a typed failure rather than a skip. And the +independent-audit property holds for **public** repositories: a private repository +signs against GitHub's own instance, which has no public transparency log. ## 0.11.0 — 2026-07-26 diff --git a/action.yml b/action.yml index de4a30d..43c37d6 100644 --- a/action.yml +++ b/action.yml @@ -32,6 +32,27 @@ inputs: detection — the gate then only records the head for a later comparison. required: false default: "" + anchor: + description: >- + Path to a tracked loop-engineer/anchor@1 file carrying a previously + attested chain head. The gate corroborates the carried head against the + attestation index and then checks it is still an ANCESTOR of this run's + chain. An attestation can only corroborate a head, never discover one — + GitHub exposes no endpoint that lists attestations without a subject + digest. Requires signer-workflow. Empty performs no anchor resolution. + Anchor trust is exactly ordinary write access to the anchor file. On a + PRIVATE repository the index read needs credentials, so pass + github-token as well; on a public repository it succeeds unauthenticated. + required: false + default: "" + signer-workflow: + description: >- + Mandatory pin when anchor is set, in the form + [host/]////. Without it a corroboration + means nothing, so the resolve step fails closed rather than resolving + unpinned. + required: false + default: "" attest: description: >- Emit a loop-engineer/verdict@1 predicate and attest it keylessly via @@ -52,6 +73,14 @@ outputs: attestation-id: description: "ID of the attestation created by this run ('' when attest is false)." value: ${{ steps.attest.outputs.attestation-id }} + anchor-outcome: + description: >- + corroborated | contradicted | unavailable ('' when anchor is empty). + Distinct for OBSERVABILITY only: anything short of a verified 200 plus a + successful gh attestation verify is non-promoting, and transport-class + failures are separately reportable but exactly as non-promoting as a + clean denial. + value: ${{ steps.anchor.outputs.anchor-outcome }} runs: using: "composite" @@ -141,13 +170,33 @@ runs: loop verdict "$LOOP_PATH" > "${RUNNER_TEMP}/verdict.json" echo "predicate-path=${RUNNER_TEMP}/verdict.json" >> "$GITHUB_OUTPUT" + - name: chain-head subject file + id: subject + if: ${{ inputs.attest == 'true' && steps.chain-head.outputs.chain-head != '' }} + shell: bash + env: + LOOP_PATH: "${{ inputs.path }}" + run: | + # Exactly 64 bytes, lowercase hex, NO trailing newline (reference §24). The one + # writer is loop.verdict.subject_bytes, reached via --emit-subject, so the attest + # side and the resolve side cannot disagree on the byte form. + # + # This file is what makes verification runnable at all: the chain head is a + # SHA-256 over a synthesized event preimage, so no retrievable bytes hash TO it, + # and `gh attestation verify` accepts only a path and hashes that file's CONTENT. + loop verdict --emit-subject "$LOOP_PATH" > "${RUNNER_TEMP}/loop-chain-head" + echo "path=${RUNNER_TEMP}/loop-chain-head" >> "$GITHUB_OUTPUT" + - name: attest verdict id: attest if: ${{ inputs.attest == 'true' && steps.chain-head.outputs.chain-head != '' }} uses: actions/attest@v4 with: - subject-name: loop-chain-head - subject-digest: sha256:${{ steps.chain-head.outputs.chain-head }} + # subject-path ONLY, with no subject-name: upstream requires subject-name only + # when the subject is identified by a raw digest input, and with a path the + # derived name equals the file's basename (loop-chain-head) — which is exactly + # the pinned SUBJECT_NAME. + subject-path: ${{ steps.subject.outputs.path }} predicate-type: urn:loop-engineer:verdict:1 predicate-path: ${{ steps.verdict.outputs.predicate-path }} push-to-registry: false @@ -160,6 +209,69 @@ runs: shell: bash run: echo "::warning::attest requested but the store has no chained events; nothing to attest." + - name: resolve the anchor attestation + id: anchor + # This step carries no error-tolerance key and no run-anyway guard: either would + # decouple its exit code from the job's outcome, which IS the gate. + # Precedence (ADR 0002 decision 5): + # a non-empty expect-chain-head input wins over anchor resolution, and the skip is + # announced in the step summary below — a silently dropped anchor is worse than a + # refused one. + if: ${{ inputs.anchor != '' && inputs.expect-chain-head == '' }} + shell: bash + env: + GH_TOKEN: ${{ inputs.github-token }} + LOOP_ANCHOR: "${{ inputs.anchor }}" + LOOP_SIGNER_WORKFLOW: "${{ inputs.signer-workflow }}" + run: | + # No shell-level error suppression around a gating call. The scorecard step below + # suppresses errors deliberately because inspect is ADVISORY — this step is not + # advisory, and its exit code is the finding. The script itself fails closed when + # --signer-workflow is empty. + python "${{ github.action_path }}/scripts/action_anchor_resolve.py" \ + --anchor "$LOOP_ANCHOR" \ + --repo "$GITHUB_REPOSITORY" \ + --signer-workflow "$LOOP_SIGNER_WORKFLOW" \ + --runner-temp "$RUNNER_TEMP" \ + --github-output "$GITHUB_OUTPUT" + + - name: compare the attested verdict + if: ${{ steps.anchor.outputs.anchor-outcome == 'corroborated' }} + shell: bash + env: + LOOP_PATH: "${{ inputs.path }}" + ANCHOR_HEAD: "${{ steps.anchor.outputs.anchor-head }}" + CURRENT_HEAD: "${{ steps.chain-head.outputs.chain-head }}" + PREDICATE: "${{ steps.anchor.outputs.predicate-path }}" + run: | + # Authenticity first (gh attestation verify, inside the resolve step above), then + # ancestry, then agreement where agreement is a meaningful question. No error + # suppression: every call here is gating. + # + # ANCESTRY IS THE CROSS-RUN GATE and runs unconditionally. + loop doctor --expect-chain-ancestor "$ANCHOR_HEAD" "$LOOP_PATH" + # AGREEMENT IS A SAME-RUN QUESTION. A verdict projects ONE run, so an attested + # head that is merely an ancestor of the current head is a different run's + # verdict — a disagreement by design (reference §24). Running --compare against + # an older attested predicate on a store that legitimately grew would therefore + # fail every time, which would make this input unusable for the cross-run + # detection it exists for. So compare only when the anchored head IS the current + # head, and say out loud when it was skipped. + if [ "$ANCHOR_HEAD" = "$CURRENT_HEAD" ]; then + loop verdict --compare "$PREDICATE" "$LOOP_PATH" + else + echo "**loop-engineer anchor:** corroborated, and still an ancestor of this run's chain." \ + "\`--compare\` was not run: the anchored head \`$ANCHOR_HEAD\` is an earlier run's," \ + "and a verdict projects one run (reference §24)." >> "$GITHUB_STEP_SUMMARY" + fi + + - name: anchor resolution skipped (explicit head wins) + if: ${{ inputs.anchor != '' && inputs.expect-chain-head != '' }} + shell: bash + run: | + echo "**loop-engineer anchor:** not resolved — the explicit expect-chain-head input wins (ADR 0002 decision 5)." \ + >> "$GITHUB_STEP_SUMMARY" + - name: loop inspect (scorecard) shell: bash env: diff --git a/docs/adr/0002-ci-attested-verdict.md b/docs/adr/0002-ci-attested-verdict.md index 5aba364..85bc899 100644 --- a/docs/adr/0002-ci-attested-verdict.md +++ b/docs/adr/0002-ci-attested-verdict.md @@ -269,3 +269,76 @@ the decisions above. 5. **`create-storage-record` and `push-to-registry` defaults.** Pass both explicitly as false so the two-permission claim is true by construction rather than by assumed default. + +## Amendment (2026-07-29, Slice 4b) + +A new dated section rather than an in-place edit of the decision text: the +2026-07-28 erratum is the precedent for a *correction*, and this is a **decision +change**, which earns its own record. Where this amendment and the decisions above +disagree, the amendment governs. Normative detail lives in +`reference/repo-os-contract.md` §24. + +**Decision 2 — mechanism change.** The subject is now a head-bearing **file** +(`subject-path`), not `subject-digest: sha256:`. Decision 2's sentence *"The +subject is the chain head alone"* **survives in spirit** — the subject still commits +to nothing but the head — and changes in **mechanism** only. The predicate bytes were +the obvious alternative subject and are **rejected**: `doctor.validation_mode` and +`tool.version` live inside the predicate, so the same run projects `873dfc87…` with +jsonschema and `8de3d88c…` in structural-fallback, and a consumer on another tool +version could never reproduce them. The head is version-independent. + +**Decision 4 — its first half was not executable as shipped.** `gh attestation +verify` accepts only `[ | oci://]` and hashes the file's +**content**; there is no digest-only input, and `--digest-alg` selects only which +algorithm to hash the artifact with. A synthesized chain head has no preimage, so no +artifact could ever be presented for a subject identified by that digest — the +decisive probe was a file *named* after the digest but empty, which made `gh` look up +the SHA-256 of the empty string and return 404. Decision 2's mechanism change makes +the authenticity step executable for the first time. + +**Decision 5 — two corrections.** + +1. *"Fetch the most recent matching attestation"* is **not implementable from the + index.** `GET /repos/{owner}/{repo}/attestations/{subject_digest}` is the only list + operation, the no-digest route 404s, there is no `gh attestation list`, GraphQL's + `Repository` type exposes no attestation fields, and no ordering guarantee is + documented. So the head is **carried** in a tracked `loop-engineer/anchor@1` file and + the attestation *corroborates* it: an attestation can never *discover* a head. Anchor + trust is exactly ordinary write access to the anchor file — the same class of limit as + "the worker can edit the verifier". +2. The cross-run check is **ancestry**, not head equality. Appending one event moves the + head (measured `9d388ae5…` seq 4 → `c336ecdc…` seq 5), so the ADR's resolve target was + incoherent: feeding run N's head to `--expect-chain-head` at run N+1 fails by + construction. `loop doctor --expect-chain-ancestor` asks the answerable question, and + answers it by **replay**, recomputing every hash rather than trusting the stored + `event_hash` column. + +**Open verification item 2 — settled.** No retention window is documented anywhere for +the attestation index (the familiar 90-day/400-day figures are workflow artifacts and +logs). Attestations are **deletable** through permission-gated endpoints, and GitHub's +own guidance recommends deleting ones no longer needed. The REST attestations route is +additionally **deprecated**, route-level, with `Sunset: Fri, 10 Mar 2028`. An absent +anchor attestation is therefore a **typed failure, never a skip** — otherwise an +availability attack on the index becomes a gate bypass. + +**Open verification item 3 — settled.** `--signer-digest` pins the signer-digest +certificate extension, populated from the `job_workflow_sha` claim, which for a +non-reusable top-level workflow **equals the triggering commit SHA**. It invalidates on +**every push**, not merely on a workflow edit — observed across all three attestations +this repository had minted, none of whose commits touched `attest.yml` or `action.yml`. +It is therefore **not** a mandatory pin. `--signer-workflow`, byte-identical across all +three certificates, is. + +**Decision 6 — the path list grew** by the anchor path (`loop-anchor.json` at any +depth), which is a gate-defining path in exactly the sense `loop/**` and `action.yml` +are: an actor who can edit it re-points the anchor at a head they had attested. +Decision 6 itself remains **documented but not in force** — the live repository ruleset +requires 0 approvals — so slice 4b landed autonomously and CODEOWNERS must not be +described as an operative control until the ruleset requires it. + +**What 4b deliberately does not add.** No in-kernel signature verification (permanently +rejected: it needs a trust root, X.509 chain validation and log inclusion proofs, which +is the dependency constraint this ADR exists to hold). No `verdict@1` field carrying an +anchor — adding a field to a permanent public log is a one-way door, and ancestry is a +doctor concern. No schema for the comparison report: it is a report, like +`doctor_report`'s, not an interchange artifact. diff --git a/loop/__main__.py b/loop/__main__.py index 2f5e1a8..acbe59b 100644 --- a/loop/__main__.py +++ b/loop/__main__.py @@ -5,7 +5,7 @@ import sys from pathlib import Path -from .contract import VALIDATION_MODES, ValidationModeError, doctor_report +from .contract import VALIDATION_MODES, ContractIssue, ValidationModeError, doctor_report from .plan import validate_plan from .runtime import RuntimeStoreError, replay_report, status_report from .runcontrol import RunControlError @@ -25,8 +25,10 @@ {_USAGE} {_PROG} metrics [--baseline] {_PROG} doctor|validate|verify [--mode basic|strict|release] - [--expect-chain-head SHA256] - {_PROG} verdict [--mode basic|strict|release] + [--expect-chain-head SHA256] + [--expect-chain-ancestor SHA256 | --anchor PATH] + {_PROG} verdict [--mode basic|strict|release] + [--compare FILE|- | --emit-subject] {_PROG} status [--mode basic|strict|release] {_PROG} replay [--mode basic|strict|release] {_PROG} simulate [--mode basic|strict|release] @@ -80,6 +82,30 @@ (doctor/validate/verify) fail unless the event store's chain head is exactly this 64-character lowercase hex hash. A missing, unreadable, unchained, or diverged store fails the gate. + --expect-chain-ancestor SHA256 + (doctor/validate/verify) fail unless this digest WAS the chain head + at some sequence, established by replaying and recomputing every + hash — never by trusting the stored event_hash column. Use this + across runs: exact head equality fails by construction once the + store grows. Composes with --expect-chain-head. + --anchor PATH (doctor/validate/verify) resolve --expect-chain-ancestor from a + tracked loop-engineer/anchor@1 file (conventionally + loop-anchor.json, and never under .loop/). Mutually exclusive with + --expect-chain-ancestor. An unreadable or non-conformant anchor is + a typed issue in the report with ok=false, never a skip. + --compare FILE|- + (verdict) compare an attested loop-engineer/verdict@1 predicate + against this workspace's projection and print an agreement report: + exit 0 agree, 1 disagree, 2 refusal. Accepts a BARE predicate only — + an in-toto Statement or a `gh --format json` envelope is refused with + the jq path to unwrap. This never verifies a signature: authenticity + is `gh attestation verify`'s job, it runs first, and neither check + implies the other (signature_checked is always false). + --emit-subject + (verdict) write the attested subject's bytes to stdout: exactly the + 64-character lowercase hex chain head, no trailing newline. One + definition of the byte form, so the signer side and the consumer + side cannot disagree. --executor ID (run) record this identity as produced_by.executor on the run's evidence records; unset records "unattributed". --verifier-identity ID (run) record this identity as verified_by.by; unset records @@ -167,6 +193,33 @@ def _extract_value_flag(argv: list[str], flag: str) -> tuple[str | None, list[st return value, remaining +def _read_compare_document(value: str) -> object: + """Load the attested document from a path or from stdin. + + ONE reader, so the file branch and the '-' branch cannot diverge in their failure + behavior. Every failure is a VerdictError, so the verdict dispatch renders it as + `verdict: …` on stderr with exit 2 and no traceback: a bare read_text() here would + let OSError escape and give the operator Python's own exit 1, which in this CLI + means "a report said not-ok" — an unreadable file would read as a disagreement. + """ + from .verdict import VerdictError + + try: + text = sys.stdin.read() if value == "-" else Path(value).read_text(encoding="utf-8") + except OSError as exc: # missing, a directory, unreadable + raise VerdictError(f"--compare could not read {value!r}: {exc}") from exc + except UnicodeDecodeError as exc: # the #107 lesson + raise VerdictError(f"--compare input is not valid UTF-8: {exc}") from exc + if not text.strip(): + # Refused explicitly rather than left to json.loads: an empty stdin is the + # error a pipeline whose upstream jq produced nothing hits. + raise VerdictError(f"--compare input is empty: {value!r}") + try: + return json.loads(text) + except json.JSONDecodeError as exc: + raise VerdictError(f"--compare input is not JSON: {exc}") from exc + + def _extract_run_stub_flags(argv: list[str]) -> tuple[list[str], list[str]]: """Remove requested but not-yet-supported run-mode flags from argv.""" flags = {"--continuous", "--approve"} @@ -273,6 +326,71 @@ def main(argv: list[str] | None = None) -> int: print(_USAGE, file=sys.stderr) return 2 + expect_chain_ancestor = anchor = None + if command in {"doctor", "validate", "verify"}: + try: + expect_chain_ancestor, argv = _extract_value_flag(argv, "--expect-chain-ancestor") + anchor, argv = _extract_value_flag(argv, "--anchor") + except ValueError as exc: + print(f"{command}: {exc}", file=sys.stderr) + print(_USAGE, file=sys.stderr) + return 2 + if (expect_chain_ancestor is not None + and re.fullmatch(r"[0-9a-f]{64}", expect_chain_ancestor) is None): + print(f"{command}: --expect-chain-ancestor must be a 64-character lowercase hex sha256", + file=sys.stderr) + return 2 + if anchor is not None and expect_chain_ancestor is not None: + # Silent precedence between an explicit digest and a resolved one is how a + # gate becomes a suggestion. The action layer, where the inputs are the + # surface, is where ADR 0002 decision 5's precedence is honored. + print(f"{command}: --anchor and --expect-chain-ancestor are mutually exclusive", + file=sys.stderr) + print(_USAGE, file=sys.stderr) + return 2 + else: + # Same reason as the --expect-chain-head guard above. + for flag in ("--expect-chain-ancestor", "--anchor"): + if any(a == flag or a.startswith(f"{flag}=") for a in argv): + print(f"{command}: {flag} is only valid for doctor/validate/verify", + file=sys.stderr) + print(_USAGE, file=sys.stderr) + return 2 + + compare_path = None + emit_subject = False + if command == "verdict": + try: + compare_path, argv = _extract_value_flag(argv, "--compare") + except ValueError as exc: + print(f"{command}: {exc}", file=sys.stderr) + print(_USAGE, file=sys.stderr) + return 2 + if "--emit-subject" in argv: + emit_subject = True + argv = [a for a in argv if a != "--emit-subject"] + if compare_path is not None and emit_subject: + print("verdict: --compare and --emit-subject are mutually exclusive", + file=sys.stderr) + print(_USAGE, file=sys.stderr) + return 2 + for flag in ("--verify-signature", "--signature", "--signer-workflow", "--signer-digest"): + if any(a == flag or a.startswith(f"{flag}=") for a in argv): + # D10.1: there is no flag to flip. Authenticity is `gh attestation + # verify`'s job and it runs first; this command establishes agreement. + print(f"verdict: {flag} is not a verdict option — verdict never verifies a " + "signature", file=sys.stderr) + print(_USAGE, file=sys.stderr) + return 2 + else: + # Same reason as the --expect-chain-head guard above: there is no generic + # unknown-flag guard, so scaffold would CREATE a directory named after the flag. + for flag in ("--compare", "--emit-subject"): + if any(a == flag or a.startswith(f"{flag}=") for a in argv): + print(f"{command}: {flag} is only valid for verdict", file=sys.stderr) + print(_USAGE, file=sys.stderr) + return 2 + executor = verifier_identity = None if command == "run": for flag, slot in (("--executor", "executor"), ("--verifier-identity", "verifier_identity")): @@ -365,17 +483,44 @@ def main(argv: list[str] | None = None) -> int: return 0 if command in {"doctor", "validate", "verify"}: + resolved_ancestor = expect_chain_ancestor + anchor_issue = None + if anchor is not None: + from .anchor import AnchorError, read_anchor + + try: + resolved_ancestor = read_anchor(anchor)["chain_head"] + except AnchorError as exc: + # A report, never a bare stderr line: the operator's CI reads the JSON, + # and a stderr line loses the code. resolved_ancestor stays None so the + # ancestry gate is not ALSO asked a question it has no digest for — + # one failure, one code. + resolved_ancestor = None + anchor_issue = ContractIssue(exc.code, str(exc), Path(anchor)) try: - return _print_json(doctor_report(target, mode=mode, expect_chain_head=expect_chain_head)) + report = doctor_report(target, mode=mode, expect_chain_head=expect_chain_head, + expect_chain_ancestor=resolved_ancestor) except ValidationModeError as exc: print(f"{command}: {exc}", file=sys.stderr) return 2 + if anchor_issue is not None: + report = {**report, "issues": [*report["issues"], anchor_issue], "ok": False} + return _print_json(report) if command == "verdict": - from .verdict import VerdictError, build_verdict + from .verdict import VerdictError, build_verdict, compare_verdict, subject_bytes from .chain import ChainHashError, canonical_json try: + if emit_subject: + # buffer.write, never print: a trailing newline would change the subject + # digest, and the 64-byte form is normative. + sys.stdout.buffer.write( + subject_bytes(build_verdict(target, mode=mode)["chain"]["head"])) + return 0 + if compare_path is not None: + return _print_json(compare_verdict( + _read_compare_document(compare_path), target, mode=mode)) print(canonical_json(build_verdict(target, mode=mode))) return 0 except (VerdictError, ChainHashError) as exc: diff --git a/loop/anchor.py b/loop/anchor.py new file mode 100644 index 0000000..4519c86 --- /dev/null +++ b/loop/anchor.py @@ -0,0 +1,138 @@ +"""Read a tracked ``anchor@1`` file that CARRIES a previously attested chain head. + +Nothing here is trusted. The anchor is a *carried claim* whose only function is to +give an attestation lookup a digest to ask about: GitHub exposes no endpoint that +lists attestations without a subject digest, so an attestation can corroborate a +head but can never discover one. Anchor trust is therefore exactly ordinary write +access to the anchor file — no better, and the same class of limit as ADR 0002's +"the worker can edit the verifier". + +Pure: no environment, no network, no signing, no subprocess. +""" + +from __future__ import annotations + +import json +import re +from pathlib import Path +from typing import Any + +from ._resources import schemas_dir + +ANCHOR_SCHEMA_ID = "loop-engineer/anchor@1" +DEFAULT_ANCHOR_FILENAME = "loop-anchor.json" + +UNREADABLE_CODE = "anchor_file_unreadable" +INVALID_CODE = "anchor_file_invalid" + +_HEAD_PATTERN = re.compile(r"[0-9a-f]{64}") +_LOOP_DIR_NAME = ".loop" + +# Mirrors schemas/anchor.schema.json. The structural leg must refuse everything the +# schema refuses, for every field the schema declares — a schema-only check would let +# a jsonschema-less environment accept a document the schema rejects (the S3 +# plan-lint mode-parity repair is the precedent). +_OPTIONAL_STRINGS = {"attestation_id": (0, 64), "run_id": (1, 256), "recorded_at": (0, 64)} +_KNOWN_KEYS = {"schema", "chain_head", "sequence", *_OPTIONAL_STRINGS} + + +class AnchorError(ValueError): + """The anchor file is absent, unreadable, or not a conformant ``anchor@1``. + + Carries the failure class as ``.code`` (the ``RuntimeStoreError`` precedent) so a + caller can pick the right doctor issue code without regexing the message: + ``anchor_file_unreadable`` when the document could not be read or parsed at all, + ``anchor_file_invalid`` when it parsed but is not a conformant ``anchor@1``. + """ + + def __init__(self, message: str, *, code: str = UNREADABLE_CODE) -> None: + super().__init__(message) + self.code = code + + +def _load_anchor_schema() -> dict[str, Any]: + return json.loads((schemas_dir() / "anchor.schema.json").read_text(encoding="utf-8")) + + +def _structural_violation(data: dict[str, Any]) -> str | None: + """The first way ``data`` departs from ``anchor@1``, or ``None``.""" + if data.get("schema") != ANCHOR_SCHEMA_ID: + return f"schema must be {ANCHOR_SCHEMA_ID!r}, found {data.get('schema')!r}" + head = data.get("chain_head") + if not isinstance(head, str): + return "chain_head is required and must be a string" + if _HEAD_PATTERN.fullmatch(head) is None: + # fullmatch, never match: `pattern` in JSON Schema is re.search semantics, so + # a prefix-anchored check alone accepts 64 hex followed by anything. + return "chain_head must be 64 lowercase hex characters" + unknown = sorted(set(data) - _KNOWN_KEYS) + if unknown: + return f"unknown field(s): {', '.join(unknown)}" + sequence = data.get("sequence") + if sequence is not None: + # bool is an int subclass in Python; the schema means integer, not True. + if isinstance(sequence, bool) or not isinstance(sequence, int): + return "sequence must be an integer" + if sequence < 0: + return "sequence must be >= 0" + for field, (minimum, maximum) in _OPTIONAL_STRINGS.items(): + value = data.get(field) + if value is None: + continue + if not isinstance(value, str): + return f"{field} must be a string" + if not minimum <= len(value) <= maximum: + return f"{field} must be {minimum}..{maximum} characters" + return None + + +def _schema_violation(data: dict[str, Any]) -> str | None: + """The first jsonschema violation, or ``None`` when jsonschema is unavailable. + + Defence in depth over the structural checks, never a replacement: ``iter_errors`` + *collects* errors and never raises, so there is no ``ValueError`` to catch here — + every jsonschema call site in this package uses the same idiom. + """ + try: + import jsonschema # type: ignore + except ImportError: + return None + validator = jsonschema.Draft202012Validator(_load_anchor_schema()) + for error in validator.iter_errors(data): + location = "/".join(str(part) for part in error.absolute_path) or "" + return f"{location}: {error.message}" + return None + + +def read_anchor(path: str | Path) -> dict[str, Any]: + """Read and validate an ``anchor@1`` document. + + Every failure is an :class:`AnchorError`. Nothing else escapes — a caller + gating on an anchor must never receive a bare ``OSError`` or + ``UnicodeDecodeError`` where it expected a typed refusal. + """ + path = Path(path) + if _LOOP_DIR_NAME in path.parts: + raise AnchorError( + f"anchor must not live under {_LOOP_DIR_NAME}/: {path} — an anchor inside " + "the tree it certifies is gitignored, so it would never land in a commit" + ) + try: + raw = path.read_bytes() + except OSError as exc: + raise AnchorError(f"anchor file is unreadable: {path}: {exc}") from exc + try: + text = raw.decode("utf-8") + except UnicodeDecodeError as exc: + raise AnchorError(f"anchor file is not valid UTF-8: {path}: {exc}") from exc + try: + data = json.loads(text) + except json.JSONDecodeError as exc: + raise AnchorError(f"anchor file is not valid JSON: {path}: {exc}") from exc + if not isinstance(data, dict): + raise AnchorError(f"anchor document must be an object: {path}", code=INVALID_CODE) + violation = _structural_violation(data) or _schema_violation(data) + if violation is not None: + raise AnchorError(f"anchor is not a conformant {ANCHOR_SCHEMA_ID}: {path}: {violation}", + code=INVALID_CODE) + return data diff --git a/loop/attestation.py b/loop/attestation.py new file mode 100644 index 0000000..5a4e3c7 --- /dev/null +++ b/loop/attestation.py @@ -0,0 +1,171 @@ +"""A pure signer-trust policy over an ALREADY-VERIFIED ``verificationResult``. + +This module evaluates claims ``gh attestation verify`` has already established. It +establishes NOTHING itself: it never signs, never verifies a signature, never reaches +the network, and never reads a process variable. It refuses — loudly — when a claim it +needs is absent, because a policy that treats a missing claim as satisfied is worse +than no policy. + +Only ``signature.certificate`` and ``verifiedTimestamps`` are unforgeable by the +workflow that produced the attestation (gh's own help text says so). Everything under +``statement.predicate`` is user-controllable metadata and is never read here. + +The three anchor-lookup outcomes never collapse: *anything short of a verified 200 plus +a successful ``gh attestation verify`` is non-promoting, and transport-class failures +(5xx, timeout, auth) are separately reportable but exactly as non-promoting as a clean +denial.* The distinction exists for observability, never for differential trust. +""" + +from __future__ import annotations + +from typing import Any, Mapping + +from .contract import ContractIssue + +# The certificate claim names gh surfaces under signature.certificate. These leaf names +# cannot be established before merge (F1: gh attestation verify is unrunnable against +# every attestation this repo has minted so far, so no --format json output exists to +# read them from), so they are PINNED and a wrong name produces a refusal rather than a +# silent pass. The live experiment in .github/workflows/attest.yml confirms them; the +# extraction lives in scripts/action_anchor_resolve.py so a correction is a one-line +# change outside loop/. Why these names and not others: repo-os-contract.md #24. +REQUIRED_CERTIFICATE_CLAIMS = ("subjectAlternativeName", "sourceRepositoryURI", + "runnerEnvironment") + +# At least one must be present. ADR 0002 decision 5 requires a `push` trigger but names +# no claim, and the two candidates are equally plausible — so require either and refuse +# when neither appears. +_TRIGGER_CLAIM_ALIASES = ("githubWorkflowTrigger", "buildTrigger") + +ANCHOR_LOOKUP_OUTCOMES = ("corroborated", "contradicted", "unavailable") + +_LOOKUP_CODES = { + "contradicted": "anchor_attestation_contradicted", + "unavailable": "anchor_attestation_unavailable", +} +_NON_PROMOTING = ( + "anything short of a verified 200 plus a successful `gh attestation verify` is " + "non-promoting, and transport-class failures (5xx, timeout, auth) are separately " + "reportable but exactly as non-promoting as a clean denial" +) + + +class AttestationPolicyError(ValueError): + """A claim the policy needs is absent, or an outcome it cannot classify was passed.""" + + +def _certificate(result: object) -> Mapping[str, Any]: + if not isinstance(result, Mapping): + raise AttestationPolicyError( + f"verificationResult must be an object, found {type(result).__name__}") + signature = result.get("signature") + certificate = signature.get("certificate") if isinstance(signature, Mapping) else None + if not isinstance(certificate, Mapping): + raise AttestationPolicyError( + "verificationResult.signature.certificate is absent or is not an object — " + "the certificate is one of only two fields the originating workflow cannot " + "forge, so its absence is a refusal, never a pass") + timestamps = result.get("verifiedTimestamps") + if not isinstance(timestamps, list) or not timestamps: + raise AttestationPolicyError( + "verificationResult.verifiedTimestamps is absent or empty — an unwitnessed " + "attestation is not a trusted one") + missing = [claim for claim in REQUIRED_CERTIFICATE_CLAIMS if claim not in certificate] + if missing: + raise AttestationPolicyError( + f"certificate is missing required claim(s): {', '.join(missing)} — a pinned " + "claim name that does not match gh's actual JSON must fail loudly, so this " + "refuses rather than treating the claim as satisfied") + if not any(alias in certificate for alias in _TRIGGER_CLAIM_ALIASES): + raise AttestationPolicyError( + "certificate carries neither trigger claim " + f"({' nor '.join(_TRIGGER_CLAIM_ALIASES)}) — the trigger cannot be checked") + return certificate + + +def _normalize_identity(value: object) -> str: + """Strip an optional scheme, an optional `@` suffix, and a trailing slash. + + Exact equality after normalization, deliberately NOT a substring test: the SAN is + `https://///@` while the pin is + `[host/]//`, and a substring match would accept a crafted + repository whose name merely ends with the pinned path. + """ + text = str(value) + text = text.split("@", 1)[0] + for scheme in ("https://", "http://"): + if text.startswith(scheme): + text = text[len(scheme):] + break + return text.rstrip("/") + + +def _identity_agrees(claim: object, pin: str) -> bool: + normalized_claim = _normalize_identity(claim) + normalized_pin = _normalize_identity(pin) + return normalized_claim in (normalized_pin, f"github.com/{normalized_pin}") + + +def check_signer_trust(result: object, *, signer_workflow: str, source_repository_uri: str, + expect_trigger: str = "push") -> dict[str, Any]: + """Evaluate the pinned signer claims. Denials are typed codes, never booleans. + + There is deliberately no ``signer_digest`` parameter (D4): the signer-digest pin + resolves to the ``job_workflow_sha`` claim, which for a non-reusable top-level + workflow equals the *triggering commit SHA* — so it invalidates on every push, not + merely on a workflow edit. ``--signer-workflow`` is the mandatory pin. + Full reasoning: reference/repo-os-contract.md #24. + + ``signature_checked`` is ``False`` here too: this evaluates a verdict gh already + reached, and re-asserting authenticity would be a claim this module cannot make. + """ + certificate = _certificate(result) + issues: list[dict[str, Any]] = [] + if not _identity_agrees(certificate["subjectAlternativeName"], signer_workflow): + issues.append(ContractIssue( + "signer_workflow_mismatch", + f"certificate signer {certificate['subjectAlternativeName']!r} is not the " + f"pinned workflow {signer_workflow!r}")) + if not _identity_agrees(certificate["sourceRepositoryURI"], source_repository_uri): + issues.append(ContractIssue( + "signer_repository_mismatch", + f"certificate sourceRepositoryURI {certificate['sourceRepositoryURI']!r} is " + f"not the pinned repository {source_repository_uri!r}")) + if certificate["runnerEnvironment"] != "github-hosted": + issues.append(ContractIssue( + "self_hosted_runner", + f"certificate runnerEnvironment is {certificate['runnerEnvironment']!r}; " + "--deny-self-hosted-runners is passed unconditionally, so gh should already " + "have refused — this check is defensive")) + trigger = next((certificate[alias] for alias in _TRIGGER_CLAIM_ALIASES + if alias in certificate), None) + if trigger != expect_trigger: + issues.append(ContractIssue( + "signer_trigger_mismatch", + f"certificate trigger is {trigger!r}, expected {expect_trigger!r}")) + return {"ok": not issues, "signature_checked": False, "issues": issues} + + +def anchor_lookup_issue(outcome: object, *, detail: str | None = None) -> dict[str, Any] | None: + """Map an anchor-lookup outcome to its issue, or ``None`` when corroborated. + + An unknown outcome RAISES rather than defaulting to anything: a silent default here + would be the difference between a gate and a suggestion. + """ + if outcome == "corroborated": + return None + code = _LOOKUP_CODES.get(outcome) if isinstance(outcome, str) else None + if code is None: + raise AttestationPolicyError( + f"unknown anchor lookup outcome {outcome!r}; expected one of " + f"{', '.join(ANCHOR_LOOKUP_OUTCOMES)}") + reason = { + "contradicted": "an attestation was found and it does not corroborate the carried " + "chain head (I looked and it said no)", + "unavailable": "no attestation was found, or the index could not be reached at all " + "(I could not look)", + }[outcome] + message = f"{reason}: {_NON_PROMOTING}" + if detail: + message = f"{message} — {detail}" + return ContractIssue(code, message) diff --git a/loop/chain.py b/loop/chain.py index dd66475..97a0440 100644 --- a/loop/chain.py +++ b/loop/chain.py @@ -11,8 +11,11 @@ import hashlib import json +import re from typing import Any, Iterable, Mapping +_DIGEST_PATTERN = re.compile(r"[0-9a-f]{64}") + _PREIMAGE_FIELDS = ( "schema", "run_id", "sequence", "event_id", "type", "actor", "ts", "causation_id", "correlation_id", "payload", "artifact_hashes", @@ -63,6 +66,37 @@ def link_issue(record: Mapping[str, Any], prev_head: Mapping[str, Any] | None) - return None +def head_sequence(events: Iterable[Mapping[str, Any]], digest: str) -> int | None: + """Sequence at which `digest` WAS the chain head, or None if it never was. + + Established by REPLAY: every link is re-checked and every hash recomputed via + link_issue/compute_event_hash. The stored event_hash column is never trusted — + an adversary who can rewrite the store can also insert a row bearing the + anchored digest, and only recomputation refuses that row. + + This is the cross-run check: --expect-chain-head is exact current-head equality, + so it fails by construction on a store that legitimately grew. Sequence 0 is a + legitimate answer, so callers must compare against None, never truthiness. + + Raises ValueError (deliberately NOT ChainHashError, which is scoped to + canonicalization failures) when `digest` is not 64 lowercase hex characters: a + silent None on a typo would read as "rewrite detected". + """ + if not isinstance(digest, str) or _DIGEST_PATTERN.fullmatch(digest) is None: + raise ValueError(f"digest must be 64 lowercase hex characters, got {digest!r}") + head: dict[str, Any] | None = None + for record in events: + if link_issue(record, head) is not None: + break + stored = record.get("event_hash") + if stored is None: + continue + if stored == digest: + return record.get("sequence") + head = {"sequence": record.get("sequence"), "event_hash": stored} + return None + + def verify_chain(events: Iterable[Mapping[str, Any]], *, expected_head: str | None = None) -> dict[str, Any]: """Verify a COMPLETE run stream's hash chain (sequence 0 onward); pure, I/O-free.""" issues: list[str] = [] diff --git a/loop/contract.py b/loop/contract.py index 54165e7..d173500 100644 --- a/loop/contract.py +++ b/loop/contract.py @@ -1101,12 +1101,14 @@ def validate_contract(target: str | Path, *, mode: str | None = None) -> dict[st def doctor_report(target: str | Path, *, mode: str | None = None, - expect_chain_head: str | None = None) -> dict[str, Any]: + expect_chain_head: str | None = None, + expect_chain_ancestor: str | None = None) -> dict[str, Any]: report = validate_contract(target, mode=mode) from .runtime import event_consistency_issues event_store, event_issues = event_consistency_issues( - target, mode=mode, expect_chain_head=expect_chain_head) + target, mode=mode, expect_chain_head=expect_chain_head, + expect_chain_ancestor=expect_chain_ancestor) issues = report["issues"] + list(event_issues) if event_issues else report["issues"] return {**report, "event_store": event_store, "issues": issues, "ok": report["ok"] and not event_issues} diff --git a/loop/runtime.py b/loop/runtime.py index 7c994e7..6c3af6b 100644 --- a/loop/runtime.py +++ b/loop/runtime.py @@ -10,6 +10,7 @@ from tempfile import TemporaryDirectory from typing import Any, Callable, Iterator, TypeVar +from .chain import head_sequence from .completion import CompletionPolicyError, criteria_satisfy_completion from .contract import ContractIssue from .events import ( @@ -279,8 +280,30 @@ def _anchor_mismatch(message: str) -> dict[str, Any]: return ContractIssue("chain_anchor_mismatch", message) +def _not_ancestor(message: str) -> dict[str, Any]: + """Deliberately NOT a reuse of chain_anchor_mismatch (D3). + + "your current head is not what I expected" and "the head you anchored is not in + my history at all" are different facts, and doctor issue codes are the population + verdict.doctor.issue_codes is drawn from — a permanent, public log. One shared + code would collapse them there forever. + """ + return ContractIssue("chain_anchor_not_ancestor", message) + + +def _ancestry(target: str | Path, mode: str | None, expect_chain_ancestor: str) -> int | None: + """Sequence at which the anchored head was the head, by replay. + + A fourth read-only fold of the store, in the same tradition as + _bound_evidence_issues (repo-os-contract.md #22). + """ + _, _run_id, events, _validation = _events(target, mode) + return head_sequence(events, expect_chain_ancestor) + + def event_consistency_issues( - target: str | Path, *, mode: str | None = None, expect_chain_head: str | None = None + target: str | Path, *, mode: str | None = None, expect_chain_head: str | None = None, + expect_chain_ancestor: str | None = None, ) -> tuple[dict[str, Any], list[dict[str, Any]]]: """Return event-store health and the existing status/replay findings.""" path = _store_path(target) @@ -295,6 +318,11 @@ def event_consistency_issues( if expect_chain_head is not None: absent_issues.append(_anchor_mismatch( "an anchored chain head was supplied but no event store is present")) + if expect_chain_ancestor is not None: + # D5: an absent store with an ancestor supplied FAILS. It never skips — + # otherwise deleting the store is a gate bypass. + absent_issues.append(_not_ancestor( + "an anchored chain ancestor was supplied but no event store is present")) if absent_issues: return {"present": False, "sidecar_residue": bool(residue)}, absent_issues return {"present": False}, [] @@ -306,11 +334,19 @@ def event_consistency_issues( # store that becomes unreadable between reads must surface as a typed finding # rather than an untyped traceback out of doctor_report (R007). bound_issues = _bound_evidence_issues(target, mode) + # Inside the same guard, for the same R007 reason: the ancestry replay is a + # FOURTH independent read, and a store that becomes unreadable between reads + # must surface as a typed finding rather than a traceback out of doctor_report. + ancestor_sequence = (_ancestry(target, mode, expect_chain_ancestor) + if expect_chain_ancestor is not None else None) except RuntimeStoreError as exc: unreadable_issues = [ContractIssue(exc.code, str(exc))] if expect_chain_head is not None: unreadable_issues.append(_anchor_mismatch( "an anchored chain head was supplied but the event store cannot be read")) + if expect_chain_ancestor is not None: + unreadable_issues.append(_not_ancestor( + "an anchored chain ancestor was supplied but the event store cannot be read")) return {"present": True, "readable": False, "error_code": exc.code}, unreadable_issues issues = list(status["divergence"]) + list(replay["findings"]) issues.extend(bound_issues) @@ -325,7 +361,12 @@ def event_consistency_issues( if actual != expect_chain_head: issues.append(_anchor_mismatch( f"chain head {actual!r} does not match expected {expect_chain_head!r}")) - return { + if expect_chain_ancestor is not None and ancestor_sequence is None: + issues.append(_not_ancestor( + f"anchored chain head {expect_chain_ancestor!r} was never the head of this " + "chain at any sequence — established by replay, so a row bearing the " + "anchored digest without a matching recomputed hash does not satisfy it")) + report = { "present": True, "readable": True, "run_id": status["run_id"], @@ -334,7 +375,12 @@ def event_consistency_issues( "deterministic": replay["deterministic"], "legal_sequence": replay["legal_sequence"], "chain": {"head": status["chain_head"], "unchained_prefix": status["unchained_prefix"]}, - }, issues + } + if expect_chain_ancestor is not None: + # Only when asked: with no ancestor supplied the report stays byte-identical + # to the pre-4b shape (the #22 habit). + report["anchor"] = {"expected": expect_chain_ancestor, "sequence": ancestor_sequence} + return report, issues def _bound_evidence_issues(target: str | Path, mode: str | None) -> list[dict[str, Any]]: diff --git a/loop/verdict.py b/loop/verdict.py index 7ae0912..eda5c8e 100644 --- a/loop/verdict.py +++ b/loop/verdict.py @@ -10,23 +10,50 @@ import hashlib import json +import re from importlib import metadata from pathlib import Path from typing import Any from ._resources import schemas_dir -from .contract import _strict_evidence_failure, doctor_report +from .contract import ContractIssue, _strict_evidence_failure, doctor_report from .paths import LoopPaths, resolve_loop_paths from .runtime import RuntimeStoreError, bound_artifact_digests VERDICT_SCHEMA_ID = "loop-engineer/verdict@1" PREDICATE_TYPE = "urn:loop-engineer:verdict:1" +SUBJECT_NAME = "loop-chain-head" + +_HEAD_PATTERN = re.compile(r"[0-9a-f]{64}") class VerdictError(ValueError): """A verdict cannot be projected from this workspace.""" +def subject_bytes(head: object) -> bytes: + """The attested subject's bytes: exactly the 64-hex chain head, nothing else. + + ONE definition, so the signer side and the consumer side cannot disagree about + 64 bytes. The signer hands this file to ``actions/attest`` as ``subject-path``; + a consumer regenerates byte-identical content from the head alone and hands it + to ``gh attestation verify``. That is what makes verification runnable at all: + the chain head is a SHA-256 over a synthesized event preimage, so no retrievable + bytes hash to it, and ``gh attestation verify`` accepts only a file path or an + OCI URI and hashes that file's *content* — so an attestation whose subject digest + IS the head can never be presented an artifact. + + No trailing newline. The byte form is normative: a stray ``\\n`` would change the + subject digest, so it is pinned by test rather than left to a shell's ``echo``. + """ + if not isinstance(head, str) or _HEAD_PATTERN.fullmatch(head) is None: + raise VerdictError( + "subject requires a 64-character lowercase hex chain head; " + f"refusing {head!r} (a store-less workspace has no subject to attest)" + ) + return head.encode("ascii") + + def _load_verdict_schema() -> dict[str, Any]: return json.loads((schemas_dir() / "verdict.schema.json").read_text(encoding="utf-8")) @@ -112,6 +139,134 @@ def _verified_evidence( ] +COMPARISON_CODES = ( + "verdict_run_id_disagreement", + "verdict_head_disagreement", + "verdict_terminal_disagreement", + "verdict_evidence_disagreement", +) + +_UNWRAP_HINT = ( + "compare accepts a BARE loop-engineer/verdict@1 predicate; extract it with " + "`jq '.[0].verificationResult.statement.predicate'`" +) +_STATEMENT_KEYS = ("_type", "subject", "predicateType", "predicate") +_ENVELOPE_KEYS = ("verificationResult", "attestation") + + +def _refuse_unless_bare_predicate(attested: object) -> dict[str, Any]: + """Typed refusals, before any comparison, so an operator who piped a `gh` envelope + is told to unwrap rather than told their heads disagree. + + Best-effort unwrapping of a vendor envelope here is exactly how a trust boundary + rots: the kernel would start depending on the shape of another tool's output. + """ + if isinstance(attested, list): + raise VerdictError( + f"attested document is a top-level array — this is a `gh --format json` " + f"envelope, not a predicate. {_UNWRAP_HINT}") + if not isinstance(attested, dict): + raise VerdictError( + f"attested document is not a JSON object (found {type(attested).__name__}). " + f"{_UNWRAP_HINT}") + found = [key for key in _STATEMENT_KEYS if key in attested] + if found: + raise VerdictError( + f"attested document carries {', '.join(found)} — this is an in-toto Statement, " + f"not a predicate. {_UNWRAP_HINT}") + wrapper = [key for key in _ENVELOPE_KEYS if key in attested] + if wrapper: + raise VerdictError( + f"attested document carries {', '.join(wrapper)} — this is a `gh --format json` " + f"envelope, not a predicate. {_UNWRAP_HINT}") + if attested.get("schema") != VERDICT_SCHEMA_ID: + raise VerdictError( + f"attested document is not a {VERDICT_SCHEMA_ID} (schema is " + f"{attested.get('schema')!r}). {_UNWRAP_HINT}") + return attested + + +def _evidence_set(entries: object) -> frozenset[tuple[Any, Any, Any]] | None: + """The comparable evidence identity: the de-duplicated three-tuple set. + + None for a malformed evidence field, which can never agree with a projection. + """ + if not isinstance(entries, list): + return None + return frozenset( + (entry.get("digest"), entry.get("code_digest"), entry.get("policy_digest")) + if isinstance(entry, dict) else ("", json.dumps(entry, default=str), None) + for entry in entries + ) + + +def _evidence_view(entries: object) -> list[dict[str, Any]]: + """Digest-only projection of an evidence list for the report's compared block.""" + items = _evidence_set(entries) + if items is None: + return [] + return [{"digest": digest, "code_digest": code, "policy_digest": policy} + for digest, code, policy in sorted(items, key=lambda i: (str(i[0]), str(i[1]), str(i[2])))] + + +def compare_verdict(attested: object, target: str | Path, *, + mode: str | None = None) -> dict[str, Any]: + """Compare an attested ``verdict@1`` predicate against this workspace's projection. + + This establishes AGREEMENT. ``gh attestation verify`` establishes AUTHENTICITY, it + runs first, and neither implies the other — so ``signature_checked`` is the literal + ``False`` on every path and no flag changes it. Four facets are compared: ``run_id``, + ``chain.head``, the whole ``terminal`` object, and the ``evidence`` digest set. + + ``doctor`` and ``tool`` are deliberately NOT compared: both live inside the + predicate and are environment-coupled (the same run projects a different + ``doctor.validation_mode`` with and without jsonschema), so comparing them would + make an honest environment difference read as tampering. Whether an attested + ``doctor.ok`` should GATE is a policy question, not an agreement question. + + Refuses (``VerdictError``) anything that is not a bare predicate; the local side + comes from :func:`build_verdict`, so its no-terminal-record refusal is inherited. + """ + document = _refuse_unless_bare_predicate(attested) + local = build_verdict(target, mode=mode) + + attested_chain = document.get("chain") if isinstance(document.get("chain"), dict) else {} + attested_terminal = (document.get("terminal") + if isinstance(document.get("terminal"), dict) else {}) + terminal_facet = ("state", "completion_policy", "false_completion") + + compared = { + "run_id": {"attested": document.get("run_id"), "local": local["run_id"]}, + "head": {"attested": attested_chain.get("head"), "local": local["chain"]["head"]}, + "terminal": { + "attested": {key: attested_terminal.get(key) for key in terminal_facet}, + "local": {key: local["terminal"][key] for key in terminal_facet}, + }, + "evidence": {"attested": _evidence_view(document.get("evidence")), + "local": _evidence_view(local["evidence"])}, + } + agreement = { + "run_id": document.get("run_id") == local["run_id"], + "head": attested_chain.get("head") == local["chain"]["head"], + "terminal": compared["terminal"]["attested"] == compared["terminal"]["local"], + "evidence": _evidence_set(document.get("evidence")) == _evidence_set(local["evidence"]), + } + for facet, agrees in agreement.items(): + compared[facet]["agrees"] = agrees + + issues: list[dict[str, Any]] = [] + for facet, code in (("run_id", "verdict_run_id_disagreement"), + ("head", "verdict_head_disagreement"), + ("terminal", "verdict_terminal_disagreement"), + ("evidence", "verdict_evidence_disagreement")): + if not agreement[facet]: + issues.append(ContractIssue( + code, + f"attested {facet} {compared[facet]['attested']!r} does not agree with the " + f"local projection {compared[facet]['local']!r}")) + return {"ok": not issues, "signature_checked": False, "compared": compared, "issues": issues} + + def build_verdict(target: str | Path, *, mode: str | None = None) -> dict[str, Any]: """Project local run state into a ``verdict@1`` predicate body. diff --git a/reference/repo-os-contract.md b/reference/repo-os-contract.md index 3a55b17..f394ffa 100644 --- a/reference/repo-os-contract.md +++ b/reference/repo-os-contract.md @@ -1613,17 +1613,31 @@ predicateType is written immutably into a public log, and a rename must not be able to orphan it. **The subject seam — read this before verifying anything.** The signer binds -`subject-name: loop-chain-head` with -`subject-digest: sha256:`. That `sha256:` key satisfies the -signer's required `algorithm:hex_digest` form, but the chain head is a SHA-256 -over a **synthesized event preimage** (§16), not over any retrievable -artifact's bytes. A consumer must never conclude "fetch the bytes, re-hash, -compare" — there are no bytes to fetch. The only meaningful comparison is -equality against a chain head recomputed locally from the store (`loop -doctor`'s `chain` block, or the anchor check via `--expect-chain-head`). -Attested-vs-local agreement is a 4b (`--compare`) concern; authenticity -(`gh attestation verify`) and agreement are separate checks, in that order, -and neither implies the other. +`subject-path`: a file whose **entire content is the chain head**, exactly 64 +lowercase hex bytes with no trailing newline, written by the single definition +`loop.verdict.subject_bytes` and reachable as `loop verdict --emit-subject`. A +consumer therefore **can** regenerate the subject's bytes from the head alone, +and `gh attestation verify` **does** succeed against them. That is the whole +point of the byte form: `gh attestation verify` accepts only a file path (or an +OCI URI) and hashes that file's **content**, so a subject identified by a bare +digest can never be presented an artifact at all. + +Two things a reader must not confuse with that. First, the chain head *itself* +is still a SHA-256 over a **synthesized event preimage** (§16): no retrievable +artifact's bytes hash **to** the head, and a consumer must not go looking for +one. The subject file *carries* the head; it is not a preimage of it, and its +own digest is `sha256()`, which is necessarily **not** the +head. Second, the predicate bytes are **not** the subject, deliberately: +`doctor.validation_mode` and `tool.version` live inside the predicate, so the +same run projects different bytes in different environments (measured: +`873dfc87…` with jsonschema, `8de3d88c…` in structural-fallback), and a +consumer on another tool version could never reproduce them. The head is +version-independent; the predicate bytes are not. + +Authenticity (`gh attestation verify`) and agreement (`loop verdict --compare`) +are separate checks, in that order, and **neither implies the other**. §24 +specifies both, together with the anchor carry-channel and the signer-trust +policy. **What an attestation buys — and does not.** The signature attests *context*: which repository, which workflow, which trigger, at what time. It never @@ -1640,3 +1654,203 @@ fabricated wholesale at authoring time is byte-valid. Detection of an unattested rewrite is at best one run late. And an attestation nothing verifies is decoration: until a consumer checks it, this section describes a publication surface, not a gate. + +## 24. Consuming an attested verdict — agreement, ancestry, and signer trust + +§23 specifies how a verdict is *published*. This section specifies how one is +*consumed*, and it is deliberately explicit about what consumption does **not** +establish. A reader who assumes otherwise has a false sense of a gate. + +**The comparison report.** `loop verdict --compare ` loads an +attested `verdict@1` predicate, projects the workspace locally, and reports +agreement over exactly **four** facets: `run_id`, `chain.head`, the whole +`terminal` object (`state`, `completion_policy`, `false_completion`), and the +`evidence` digest set (set equality over the `digest`/`code_digest`/`policy_digest` +triple). Exit **0** on agreement, **1** on disagreement, **2** on refusal. The +report's field set is `{ok, signature_checked, compared, issues}`, and the +`compared` block carries digests, enums and `run_id` only — no free text. +`issues[].message` is the one exempt surface: a local report may explain itself; a +predicate may not. + +`doctor` and `tool` are **deliberately not compared.** Both live inside the +predicate and are environment-coupled — the same run projects `873dfc87…` with +jsonschema and `8de3d88c…` in structural-fallback — so comparing them would make an +honest environment difference read as tampering. Whether an attested `doctor.ok` +should *gate* is a policy question, not an agreement question, and it is out of +scope here. + +**`signature_checked` is the literal `false` on every path, and there is no flag to +flip it.** The kernel establishes agreement; `gh attestation verify` establishes +authenticity; it runs first; neither implies the other. `verdict` rejects +`--verify-signature`, `--signature`, `--signer-workflow` and `--signer-digest` +outright, so the absence is a contract rather than an omission. + +**Typed refusals: a bare predicate only.** An in-toto Statement (any of `_type`, +`subject`, `predicateType`, `predicate`) and a `gh --format json` envelope (a +top-level array, or `verificationResult` / `attestation`) are refused by name, with +the documented unwrapping path `.[0].verificationResult.statement.predicate` in the +message. Best-effort parsing of a vendor envelope inside the kernel is exactly how a +trust boundary rots. + +**The subject byte form (normative).** The file a consumer presents to +`gh attestation verify` is **exactly 64 bytes**: the chain head as lowercase hex, +with **no trailing newline** and nothing else. There is one definition — +`loop.verdict.subject_bytes`, reached from either side as +`loop verdict --emit-subject` — so the signer side and the consumer side cannot +disagree about those bytes. A stray `\n` would change the subject digest, which is +why the form is pinned by test rather than left to a shell's `echo`. The subject +file's own digest is `sha256()` and is therefore **never** equal +to the head itself; that inequality is the crispest check that this mechanism, and +not §23's retired digest form, is what produced an attestation. + +**`loop-engineer/anchor@1` — the carry channel.** A tracked JSON document +(conventionally `loop-anchor.json` at the workspace root) whose required fields are +`schema` and `chain_head`; `sequence`, `attestation_id`, `run_id` and `recorded_at` +are optional provenance. It **must be tracked and must not live under `.loop/`** — +that directory is gitignored here, so an anchor inside the tree it certifies would +never land in a commit; `read_anchor` refuses a path with a `.loop` component +outright. Every `pattern` in the schema carries a sibling `maxLength`, because +jsonschema's `pattern` is `re.search` semantics and a bare anchored pattern accepts a +trailing newline. + +**The inversion that makes the anchor necessary: an attestation can corroborate a +carried head, but it can never discover one.** `GET +/repos/{owner}/{repo}/attestations/{subject_digest}` is the only list operation; the +no-digest route is a 404; there is no `gh attestation list`; GraphQL's `Repository` +type exposes no attestation fields; and **no ordering guarantee is documented +anywhere**. You must already know the digest to look anything up. So the head is +*carried* in a tracked file and the attestation proves that carried head was +notarized. + +**Anchor trust is exactly ordinary write access — no better.** An actor who can edit +the anchor file re-points it at a head they had attested. That is the same class of +limit as "the worker can edit the verifier" in §23. The anchor path joins +CODEOWNERS, and CODEOWNERS is a control only while the repository ruleset enforces +it. + +**Ancestry, not head equality.** `--expect-chain-head` is exact *current-head* +equality, so it fails **by construction** on any store that legitimately grew: +appending one event moves the head (measured: `9d388ae5…` at sequence 4 → +`c336ecdc…` at sequence 5). Feeding run N's head to `--expect-chain-head` at run N+1 +therefore always fails. The meaningful cross-run question is *"was this digest ever +my head?"*, and `loop doctor --expect-chain-ancestor ` (or `--anchor `, +which resolves the digest from an `anchor@1` file) asks it via +`loop.chain.head_sequence`. + +Ancestry is **established by replay, recomputing every hash — never by trusting the +stored `event_hash` column.** A tamperer who can rewrite the store can also insert a +row bearing the anchored digest, and only recomputation refuses that row. Sequence +`0` is a legitimate answer, so callers compare against `None`, never truthiness. +`--expect-chain-head` and `--expect-chain-ancestor` **compose** — equality and +ancestry are different questions and both may be asked — while `--anchor` and +`--expect-chain-ancestor` are **mutually exclusive** at the CLI, because silent +precedence between an explicit digest and a resolved one is how a gate becomes a +suggestion. Precedence between the action's `expect-chain-head` and `anchor` inputs is +resolved in `action.yml`, where the inputs are the surface, and a dropped anchor is +announced in the step summary rather than silently ignored. + +**The five new codes.** + +| Code | Meaning | +|---|---| +| `chain_anchor_not_ancestor` | The supplied ancestor digest was **never** the head at any sequence of the replayed chain. Also raised — never skipped — when the store is absent, empty or unreadable. | +| `anchor_file_unreadable` | The `--anchor` path is absent, unreadable, not UTF-8, or not JSON. | +| `anchor_file_invalid` | It parsed, but is not a conformant `anchor@1`. | +| `anchor_attestation_contradicted` | The index was reached, an attestation was found, and it does not corroborate the carried head. *I looked and it said no.* | +| `anchor_attestation_unavailable` | Nothing was found (404), **or** the index could not be reached at all (5xx, timeout, auth), **or** the classifier could not confidently classify what it saw. *I could not look.* | + +`chain_anchor_not_ancestor` is deliberately **not** a reuse of +`chain_anchor_mismatch`. "Your current head is not what I expected" and "the head you +anchored is not in my history at all" are different facts, and doctor issue codes are +the population `verdict.doctor.issue_codes` is drawn from — a permanent, public, +append-only log. One shared code would collapse them there forever. + +**The sentence that keeps this a gate.** *Anything short of a verified 200 plus a +successful `gh attestation verify` is non-promoting, and transport-class failures +(5xx, timeout, auth) are separately reportable but exactly as non-promoting as a +clean denial.* The two lookup codes exist for **observability**, never for +differential trust. + +This matters because the anchor is a **deletable dependency**. Attestations can be +deleted — user- and org-scoped delete, bulk-delete and delete-request endpoints all +exist, permission-gated — and GitHub's own guidance recommends deleting attestations +that are no longer needed. **No retention window is documented anywhere**; the +familiar 90-day and 400-day figures are workflow *artifacts and logs*, not +attestations, and the roadmap issue tracking expiry records none. So a missing anchor +attestation must be a typed failure: otherwise an availability attack on the index +becomes a gate bypass. + +**Do not over-read the codes.** A 404 is consistent with never-attested, +attested-then-deleted, and a transient index fault; HTTP status alone cannot separate +them. And do not key logic on response *body* text — the no-digest route returns a +generic `documentation_url` while the digest-present-but-non-matching family returns +one pointing at the list-attestations reference. + +**The signer-trust policy.** `loop.attestation.check_signer_trust` is a pure function +over an **already-verified** `verificationResult`. It reads only +`signature.certificate` and `verifiedTimestamps` — per `gh`'s own help, those are the +only fields the originating workflow cannot manipulate — and treats everything under +`statement.predicate` as data to compare, never to trust. It **refuses**, loudly and +typed, when a claim it needs is absent, because a policy that treats a missing claim +as satisfied is worse than no policy; an unwitnessed attestation (absent or empty +`verifiedTimestamps`) is likewise refused. Denials are typed codes — +`signer_workflow_mismatch`, `signer_repository_mismatch`, `self_hosted_runner`, +`signer_trigger_mismatch` — never booleans, and the returned verdict carries +`signature_checked: false` too, because it evaluates a conclusion `gh` already +reached. + +**`--signer-digest` is deliberately not required.** It pins the signer-digest +certificate extension, which is populated from the `job_workflow_sha` claim; for a +non-reusable top-level workflow that value **equals the triggering commit SHA**. It +therefore does not merely invalidate on a workflow edit — **it invalidates on every push**. +That was observed across all three attestations this repository minted before this +slice, even though none of those commits touched `attest.yml` or `action.yml`. +`--signer-workflow` (whose value was byte-identical across all three) is the mandatory +pin; `--signer-digest` is offered only as an optional, human-invoked one-off. +`check_signer_trust` has no `signer_digest` parameter at all. + +**The REST attestations route is deprecated.** Measured on both the 200 and the 404 +route and absent from four control endpoints, so it is route-level: +`Deprecation: Tue, 10 Mar 2026`, **`Sunset: Fri, 10 Mar 2028`**. The fetch/verify path +therefore goes through `gh attestation verify`, a GitHub-maintained abstraction that +will be migrated over whatever replaces the raw route, and the single `gh` call site +lives in `scripts/action_anchor_resolve.py` so a migration is a one-line change. The +sunset date is recorded here so a future maintainer meets it as a documented fact +rather than as an outage. + +**The `[0]` selection is an assumption, and is stated as one.** No ordering guarantee +is documented for the underlying endpoint, so `[0]` means *"some verified attestation +for this subject"* — never *"the newest"*. It is sound only because every entry has +already passed the same signer-trust policy over the same subject digest, so two +entries cannot materially disagree about the head. **If the policy is ever relaxed to +accept more than one signer, this assumption breaks** and the step must compare every +entry rather than indexing. + +**Public/private asymmetry.** Public repositories sign against the Sigstore Public +Good instance and its public transparency log. **Private repositories use GitHub's own +signing instance, which has no transparency log** and federates only with Actions. The +independent-audit property this design leans on exists for public repositories and does +**not** exist for private ones. For a public repository the attestations read also +succeeds unauthenticated, so an `attestations: read` permission is defensive +future-proofing rather than a requirement. + +**Honest limits.** + +1. **This repository cannot dogfood cross-run ancestry.** `.github/workflows/attest.yml` + seeds an ephemeral `$RUNNER_TEMP` workspace on every run, so its chain head is new + by construction and there is no persistent store to anchor. Coverage is therefore + (a) synthetic, through a fake `gh` on `PATH`, and (b) a real *within-run* grown-store + ancestry exercise. Do not read that CI green as a cross-run proof. +2. **Detection of an unattested rewrite is at best one run late**, unchanged from §23. + An actor who can land a commit lets CI run once, which mints a genuine attestation + over the rewritten chain. Opt-in changes operator consent; it does not change that. +3. **Mode parity is now CI-covered, and was not before.** Every other CI job installs + jsonschema, so the structural hand-checks that back `anchor@1` in a jsonschema-less + environment ran nowhere in CI: loosening a `fullmatch` to a `match` in `loop/anchor.py` + kills four tests in the pyyaml-only leg but only two with jsonschema installed, + because the schema layer masks the rest. The `gates-fallback` job closes that class, + and asserts jsonschema is genuinely absent so it cannot decay into a duplicate. +4. **`--source-ref refs/heads/main` is a repo-specific constant, not a placeholder.** An + adopter whose default branch differs must change it. It is deliberately **not** + parameterized: a `--source-ref` taken from an untrusted input would let a caller widen + the pin to any ref and defeat the control. diff --git a/schemas/anchor.schema.json b/schemas/anchor.schema.json new file mode 100644 index 0000000..45f716a --- /dev/null +++ b/schemas/anchor.schema.json @@ -0,0 +1,17 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "loop-engineer/anchor@1", + "title": "Loop Engineer Anchor @1", + "description": "A tracked carry channel for a previously attested chain head. Nothing here is trusted: the anchor is a carried claim whose only function is to give an attestation lookup a digest to ask about, because GitHub exposes no endpoint that lists attestations without one. Anchor trust is exactly ordinary write access to this file.", + "type": "object", + "additionalProperties": false, + "required": ["schema", "chain_head"], + "properties": { + "schema": { "const": "loop-engineer/anchor@1" }, + "chain_head": { "type": "string", "pattern": "^[0-9a-f]{64}$", "maxLength": 64 }, + "sequence": { "type": "integer", "minimum": 0 }, + "attestation_id": { "type": "string", "maxLength": 64 }, + "run_id": { "type": "string", "minLength": 1, "maxLength": 256 }, + "recorded_at": { "type": "string", "maxLength": 64 } + } +} diff --git a/scripts/action_anchor_resolve.py b/scripts/action_anchor_resolve.py new file mode 100644 index 0000000..258622c --- /dev/null +++ b/scripts/action_anchor_resolve.py @@ -0,0 +1,267 @@ +#!/usr/bin/env python3 +"""Resolve a carried anchor@1 head against the attestation index, via `gh`. + +The ONLY place in this repo that invokes `gh`. It lives in scripts/, not loop/, because +it reads the environment and touches the network — the tool layer, following the +scripts/action_scorecard.py precedent of an extracted, TESTED script the composite +action calls. + +WHY THE CLASSIFIER PARSES STDERR (normative, from `gh help exit-codes`, gh 2.92.0): + + 0 success 1 any failure 2 cancelled 4 authentication required + +There is NO distinct exit code separating "no attestation exists" from "an attestation +was found but the signer policy denied it" from "the index was unreachable". All three +arrive as exit 1 — measured live: a 64-hex subject with no matching attestation prints +`Error: HTTP 404: Not Found (…)` and exits 1, the same code a signature failure exits +with. So the classifier must read a vendor string that has no stability contract and +can drift. + +Three consequences, all normative: + +1. The fallback rule is fail-closed and ABSOLUTE: any output the classifier cannot + confidently classify becomes `unavailable`. Never `corroborated`. Never a skip. An + unrecognized stderr shape is the MOST suspicious case, not the most benign one. +2. Exit 4 (auth) and exit 2 (cancelled) are transport-class -> `unavailable`. Only + exit 0 PLUS a parseable payload PLUS a passing signer-trust policy reaches + `corroborated`. +3. The 404 branch is driven by a VERBATIM captured fixture + (scripts/fixtures/gh_attestation_verify/no_attestation_404.txt), not a paraphrase. + The DENIAL shape cannot be captured before this ships — no attestation this repo has + minted is verifiable yet — so it is captured in the first post-merge run. + +Anything short of a verified 200 plus a successful `gh attestation verify` is +non-promoting, and transport-class failures are separately reportable but exactly as +non-promoting as a clean denial. + +Exit codes: 0 corroborated, 1 contradicted-or-unavailable, 2 usage/refusal. +""" + +from __future__ import annotations + +import argparse +import json +import subprocess +import sys +from pathlib import Path + +_REPO_ROOT = Path(__file__).resolve().parent.parent +if str(_REPO_ROOT) not in sys.path: + # A path-invoked script has no installed package in CI (the slice-3b lesson). + sys.path.insert(0, str(_REPO_ROOT)) + +from loop.anchor import AnchorError, read_anchor # noqa: E402 +from loop.attestation import (AttestationPolicyError, anchor_lookup_issue, # noqa: E402 + check_signer_trust) +from loop.verdict import PREDICATE_TYPE, SUBJECT_NAME, subject_bytes # noqa: E402 + +# refs/heads/main is a deliberate repo-specific constant, NOT a placeholder: ADR 0002 +# decision 5 requires a push trigger on the default branch, and this repo's default +# branch is main. It is not parameterized because a --source-ref taken from an untrusted +# input would let a caller widen the pin to any ref and defeat the control. +SOURCE_REF = "refs/heads/main" +PREDICATE_FILENAME = "attested-verdict.json" + +# Checked in order. "nothing was found" and transport faults are read FIRST, because +# `HTTP 404: Not Found` would otherwise fall through to a denial pattern. +_UNAVAILABLE_MARKERS = ( + "http 404", "not found", "no attestation", "no matching attestation", + "http 500", "http 502", "http 503", "http 504", "timeout", "timed out", + "authentication", "gh auth", "connection refused", "connection reset", + "temporary failure", "cancelled", "canceled", +) +# Only these reach `contradicted`: an attestation WAS found and did not survive. +_CONTRADICTED_MARKERS = ( + "verification failed", "failed to verify", "signature", "does not match", + "not signed by", "unable to verify", "policy", "mismatch", +) + + +class ResolveUsageError(Exception): + """The step was invoked in a way that can never produce a trustworthy answer.""" + + +def _classify_failure(exit_code: int, stderr: str) -> tuple[str, str]: + """(outcome, detail) for a non-zero `gh` invocation. Never returns corroborated.""" + if exit_code in (2, 4): + return "unavailable", (f"gh exited {exit_code} (transport class: " + f"{'cancelled' if exit_code == 2 else 'authentication'})") + haystack = stderr.lower() + for marker in _UNAVAILABLE_MARKERS: + if marker in haystack: + return "unavailable", f"gh exited {exit_code}: {stderr.strip()[:400]}" + for marker in _CONTRADICTED_MARKERS: + if marker in haystack: + return "contradicted", f"gh exited {exit_code}: {stderr.strip()[:400]}" + # Fail-closed: an unrecognized shape is the most suspicious case, not the most benign. + return "unavailable", (f"gh exited {exit_code} with output this classifier cannot " + f"confidently classify: {stderr.strip()[:400]}") + + +def _verification_result(stdout: str) -> tuple[dict | None, str | None]: + """The first entry's verificationResult, or (None, detail) naming the shape that failed. + + Each caught class is named deliberately: a blanket catch-all handler would also + swallow a genuine bug in this file and report it as a clean "index unavailable", + which is a false-negative gate. + """ + try: + payload = json.loads(stdout) # JSONDecodeError: banner/empty + except json.JSONDecodeError as exc: + return None, f"gh stdout was not JSON: {exc}" + if not isinstance(payload, list): + return None, (f"gh stdout was not a list (found {type(payload).__name__}) — the " + "--format json contract is an array of attestations") + if not payload: + return None, "gh stdout was an empty array: no attestation to evaluate" + try: + result = payload[0]["verificationResult"] # KeyError / TypeError + except (KeyError, TypeError) as exc: + return None, f"gh stdout is missing [0].verificationResult: {exc}" + if not isinstance(result, dict): + return None, (f"[0].verificationResult is not an object " + f"(found {type(result).__name__})") + return result, None + + +def _bare_predicate(result: dict) -> tuple[dict | None, str | None]: + """`.[0].verificationResult.statement.predicate`, so `loop verdict --compare` + receives a bare verdict@1 and not the envelope it is required to refuse.""" + try: + predicate = result["statement"]["predicate"] + except (KeyError, TypeError) as exc: + return None, f"gh stdout is missing [0].verificationResult.statement.predicate: {exc}" + if not isinstance(predicate, dict): + return None, f"the extracted predicate is not an object (found {type(predicate).__name__})" + return predicate, None + + +def _run_gh(subject_path: Path, repo: str, signer_workflow: str) -> tuple[int, str, str] | str: + """Invoke gh as an argv list with shell=False. Returns (code, stdout, stderr), or a + detail string when the process never started at all — categorically different from a + bad exit code, because there is no stderr to classify. + + --predicate-type is MANDATORY: it defaults to the SLSA provenance type and would + reject every verdict@1 attestation. --deny-self-hosted-runners is passed + unconditionally and there is deliberately no input to disable it. + --signer-digest is NEVER passed (D4): it resolves to job_workflow_sha, which for a + non-reusable top-level workflow equals the triggering commit SHA, so it would + invalidate on every push. + """ + argv = [ + "gh", "attestation", "verify", str(subject_path), + "--repo", repo, + "--predicate-type", PREDICATE_TYPE, + "--signer-workflow", signer_workflow, + "--deny-self-hosted-runners", + "--source-ref", SOURCE_REF, + "--format", "json", + ] + try: + proc = subprocess.run(argv, capture_output=True, text=True, shell=False) + except FileNotFoundError: + return "gh was not found on PATH: the process never started, so there is no exit code to classify" + except OSError as exc: + return f"gh could not be executed: {exc}" + return proc.returncode, proc.stdout, proc.stderr + + +def _emit(outputs: dict[str, str], github_output: str | None) -> None: + lines = [f"{name}={value}" for name, value in outputs.items()] + if github_output: + with open(github_output, "a", encoding="utf-8") as handle: + handle.write("\n".join(lines) + "\n") + else: + print("\n".join(lines)) + + +def resolve(args: argparse.Namespace) -> int: + if not args.signer_workflow or not args.signer_workflow.strip(): + raise ResolveUsageError( + "--signer-workflow is mandatory: it is the pin that makes a corroboration " + "mean anything. Expected the form [host/]////") + try: + anchor = read_anchor(args.anchor) + except AnchorError as exc: + raise ResolveUsageError(f"anchor is unusable, refusing to resolve: {exc}") from exc + + head = anchor["chain_head"] + runner_temp = Path(args.runner_temp) + runner_temp.mkdir(parents=True, exist_ok=True) + subject_path = runner_temp / SUBJECT_NAME + # Regenerated from the carried head alone. Only possible because the subject is a + # head-bearing FILE: under a synthesized subject-digest there is no preimage, so no + # file could ever be presented to `gh attestation verify`. + subject_path.write_bytes(subject_bytes(head)) + + outputs = {"anchor-outcome": "unavailable", "anchor-head": head, + "subject-path": str(subject_path), "predicate-path": ""} + + invocation = _run_gh(subject_path, args.repo, args.signer_workflow) + if isinstance(invocation, str): + return _finish("unavailable", invocation, outputs, args.github_output) + code, stdout, stderr = invocation + if code != 0: + outcome, detail = _classify_failure(code, stderr) + return _finish(outcome, detail, outputs, args.github_output) + + result, shape_detail = _verification_result(stdout) + if result is None: + return _finish("unavailable", shape_detail, outputs, args.github_output) + + try: + trust = check_signer_trust(result, signer_workflow=args.signer_workflow, + source_repository_uri=f"https://github.com/{args.repo}") + except AttestationPolicyError as exc: + # A claim name that does not match reality is a FAILURE, never a skip. It is + # `unavailable` rather than `contradicted`: we could not evaluate the claims, so + # we cannot honestly say the index said no. + return _finish("unavailable", f"signer-trust policy refused: {exc}", + outputs, args.github_output) + if not trust["ok"]: + codes = ", ".join(issue["code"] for issue in trust["issues"]) + return _finish("contradicted", f"signer-trust policy denied: {codes}", + outputs, args.github_output) + + predicate, predicate_detail = _bare_predicate(result) + if predicate is None: + return _finish("unavailable", predicate_detail, outputs, args.github_output) + predicate_path = runner_temp / PREDICATE_FILENAME + predicate_path.write_text(json.dumps(predicate), encoding="utf-8") + outputs["predicate-path"] = str(predicate_path) + return _finish("corroborated", None, outputs, args.github_output) + + +def _finish(outcome: str, detail: str | None, outputs: dict[str, str], + github_output: str | None) -> int: + outputs["anchor-outcome"] = outcome + _emit(outputs, github_output) + issue = anchor_lookup_issue(outcome, detail=detail) + if issue is None: + print(f"::notice::anchor corroborated for {outputs['anchor-head']}") + return 0 + print(f"::error::{issue['code']}: {issue['message']}") + return 1 + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + prog="action_anchor_resolve.py", + description="Corroborate a carried anchor@1 chain head against the attestation index.") + parser.add_argument("--anchor", required=True, help="path to a tracked anchor@1 file") + parser.add_argument("--repo", required=True, help="owner/repo") + parser.add_argument("--signer-workflow", default="", + help="[host/]//// (mandatory)") + parser.add_argument("--runner-temp", required=True, help="scratch directory") + parser.add_argument("--github-output", default=None, + help="append step outputs here instead of stdout") + args = parser.parse_args(argv) + try: + return resolve(args) + except ResolveUsageError as exc: + print(f"action_anchor_resolve: {exc}", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/fixtures/gh_attestation_verify/no_attestation_404.txt b/scripts/fixtures/gh_attestation_verify/no_attestation_404.txt new file mode 100644 index 0000000..f7e0f28 --- /dev/null +++ b/scripts/fixtures/gh_attestation_verify/no_attestation_404.txt @@ -0,0 +1,2 @@ + +Error: HTTP 404: Not Found (https://api.github.com/repos/SollanSystems/loop-engineer/attestations/sha256:60e05bd1b195af2f94112fa7197a5c88289058840ce7c6df9693756bc6250f55?per_page=30&predicate_type=urn:loop-engineer:verdict:1) diff --git a/scripts/test_action_anchor_resolve.py b/scripts/test_action_anchor_resolve.py new file mode 100644 index 0000000..57b0a27 --- /dev/null +++ b/scripts/test_action_anchor_resolve.py @@ -0,0 +1,313 @@ +"""scripts/test_action_anchor_resolve.py — the resolve step, against a fake `gh`. + +The highest-risk new module in the slice: it shells out and parses another program's +stdout. Every failure must land as a TYPED outcome naming which shape assumption failed +— never a traceback, never a silent pass, and never `corroborated`. + +`gh`'s exit codes cannot tell D5's three outcomes apart (all arrive as exit 1), so the +classifier reads stderr. The 404 branch is therefore driven by a VERBATIM captured +vendor string, not a paraphrase: a remembered approximation is exactly the thing that +passes review and fails in production. +""" +from __future__ import annotations + +import ast +import json +import os +import pathlib +import subprocess +import sys + +import pytest + +ROOT = pathlib.Path(__file__).resolve().parent.parent +SCRIPT = ROOT / "scripts" / "action_anchor_resolve.py" +FIXTURE_404 = ROOT / "scripts" / "fixtures" / "gh_attestation_verify" / "no_attestation_404.txt" + +_HEAD = "a1" * 32 +_REPO = "SollanSystems/loop-engineer" +_WORKFLOW = "SollanSystems/loop-engineer/.github/workflows/attest.yml" + +_CERTIFICATE = { + "subjectAlternativeName": f"https://github.com/{_WORKFLOW}@refs/heads/main", + "sourceRepositoryURI": f"https://github.com/{_REPO}", + "runnerEnvironment": "github-hosted", + "githubWorkflowTrigger": "push", +} +_PREDICATE = {"schema": "loop-engineer/verdict@1", "run_id": "coverage-repair", + "chain": {"head": _HEAD, "sequence": 3, "unchained_prefix": 0}} + + +def _payload(*, certificate=None, statement=True): + result = {"signature": {"certificate": dict(certificate or _CERTIFICATE)}, + "verifiedTimestamps": [{"type": "Tlog", "timestamp": "2026-07-29T00:00:00Z"}]} + if statement: + result["statement"] = {"predicateType": "urn:loop-engineer:verdict:1", + "predicate": _PREDICATE} + return json.dumps([{"attestation": {}, "verificationResult": result}]) + + +def _anchor(tmp_path, head=_HEAD): + path = tmp_path / "loop-anchor.json" + path.write_text(json.dumps({"schema": "loop-engineer/anchor@1", "chain_head": head}), + encoding="utf-8") + return path + + +def _shim(tmp_path, *, stdout="", stderr="", exit_code=0): + """An executable `gh` on PATH that logs its real argv and emits canned output.""" + bin_dir = tmp_path / "bin" + bin_dir.mkdir(parents=True, exist_ok=True) + log = tmp_path / "argv.log" + (bin_dir / "gh").write_text( + "#!/bin/sh\n" + f'for arg in "$@"; do echo "$arg" >> {log}; done\n' + f"cat {tmp_path / 'stdout.txt'}\n" + f"cat {tmp_path / 'stderr.txt'} >&2\n" + f"exit {exit_code}\n", + encoding="utf-8") + (bin_dir / "gh").chmod(0o755) + (tmp_path / "stdout.txt").write_text(stdout, encoding="utf-8") + (tmp_path / "stderr.txt").write_text(stderr, encoding="utf-8") + return bin_dir, log + + +def _resolve(tmp_path, *, bin_dir=None, anchor=None, signer_workflow=_WORKFLOW, + path_override=None): + env = dict(os.environ) + if path_override is not None: + env["PATH"] = str(path_override) + elif bin_dir is not None: + env["PATH"] = f"{bin_dir}{os.pathsep}{env.get('PATH', '')}" + runner_temp = tmp_path / "runner-temp" + outputs = tmp_path / "github-output.txt" + proc = subprocess.run( + [sys.executable, "-B", str(SCRIPT), + "--anchor", str(anchor if anchor is not None else _anchor(tmp_path)), + "--repo", _REPO, + "--signer-workflow", signer_workflow, + "--runner-temp", str(runner_temp), + "--github-output", str(outputs)], + capture_output=True, text=True, env=env, cwd=tmp_path) + parsed = {} + if outputs.exists(): + for line in outputs.read_text(encoding="utf-8").splitlines(): + if "=" in line: + name, _, value = line.partition("=") + parsed[name] = value + return proc, parsed, runner_temp + + +def _argv(log: pathlib.Path) -> list[str]: + return log.read_text(encoding="utf-8").splitlines() if log.exists() else [] + + +# --- the three outcomes ------------------------------------------------------ + + +def test_resolve_corroborates_with_a_fake_gh(tmp_path): + bin_dir, _log = _shim(tmp_path, stdout=_payload()) + proc, outputs, _temp = _resolve(tmp_path, bin_dir=bin_dir) + assert proc.returncode == 0, proc.stderr + proc.stdout + assert outputs["anchor-outcome"] == "corroborated" + + +def test_resolve_reports_contradicted_when_verify_denies(tmp_path): + """A signer the policy denies: an attestation WAS found and did not survive.""" + denied = dict(_CERTIFICATE, + subjectAlternativeName="https://github.com/other/repo/.github/workflows/x.yml@refs/heads/main") + bin_dir, _log = _shim(tmp_path, stdout=_payload(certificate=denied)) + proc, outputs, _temp = _resolve(tmp_path, bin_dir=bin_dir) + assert proc.returncode == 1 + assert outputs["anchor-outcome"] == "contradicted" + assert "anchor_attestation_contradicted" in proc.stdout + + +def test_resolve_reports_unavailable_on_a_404(tmp_path): + bin_dir, _log = _shim(tmp_path, stderr="Error: HTTP 404: Not Found (…)", exit_code=1) + proc, outputs, _temp = _resolve(tmp_path, bin_dir=bin_dir) + assert proc.returncode == 1 + assert outputs["anchor-outcome"] == "unavailable" + assert "anchor_attestation_unavailable" in proc.stdout + + +def test_resolve_reports_unavailable_on_a_transport_failure(tmp_path): + """A 5xx is 'I could not look', NOT 'it said no'.""" + bin_dir, _log = _shim(tmp_path, stderr="Error: HTTP 503 Service Unavailable", exit_code=7) + proc, outputs, _temp = _resolve(tmp_path, bin_dir=bin_dir) + assert proc.returncode == 1 + assert outputs["anchor-outcome"] == "unavailable" + + +def test_resolve_classifies_the_real_captured_404_stderr(tmp_path): + """M2 — driven by the committed verbatim capture, not a paraphrase.""" + assert FIXTURE_404.is_file(), "the captured vendor string is the source of truth here" + captured = FIXTURE_404.read_text(encoding="utf-8") + assert "HTTP 404" in captured and "attestations/sha256:" in captured + bin_dir, _log = _shim(tmp_path, stderr=captured, exit_code=1) + proc, outputs, _temp = _resolve(tmp_path, bin_dir=bin_dir) + assert proc.returncode == 1 + assert outputs["anchor-outcome"] == "unavailable" + + +def test_resolve_maps_an_unclassifiable_failure_to_unavailable(tmp_path): + """M2's fallback rule: an unrecognized shape is the most suspicious case.""" + bin_dir, _log = _shim(tmp_path, stderr="weasel", exit_code=1) + proc, outputs, _temp = _resolve(tmp_path, bin_dir=bin_dir) + assert proc.returncode == 1 + assert outputs["anchor-outcome"] == "unavailable" + assert "cannot" in proc.stdout # says it could not classify + + +@pytest.mark.parametrize("exit_code", [4, 2]) +def test_resolve_maps_gh_auth_and_cancel_exits_to_unavailable(tmp_path, exit_code): + bin_dir, _log = _shim(tmp_path, stderr="", exit_code=exit_code) + proc, outputs, _temp = _resolve(tmp_path, bin_dir=bin_dir) + assert proc.returncode == 1 + assert outputs["anchor-outcome"] == "unavailable" + + +def test_resolve_reports_unavailable_when_gh_is_not_on_path(tmp_path): + """M3 — no shim can reach this: subprocess.run raises FileNotFoundError before any + exit code or stderr exists, so the process never started.""" + empty_bin = tmp_path / "empty-bin" + empty_bin.mkdir() + proc, outputs, _temp = _resolve(tmp_path, path_override=empty_bin) + assert proc.returncode == 1 + assert outputs["anchor-outcome"] == "unavailable" + assert "gh was not found on PATH" in proc.stdout + assert "Traceback" not in proc.stderr + + +@pytest.mark.parametrize("stdout,marker", [ + ("this is not json at all", "not JSON"), + ("[]", "empty array"), + ('{"verificationResult": {}}', "not a list"), +]) +def test_resolve_reports_unavailable_on_unparseable_gh_stdout(tmp_path, stdout, marker): + """M1 — each shape assumption names itself when it fails.""" + bin_dir, _log = _shim(tmp_path, stdout=stdout, exit_code=0) + proc, outputs, _temp = _resolve(tmp_path, bin_dir=bin_dir) + assert proc.returncode == 1 + assert outputs["anchor-outcome"] == "unavailable" + assert marker in proc.stdout + assert "Traceback" not in proc.stderr + + +# --- fail-closed refusals ---------------------------------------------------- + + +def test_resolve_fails_closed_when_signer_workflow_is_empty(tmp_path): + bin_dir, _log = _shim(tmp_path, stdout=_payload()) + proc, _outputs, _temp = _resolve(tmp_path, bin_dir=bin_dir, signer_workflow="") + assert proc.returncode == 2 + assert "--signer-workflow" in proc.stderr + assert "/" in proc.stderr + + +def test_resolve_fails_closed_on_an_unreadable_anchor(tmp_path): + bin_dir, _log = _shim(tmp_path, stdout=_payload()) + proc, _outputs, _temp = _resolve(tmp_path, bin_dir=bin_dir, + anchor=tmp_path / "absent.json") + assert proc.returncode == 2 + assert "anchor" in proc.stderr + assert "Traceback" not in proc.stderr + + +def test_resolve_refuses_when_the_policy_claims_are_absent(tmp_path): + """A pinned claim name that does not match reality is a FAILURE, not a skip.""" + incomplete = {k: v for k, v in _CERTIFICATE.items() if k != "runnerEnvironment"} + bin_dir, _log = _shim(tmp_path, stdout=_payload(certificate=incomplete)) + proc, outputs, _temp = _resolve(tmp_path, bin_dir=bin_dir) + assert proc.returncode == 1 + assert outputs["anchor-outcome"] != "corroborated" + assert "runnerEnvironment" in proc.stdout + + +# --- the subject file, the extraction, and the argv -------------------------- + + +def test_resolve_writes_the_subject_file_bytes_from_the_anchor(tmp_path): + bin_dir, _log = _shim(tmp_path, stdout=_payload()) + _proc, outputs, runner_temp = _resolve(tmp_path, bin_dir=bin_dir) + subject = pathlib.Path(outputs["subject-path"]) + assert subject.parent == runner_temp + assert subject.name == "loop-chain-head" + assert subject.read_bytes() == _HEAD.encode("ascii") + assert len(subject.read_bytes()) == 64 + + +def test_resolve_extracts_a_bare_predicate_for_compare(tmp_path): + bin_dir, _log = _shim(tmp_path, stdout=_payload()) + _proc, outputs, _temp = _resolve(tmp_path, bin_dir=bin_dir) + extracted = json.loads(pathlib.Path(outputs["predicate-path"]).read_text(encoding="utf-8")) + assert extracted["schema"] == "loop-engineer/verdict@1" + assert not {"_type", "subject", "predicateType", "predicate"} & extracted.keys() + + +def test_resolve_never_passes_signer_digest(tmp_path): + """D4: it resolves to job_workflow_sha, so it invalidates on every push.""" + bin_dir, log = _shim(tmp_path, stdout=_payload()) + _resolve(tmp_path, bin_dir=bin_dir) + assert "--signer-digest" not in _argv(log) + assert "--signer-digest" not in SCRIPT.read_text(encoding="utf-8").replace( + "--signer-digest is NEVER passed", "") + + +def test_resolve_always_passes_deny_self_hosted_runners(tmp_path): + bin_dir, log = _shim(tmp_path, stdout=_payload()) + _resolve(tmp_path, bin_dir=bin_dir) + assert "--deny-self-hosted-runners" in _argv(log) + + +def test_resolve_passes_the_predicate_type(tmp_path): + """Without it gh enforces the SLSA default and rejects everything.""" + bin_dir, log = _shim(tmp_path, stdout=_payload()) + _resolve(tmp_path, bin_dir=bin_dir) + argv = _argv(log) + assert "--predicate-type" in argv + assert argv[argv.index("--predicate-type") + 1] == "urn:loop-engineer:verdict:1" + assert "--source-ref" in argv + assert argv[argv.index("--source-ref") + 1] == "refs/heads/main" + + +def test_resolve_invokes_gh_with_shell_false_argv(tmp_path): + """The anchor head is never interpolated into a shell string.""" + bin_dir, log = _shim(tmp_path, stdout=_payload()) + _resolve(tmp_path, bin_dir=bin_dir) + argv = _argv(log) + assert all(" " not in token for token in argv), argv + tree = ast.parse(SCRIPT.read_text(encoding="utf-8")) + runs = [node for node in ast.walk(tree) + if isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute) + and node.func.attr == "run"] + assert len(runs) == 1 + shell = [kw for kw in runs[0].keywords if kw.arg == "shell"] + assert shell and isinstance(shell[0].value, ast.Constant) and shell[0].value.value is False + assert isinstance(runs[0].args[0], ast.Name) # an argv list, not a string + + +def test_resolve_isolates_the_single_gh_invocation(tmp_path): + """D6 — one call site, so a migration off the deprecated route is a one-line change.""" + tree = ast.parse(SCRIPT.read_text(encoding="utf-8")) + call_sites = [node for node in ast.walk(tree) + if isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute) + and node.func.attr == "run" + and isinstance(node.func.value, ast.Name) and node.func.value.id == "subprocess"] + assert len(call_sites) == 1 + # AST-level, not a string count: prose ABOUT not using a bare catch must not fail a + # test whose subject is the code. Every handler names the classes it means. + blanket = [ + handler for node in ast.walk(tree) if isinstance(node, ast.Try) + for handler in node.handlers + if handler.type is None + or (isinstance(handler.type, ast.Name) and handler.type.id in ("Exception", "BaseException")) + ] + assert blanket == [], [ast.dump(h) for h in blanket] + + +def test_resolve_emits_all_four_step_outputs(tmp_path): + bin_dir, _log = _shim(tmp_path, stdout=_payload()) + _proc, outputs, _temp = _resolve(tmp_path, bin_dir=bin_dir) + assert set(outputs) == {"anchor-outcome", "anchor-head", "predicate-path", "subject-path"} + assert outputs["anchor-head"] == _HEAD diff --git a/scripts/test_action_attest_surface.py b/scripts/test_action_attest_surface.py new file mode 100644 index 0000000..25fab88 --- /dev/null +++ b/scripts/test_action_attest_surface.py @@ -0,0 +1,152 @@ +"""scripts/test_action_attest_surface.py — action.yml's attest and anchor surface. + +D1 is the load-bearing change of the slice: the subject becomes a head-bearing FILE, so +`gh attestation verify` is executable against a verdict@1 attestation for the first time. + +A composite action that swallows the resolve step's exit code turns the whole gate into +decoration — Task 8's tests exercise the SCRIPT, and nothing there pins that action.yml +lets its failure reach the job. That wiring is pinned here. +""" +from __future__ import annotations + +import pathlib + +import pytest +import yaml + +ROOT = pathlib.Path(__file__).resolve().parent.parent +ACTION = ROOT / "action.yml" + +# Scoped by step id/name, not a blanket string ban: this repo contains three DELIBERATE +# advisory uses that must keep passing — `set +e` in "loop inspect (scorecard)" (inspect +# is advisory) and in "PR scorecard comment (optional)" (non-fatal on any API failure), +# and `if: always()` on "chain head (anchor surface)" (a mismatch is exactly the run whose +# head an operator needs recorded). A blanket ban would fail on correct existing code and +# be deleted by the next implementer instead of fixed. +_GATING_STEPS = ("resolve the anchor attestation", "compare the attested verdict") + + +@pytest.fixture(scope="module") +def action() -> dict: + return yaml.safe_load(ACTION.read_text(encoding="utf-8")) + + +@pytest.fixture(scope="module") +def steps(action) -> dict[str, dict]: + return {step.get("name"): step for step in action["runs"]["steps"] if step.get("name")} + + +def test_action_attests_a_subject_path_not_a_subject_digest(steps): + """D1. Under the retired form the subject digest was a SHA-256 over a synthesized + event preimage: no bytes hash to it, so no artifact could ever be presented.""" + attest = steps["attest verdict"] + assert attest["uses"].startswith("actions/attest@") + assert "subject-path" in attest["with"] + assert "subject-digest" not in attest["with"] + assert "subject-name" not in attest["with"] + + +def test_action_never_passes_subject_digest_anywhere(): + assert "subject-digest" not in ACTION.read_text(encoding="utf-8") + + +def test_subject_file_basename_is_the_pinned_subject_name(steps): + from loop.verdict import SUBJECT_NAME + + body = steps["chain-head subject file"]["run"] + assert f'"${{RUNNER_TEMP}}/{SUBJECT_NAME}"' in body + assert steps["attest verdict"]["with"]["subject-path"] == "${{ steps.subject.outputs.path }}" + + +def test_subject_file_is_written_by_emit_subject(steps): + """One definition of the byte form, so the attest side and the resolve side cannot + disagree about 64 bytes.""" + assert "loop verdict --emit-subject" in steps["chain-head subject file"]["run"] + + +def test_action_pins_push_to_registry_and_create_storage_record_false(steps): + """create-storage-record's live default is true (F5), so the explicit pin does work.""" + with_block = steps["attest verdict"]["with"] + assert with_block["push-to-registry"] is False + assert with_block["create-storage-record"] is False + + +def test_action_declares_the_anchor_and_signer_workflow_inputs(action): + for name in ("anchor", "signer-workflow"): + assert action["inputs"][name]["default"] == "" + assert action["inputs"][name]["required"] is False + + +def test_action_requires_signer_workflow_when_anchor_is_set(steps): + """The script fails closed; the action must actually hand it the input.""" + resolve = steps["resolve the anchor attestation"] + assert "--signer-workflow" in resolve["run"] + assert resolve["env"]["LOOP_SIGNER_WORKFLOW"] == "${{ inputs.signer-workflow }}" + assert "scripts/action_anchor_resolve.py" in resolve["run"] + + +def test_action_outputs_the_anchor_outcome(action): + assert "anchor-outcome" in action["outputs"] + assert action["outputs"]["anchor-outcome"]["value"] == \ + "${{ steps.anchor.outputs.anchor-outcome }}" + + +def test_explicit_expect_chain_head_wins_over_the_resolved_anchor(steps): + """ADR decision 5's precedence, expressed in the guard rather than left implicit — + and the drop is announced, because a silently dropped anchor is worse than a refused + one.""" + assert steps["resolve the anchor attestation"]["if"] == \ + "${{ inputs.anchor != '' && inputs.expect-chain-head == '' }}" + skip = steps["anchor resolution skipped (explicit head wins)"] + assert skip["if"] == "${{ inputs.anchor != '' && inputs.expect-chain-head != '' }}" + assert "GITHUB_STEP_SUMMARY" in skip["run"] + + +def _effective(run: str) -> str: + """The shell body with comment-only lines removed. + + The comments in these steps explain WHY they must not swallow an exit code, and they + name the constructs they avoid. Matching against raw text would let accurate prose + fail a test whose subject is the executed script. + """ + return "\n".join(line for line in run.splitlines() if not line.strip().startswith("#")) + + +@pytest.mark.parametrize("name", _GATING_STEPS) +def test_resolve_step_and_downstream_checks_have_no_continue_on_error(steps, name): + """B2 — a swallowed exit code here makes the gate decoration.""" + step = steps[name] + assert "continue-on-error" not in step # not even `false` + body = _effective(step["run"]) + assert "set +e" not in body + assert "|| true" not in body + assert body.strip(), "the step must still actually run something" + + +def test_compare_is_guarded_on_head_equality_while_ancestry_is_unconditional(steps): + """Found by whole-branch review: `--compare` treats an ancestor head as a + DISAGREEMENT (a verdict projects one run), so running it against an older attested + predicate on a store that legitimately grew fails every time — which would make the + `anchor` input unusable for the cross-run detection it exists for. Ancestry is the + cross-run gate and must be unconditional; agreement is a same-run question and must + be guarded on head equality, with the skip announced.""" + body = steps["compare the attested verdict"]["run"] + ancestry, _, remainder = body.partition("loop doctor --expect-chain-ancestor") + assert remainder, "the ancestry gate must be present" + assert "loop verdict --compare" not in ancestry, "ancestry must run FIRST, unguarded" + assert 'if [ "$ANCHOR_HEAD" = "$CURRENT_HEAD" ]' in remainder + assert "loop verdict --compare" in remainder + assert "GITHUB_STEP_SUMMARY" in remainder, "a skipped compare must be announced" + env = steps["compare the attested verdict"]["env"] + assert env["CURRENT_HEAD"] == "${{ steps.chain-head.outputs.chain-head }}" + + +def test_no_gating_step_is_marked_if_always(steps): + """always() runs a step after an upstream failure and is the standard way a gate's + red goes unseen. The pre-existing `if: always()` on "chain head (anchor surface)" is + deliberate and is excluded by name.""" + for name in _GATING_STEPS: + assert "always()" not in str(steps[name].get("if", "")) + # Non-vacuous: the one deliberate always() really is present, so the exclusion is a + # scoping decision rather than an absence of any always() to find. + assert "always()" in str(steps["chain head (anchor surface)"]["if"]) diff --git a/scripts/test_anchor.py b/scripts/test_anchor.py new file mode 100644 index 0000000..5d3b13e --- /dev/null +++ b/scripts/test_anchor.py @@ -0,0 +1,186 @@ +import hashlib +import json + +import pytest + +from loop import contract +from loop._resources import schemas_dir +from loop.anchor import ( + ANCHOR_SCHEMA_ID, + DEFAULT_ANCHOR_FILENAME, + AnchorError, + read_anchor, +) +from loop.verdict import SUBJECT_NAME, VerdictError, subject_bytes + +# A real chain head this repo actually attested (run 30472017488), so the fixture is +# a value the live signer produced rather than a hand-typed 64 characters. +HEAD = "c2333cd250122cc111d5e0e97322a2458abbd1b890876d19efde750c921d6ae9" + +# The five ways a 64-lowercase-hex field is malformed. Shared by the anchor reader and +# subject_bytes so the two cannot drift on what "a head" means. +MALFORMED_HEADS = [ + pytest.param(HEAD.upper(), id="uppercase"), + pytest.param(HEAD[:63], id="63-chars"), + pytest.param(HEAD + "a", id="65-chars"), + pytest.param(HEAD + "\n", id="trailing-newline"), + pytest.param("z" * 64, id="non-hex"), +] + + +def _anchor(**overrides): + document = {"schema": ANCHOR_SCHEMA_ID, "chain_head": HEAD, "sequence": 41, + "attestation_id": "37747063", "run_id": "coverage-repair", + "recorded_at": "2026-07-29T00:00:00+00:00"} + document.update(overrides) + return document + + +def _write(path, document): + path.parent.mkdir(parents=True, exist_ok=True) + payload = document if isinstance(document, str) else json.dumps(document) + path.write_text(payload, encoding="utf-8") + return path + + +def _load_schema(): + return json.loads((schemas_dir() / "anchor.schema.json").read_text(encoding="utf-8")) + + +def _patterns_without_maxlength(node, trail=()): + """Every subschema carrying a `pattern` but no sibling `maxLength`.""" + offenders = [] + if isinstance(node, dict): + if "pattern" in node and "maxLength" not in node: + offenders.append("/".join(trail) or "") + for key, value in node.items(): + offenders.extend(_patterns_without_maxlength(value, trail + (str(key),))) + elif isinstance(node, list): + for index, value in enumerate(node): + offenders.extend(_patterns_without_maxlength(value, trail + (str(index),))) + return offenders + + +def test_anchor_schema_id_and_default_filename_are_pinned(): + assert ANCHOR_SCHEMA_ID == "loop-engineer/anchor@1" + assert DEFAULT_ANCHOR_FILENAME == "loop-anchor.json" + + +def test_anchor_schema_file_declares_the_matching_id(): + schema = _load_schema() + assert schema["$id"] == ANCHOR_SCHEMA_ID + assert schema["$schema"] == "https://json-schema.org/draft/2020-12/schema" + + +def test_anchor_schema_is_not_a_contract_artifact(): + # anchor@1 is a carry channel, not a contract object; SCHEMA_IDS is what doctor + # validates a workspace against, and an anchor is not part of a workspace. + assert ANCHOR_SCHEMA_ID not in contract.SCHEMA_IDS + + +def test_anchor_schema_pattern_carries_a_maxlength(): + # jsonschema's `pattern` is re.search semantics, so an anchored pattern alone + # still accepts a trailing newline — the hole PR #89 closed for the other schemas. + assert _patterns_without_maxlength(_load_schema()) == [] + + +def test_anchor_schema_validates_the_shipped_example_anchor(): + jsonschema = pytest.importorskip("jsonschema") + validator = jsonschema.Draft202012Validator(_load_schema()) + assert list(validator.iter_errors(_anchor())) == [] + + +def test_read_anchor_returns_the_carried_head(tmp_path): + anchor = read_anchor(_write(tmp_path / DEFAULT_ANCHOR_FILENAME, _anchor())) + assert anchor["chain_head"] == HEAD + assert anchor["sequence"] == 41 + assert anchor["attestation_id"] == "37747063" + + +def test_read_anchor_refuses_a_missing_file(tmp_path): + missing = tmp_path / DEFAULT_ANCHOR_FILENAME + with pytest.raises(AnchorError) as excinfo: + read_anchor(missing) + assert DEFAULT_ANCHOR_FILENAME in str(excinfo.value) + + +def test_read_anchor_refuses_a_non_object(tmp_path): + with pytest.raises(AnchorError): + read_anchor(_write(tmp_path / DEFAULT_ANCHOR_FILENAME, json.dumps([_anchor()]))) + + +def test_read_anchor_refuses_undecodable_bytes(tmp_path): + path = tmp_path / DEFAULT_ANCHOR_FILENAME + path.write_bytes(b"\xff\xfe\xfd") + # The #107 lesson: a decode failure is a typed finding, never an escaped + # UnicodeDecodeError. pytest.raises(AnchorError) would not catch one. + with pytest.raises(AnchorError): + read_anchor(path) + + +def test_read_anchor_refuses_a_wrong_schema_id(tmp_path): + document = _anchor(schema="loop-engineer/verdict@1") + with pytest.raises(AnchorError): + read_anchor(_write(tmp_path / DEFAULT_ANCHOR_FILENAME, document)) + + +def test_read_anchor_refuses_a_missing_chain_head(tmp_path): + document = _anchor() + del document["chain_head"] + with pytest.raises(AnchorError): + read_anchor(_write(tmp_path / DEFAULT_ANCHOR_FILENAME, document)) + + +@pytest.mark.parametrize("overrides", [ + pytest.param({"sequence": -1}, id="negative-sequence"), + pytest.param({"sequence": "41"}, id="stringly-sequence"), +]) +def test_read_anchor_refuses_a_schema_invalid_but_json_valid_document(tmp_path, overrides): + # Parses, carries the right schema id, carries a well-formed chain_head, and + # violates anchor@1 elsewhere. Deliberately NOT importorskip-gated: mode parity + # is the point, and gating it would let the fallback leg accept a document the + # schema rejects (the S3 plan-lint mode-parity repair is the precedent). + with pytest.raises(AnchorError): + read_anchor(_write(tmp_path / DEFAULT_ANCHOR_FILENAME, _anchor(**overrides))) + + +@pytest.mark.parametrize("head", MALFORMED_HEADS) +def test_read_anchor_refuses_a_malformed_chain_head(tmp_path, head): + with pytest.raises(AnchorError): + read_anchor(_write(tmp_path / DEFAULT_ANCHOR_FILENAME, _anchor(chain_head=head))) + + +def test_read_anchor_refuses_an_anchor_under_a_loop_dir(tmp_path): + # D2 made mechanical: an anchor inside the tree it certifies is gitignored here, + # so an adopter would silently carry an anchor that never lands in a commit. + inside = _write(tmp_path / ".loop" / DEFAULT_ANCHOR_FILENAME, _anchor()) + with pytest.raises(AnchorError) as excinfo: + read_anchor(inside) + assert ".loop" in str(excinfo.value) + + +def test_subject_name_is_pinned(): + assert SUBJECT_NAME == "loop-chain-head" + + +def test_subject_bytes_is_exactly_64_lowercase_hex_with_no_trailing_newline(): + payload = subject_bytes(HEAD) + assert isinstance(payload, bytes) + assert len(payload) == 64 + assert payload == payload.lower() + assert not payload.endswith(b"\n") + assert payload.decode("ascii") == HEAD + + +@pytest.mark.parametrize("head", MALFORMED_HEADS) +def test_subject_bytes_refuses_a_malformed_head(head): + with pytest.raises(VerdictError): + subject_bytes(head) + + +def test_subject_bytes_digest_is_not_the_head_itself(): + # The property that distinguishes a D1 attestation from the three already minted: + # the subject digest is now sha256 OF the head's bytes, not the head. A consumer + # can regenerate those bytes from the head alone, which is what makes + # `gh attestation verify` runnable at all. + assert hashlib.sha256(subject_bytes(HEAD)).hexdigest() != HEAD diff --git a/scripts/test_attest_workflow.py b/scripts/test_attest_workflow.py new file mode 100644 index 0000000..065d40e --- /dev/null +++ b/scripts/test_attest_workflow.py @@ -0,0 +1,123 @@ +"""scripts/test_attest_workflow.py — the live experiment's shape. + +Task 10 cannot be validated before merge: attest.yml fires on push to main only, so a +real attestation mints only after landing. The FIRST post-merge run IS the experiment, +and its falsifiable check is `subject[0].digest.sha256 != predicate.chain.head` — an +inequality that was false in all three attestations this repo minted before this slice. + +What IS pinnable now is that the experiment resolves through the SHIPPED script rather +than a hand-written duplicate. Without that, the only code getting real-gh mileage would +be one no adopter ever runs. +""" +from __future__ import annotations + +import pathlib + +import pytest +import yaml + +ROOT = pathlib.Path(__file__).resolve().parent.parent +ATTEST = ROOT / ".github" / "workflows" / "attest.yml" +CI = ROOT / ".github" / "workflows" / "ci.yml" +_SIGNER_WORKFLOW = "SollanSystems/loop-engineer/.github/workflows/attest.yml" + + +@pytest.fixture(scope="module") +def attest_text() -> str: + return ATTEST.read_text(encoding="utf-8") + + +@pytest.fixture(scope="module") +def attest_steps(attest_text) -> dict[str, dict]: + document = yaml.safe_load(attest_text) + return {step.get("name"): step + for step in document["jobs"]["verdict"]["steps"] if step.get("name")} + + +def test_attest_workflow_resolves_through_the_shipped_script(attest_steps): + """M4 — the same entry point action.yml's `anchor` input calls.""" + body = attest_steps["resolve the attestation we just minted"]["run"] + assert "scripts/action_anchor_resolve.py" in body + assert (ROOT / "scripts" / "action_anchor_resolve.py").is_file() + + +def test_attest_workflow_contains_no_inline_gh_attestation_call(attest_text): + """M4's teeth. Without this pin an inline duplicate can be reintroduced beside the + script and the two drift silently.""" + assert "gh attestation" not in attest_text + + +def test_attest_workflow_writes_a_real_anchor_file_for_the_resolve(attest_steps): + """So the anchor READ path is exercised, not bypassed.""" + body = attest_steps["resolve the attestation we just minted"]["run"] + assert "loop-engineer/anchor@1" in body + assert "chain_head" in body + assert "--anchor" in body + + +def test_attest_workflow_pins_the_signer_workflow(attest_steps): + """D4: --signer-workflow is the mandatory pin; --signer-digest invalidates on every + push, so it is deliberately absent.""" + body = attest_steps["resolve the attestation we just minted"]["run"] + assert f"--signer-workflow {_SIGNER_WORKFLOW}" in body + assert "--signer-digest" not in body + + +def test_attest_workflow_pins_the_predicate_type_and_denies_self_hosted(): + """Both are the SCRIPT's responsibility, so this cross-checks that the script the + workflow invokes really carries them unconditionally — without --predicate-type gh + enforces the SLSA default and rejects every verdict@1 attestation.""" + script = (ROOT / "scripts" / "action_anchor_resolve.py").read_text(encoding="utf-8") + assert '"--predicate-type", PREDICATE_TYPE' in script + assert '"--deny-self-hosted-runners"' in script + body = ATTEST.read_text(encoding="utf-8") + for disabling in ("--no-deny-self-hosted", "--predicate-type "): + assert disabling not in body, disabling + + +def test_attest_workflow_asserts_the_subject_name_and_digest_inequality(attest_steps): + body = attest_steps["assert the resolved attestation proves D1 landed"]["run"] + assert 'subject.name == "loop-chain-head"' in body + assert "len(raw) == 64" in body + assert "subject_digest != head" in body + + +def test_attest_workflow_runs_compare_against_the_resolved_predicate(attest_steps): + """Consumes the script's predicate-path output, not a hand-rolled jq extraction.""" + steps = attest_steps["assert the resolved attestation proves D1 landed"] + assert "loop verdict --compare" in steps["run"] + assert steps["env"]["PREDICATE"] == "${{ steps.resolve.outputs.predicate-path }}" + assert "jq" not in steps["run"] + + +def test_attest_workflow_bounds_the_index_consistency_retry(attest_steps): + """An unbounded or silently-passing wait would make the step unfalsifiable.""" + body = attest_steps["resolve the attestation we just minted"]["run"] + assert "for attempt in 1 2 3 4 5 6" in body + assert 'if [ "$attempt" = "6" ]' in body + assert "exit 1" in body + + +def test_attest_workflow_still_runs_only_on_push_to_main(attest_text): + """ADR decision 5's confused-deputy guard, unchanged: attesting on a PR would mint a + signed verdict under the repository's identity before review.""" + document = yaml.safe_load(attest_text) + triggers = document[True] if True in document else document["on"] + assert set(triggers) == {"push"} + assert triggers["push"]["branches"] == ["main"] + + +def test_ci_exercises_ancestry_on_a_grown_store_and_a_fallback_leg(): + """The within-run ancestry pair (the only ancestry coverage this repo can honestly + claim) plus the structural-fallback leg that gives mode parity its CI teeth.""" + document = yaml.safe_load(CI.read_text(encoding="utf-8")) + anchor_steps = {step.get("name"): step + for step in document["jobs"]["anchor-live"]["steps"] if step.get("name")} + body = anchor_steps["Ancestry survives a grown store where head equality cannot"]["run"] + assert "--expect-chain-ancestor" in body and "--expect-chain-head" in body + assert "chain_anchor_not_ancestor" in body and "chain_anchor_mismatch" in body + + fallback = document["jobs"]["gates-fallback"] + installs = [step["run"] for step in fallback["steps"] if "run" in step] + assert not any("jsonschema" in line for line in installs if "pip install" in line) + assert any("find_spec" in line for line in installs), "the leg must prove it is the leg" diff --git a/scripts/test_attestation_policy.py b/scripts/test_attestation_policy.py new file mode 100644 index 0000000..1c44687 --- /dev/null +++ b/scripts/test_attestation_policy.py @@ -0,0 +1,208 @@ +"""scripts/test_attestation_policy.py — the signer-trust policy establishes NOTHING. + +It is a pure function over a `verificationResult` gh has ALREADY verified. Only +`signature.certificate` and `verifiedTimestamps` are unforgeable by the workflow that +produced the attestation (gh's own help says so); everything under `statement.predicate` +is user-controllable metadata and is never read here. + +It must REFUSE — loudly — when a claim it needs is absent, because a policy that treats +a missing claim as satisfied is worse than no policy. The exact leaf claim names cannot +be established before merge (F1), so a wrong name yields a refusal, not a silent pass. +""" +from __future__ import annotations + +import copy +import inspect +import re + +import pytest + +from loop.attestation import (ANCHOR_LOOKUP_OUTCOMES, REQUIRED_CERTIFICATE_CLAIMS, + _TRIGGER_CLAIM_ALIASES, AttestationPolicyError, + anchor_lookup_issue, check_signer_trust) + +_WORKFLOW = "SollanSystems/loop-engineer/.github/workflows/attest.yml" +_REPO_URI = "https://github.com/SollanSystems/loop-engineer" +_SAN = f"https://github.com/{_WORKFLOW}@refs/heads/main" + + +def _result(*, certificate=None, drop=(), **overrides): + base_certificate = { + "subjectAlternativeName": _SAN, + "sourceRepositoryURI": _REPO_URI, + "runnerEnvironment": "github-hosted", + "githubWorkflowTrigger": "push", + } + # drop FIRST, then apply overrides — otherwise dropping the alias defaults would + # also remove the single alias a caller just asked for. + for key in drop: + base_certificate.pop(key, None) + if certificate is not None: + base_certificate.update(certificate) + result = { + "signature": {"certificate": base_certificate}, + "verifiedTimestamps": [{"type": "Tlog", "timestamp": "2026-07-29T00:00:00Z"}], + } + result.update(overrides) + return result + + +def _check(result): + return check_signer_trust(result, signer_workflow=_WORKFLOW, + source_repository_uri=_REPO_URI) + + +def _codes(verdict): + return {issue["code"] for issue in verdict["issues"]} + + +def test_required_certificate_claims_are_pinned(): + assert REQUIRED_CERTIFICATE_CLAIMS == ("subjectAlternativeName", "sourceRepositoryURI", + "runnerEnvironment") + assert _TRIGGER_CLAIM_ALIASES == ("githubWorkflowTrigger", "buildTrigger") + assert ANCHOR_LOOKUP_OUTCOMES == ("corroborated", "contradicted", "unavailable") + + +def test_signer_trust_passes_on_a_conformant_result(): + verdict = _check(_result()) + assert verdict["ok"] is True + assert verdict["issues"] == [] + + +def test_signer_trust_refuses_when_signature_certificate_is_absent(): + for result in ({"verifiedTimestamps": [1]}, + {"signature": {}, "verifiedTimestamps": [1]}, + {"signature": {"certificate": "not-an-object"}, "verifiedTimestamps": [1]}): + with pytest.raises(AttestationPolicyError): + _check(result) + + +@pytest.mark.parametrize("missing", ["subjectAlternativeName", "sourceRepositoryURI", + "runnerEnvironment", ""]) +def test_signer_trust_refuses_when_a_required_claim_is_absent(missing): + """D10.8 — an absent claim is a refusal, never a pass.""" + if missing == "": + result = _result(drop=REQUIRED_CERTIFICATE_CLAIMS) + else: + result = _result(drop=(missing,)) + with pytest.raises(AttestationPolicyError) as excinfo: + _check(result) + if missing != "": + assert missing in str(excinfo.value) + + +@pytest.mark.parametrize("timestamps", [None, []]) +def test_signer_trust_refuses_when_verified_timestamps_are_absent_or_empty(timestamps): + """An unwitnessed attestation is not a trusted one.""" + result = _result() + if timestamps is None: + del result["verifiedTimestamps"] + else: + result["verifiedTimestamps"] = timestamps + with pytest.raises(AttestationPolicyError) as excinfo: + _check(result) + assert "verifiedTimestamps" in str(excinfo.value) + + +@pytest.mark.parametrize("alias", ["githubWorkflowTrigger", "buildTrigger"]) +def test_signer_trust_accepts_either_trigger_claim_alias(alias): + result = _result(drop=_TRIGGER_CLAIM_ALIASES, certificate={alias: "push"}) + assert _check(result)["ok"] is True + + +def test_signer_trust_refuses_when_neither_trigger_alias_is_present(): + with pytest.raises(AttestationPolicyError) as excinfo: + _check(_result(drop=_TRIGGER_CLAIM_ALIASES)) + assert "trigger" in str(excinfo.value) + + +def test_signer_workflow_mismatch_is_denied(): + result = _result(certificate={ + "subjectAlternativeName": "https://github.com/other/repo/.github/workflows/x.yml@refs/heads/main"}) + verdict = _check(result) + assert verdict["ok"] is False + assert "signer_workflow_mismatch" in _codes(verdict) + + +def test_source_repository_mismatch_is_denied(): + verdict = _check(_result(certificate={"sourceRepositoryURI": "https://github.com/other/repo"})) + assert "signer_repository_mismatch" in _codes(verdict) + + +def test_self_hosted_runner_is_denied(): + verdict = _check(_result(certificate={"runnerEnvironment": "self-hosted"})) + assert "self_hosted_runner" in _codes(verdict) + + +def test_non_push_trigger_is_denied(): + verdict = _check(_result(certificate={"githubWorkflowTrigger": "workflow_dispatch"})) + assert "signer_trigger_mismatch" in _codes(verdict) + + +def test_signer_trust_ignores_statement_predicate_entirely(): + """Rule 2, behavioral — the stronger of the two. statement.predicate is + user-controllable metadata; contradicting the certificate from there changes nothing.""" + honest = _check(_result()) + lying = copy.deepcopy(_result()) + lying["statement"] = { + "predicateType": "urn:loop-engineer:verdict:1", + "predicate": {"schema": "loop-engineer/verdict@1", "run_id": "whatever", + "subjectAlternativeName": "https://github.com/other/repo/x.yml", + "sourceRepositoryURI": "https://github.com/other/repo", + "runnerEnvironment": "self-hosted", "githubWorkflowTrigger": "pull_request"}, + } + assert _check(lying) == honest + + +def test_signer_trust_source_never_reads_statement(): + """Rule 2, structural.""" + assert "statement" not in inspect.getsource(check_signer_trust) + + +def test_signer_trust_reports_signature_checked_false(): + """The policy evaluates a verdict gh already reached; it does not re-establish it.""" + assert _check(_result())["signature_checked"] is False + + +def test_signer_trust_has_no_signer_digest_parameter(): + """D10.10's code half: --signer-digest pins job_workflow_sha, which for a + non-reusable top-level workflow equals the triggering commit SHA — it invalidates + on EVERY push, not merely on a workflow edit.""" + assert "signer_digest" not in inspect.signature(check_signer_trust).parameters + + +def test_anchor_lookup_corroborated_yields_no_issue(): + assert anchor_lookup_issue("corroborated") is None + + +def test_anchor_lookup_contradicted_has_its_own_code(): + issue = anchor_lookup_issue("contradicted", detail="signature did not verify") + assert issue["code"] == "anchor_attestation_contradicted" + assert "signature did not verify" in issue["message"] + + +def test_anchor_lookup_unavailable_has_a_distinct_code(): + """D10.6 — 'I could not look' must not collapse into 'it said no'.""" + unavailable = anchor_lookup_issue("unavailable", detail="HTTP 404: Not Found") + contradicted = anchor_lookup_issue("contradicted", detail="denied") + assert unavailable["code"] == "anchor_attestation_unavailable" + assert unavailable["code"] != contradicted["code"] + + +def test_anchor_lookup_transport_error_is_unavailable_not_contradicted(): + issue = anchor_lookup_issue("unavailable", detail="HTTP 503 from the attestations index") + assert issue["code"] == "anchor_attestation_unavailable" + # Distinct for OBSERVABILITY only — equally non-promoting. + assert "non-promoting" in issue["message"] + + +def test_anchor_lookup_refuses_an_unknown_outcome(): + """No silent default: an unknown outcome is a programming error, not a pass.""" + for outcome in ("promoted", "", None, "CORROBORATED"): + with pytest.raises(AttestationPolicyError): + anchor_lookup_issue(outcome) + + +def test_anchor_lookup_codes_match_the_public_issue_code_pattern(): + for outcome in ("contradicted", "unavailable"): + assert re.fullmatch(r"[a-z0-9_]{1,64}", anchor_lookup_issue(outcome)["code"]) diff --git a/scripts/test_chain_ancestry.py b/scripts/test_chain_ancestry.py new file mode 100644 index 0000000..c8b3f85 --- /dev/null +++ b/scripts/test_chain_ancestry.py @@ -0,0 +1,130 @@ +"""scripts/test_chain_ancestry.py — head_sequence: ancestry established by replay. + +D3: --expect-chain-head is exact current-head equality, so feeding run N's head to +run N+1 fails by construction on any growing store. The meaningful cross-run check +is "the previously attested head still appears in my chain" — and it is established +by REPLAY (recomputing every hash), never by trusting the stored event_hash column. +""" +from types import MappingProxyType + +import pytest + +from loop.chain import compute_event_hash, head_sequence, verify_chain + + +def _record(**overrides): + base = { + "schema": "loop-engineer/event@1", "event_id": "e1", "run_id": "r1", + "sequence": 0, "type": "contract_opened", "actor": "operator", + "causation_id": None, "correlation_id": None, "ts": "2026-07-24T00:00:00+00:00", + "payload": {"workspace": "ws"}, "artifact_hashes": [], "prev_event_hash": None, + } + base.update(overrides) + return base + + +def _chain(count, *, run_id="r1"): + """A well-formed chained stream of `count` events, sequence 0..count-1.""" + events = [] + prev = None + for seq in range(count): + record = _record(sequence=seq, event_id=f"e{seq}", run_id=run_id, + prev_event_hash=prev) + record["event_hash"] = compute_event_hash(record) + prev = record["event_hash"] + events.append(record) + return events + + +def test_head_sequence_finds_the_current_head(): + events = _chain(4) + assert head_sequence(events, events[3]["event_hash"]) == 3 + + +def test_head_sequence_finds_an_earlier_head_after_the_chain_grew(): + events = _chain(7) + anchored = events[3]["event_hash"] + # The whole point: the anchored head is still an ancestor after 4-6 were appended, + # while exact head equality (verify_chain's expected_head) fails by construction. + assert head_sequence(events[:4], anchored) == 3 + assert head_sequence(events, anchored) == 3 + assert events[6]["event_hash"] != anchored + assert verify_chain(events, expected_head=anchored)["ok"] is False + + +def test_head_sequence_returns_none_for_an_unknown_digest(): + assert head_sequence(_chain(3), "b" * 64) is None + + +def test_head_sequence_returns_none_for_an_empty_stream(): + assert head_sequence([], "b" * 64) is None + + +def test_head_sequence_ignores_an_unchained_prefix(): + # A migrated store: legacy rows carry event_hash None, then chaining begins. + legacy = [_record(sequence=seq, event_id=f"legacy-e{seq}") for seq in range(2)] + for record in legacy: + record["event_hash"] = None + chained = [] + prev = None + for seq in (2, 3, 4): + record = _record(sequence=seq, event_id=f"e{seq}", prev_event_hash=prev) + record["event_hash"] = compute_event_hash(record) + prev = record["event_hash"] + chained.append(record) + events = [*legacy, *chained] + report = verify_chain(events) + assert report["ok"] is True and report["unchained_prefix"] == 2 + # The prefix contributes no sequence and does not abort the walk. + assert head_sequence(events, chained[0]["event_hash"]) == 2 + assert head_sequence(events, chained[2]["event_hash"]) == 4 + + +def test_head_sequence_recomputes_and_refuses_a_forged_event_hash_column(): + # D10.5: a tamperer who can rewrite the store can also insert a row bearing the + # anchored digest. Only recomputation refuses that row. + events = _chain(4) + anchored = "c" * 64 + events[2]["event_hash"] = anchored # forged column, hash NOT recomputed + assert head_sequence(events, anchored) is None + + +def test_head_sequence_stops_at_a_broken_link(): + events = _chain(5) + later_head = events[4]["event_hash"] + earlier_head = events[0]["event_hash"] + events[1] = dict(events[1], prev_event_hash="a" * 64) # splice, hash left stale + # The walk stops exactly where verify_chain stops. + report = verify_chain(events) + assert report["ok"] is False and "sequence 1" in report["issues"][0] + assert head_sequence(events, later_head) is None + assert head_sequence(events, earlier_head) == 0 + + +def test_head_sequence_accepts_any_ordered_mapping_sequence(): + # The module's portability contract: works over any ordered Mapping stream + # (a JSONL export), no store and no dict-specific behavior involved. + events = tuple(MappingProxyType(record) for record in _chain(3)) + assert head_sequence(events, events[2]["event_hash"]) == 2 + assert head_sequence(iter(events), events[1]["event_hash"]) == 1 + + +def test_head_sequence_finds_the_digest_at_sequence_zero(): + # The boundary a `if seq:` truth test would silently swallow: 0 is a legitimate + # ancestor, so callers must compare against None, never truthiness. + events = _chain(3) + found = head_sequence(events, events[0]["event_hash"]) + assert found == 0 + assert found is not None + + +def test_head_sequence_refuses_a_malformed_digest_argument(): + # A silent None on a typo would read as "rewrite detected". Exactly ValueError — + # NOT ChainHashError, which its own docstring scopes to canonicalization failures. + # Because ChainHashError IS a ValueError, a bare pytest.raises(ValueError) would + # pass for either class and pin nothing. + malformed = ["A" * 64, "a" * 63, "a" * 65, "z" * 64, "", None, 42] + for digest in malformed: + with pytest.raises(ValueError) as excinfo: + head_sequence(_chain(2), digest) + assert type(excinfo.value) is ValueError, digest diff --git a/scripts/test_docs_slice4b.py b/scripts/test_docs_slice4b.py new file mode 100644 index 0000000..4f7235a --- /dev/null +++ b/scripts/test_docs_slice4b.py @@ -0,0 +1,150 @@ +"""scripts/test_docs_slice4b.py — doc parity for slice 4b. + +Every assertion here was demonstrated to FAIL against the tree as it stood before the +documentation commit. A pin that passes both before and after documents nothing. +""" +from __future__ import annotations + +import json +import pathlib +import re + +import pytest + +ROOT = pathlib.Path(__file__).resolve().parent.parent +REFERENCE = ROOT / "reference" / "repo-os-contract.md" +ADR = ROOT / "docs" / "adr" / "0002-ci-attested-verdict.md" +CODEOWNERS = ROOT / ".github" / "CODEOWNERS" +CHANGELOG = ROOT / "CHANGELOG.md" +STRUCTURAL = ROOT / "evals" / "cases" / "structural.json" + +# The version at c493804, the slice's base. Tasks 1-11 land as one feature PR and every +# version surface moves only in a separate release cut. +_BASE_VERSION = "0.11.0" +_NEW_CODES = ("chain_anchor_not_ancestor", "anchor_file_unreadable", "anchor_file_invalid", + "anchor_attestation_contradicted", "anchor_attestation_unavailable") + + +@pytest.fixture(scope="module") +def reference() -> str: + return REFERENCE.read_text(encoding="utf-8") + + +@pytest.fixture(scope="module") +def section_24(reference) -> str: + start = reference.index("## 24.") + return reference[start:] + + +@pytest.fixture(scope="module") +def amendment() -> str: + text = ADR.read_text(encoding="utf-8") + return text[text.index("## Amendment (2026-07-29, Slice 4b)"):] + + +def test_section_24_exists(reference): + assert re.search(r"(?m)^## 24\. ", reference) + + +@pytest.mark.parametrize("code", _NEW_CODES) +def test_section_24_documents_every_new_issue_code(section_24, code): + assert code in section_24 + + +def test_section_24_documents_the_subject_file_byte_form(section_24): + assert "64" in section_24 + assert "no trailing newline" in section_24 + + +def test_section_23_subject_seam_paragraph_was_rewritten(reference): + """D1 makes the retired framing false: `gh attestation verify` DOES succeed against + the subject file's bytes now. The unqualified 'there are no bytes to fetch' claim + must be gone.""" + seam = reference[reference.index("**The subject seam"):reference.index("## 24.")] + assert not re.search(r"never conclude\s+.fetch the bytes, re-hash,\s*compare.", seam, + re.IGNORECASE) + assert "there are no bytes to fetch" not in seam.lower() + assert "subject-path" in seam + assert "hashes that file's **content**" in seam + + +def test_section_24_records_the_rest_sunset_date(section_24): + assert "10 Mar 2028" in section_24 + + +def test_section_24_documents_the_public_private_asymmetry(section_24): + assert "no transparency log" in section_24 + assert re.search(r"[Pp]rivate repositor", section_24) + + +def test_section_24_documents_signer_digest_as_deliberately_not_required(section_24): + assert "signer-digest" in section_24 + assert "job_workflow_sha" in section_24 + assert "every push" in section_24 + + +def test_section_24_carries_the_non_promoting_sentence(section_24): + assert "non-promoting" in section_24 + assert "exactly as non-promoting as a" in section_24 + + +def test_section_24_states_that_ancestry_is_established_by_replay(section_24): + assert "replay" in section_24.lower() + assert "never by trusting the\nstored `event_hash` column" in section_24 \ + or "never by trusting the stored `event_hash` column" in section_24 + + +def test_adr_0002_carries_the_slice_4b_amendment(amendment): + assert amendment.startswith("## Amendment (2026-07-29, Slice 4b)") + + +@pytest.mark.parametrize("decision", ["Decision 2", "Decision 4", "Decision 5"]) +def test_amendment_names_the_three_overridden_decisions(amendment, decision): + assert f"**{decision} —" in amendment + + +def test_codeowners_covers_the_anchor_path(): + text = CODEOWNERS.read_text(encoding="utf-8") + owned = [line for line in text.splitlines() + if line.strip() and not line.lstrip().startswith("#") + and line.split()[0].endswith("loop-anchor.json")] + assert owned, text + assert owned[0].split()[1:] == ["@SollanSystems"] + + +def test_reference_file_count_is_still_eight(): + """§24 was APPENDED, not added as a file: structural.json pins the count at 8.""" + pinned = json.loads(STRUCTURAL.read_text(encoding="utf-8"))["reference_filenames"] + assert len(pinned) == 8 + live = sorted(path.name for path in (ROOT / "reference").iterdir() if path.is_file()) + assert live == sorted(pinned) + + +def test_changelog_has_an_unreleased_slice_4b_entry(): + text = CHANGELOG.read_text(encoding="utf-8") + unreleased = text[text.index("## Unreleased"):text.index("## 0.11.0")] + assert "--compare" in unreleased + assert "slice 4b" in unreleased.lower() + assert "subject-path" in unreleased or "head-bearing file" in unreleased + + +def test_no_shipped_surface_still_advertises_the_retired_subject_form(): + """Fails against the tree as it stood today: the Unreleased section advertised + `subject-digest: sha256:` as the shipped form and claimed 4b "does not + ship here". Both became false the moment this slice landed.""" + unreleased_text = CHANGELOG.read_text(encoding="utf-8") + unreleased = unreleased_text[unreleased_text.index("## Unreleased"): + unreleased_text.index("## 0.11.0")] + assert "subject-digest" not in unreleased + assert "does not ship here" not in unreleased + assert "subject-digest" not in (ROOT / "action.yml").read_text(encoding="utf-8") + # 4a's code-owner sentence stays TRUE and must be KEPT: the ruleset still requires 0 + # approvals, so CODEOWNERS is not yet an operative control. + assert "in force only once the repository ruleset requires it" in unreleased + + +def test_no_version_bump_in_this_slice(): + pyproject = re.search(r'(?m)^version\s*=\s*"([^"]+)"', + (ROOT / "pyproject.toml").read_text(encoding="utf-8")).group(1) + plugin = json.loads((ROOT / ".claude-plugin" / "plugin.json").read_text(encoding="utf-8")) + assert pyproject == plugin["version"] == _BASE_VERSION diff --git a/scripts/test_doctor_anchor_ancestry.py b/scripts/test_doctor_anchor_ancestry.py new file mode 100644 index 0000000..9ccf2d5 --- /dev/null +++ b/scripts/test_doctor_anchor_ancestry.py @@ -0,0 +1,393 @@ +"""scripts/test_doctor_anchor_ancestry.py — the doctor ancestry gate (D3/D5). + +`--expect-chain-head` is exact current-head equality, so it fails by construction on +a store that legitimately grew (F3). `--expect-chain-ancestor` asks the cross-run +question instead — "was this digest ever my head?" — established by replay, and +`--anchor` resolves that digest from a tracked anchor@1 file. + +An absent, unreadable or empty store with an ancestor supplied FAILS; it never skips. +""" + +import json +import re +import sqlite3 +import subprocess +import sys +from pathlib import Path + +import pytest + +from chain_fixtures import drop_triggers, restore_triggers +from loop.chain import canonical_json, compute_event_hash +from loop.contract import doctor_report +from loop.events import SQLiteEventStore +from loop.scaffold import scaffold + +ROOT = Path(__file__).resolve().parent.parent +_EVENT_SCHEMA_ID = "loop-engineer/event@1" +_ANCHOR_CODES = ("chain_anchor_not_ancestor", "anchor_file_unreadable", "anchor_file_invalid") + +# Captured from the tree BEFORE the runtime edit that added the ancestry gate, with +# only the genuinely volatile values placeheld (tmp paths, the run's own hashes, and +# the jsonschema-vs-fallback mode fields). Regenerating this after the change would +# pin nothing; it is a literal on purpose. +_PRE_CHANGE_DOCTOR_REPORT = ( + '{"event_store":{"chain":{"head":{"event_hash":"","sequence":3},' + '"unchained_prefix":0},"deterministic":true,"event_count":4,"legal_sequence":true,' + '"present":true,"readable":true,"run_id":"run-1","state_json_agrees":true},' + '"issues":[],"lifecycle":"running","ok":true,"paths":"",' + '"requested_mode":"auto","schemas_checked":"","validation_mode":""}' +) +_HEX64 = re.compile(r"[0-9a-f]{64}") + + +def _run(*args: str) -> subprocess.CompletedProcess[str]: + return subprocess.run([sys.executable, "-B", "-m", "loop", *args], + cwd=ROOT, text=True, capture_output=True) + + +def _codes(report): + return {issue["code"] for issue in report["issues"]} + + +def _store_path(target): + return Path(target) / ".loop" / "events.db" + + +def _sync_state(target, **fields): + path = Path(target) / ".loop" / "state.json" + state = json.loads(path.read_text(encoding="utf-8")) + state.update(fields) + path.write_text(json.dumps(state), encoding="utf-8") + + +def _chained_workspace(tmp_path, name="workspace"): + """A synced workspace over a 4-event chained store (spliceable middle).""" + target = tmp_path / name + scaffold(target) + store = SQLiteEventStore(_store_path(target)) + store.append("run-1", "contract_opened", {"workspace": name}, actor="test") + store.append("run-1", "iteration_appended", {"iteration_id": 1, "outcome": "task_failed"}, actor="test") + store.append("run-1", "iteration_appended", {"iteration_id": 2, "outcome": "task_passed"}, actor="test") + store.append("run-1", "receipt_appended", + {"iteration_id": 2, "role": "write", "model": "test-model", "outcome": "ok"}, actor="test") + _sync_state(target, iteration_id=2, active_task=None) + return target + + +def _grow(target, iterations=(3, 4)): + """Append more chained events, keeping the contract doctor-clean.""" + store = SQLiteEventStore(_store_path(target)) + for iteration_id in iterations: + store.append("run-1", "iteration_appended", + {"iteration_id": iteration_id, "outcome": "task_passed"}, actor="test") + _sync_state(target, iteration_id=iterations[-1]) + return _head(target) + + +def _head(target): + return ((doctor_report(target)["event_store"]["chain"] or {}).get("head") or {}).get("event_hash") + + +def _record_at(conn, sequence, prev_event_hash): + row = conn.execute( + "SELECT run_id, sequence, event_id, type, actor, causation_id, correlation_id, ts, " + "payload, artifact_hashes FROM events WHERE sequence = ?", (sequence,)).fetchone() + return {"schema": _EVENT_SCHEMA_ID, "run_id": row[0], "sequence": row[1], "event_id": row[2], + "type": row[3], "actor": row[4], "causation_id": row[5], "correlation_id": row[6], + "ts": row[7], "payload": json.loads(row[8]), "artifact_hashes": json.loads(row[9]), + "prev_event_hash": prev_event_hash} + + +def _rewrite_history(target, *, forge_event_hash_at=None, forged_digest=None): + """The competent adversary: rewrite payloads and re-chain from genesis. + + With forge_event_hash_at, additionally plant `forged_digest` in one row's + event_hash column WITHOUT recomputing — the attack only replay refuses. + """ + store_path = _store_path(target) + drop_triggers(store_path) + conn = sqlite3.connect(str(store_path)) + try: + conn.execute("UPDATE events SET payload = replace(payload, '\"task_failed\"', " + "'\"task_passed\"') WHERE type = 'iteration_appended'") + prev = None + for row in conn.execute("SELECT sequence FROM events ORDER BY sequence ASC").fetchall(): + record = _record_at(conn, row[0], prev) + digest = compute_event_hash(record) + conn.execute("UPDATE events SET prev_event_hash = ?, event_hash = ? WHERE sequence = ?", + (prev, digest, row[0])) + prev = digest + if forge_event_hash_at is not None: + conn.execute("UPDATE events SET event_hash = ? WHERE sequence = ?", + (forged_digest, forge_event_hash_at)) + conn.commit() + finally: + conn.close() + restore_triggers(store_path) + + +def _write_anchor(path, head, **extra): + document = {"schema": "loop-engineer/anchor@1", "chain_head": head, **extra} + Path(path).write_text(json.dumps(document), encoding="utf-8") + return path + + +# --- the ancestry gate itself ------------------------------------------------- + + +def test_ancestor_flag_passes_when_the_anchored_head_is_in_the_chain(tmp_path): + ws = _chained_workspace(tmp_path) + report = doctor_report(ws, expect_chain_ancestor=_head(ws)) + assert report["ok"] is True + assert not _codes(report) & set(_ANCHOR_CODES) + + +def test_ancestor_flag_passes_after_the_chain_grew(tmp_path): + ws = _chained_workspace(tmp_path) + anchored = _head(ws) + grown = _grow(ws) + assert grown != anchored + report = doctor_report(ws, expect_chain_ancestor=anchored) + assert report["ok"] is True, report["issues"] + assert report["event_store"]["anchor"] == {"expected": anchored, "sequence": 3} + + +def test_expect_chain_head_fails_on_the_same_grown_store(tmp_path): + """F3, mechanical: the negative control proving ancestry is strictly more useful.""" + ws = _chained_workspace(tmp_path) + anchored = _head(ws) + _grow(ws) + report = doctor_report(ws, expect_chain_head=anchored) + assert report["ok"] is False + assert "chain_anchor_mismatch" in _codes(report) + + +def test_ancestor_flag_fails_with_chain_anchor_not_ancestor_on_an_unknown_head(tmp_path): + ws = _chained_workspace(tmp_path) + report = doctor_report(ws, expect_chain_ancestor="b" * 64) + assert report["ok"] is False + assert "chain_anchor_not_ancestor" in _codes(report) + + +def test_ancestry_detects_a_wholesale_rewrite_that_self_verifies_clean(tmp_path): + """D10.4: history rewritten, re-chained from genesis, then grown. The chain alone + reports nothing — the anchored ancestor is the only control left.""" + ws = _chained_workspace(tmp_path) + anchored = _head(ws) + _rewrite_history(ws) + _grow(ws) + unanchored = doctor_report(ws) + assert "event_chain_broken" not in _codes(unanchored) # PINNED LIMITATION + assert unanchored["event_store"]["chain"]["head"] is not None + anchored_report = doctor_report(ws, expect_chain_ancestor=anchored) + assert "chain_anchor_not_ancestor" in _codes(anchored_report) + + +def test_ancestry_refuses_a_forged_row_bearing_the_anchored_digest_end_to_end(tmp_path): + """D10.5 at doctor level: the rewrite plants the anchored digest in a row's + event_hash column. A column-trusting gate would call that ancestry satisfied.""" + ws = _chained_workspace(tmp_path) + anchored = _head(ws) + _rewrite_history(ws, forge_event_hash_at=1, forged_digest=anchored) + with sqlite3.connect(str(_store_path(ws))) as conn: + planted = conn.execute("SELECT event_hash FROM events WHERE sequence = 1").fetchone()[0] + assert planted == anchored, "the forged column must really carry the anchored digest" + report = doctor_report(ws, expect_chain_ancestor=anchored) + assert "chain_anchor_not_ancestor" in _codes(report) + assert report["ok"] is False + + +# --- D5: never skip ---------------------------------------------------------- + + +def test_absent_store_with_an_ancestor_fails_and_never_skips(tmp_path): + target = tmp_path / "storeless" + scaffold(target) + report = doctor_report(target, expect_chain_ancestor="b" * 64) + assert report["event_store"]["present"] is False + assert "chain_anchor_not_ancestor" in _codes(report) + assert report["ok"] is False + + +def test_unreadable_store_with_an_ancestor_fails_and_never_skips(tmp_path): + ws = _chained_workspace(tmp_path) + _store_path(ws).write_bytes(b"this is not a sqlite database") + report = doctor_report(ws, expect_chain_ancestor="b" * 64) + codes = _codes(report) + assert "chain_anchor_not_ancestor" in codes + assert codes & {"corrupt_store", "invalid_event", "ambiguous_run_id"} + assert report["event_store"]["readable"] is False + + +def test_empty_store_with_an_ancestor_fails_and_never_skips(tmp_path): + target = tmp_path / "emptystore" + scaffold(target) + # A read materializes the DDL (the constructor alone does not touch the disk), + # leaving a real but eventless store — distinct from an absent one. + SQLiteEventStore(_store_path(target)).read("run-1") + assert _store_path(target).exists() + report = doctor_report(target, expect_chain_ancestor="b" * 64) + codes = _codes(report) + assert "chain_anchor_not_ancestor" in codes + assert "empty_store" in codes + + +def test_ancestor_code_is_distinct_from_chain_anchor_mismatch(tmp_path): + """D3: 'your head is not what I expected' and 'the head you anchored is not in my + history at all' are different facts; one shared code would collapse them.""" + ws = _chained_workspace(tmp_path) + anchored = _head(ws) + _grow(ws) + report = doctor_report(ws, expect_chain_head=anchored, expect_chain_ancestor=anchored) + codes = _codes(report) + assert "chain_anchor_mismatch" in codes + assert "chain_anchor_not_ancestor" not in codes + + +# --- --anchor resolution (CLI layer) ----------------------------------------- + + +def test_anchor_file_resolves_the_expected_ancestor(tmp_path): + ws = _chained_workspace(tmp_path) + anchored = _head(ws) + _grow(ws) + anchor = _write_anchor(tmp_path / "loop-anchor.json", anchored, sequence=3) + resolved = _run("doctor", "--anchor", str(anchor), str(ws)) + explicit = _run("doctor", "--expect-chain-ancestor", anchored, str(ws)) + assert resolved.returncode == 0, resolved.stderr + assert json.loads(resolved.stdout) == json.loads(explicit.stdout) + + +def test_unreadable_anchor_file_fails_with_anchor_file_unreadable(tmp_path): + ws = _chained_workspace(tmp_path) + result = _run("doctor", "--anchor", str(tmp_path / "absent.json"), str(ws)) + assert result.returncode == 1, result.stderr # a report, not exit 2 + assert "anchor_file_unreadable" in _codes(json.loads(result.stdout)) + + +def test_invalid_anchor_file_fails_with_anchor_file_invalid(tmp_path): + ws = _chained_workspace(tmp_path) + anchor = tmp_path / "loop-anchor.json" + anchor.write_text(json.dumps({"schema": "loop-engineer/anchor@1"}), encoding="utf-8") + result = _run("doctor", "--anchor", str(anchor), str(ws)) + assert result.returncode == 1, result.stderr + assert "anchor_file_invalid" in _codes(json.loads(result.stdout)) + + +@pytest.mark.parametrize("flavor", ["unreadable", "invalid"]) +def test_anchor_file_failure_emits_a_full_doctor_report_shape(tmp_path, flavor): + """Design rule 6's pinned shape: a consumer parsing validation_mode or + event_store must not crash on the anchor-failure path.""" + ws = _chained_workspace(tmp_path) + anchor = tmp_path / "loop-anchor.json" + if flavor == "invalid": + anchor.write_text('{"schema":"loop-engineer/anchor@1","chain_head":"nope"}', encoding="utf-8") + result = _run("doctor", "--anchor", str(anchor), str(ws)) + report = json.loads(result.stdout) + assert set(report) == {"paths", "ok", "validation_mode", "requested_mode", + "schemas_checked", "lifecycle", "issues", "event_store"} + assert report["ok"] is False + assert len(report["issues"]) == 1 + assert report["issues"][0]["code"] == f"anchor_file_{flavor}" + # No digest was ever resolved, so the gate was asked nothing: one failure, one code. + assert "chain_anchor_not_ancestor" not in _codes(report) + assert "anchor" not in report["event_store"] + + +# --- the §22 habit: no flag, no change --------------------------------------- + + +def test_doctor_report_is_byte_identical_when_no_anchor_flag_is_supplied(tmp_path): + def normalize(value, key=None): + if key in ("paths", "schemas_checked", "validation_mode"): + return {"paths": "", "schemas_checked": "", + "validation_mode": ""}[key] + if isinstance(value, dict): + return {k: normalize(v, k) for k, v in value.items()} + if isinstance(value, list): + return [normalize(v) for v in value] + if isinstance(value, str) and _HEX64.fullmatch(value): + return "" + return value + + ws = _chained_workspace(tmp_path) + assert canonical_json(normalize(doctor_report(ws))) == _PRE_CHANGE_DOCTOR_REPORT + + +def test_anchor_block_appears_only_when_an_ancestor_was_supplied(tmp_path): + ws = _chained_workspace(tmp_path) + assert "anchor" not in doctor_report(ws)["event_store"] + supplied = doctor_report(ws, expect_chain_ancestor=_head(ws))["event_store"]["anchor"] + assert set(supplied) == {"expected", "sequence"} + + +def test_expect_chain_head_and_expect_chain_ancestor_compose(tmp_path): + """Equality and ancestry are different questions; both may be asked.""" + ws = _chained_workspace(tmp_path) + anchored = _head(ws) + grown = _grow(ws) + both_satisfied = doctor_report(ws, expect_chain_head=grown, expect_chain_ancestor=anchored) + assert both_satisfied["ok"] is True, both_satisfied["issues"] + both_violated = doctor_report(ws, expect_chain_head="f" * 64, expect_chain_ancestor="e" * 64) + assert _codes(both_violated) >= {"chain_anchor_mismatch", "chain_anchor_not_ancestor"} + + +# --- CLI guards -------------------------------------------------------------- + + +def test_anchor_and_expect_chain_ancestor_are_mutually_exclusive(tmp_path): + ws = _chained_workspace(tmp_path) + anchor = _write_anchor(tmp_path / "loop-anchor.json", _head(ws)) + result = _run("doctor", "--anchor", str(anchor), + "--expect-chain-ancestor", _head(ws), str(ws)) + assert result.returncode == 2 + assert result.stdout == "" + assert "mutually exclusive" in result.stderr + + +@pytest.mark.parametrize("command", ["scaffold", "verdict", "status"]) +def test_ancestor_flag_rejected_for_non_doctor_commands(tmp_path, command): + target = tmp_path / f"{command}-target" + result = _run(command, "--expect-chain-ancestor", "a" * 64, str(target)) + assert result.returncode == 2 + assert "--expect-chain-ancestor is only valid for doctor" in result.stderr + # scaffold resolves a relative target against its CWD, so an unguarded flag + # creates the directory HERE, not under tmp_path. + assert not (ROOT / "--expect-chain-ancestor").exists() + + +@pytest.mark.parametrize("command", ["scaffold", "verdict", "status"]) +def test_anchor_flag_rejected_for_non_doctor_commands(tmp_path, command): + target = tmp_path / f"{command}-target" + result = _run(command, "--anchor", str(tmp_path / "loop-anchor.json"), str(target)) + assert result.returncode == 2 + assert "--anchor is only valid for doctor" in result.stderr + assert not (ROOT / "--anchor").exists() + + +@pytest.mark.parametrize("value", ["A" * 64, "a" * 63, "a" * 65, "z" * 64]) +def test_expect_chain_ancestor_must_be_64_lowercase_hex(tmp_path, value): + ws = _chained_workspace(tmp_path) + result = _run("doctor", "--expect-chain-ancestor", value, str(ws)) + assert result.returncode == 2 + assert "64-character lowercase hex" in result.stderr + + +def test_anchor_resolution_works_in_release_mode(tmp_path): + pytest.importorskip("jsonschema") + ws = _chained_workspace(tmp_path) + anchor = _write_anchor(tmp_path / "loop-anchor.json", _head(ws)) + result = _run("doctor", "--mode", "release", "--anchor", str(anchor), str(ws)) + assert result.returncode == 0, result.stderr + report = json.loads(result.stdout) + assert report["validation_mode"] == "jsonschema" + # The anchor must have been CONSUMED and resolved, not silently ignored. + assert report["event_store"]["anchor"]["expected"] == _head(ws) + + +def test_new_doctor_codes_match_the_public_issue_code_pattern(): + """Doctor issue codes are the population verdict.doctor.issue_codes is drawn + from — a permanent, public, append-only log.""" + for code in _ANCHOR_CODES: + assert re.fullmatch(r"[a-z0-9_]{1,64}", code), code diff --git a/scripts/test_verdict_cli.py b/scripts/test_verdict_cli.py index 14940ac..0c55e36 100644 --- a/scripts/test_verdict_cli.py +++ b/scripts/test_verdict_cli.py @@ -127,3 +127,177 @@ def test_verdict_rejects_expect_chain_head(repo_root: pathlib.Path): ) assert result.returncode == 2 assert "only valid for doctor/validate/verify" in result.stderr + + +# --- slice 4b: --compare and --emit-subject ---------------------------------- +# +# The exit-code contract is normative: 0 agreement, 1 disagreement, 2 refusal. Exit 1 +# in this CLI MEANS "a report said not-ok", so an unreadable --compare input must never +# land there — it would read as a genuine disagreement. + +sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent)) +from ci_anchor_probe import seed as _seed_chained # noqa: E402 + +from loop.verdict import subject_bytes as _subject_bytes # noqa: E402 + +_SIGNATURE_FLAGS = ("--verify-signature", "--signature", "--signer-workflow") + + +def _run_stdin(repo_root: pathlib.Path, stdin: str, *args: str) -> subprocess.CompletedProcess[str]: + return subprocess.run([sys.executable, "-B", "-m", "loop", *args], + capture_output=True, text=True, cwd=repo_root, input=stdin) + + +def _chained_terminal(tmp_path: pathlib.Path) -> tuple[pathlib.Path, str]: + """A chained workspace with a terminal record, so a verdict carries a real head.""" + workspace = tmp_path / "ws" + head = _seed_chained(workspace) + (workspace / ".loop" / "terminal_state.json").write_text(json.dumps({ + "schema": "loop-engineer/terminal@1", "state": "Succeeded", + "completion_policy": {"mode": "all_required"}, "criteria_met": {"T1": True}, + "evidence": [], "false_completion": False, + }), encoding="utf-8") + return workspace, head + + +def test_compare_exits_zero_on_agreement(repo_root: pathlib.Path, tmp_path: pathlib.Path): + workspace, _head = _chained_terminal(tmp_path) + projected = _run(repo_root, "verdict", str(workspace)) + assert projected.returncode == 0, projected.stderr + document = tmp_path / "attested.json" + document.write_text(projected.stdout, encoding="utf-8") + result = _run(repo_root, "verdict", "--compare", str(document), str(workspace)) + assert result.returncode == 0, result.stderr + report = json.loads(result.stdout) + assert report["ok"] is True and report["signature_checked"] is False + + +def test_compare_exits_one_on_disagreement(repo_root: pathlib.Path, tmp_path: pathlib.Path): + workspace, _head = _chained_terminal(tmp_path) + projected = json.loads(_run(repo_root, "verdict", str(workspace)).stdout) + projected["chain"]["head"] = "b" * 64 + document = tmp_path / "attested.json" + document.write_text(json.dumps(projected), encoding="utf-8") + result = _run(repo_root, "verdict", "--compare", str(document), str(workspace)) + assert result.returncode == 1 + report = json.loads(result.stdout) + assert {issue["code"] for issue in report["issues"]} == {"verdict_head_disagreement"} + + +def test_compare_exits_two_on_a_wrapper_shape(repo_root: pathlib.Path, tmp_path: pathlib.Path): + workspace, _head = _chained_terminal(tmp_path) + document = tmp_path / "wrapped.json" + document.write_text(json.dumps([{"verificationResult": {"statement": {}}}]), encoding="utf-8") + result = _run(repo_root, "verdict", "--compare", str(document), str(workspace)) + assert result.returncode == 2 + assert result.stdout == "" + assert result.stderr.startswith("verdict:") + assert "Traceback" not in result.stderr + + +def test_compare_reads_stdin_with_a_dash(repo_root: pathlib.Path, tmp_path: pathlib.Path): + workspace, _head = _chained_terminal(tmp_path) + projected = _run(repo_root, "verdict", str(workspace)).stdout + result = _run_stdin(repo_root, projected, "verdict", "--compare", "-", str(workspace)) + assert result.returncode == 0, result.stderr + assert json.loads(result.stdout)["ok"] is True + + +def test_compare_missing_value_is_a_usage_error(repo_root: pathlib.Path, tmp_path: pathlib.Path): + workspace, _head = _chained_terminal(tmp_path) + result = _run(repo_root, "verdict", str(workspace), "--compare") + assert result.returncode == 2 + assert result.stdout == "" + + +def test_compare_refuses_a_nonexistent_file(repo_root: pathlib.Path, tmp_path: pathlib.Path): + """Exit 1 here would read as a genuine disagreement, so it must be exit 2.""" + workspace, _head = _chained_terminal(tmp_path) + result = _run(repo_root, "verdict", "--compare", str(tmp_path / "absent.json"), str(workspace)) + assert result.returncode == 2 + assert result.stdout == "" + assert result.stderr.startswith("verdict:") + assert "Traceback" not in result.stderr + + +def test_compare_refuses_a_directory_path(repo_root: pathlib.Path, tmp_path: pathlib.Path): + workspace, _head = _chained_terminal(tmp_path) + result = _run(repo_root, "verdict", "--compare", str(tmp_path), str(workspace)) + assert result.returncode == 2 + assert result.stdout == "" + assert "Traceback" not in result.stderr + + +def test_compare_refuses_empty_stdin(repo_root: pathlib.Path, tmp_path: pathlib.Path): + """The pipeline whose upstream jq produced nothing deserves a message that says so, + never a comparison against None.""" + workspace, _head = _chained_terminal(tmp_path) + result = _run_stdin(repo_root, "", "verdict", "--compare", "-", str(workspace)) + assert result.returncode == 2 + assert result.stdout == "" + assert "empty" in result.stderr + + +@pytest.mark.parametrize("command", ["scaffold", "doctor", "status"]) +def test_compare_rejected_for_non_verdict_commands(repo_root: pathlib.Path, + tmp_path: pathlib.Path, command: str): + target = tmp_path / f"{command}-target" + result = _run(repo_root, command, "--compare", str(tmp_path / "x.json"), str(target)) + assert result.returncode == 2 + assert "--compare is only valid for verdict" in result.stderr + # An unguarded flag makes scaffold CREATE a directory named after it, in its CWD. + assert not (repo_root / "--compare").exists() + + +def test_emit_subject_writes_exactly_64_bytes_no_newline(repo_root: pathlib.Path, + tmp_path: pathlib.Path): + workspace, head = _chained_terminal(tmp_path) + result = subprocess.run([sys.executable, "-B", "-m", "loop", "verdict", "--emit-subject", + str(workspace)], capture_output=True, cwd=repo_root) + assert result.returncode == 0, result.stderr + assert len(result.stdout) == 64 + assert not result.stdout.endswith(b"\n") + assert result.stdout == head.encode("ascii") + + +def test_emit_subject_bytes_equal_subject_bytes_of_the_projected_head(repo_root: pathlib.Path, + tmp_path: pathlib.Path): + """The CLI and loop.verdict.subject_bytes cannot drift: one writer, one byte form.""" + workspace, _head = _chained_terminal(tmp_path) + projected = json.loads(_run(repo_root, "verdict", str(workspace)).stdout) + result = subprocess.run([sys.executable, "-B", "-m", "loop", "verdict", "--emit-subject", + str(workspace)], capture_output=True, cwd=repo_root) + assert result.stdout == _subject_bytes(projected["chain"]["head"]) + + +def test_emit_subject_refuses_a_null_head(repo_root: pathlib.Path): + """A store-less workspace has no subject to attest.""" + result = subprocess.run([sys.executable, "-B", "-m", "loop", "verdict", "--emit-subject", + "examples/flaky-test-triage"], capture_output=True, cwd=repo_root) + assert result.returncode == 2 + assert result.stdout == b"" + assert result.stderr.decode().startswith("verdict:") + + +def test_compare_and_emit_subject_are_mutually_exclusive(repo_root: pathlib.Path, + tmp_path: pathlib.Path): + workspace, _head = _chained_terminal(tmp_path) + result = _run(repo_root, "verdict", "--compare", "-", "--emit-subject", str(workspace)) + assert result.returncode == 2 + assert result.stdout == "" + assert "mutually exclusive" in result.stderr + + +def test_help_documents_compare_and_emit_subject(repo_root: pathlib.Path): + result = _run(repo_root, "--help") + assert result.returncode == 0 + assert "--compare" in result.stdout and "--emit-subject" in result.stdout + assert "never verifies a signature" in result.stdout + + +@pytest.mark.parametrize("flag", _SIGNATURE_FLAGS) +def test_verdict_never_advertises_a_signature_flag(repo_root: pathlib.Path, flag: str): + """D10.1's third leg: there is no flag to flip.""" + result = _run(repo_root, "verdict", flag, "x", "examples/flaky-test-triage") + assert result.returncode == 2 + assert result.stdout == "" diff --git a/scripts/test_verdict_compare.py b/scripts/test_verdict_compare.py new file mode 100644 index 0000000..e7524f2 --- /dev/null +++ b/scripts/test_verdict_compare.py @@ -0,0 +1,294 @@ +"""scripts/test_verdict_compare.py — compare_verdict(): agreement, never authenticity. + +`gh attestation verify` establishes authenticity and runs FIRST; this establishes +agreement. Neither implies the other, `signature_checked` is the literal False on +every branch, and there is no flag to flip it (D7/D10.1). + +The function accepts a BARE verdict@1 predicate only. An in-toto Statement or a +`gh --format json` envelope is a typed refusal that names the unwrapping step — +best-effort parsing of a vendor envelope inside the kernel is how a trust boundary rots. +""" +from __future__ import annotations + +import ast +import copy +import inspect +import json +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from loop import emit # noqa: E402 +from loop import verdict as verdict_module # noqa: E402 +from loop.completion import VERIFIED_EVIDENCE_MODE # noqa: E402 +from loop.events import SQLiteEventStore # noqa: E402 +from loop.runner import dispatch_once # noqa: E402 +from loop.verdict import (COMPARISON_CODES, VerdictError, build_verdict, # noqa: E402 + compare_verdict) + +_RUN_ID = "run-1" +_VERIFY = "./scripts/verify-fast.sh" +_RAMP = ("plan", "critique-plan", "queue-tasks", "execute-task") +_JQ_PATH = ".[0].verificationResult.statement.predicate" + + +def _task(task_id="T-1"): + return {"id": task_id, "title": task_id, "status": "pending", "criterion_ref": task_id, + "verify": _VERIFY, "depends_on": [], "attempts": 0, "evidence": None} + + +def _workspace(tmp_path, name="workspace"): + """A dispatched contract carrying one chain-bound verified-evidence record, + then made terminal so a verdict can be projected from it.""" + workspace = tmp_path / name + emit.open_contract(workspace) + (workspace / "TASKS.json").write_text( + json.dumps({"schema": "loop-engineer/tasks@1", "tasks": [_task()]}), encoding="utf-8") + script = workspace / "scripts" / "verify-fast.sh" + script.parent.mkdir(parents=True, exist_ok=True) + script.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") + script.chmod(0o755) + events = SQLiteEventStore(workspace / ".loop" / "events.db") + events.append(_RUN_ID, "contract_opened", {"workspace": name}, actor="test") + for state in _RAMP: + events.append(_RUN_ID, "iteration_appended", + {"iteration_id": 0, "outcome": "replanned", "state": state}, actor="test") + state_path = workspace / ".loop" / "state.json" + state = json.loads(state_path.read_text(encoding="utf-8")) + state["state"] = _RAMP[-1] + state_path.write_text(json.dumps(state, indent=2) + "\n", encoding="utf-8") + dispatch_once(workspace) + (workspace / ".loop" / "terminal_state.json").write_text(json.dumps({ + "schema": "loop-engineer/terminal@1", "state": "Succeeded", + "completion_policy": {"mode": VERIFIED_EVIDENCE_MODE}, + "criteria_met": {"T-1": True}, + "evidence": [".loop/evidence/evidence-iter1.json"], + "false_completion": False, + }), encoding="utf-8") + return workspace + + +def _codes(report): + return {issue["code"] for issue in report["issues"]} + + +def _mutate(local, path, value): + """A deep copy of `local` with one dotted path replaced.""" + document = copy.deepcopy(local) + cursor = document + parts = path.split(".") + for part in parts[:-1]: + cursor = cursor[part] + cursor[parts[-1]] = value + return document + + +# --- the negative control ---------------------------------------------------- + + +def test_compare_passes_on_a_self_projected_verdict(tmp_path): + """D10.2's negative control: without it, every disagreement test below could be + passing because the comparison is broken rather than because it works.""" + ws = _workspace(tmp_path) + local = build_verdict(ws) + assert local["evidence"], "the fixture must carry verified evidence to compare" + report = compare_verdict(local, ws) + assert report["ok"] is True + assert report["issues"] == [] + assert all(facet["agrees"] is True for facet in report["compared"].values()) + + +# --- D10.1: signature_checked is false everywhere, with no flag to flip ------ + + +@pytest.mark.parametrize("path,value", [ + (None, None), # agreement + ("chain.head", "b" * 64), # head disagreement + ("terminal.state", "FailedBlocked"), # terminal disagreement + ("run_id", "some-other-run"), # run_id disagreement + ("evidence", [{"digest": "c" * 64, "code_digest": None, "policy_digest": None}]), +]) +def test_compare_reports_signature_checked_false_on_every_report_branch(tmp_path, path, value): + ws = _workspace(tmp_path) + local = build_verdict(ws) + attested = local if path is None else _mutate(local, path, value) + assert compare_verdict(attested, ws)["signature_checked"] is False + + +def test_signature_checked_is_never_assigned_true_in_source(): + """AST-level, not a substring scan: every signature_checked value in the module + is the constant False.""" + tree = ast.parse(Path(inspect.getsourcefile(verdict_module)).read_text(encoding="utf-8")) + seen = 0 + for node in ast.walk(tree): + if isinstance(node, ast.Dict): + for key, value in zip(node.keys, node.values): + if isinstance(key, ast.Constant) and key.value == "signature_checked": + seen += 1 + assert isinstance(value, ast.Constant) and value.value is False + if isinstance(node, ast.Assign): + for target in node.targets: + if isinstance(target, ast.Name) and target.id == "signature_checked": + seen += 1 + assert isinstance(node.value, ast.Constant) and node.value.value is False + assert seen >= 1, "no signature_checked literal found — the pin would be vacuous" + + +# --- typed disagreements ----------------------------------------------------- + + +def test_compare_head_disagreement_is_typed(tmp_path): + ws = _workspace(tmp_path) + local = build_verdict(ws) + report = compare_verdict(_mutate(local, "chain.head", "b" * 64), ws) + assert report["ok"] is False + assert "verdict_head_disagreement" in _codes(report) + assert report["compared"]["head"]["agrees"] is False + + +def test_compare_terminal_state_disagreement_is_typed(tmp_path): + ws = _workspace(tmp_path) + local = build_verdict(ws) + report = compare_verdict(_mutate(local, "terminal.state", "FailedBudget"), ws) + assert "verdict_terminal_disagreement" in _codes(report) + + +def test_compare_terminal_policy_disagreement_is_typed(tmp_path): + """all_required vs all_required_verified_evidence must never read as agreement.""" + ws = _workspace(tmp_path) + local = build_verdict(ws) + assert local["terminal"]["completion_policy"] == VERIFIED_EVIDENCE_MODE + report = compare_verdict(_mutate(local, "terminal.completion_policy", "all_required"), ws) + assert "verdict_terminal_disagreement" in _codes(report) + + +def test_compare_false_completion_disagreement_is_typed(tmp_path): + ws = _workspace(tmp_path) + local = build_verdict(ws) + report = compare_verdict(_mutate(local, "terminal.false_completion", True), ws) + assert "verdict_terminal_disagreement" in _codes(report) + + +def test_compare_run_id_disagreement_is_typed(tmp_path): + ws = _workspace(tmp_path) + local = build_verdict(ws) + report = compare_verdict(_mutate(local, "run_id", "not-this-run"), ws) + assert "verdict_run_id_disagreement" in _codes(report) + + +def test_compare_evidence_digest_disagreement_is_typed(tmp_path): + """Set equality, so BOTH directions are a disagreement.""" + ws = _workspace(tmp_path) + local = build_verdict(ws) + extra = [*local["evidence"], + {"digest": "d" * 64, "code_digest": None, "policy_digest": None}] + attested_superset = compare_verdict(_mutate(local, "evidence", extra), ws) + assert "verdict_evidence_disagreement" in _codes(attested_superset) + attested_subset = compare_verdict(_mutate(local, "evidence", []), ws) + assert "verdict_evidence_disagreement" in _codes(attested_subset) + + +def test_compare_ignores_doctor_and_tool_differences(tmp_path): + """F4, mechanical: doctor.validation_mode and tool.version live INSIDE the + predicate, so the same run projects different bytes across environments. Comparing + them would make an honest environment difference read as tampering.""" + ws = _workspace(tmp_path) + local = build_verdict(ws) + attested = _mutate(local, "tool", {"name": "loop-engineer", "version": "0.0.1-other"}) + attested = _mutate(attested, "doctor", {**attested["doctor"], + "validation_mode": "structural-fallback", + "ok": not attested["doctor"]["ok"]}) + report = compare_verdict(attested, ws) + assert report["ok"] is True, report["issues"] + assert "doctor" not in report["compared"] and "tool" not in report["compared"] + + +# --- typed refusals: bare predicate only ------------------------------------- + + +@pytest.mark.parametrize("key", ["_type", "subject", "predicateType", "predicate"]) +def test_compare_refuses_an_in_toto_statement(tmp_path, key): + ws = _workspace(tmp_path) + statement = {**build_verdict(ws), key: "anything"} + with pytest.raises(VerdictError) as excinfo: + compare_verdict(statement, ws) + assert key in str(excinfo.value) + assert "Statement" in str(excinfo.value) + + +def test_compare_refuses_a_gh_format_json_array_wrapper(tmp_path): + ws = _workspace(tmp_path) + with pytest.raises(VerdictError) as excinfo: + compare_verdict([{"verificationResult": {}}], ws) + assert "array" in str(excinfo.value) + + +def test_compare_refuses_a_gh_verification_result_object(tmp_path): + ws = _workspace(tmp_path) + for key in ("verificationResult", "attestation"): + with pytest.raises(VerdictError) as excinfo: + compare_verdict({key: {"statement": {}}}, ws) + assert key in str(excinfo.value) + + +def test_compare_refuses_an_unrecognized_schema(tmp_path): + ws = _workspace(tmp_path) + local = build_verdict(ws) + for document in ({**local, "schema": "loop-engineer/evidence@1"}, + {k: v for k, v in local.items() if k != "schema"}): + with pytest.raises(VerdictError): + compare_verdict(document, ws) + + +def test_compare_refusal_names_the_unwrapping_step(tmp_path): + """Every refusal tells the operator the documented jq path they skipped.""" + ws = _workspace(tmp_path) + documents = [ + {**build_verdict(ws), "predicateType": "urn:loop-engineer:verdict:1"}, + [{"verificationResult": {}}], + {"verificationResult": {}}, + {"schema": "something-else"}, + "not an object", + ] + for document in documents: + with pytest.raises(VerdictError) as excinfo: + compare_verdict(document, ws) + assert _JQ_PATH in str(excinfo.value), document + + +def test_compare_refuses_a_non_object_document(tmp_path): + ws = _workspace(tmp_path) + for document in ("a string", 17, None, True): + with pytest.raises(VerdictError): + compare_verdict(document, ws) + + +def test_compare_treats_an_ancestor_head_as_a_disagreement(tmp_path): + """Rule 4: a verdict projects ONE run. An attested head that is merely an ancestor + of the local head is a different run's verdict — a fact to report, not to excuse. + Ancestry is the doctor gate's question, where it is the question being asked.""" + ws = _workspace(tmp_path) + local = build_verdict(ws) + events = SQLiteEventStore(ws / ".loop" / "events.db") + previous_head = local["chain"]["head"] + events.append(_RUN_ID, "receipt_appended", + {"iteration_id": 1, "role": "write", "model": "m", "outcome": "ok"}, + actor="test") + grown = build_verdict(ws) + assert grown["chain"]["head"] != previous_head + report = compare_verdict(local, ws) # local doc now carries the ANCESTOR head + assert report["ok"] is False + assert "verdict_head_disagreement" in _codes(report) + + +def test_compare_report_field_allowlist_holds(tmp_path): + ws = _workspace(tmp_path) + local = build_verdict(ws) + report = compare_verdict(local, ws) + assert set(report) == {"ok", "signature_checked", "compared", "issues"} + assert set(report["compared"]) == {"run_id", "head", "terminal", "evidence"} + assert set(COMPARISON_CODES) == {"verdict_head_disagreement", "verdict_terminal_disagreement", + "verdict_run_id_disagreement", "verdict_evidence_disagreement"} diff --git a/scripts/test_verdict_purity.py b/scripts/test_verdict_purity.py index 7012c6f..f50bcdd 100644 --- a/scripts/test_verdict_purity.py +++ b/scripts/test_verdict_purity.py @@ -109,3 +109,163 @@ def test_predicate_validates_against_its_own_schema(): from loop.verdict import _load_verdict_schema jsonschema.validate(_predicate(), _load_verdict_schema()) + + +# --- slice 4b: the boundary extended to the consumption surfaces -------------- + +# The network and the shell live in scripts/, never in loop/. ONE deliberate exception: +# loop/runner.py runs the contract's own verify command through subprocess (the slice-3b +# subprocess-ISOLATED verifier — shlex argv, shell=False, cwd=workspace, wall-clock cap). +# `loop run` cannot execute a verifier without it, so it is named rather than banned; +# every other module under loop/ must still be shell-free, and a new shell-out site +# anywhere else fails this test. +_SUBPROCESS_ALLOWED = frozenset({"runner.py"}) +_NETWORK_MODULES = ("socket", "urllib", "http", "requests", "httpx", "ftplib", "smtplib") +_SHELL_MODULES = ("subprocess", "pty") + + +def _imported_modules(path: pathlib.Path) -> set[str]: + tree = ast.parse(path.read_text(encoding="utf-8")) + modules: set[str] = set() + for node in ast.walk(tree): + if isinstance(node, ast.Import): + modules.update(alias.name for alias in node.names) + elif isinstance(node, ast.ImportFrom) and node.level == 0 and node.module: + modules.add(node.module) + return modules + + +def _catches_import_error(node: ast.Try) -> bool: + return any( + (isinstance(handler.type, ast.Name) and handler.type.id == "ImportError") + or (isinstance(handler.type, ast.Tuple) + and any(isinstance(elt, ast.Name) and elt.id == "ImportError" + for elt in handler.type.elts)) + for handler in node.handlers + ) + + +def _third_party_import_sites(path: pathlib.Path) -> list[tuple[str, bool]]: + """(module, is_guarded) for every non-stdlib import SITE in the file. + + Per site, never per name: a module that is imported once inside a + try/except ImportError and once at module level is NOT optional, and a + name-keyed check would wave the unguarded site through. + """ + tree = ast.parse(path.read_text(encoding="utf-8")) + guarded_sites = { + id(inner) + for node in ast.walk(tree) if isinstance(node, ast.Try) and _catches_import_error(node) + for statement in node.body + for inner in ast.walk(statement) if isinstance(inner, (ast.Import, ast.ImportFrom)) + } + sites: list[tuple[str, bool]] = [] + for node in ast.walk(tree): + if isinstance(node, ast.Import): + names = [alias.name for alias in node.names] + elif isinstance(node, ast.ImportFrom) and node.level == 0 and node.module: + names = [node.module] + else: + continue + for name in names: + if name.split(".")[0] not in sys.stdlib_module_names: + sites.append((name, id(node) in guarded_sites)) + return sites + + +@pytest.mark.parametrize("module", ["anchor.py", "attestation.py", "verdict.py"]) +def test_new_modules_import_only_stdlib_and_loop(module): + """Zero NEW runtime dependencies. `jsonschema` is the one declared optional extra and + is permitted only at a site a try/except ImportError guards — the idiom every + schema-validating module here already uses. An UNGUARDED jsonschema import would + silently promote an optional extra into a hard requirement.""" + offenders = [ + name for name, guarded in _third_party_import_sites(LOOP / module) + if not (name.split(".")[0] == "jsonschema" and guarded) + ] + assert offenders == [], offenders + + +def test_no_module_under_loop_reaches_the_network_or_shells_out(): + """D10.9's new leg. Scoped to import sites, so accurate prose about what the kernel + does NOT do cannot fail it — and so the one named exception stays visible.""" + network_offenders, shell_offenders = [], [] + for path in sorted(LOOP.rglob("*.py")): + roots = {name.split(".")[0] for name in _imported_modules(path)} + network_offenders.extend(f"{path.relative_to(REPO)}:{name}" + for name in _NETWORK_MODULES if name in roots) + if path.name not in _SUBPROCESS_ALLOWED: + shell_offenders.extend(f"{path.relative_to(REPO)}:{name}" + for name in _SHELL_MODULES if name in roots) + assert network_offenders == [], network_offenders + assert shell_offenders == [], shell_offenders + # The exception is real, not aspirational: if runner.py ever stops shelling out, + # tighten _SUBPROCESS_ALLOWED rather than leaving a stale carve-out standing. + assert "subprocess" in _imported_modules(LOOP / "runner.py") + + +def _compare_report(document: pathlib.Path) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [sys.executable, "-B", "-m", "loop", "verdict", "--compare", str(document), + "examples/flaky-test-triage"], capture_output=True, text=True, cwd=REPO) + + +@pytest.mark.parametrize("surface", ["predicate", "comparison"]) +def test_kernel_emits_no_statement_key_anywhere(tmp_path, surface): + """Neither the predicate nor the comparison report is ever an envelope.""" + predicate = _predicate() + if surface == "predicate": + emitted = predicate + else: + document = tmp_path / "attested.json" + document.write_text(json.dumps(predicate), encoding="utf-8") + proc = _compare_report(document) + assert proc.returncode == 0, proc.stderr + emitted = json.loads(proc.stdout) + assert not {"_type", "subject", "predicateType", "predicate"} & emitted.keys() + + +def test_compare_report_compared_block_carries_no_free_text(tmp_path): + """Scoped to report["compared"]: digests, enums and run_id only. issues[].message is + deliberately exempt — a local report may explain itself; a PREDICATE may not.""" + predicate = _predicate() + predicate["chain"]["head"] = "b" * 64 # force a disagreement + document = tmp_path / "attested.json" + document.write_text(json.dumps(predicate), encoding="utf-8") + proc = _compare_report(document) + assert proc.returncode == 1, proc.stderr # a disagreement, as arranged + report = json.loads(proc.stdout) + + def strings(value): + if isinstance(value, str): + yield value + elif isinstance(value, dict): + for item in value.values(): + yield from strings(item) + elif isinstance(value, list): + for item in value: + yield from strings(item) + + prose = [text for text in strings(report["compared"]) if " " in text] + assert prose == [], prose + # Non-vacuous: the exempt surface really does carry prose, so the scoping matters. + assert any(" " in issue["message"] for issue in report["issues"]) + + +def test_signature_checked_literal_is_always_false(): + """AST-level over loop/: no assignment and no dict value is anything but False.""" + seen = 0 + for path in sorted(LOOP.rglob("*.py")): + tree = ast.parse(path.read_text(encoding="utf-8")) + for node in ast.walk(tree): + if isinstance(node, ast.Dict): + for key, value in zip(node.keys, node.values): + if isinstance(key, ast.Constant) and key.value == "signature_checked": + seen += 1 + assert isinstance(value, ast.Constant) and value.value is False, path + elif isinstance(node, ast.Assign): + for target in node.targets: + if isinstance(target, ast.Name) and target.id == "signature_checked": + seen += 1 + assert isinstance(node.value, ast.Constant) and node.value.value is False + assert seen >= 2, f"expected the compare report and the policy verdict, found {seen}"