From 5057943483ed964bc25638e23c136cfb51517447 Mon Sep 17 00:00:00 2001 From: ci Date: Fri, 4 Sep 2026 18:12:47 -0400 Subject: [PATCH 01/20] Add resumable offline delivery replay --- README.md | 12 + RESTORE.md | 14 + ci/required-files.txt | 5 + delivery/v1/replay.py | 369 +++++++++++++++++++++++++++ scripts/test/delivery-replay.test.sh | 189 ++++++++++++++ work/delivery-loop-first/plan.md | 23 ++ 6 files changed, 612 insertions(+) create mode 100755 delivery/v1/replay.py create mode 100755 scripts/test/delivery-replay.test.sh create mode 100644 work/delivery-loop-first/plan.md diff --git a/README.md b/README.md index 5a7015f..bdf5410 100644 --- a/README.md +++ b/README.md @@ -53,6 +53,18 @@ mismatches remain valid only when all evidence is non-passing. The stable wrappe inactive resolver, Control closure, and scanner select it together. The prior generation remains immutable and restorable. +## Inactive offline delivery replay + +`delivery/v1/replay.py` replays one already-supplied local materialization input +through the existing local Git materializer. It then checks one repo-relative +candidate blob against a supplied SHA-256 and records a private, resumable state. +It never executes candidate code or a user command string. + +Review and publisher records are supplied offline test observations. They bind the +exact request and candidate but do not authenticate an actor or authorize a real +publication. Missing review stays waiting; a completed receipt is explicitly an +offline simulation with no authority or qualification. + ## Inactive fake adapter contract matrix `adapter-tests/v1/` runs a fixed 2×2 producer/forge matrix against one unrelated diff --git a/RESTORE.md b/RESTORE.md index dce165d..6326d8a 100644 --- a/RESTORE.md +++ b/RESTORE.md @@ -55,6 +55,20 @@ with the restored `scripts/core-contract.sh`. No compiled helper is installed or restored. A future activation must separately qualify and bind a production trusted parent; restoring these files does not select a live profile. +### Restore the inactive offline delivery replay + +Restore the three paths listed under “Inactive offline delivery replay” in +[`ci/required-files.txt`](ci/required-files.txt), then run: + +```sh +bash scripts/test/delivery-replay.test.sh +``` + +The replay is a local offline simulation. It materializes only a caller-owned +candidate, reads one fixed candidate blob, and records test observations. It does +not execute candidate code, select a profile, authenticate review, publish, merge, +deploy, or contact a provider or target. + --- ## 1. Recreate yshifu (the manager) diff --git a/ci/required-files.txt b/ci/required-files.txt index c7dac9b..c53601a 100644 --- a/ci/required-files.txt +++ b/ci/required-files.txt @@ -257,6 +257,11 @@ adapters/local-git-materializer/v1/materialize.sh adapters/local-git-materializer/v1/object-closure.c scripts/test/local-git-materializer-adapter.test.sh +# Inactive offline delivery replay +delivery/v1/replay.py +scripts/test/delivery-replay.test.sh +work/delivery-loop-first/plan.md + # Inactive Claude Code producer normalizer payload adapters/claude-code-producer/v1/normalize.jq scripts/test/default-claude-code-producer-adapter.test.sh diff --git a/delivery/v1/replay.py b/delivery/v1/replay.py new file mode 100755 index 0000000..0ff5a28 --- /dev/null +++ b/delivery/v1/replay.py @@ -0,0 +1,369 @@ +#!/usr/bin/env python3 +"""Run one inactive, offline delivery replay without executing candidate code.""" + +import argparse +import fcntl +import hashlib +import json +import os +from pathlib import Path +import re +import signal +import subprocess +import sys +import tempfile + + +MAX_INPUT_BYTES = 8 * 1024 * 1024 +MAX_OBSERVATION_BYTES = 64 * 1024 +MAX_VERIFIED_BLOB_BYTES = 1024 * 1024 +OID = re.compile(r"[0-9a-f]{40}|[0-9a-f]{64}\Z") +ACTOR = re.compile(r"[a-z0-9][a-z0-9._:-]{0,127}\Z") + + +class ReplayError(Exception): + pass + + +def digest_bytes(value): + return hashlib.sha256(value).hexdigest() + + +def canonical(value): + return json.dumps(value, sort_keys=True, separators=(",", ":")).encode() + + +def read_json(path, limit): + descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)) + try: + chunks = [] + remaining = limit + 1 + while remaining: + chunk = os.read(descriptor, remaining) + if not chunk: + break + chunks.append(chunk) + remaining -= len(chunk) + data = b"".join(chunks) + finally: + os.close(descriptor) + if len(data) > limit: + raise ReplayError("input exceeds its size limit") + try: + return json.loads(data), digest_bytes(data) + except json.JSONDecodeError as error: + raise ReplayError("input is not JSON") from error + + +def private_directory(path): + value = Path(path) + stat = value.stat() + if value.is_symlink() or not value.is_dir() or stat.st_uid != os.getuid(): + raise ReplayError("state directory is not a caller-owned directory") + if stat.st_mode & 0o077: + raise ReplayError("state directory is not private") + return value.resolve() + + +def trusted_file(path): + value = Path(path) + stat = value.stat() + if value.is_symlink() or not value.is_file() or stat.st_size > MAX_INPUT_BYTES: + raise ReplayError("trusted tool is unavailable") + return value.resolve() + + +def disjoint(*paths): + resolved = [Path(path).resolve() for path in paths] + for index, left in enumerate(resolved): + for right in resolved[index + 1:]: + if left == right or left in right.parents or right in left.parents: + raise ReplayError("caller-owned boundaries overlap") + + +def atomic_json(path, value): + encoded = canonical(value) + b"\n" + descriptor, temporary = tempfile.mkstemp(prefix=".replay-", dir=path.parent) + try: + with os.fdopen(descriptor, "wb") as handle: + handle.write(encoded) + handle.flush() + os.fsync(handle.fileno()) + os.replace(temporary, path) + directory = os.open(path.parent, os.O_DIRECTORY) + try: + os.fsync(directory) + finally: + os.close(directory) + finally: + if os.path.exists(temporary): + os.unlink(temporary) + + +def load_state(path): + if not path.exists(): + return None + value, _ = read_json(path, MAX_OBSERVATION_BYTES) + if not isinstance(value, dict): + raise ReplayError("state journal is malformed") + return value + + +def safe_path(value): + if not isinstance(value, str) or not value or len(value) > 4096: + raise ReplayError("verification path is invalid") + parts = value.split("/") + if any(part in {"", ".", "..", ".git"} or part.endswith((".", " ")) for part in parts): + raise ReplayError("verification path is invalid") + if any("\\" in part or any(ord(char) < 32 for char in part) for part in parts): + raise ReplayError("verification path is invalid") + return value + + +def input_identity(input_value, input_sha, arguments, materializer): + try: + request = input_value["stage_request"] + request_sha = request["sha256"] + body = request["content"]["body"] + source = body["target_revision"]["value"] + source_tree_id = next( + item["value"]["value"]["value"]["object_id"] + for item in body["inputs"] + if item["input_id"] == body["operation"]["arguments"]["source_tree_input_id"] + ) + except (KeyError, StopIteration, TypeError) as error: + raise ReplayError("materialization input lacks an exact source identity") from error + if not isinstance(request_sha, str) or not re.fullmatch(r"[0-9a-f]{64}", request_sha): + raise ReplayError("materialization input request identity is invalid") + if not isinstance(source_tree_id, str) or not OID.fullmatch(source_tree_id): + raise ReplayError("materialization input tree identity is invalid") + if not isinstance(source, dict) or not OID.fullmatch(str(source.get("commit_id", ""))): + raise ReplayError("materialization input commit identity is invalid") + expected = arguments.expected_sha256 + if not re.fullmatch(r"[0-9a-f]{64}", expected): + raise ReplayError("expected verifier digest is invalid") + closure = trusted_file(arguments.closure_helper) + jq_bin = trusted_file(arguments.jq_bin) + materializer_sha = digest_bytes(materializer.read_bytes()) + identity = { + "input_sha256": input_sha, + "request_sha256": request_sha, + "source_commit_id": source["commit_id"], + "source_tree_id": source_tree_id, + "verifier": { + "id": "delivery.fixed-content-sha256.v1", + "path": safe_path(arguments.verify_path), + "expected_sha256": expected, + }, + "materializer_sha256": materializer_sha, + "closure_helper_sha256": digest_bytes(closure.read_bytes()), + "jq_sha256": digest_bytes(jq_bin.read_bytes()), + "source_repository_id": arguments.source_repository_id, + } + identity["run_key"] = digest_bytes(canonical(identity)) + return identity + + +def run_materializer(arguments, materializer): + command = [ + str(materializer), "materialize", str(Path(arguments.input).resolve()), + arguments.source_repository_id, str(Path(arguments.source_git_dir).resolve()), + str(Path(arguments.candidate_root).resolve()), str(Path(arguments.scratch_root).resolve()), + str(Path(arguments.closure_helper).resolve()), str(Path(arguments.jq_bin).resolve()), + ] + environment = {"PATH": "/usr/bin:/bin", "LC_ALL": "C"} + result = subprocess.run(command, env=environment, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, check=False) + if result.returncode != 0 or len(result.stdout) > MAX_INPUT_BYTES: + raise ReplayError("materialization did not complete") + try: + response = json.loads(result.stdout) + receipt_text = response["payloads"][0]["data"] + receipt = json.loads(receipt_text) + candidate = receipt["candidate"] + return { + "response_sha256": digest_bytes(result.stdout), + "receipt_sha256": response["payloads"][0]["sha256"], + "candidate_commit_id": candidate["commit_id"], + "candidate_tree_id": candidate["tree_id"], + } + except (KeyError, IndexError, TypeError, json.JSONDecodeError) as error: + raise ReplayError("materializer response is malformed") from error + + +def verify_candidate(candidate_root, candidate_tree, path, expected): + repository = Path(candidate_root).resolve() / "repository.git" + if not repository.is_dir() or repository.is_symlink() or not OID.fullmatch(candidate_tree): + raise ReplayError("candidate repository identity is unavailable") + environment = {"PATH": "/usr/bin:/bin", "LC_ALL": "C", "GIT_CONFIG_NOSYSTEM": "1", + "GIT_CONFIG_GLOBAL": "/dev/null", "GIT_NO_REPLACE_OBJECTS": "1", + "GIT_NO_LAZY_FETCH": "1", "GIT_TERMINAL_PROMPT": "0"} + object_name = f"{candidate_tree}:{path}" + size = subprocess.run(["/usr/bin/git", f"--git-dir={repository}", "cat-file", "-s", object_name], + env=environment, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, check=False) + if size.returncode != 0 or not size.stdout.strip().isdigit() or int(size.stdout) > MAX_VERIFIED_BLOB_BYTES: + raise ReplayError("fixed verifier cannot read the candidate blob") + blob = subprocess.run(["/usr/bin/git", f"--git-dir={repository}", "cat-file", "blob", object_name], + env=environment, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, check=False) + if blob.returncode != 0 or len(blob.stdout) != int(size.stdout): + raise ReplayError("fixed verifier could not read the candidate blob") + actual = digest_bytes(blob.stdout) + if actual != expected: + raise ReplayError("fixed verifier digest mismatch") + return actual + + +def observation(path, kind, identity, field): + if path is None: + return None + value, source_sha = read_json(path, MAX_OBSERVATION_BYTES) + if not isinstance(value, dict) or value.get("schema_version") != 1 or value.get("kind") != kind: + raise ReplayError("offline observation is malformed") + if not ACTOR.fullmatch(str(value.get("actor_id", ""))): + raise ReplayError("offline observation actor is invalid") + if value.get("request_sha256") != identity["request_sha256"] or value.get("candidate_tree_id") != identity["candidate_tree_id"]: + raise ReplayError("offline observation does not match this candidate") + return {"actor_id": value["actor_id"], field: value.get(field), "sha256": source_sha} + + +def result(state): + print(json.dumps({"kind": "delivery_replay_receipt", "authority": "none", + "qualification": "unavailable", "offline_simulation": True, + "state": state}, sort_keys=True, separators=(",", ":"))) + + +def replay(arguments): + repository = Path(__file__).resolve().parents[2] + materializer = trusted_file(repository / "adapters/local-git-materializer/v1/materialize.sh") + input_value, input_sha = read_json(arguments.input, MAX_INPUT_BYTES) + identity = input_identity(input_value, input_sha, arguments, materializer) + state_dir = private_directory(arguments.state_dir) + disjoint(state_dir, arguments.source_git_dir, arguments.candidate_root, arguments.scratch_root) + state_path = state_dir / "run.json" + lock_path = state_dir / "replay.lock" + interrupted = {"value": False} + previous_term = signal.getsignal(signal.SIGTERM) + previous_int = signal.getsignal(signal.SIGINT) + signal.signal(signal.SIGTERM, lambda *_: interrupted.__setitem__("value", True)) + signal.signal(signal.SIGINT, lambda *_: interrupted.__setitem__("value", True)) + try: + lock_descriptor = os.open(lock_path, os.O_RDWR | os.O_CREAT | getattr(os, "O_NOFOLLOW", 0), 0o600) + with os.fdopen(lock_descriptor, "a+b") as lock: + fcntl.flock(lock, fcntl.LOCK_EX) + state = load_state(state_path) + if state is not None and state.get("identity", {}).get("run_key") != identity["run_key"]: + result({"phase": "stale", "reason": "run identity changed"}) + return 2 + if state is None: + state = {"schema_version": 1, "kind": "delivery_replay_state", "identity": identity, + "phase": "materializing", "authority": "none", "qualification": "unavailable"} + atomic_json(state_path, state) + if state["phase"] == "failed": + if state.get("recoverable"): + state["recovery"] = "start a new replay with fresh empty candidate, scratch, and state directories" + atomic_json(state_path, state) + result(state) + return 1 + if state["phase"] == "completed-offline": + verify_candidate(arguments.candidate_root, state["identity"]["candidate_tree_id"], + identity["verifier"]["path"], identity["verifier"]["expected_sha256"]) + for supplied, kind, field, recorded in ( + (arguments.review_observation, "delivery_replay_review_observation", "verdict", state.get("review")), + (arguments.publisher_observation, "delivery_replay_publisher_observation", "disposition", state.get("publisher")), + ): + if supplied is not None and observation(supplied, kind, state["identity"], field) != recorded: + raise ReplayError("supplied offline observation changed after completion") + result(state) + return 0 + if state["phase"] == "materializing": + try: + state["materialization"] = run_materializer(arguments, materializer) + except ReplayError as error: + state.update({"phase": "failed", "recoverable": True, "reason": str(error)}) + atomic_json(state_path, state) + result(state) + return 1 + state["identity"].update({ + "candidate_commit_id": state["materialization"]["candidate_commit_id"], + "candidate_tree_id": state["materialization"]["candidate_tree_id"], + }) + state["phase"] = "verifying" + atomic_json(state_path, state) + if interrupted["value"]: + result(state) + return 75 + if state["phase"] == "verifying": + try: + state["verification"] = {"id": identity["verifier"]["id"], "path": identity["verifier"]["path"], + "sha256": verify_candidate(arguments.candidate_root, state["identity"]["candidate_tree_id"], + identity["verifier"]["path"], identity["verifier"]["expected_sha256"])} + except ReplayError as error: + state.update({"phase": "failed", "recoverable": False, "reason": str(error)}) + atomic_json(state_path, state) + result(state) + return 1 + state["phase"] = "review-wait" + atomic_json(state_path, state) + if interrupted["value"]: + result(state) + return 75 + if state["phase"] == "review-wait": + verify_candidate(arguments.candidate_root, state["identity"]["candidate_tree_id"], + identity["verifier"]["path"], identity["verifier"]["expected_sha256"]) + review = observation(arguments.review_observation, "delivery_replay_review_observation", state["identity"], "verdict") + if review is None: + result(state) + return 0 + if review["verdict"] != "clean": + state.update({"phase": "failed", "recoverable": False, "reason": "offline review did not report clean"}) + atomic_json(state_path, state) + result(state) + return 1 + state["review"] = review + state["phase"] = "publish-wait" + atomic_json(state_path, state) + if state["phase"] == "publish-wait": + publisher = observation(arguments.publisher_observation, "delivery_replay_publisher_observation", state["identity"], "disposition") + if publisher is None: + result(state) + return 0 + if publisher["disposition"] != "offline-simulated": + state.update({"phase": "failed", "recoverable": False, "reason": "offline publisher disposition is invalid"}) + atomic_json(state_path, state) + result(state) + return 1 + state["publisher"] = publisher + state["phase"] = "completed-offline" + atomic_json(state_path, state) + result(state) + return 0 + result(state) + return 1 + finally: + signal.signal(signal.SIGTERM, previous_term) + signal.signal(signal.SIGINT, previous_int) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--input", required=True) + parser.add_argument("--source-repository-id", required=True) + parser.add_argument("--source-git-dir", required=True) + parser.add_argument("--candidate-root", required=True) + parser.add_argument("--scratch-root", required=True) + parser.add_argument("--state-dir", required=True) + parser.add_argument("--closure-helper", required=True) + parser.add_argument("--jq-bin", required=True) + parser.add_argument("--verify-path", required=True) + parser.add_argument("--expected-sha256", required=True) + parser.add_argument("--review-observation") + parser.add_argument("--publisher-observation") + try: + return replay(parser.parse_args()) + except (OSError, ReplayError) as error: + print(f"delivery replay: {error}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/test/delivery-replay.test.sh b/scripts/test/delivery-replay.test.sh new file mode 100755 index 0000000..3f1c147 --- /dev/null +++ b/scripts/test/delivery-replay.test.sh @@ -0,0 +1,189 @@ +#!/usr/bin/env bash +set -euo pipefail +export LC_ALL=C +export PYTHONDONTWRITEBYTECODE=1 +umask 077 + +root=$(CDPATH='' cd -P -- "${BASH_SOURCE[0]%/*}/../.." && pwd -P) +replay="$root/delivery/v1/replay.py" +fixture_builder="$root/scripts/test/local-git-materializer-fixtures.sh" +closure_source="$root/adapters/local-git-materializer/v1/object-closure.c" +tmp=$(/usr/bin/mktemp -d "${TMPDIR:-/tmp}/ystack-delivery-replay.XXXXXX") +cleanup() { /bin/rm -rf -- "$tmp"; } +trap cleanup EXIT + +sha_file() { /usr/bin/shasum -a 256 "$1" | /usr/bin/awk '{print $1}'; } +fail() { printf 'FAIL: %s\n' "$1" >&2; exit 1; } +passed=0 +pass() { passed=$((passed + 1)); printf 'ok %s - %s\n' "$passed" "$1"; } + +platform=$(/usr/bin/uname -s):$(/usr/bin/uname -m) +case "$platform" in + Linux:x86_64) asset=jq-linux64; asset_sha=af986793a515d500ab2d35f8d2aecd656e764504b789b66d7e1a0b727a124c44 ;; + Darwin:x86_64|Darwin:arm64) asset=jq-osx-amd64; asset_sha=5c0a0a3ea600f302ee458b30317425dd9632d1ad8882259fcaf4e9b868b2b1ef ;; + *) fail "unsupported host $platform" ;; +esac +jq_bin="${TMPDIR:-/tmp}/ystack-portable-core-jq16/$asset" +[ -f "$jq_bin" ] && [ ! -L "$jq_bin" ] && [ "$(sha_file "$jq_bin")" = "$asset_sha" ] || + fail 'pinned jq 1.6 is required' + +runtime="$tmp/runtime" +/bin/mkdir -m 700 "$runtime" "$tmp/home" +if [ "$platform" = Darwin:arm64 ]; then + printf '%s\n' '#!/bin/sh' "exec /usr/bin/arch -x86_64 '$jq_bin' \"\$@\"" > "$runtime/jq" +else + /bin/cp "$jq_bin" "$runtime/jq" +fi +/bin/chmod 0555 "$runtime/jq" +jq_bin="$runtime/jq" +/usr/bin/cc -std=c11 -Wall -Wextra -Werror -O2 "$closure_source" -o "$runtime/object-closure" +/bin/chmod 0555 "$runtime/object-closure" + +git_clean() { + /usr/bin/env -i HOME="$tmp/home" TMPDIR="$tmp" PATH=/usr/bin:/bin LC_ALL=C \ + GIT_CONFIG_NOSYSTEM=1 GIT_CONFIG_GLOBAL=/dev/null GIT_NO_REPLACE_OBJECTS=1 \ + GIT_NO_LAZY_FETCH=1 GIT_TERMINAL_PROMPT=0 /usr/bin/git --no-replace-objects "$@" +} +make_source() { + local destination=$1 blob tree commit + /bin/mkdir -m 700 "$destination" + git_clean init -q --bare "$destination" + blob=$(printf '%s\n' alpha beta | git_clean --git-dir="$destination" hash-object -w --stdin) + tree=$(printf '100644 blob %s\tsource.txt\n' "$blob" | git_clean --git-dir="$destination" mktree) + commit=$(printf '%s\n' source | /usr/bin/env -i HOME="$tmp/home" PATH=/usr/bin:/bin LC_ALL=C \ + GIT_CONFIG_NOSYSTEM=1 GIT_CONFIG_GLOBAL=/dev/null GIT_AUTHOR_NAME=fixture \ + GIT_AUTHOR_EMAIL=fixture@example.invalid GIT_COMMITTER_NAME=fixture \ + GIT_COMMITTER_EMAIL=fixture@example.invalid GIT_AUTHOR_DATE=2000-01-01T00:00:00Z \ + GIT_COMMITTER_DATE=2000-01-01T00:00:00Z /usr/bin/git --git-dir="$destination" commit-tree "$tree") + git_clean --git-dir="$destination" update-ref refs/heads/main "$commit" + printf '%s %s\n' "$commit" "$tree" +} + +read -r source_commit source_tree < <(make_source "$tmp/source.git") +"$fixture_builder" build "$tmp/fixture" "$jq_bin" sha1 "$source_commit" "$source_tree" +base_input="$tmp/fixture/input.json" +expected_changed=$(printf '%s\n' alpha beta gamma | /usr/bin/shasum -a 256 | /usr/bin/awk '{print $1}') + +run_replay() { + local name=$1 input=$2 expected=$3 + local state="$tmp/$name-state" candidate="$tmp/$name-candidate" scratch="$tmp/$name-scratch" + /bin/mkdir -m 700 "$state" "$candidate" "$scratch" + python3 "$replay" --input "$input" --source-repository-id fixture.target \ + --source-git-dir "$tmp/source.git" --candidate-root "$candidate" --scratch-root "$scratch" \ + --state-dir "$state" --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" \ + --verify-path source.txt --expected-sha256 "$expected" +} + +run_replay changed "$base_input" "$expected_changed" >"$tmp/changed.out" +jq -e '.state.phase=="review-wait" and .authority=="none" and .offline_simulation==true' "$tmp/changed.out" >/dev/null || + fail missing-review-waits +request_sha=$(jq -r '.identity.request_sha256' "$tmp/changed-state/run.json") +candidate_tree=$(jq -r '.identity.candidate_tree_id' "$tmp/changed-state/run.json") +pass 'changed materialization and fixed read-only verification wait for review' + +printf '%s\n' '{"schema_version":1,"kind":"delivery_replay_review_observation","actor_id":"test.reviewer","request_sha256":"'"$request_sha"'","candidate_tree_id":"'"$candidate_tree"'","verdict":"clean"}' >"$tmp/review.json" +printf '%s\n' '{"schema_version":1,"kind":"delivery_replay_publisher_observation","actor_id":"test.publisher","request_sha256":"'"$request_sha"'","candidate_tree_id":"'"$candidate_tree"'","disposition":"offline-simulated"}' >"$tmp/publisher.json" +python3 "$replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ + --candidate-root "$tmp/changed-candidate" --scratch-root "$tmp/changed-scratch" --state-dir "$tmp/changed-state" \ + --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" --verify-path source.txt --expected-sha256 "$expected_changed" \ + --review-observation "$tmp/review.json" >"$tmp/publish-wait.out" +jq -e '.state.phase=="publish-wait"' "$tmp/publish-wait.out" >/dev/null || fail missing-publisher-waits +python3 "$replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ + --candidate-root "$tmp/changed-candidate" --scratch-root "$tmp/changed-scratch" --state-dir "$tmp/changed-state" \ + --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" --verify-path source.txt --expected-sha256 "$expected_changed" \ + --review-observation "$tmp/review.json" --publisher-observation "$tmp/publisher.json" >"$tmp/completed.out" +jq -e '.state.phase=="completed-offline" and .state.publisher.disposition=="offline-simulated"' "$tmp/completed.out" >/dev/null || + fail completed-offline +cp "$tmp/completed.out" "$tmp/completed-first.out" +python3 "$replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ + --candidate-root "$tmp/changed-candidate" --scratch-root "$tmp/changed-scratch" --state-dir "$tmp/changed-state" \ + --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" --verify-path source.txt --expected-sha256 "$expected_changed" >"$tmp/completed-repeat.out" +cmp "$tmp/completed-first.out" "$tmp/completed-repeat.out" || fail duplicate-completion-output +pass 'offline review and publisher observations complete once and replay deterministically' + +if run_replay verifier-failure "$base_input" "$(printf '0%.0s' {1..64})" >"$tmp/verifier-failure.out" 2>&1; then + fail fixed-verifier-failure +fi +jq -e '.state.phase=="failed" and (.state.reason|contains("digest mismatch"))' "$tmp/verifier-failure.out" >/dev/null || + fail fixed-verifier-failure-state +pass 'fixed verifier failure is terminal and explicit' + +mkdir -m 700 "$tmp/mismatch-state" "$tmp/mismatch-candidate" "$tmp/mismatch-scratch" +python3 "$replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ + --candidate-root "$tmp/mismatch-candidate" --scratch-root "$tmp/mismatch-scratch" --state-dir "$tmp/mismatch-state" \ + --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" --verify-path source.txt --expected-sha256 "$expected_changed" > /dev/null +printf '%s\n' '{"schema_version":1,"kind":"delivery_replay_review_observation","actor_id":"test.reviewer","request_sha256":"'"$request_sha"'","candidate_tree_id":"'"$(printf '0%.0s' {1..40})"'","verdict":"clean"}' >"$tmp/mismatch-review.json" +if python3 "$replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ + --candidate-root "$tmp/mismatch-candidate" --scratch-root "$tmp/mismatch-scratch" --state-dir "$tmp/mismatch-state" \ + --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" --verify-path source.txt --expected-sha256 "$expected_changed" \ + --review-observation "$tmp/mismatch-review.json" >"$tmp/mismatch.out" 2>&1; then fail mismatched-review; fi +grep -Fq 'does not match this candidate' "$tmp/mismatch.out" || fail mismatched-review-error +pass 'mismatched supplied review cannot complete the replay' + +empty_input="$tmp/empty-input.json" +jq -S -c '(.stage_request.content.body.inputs[] | select(.input_id=="input.producer-patch") | .value.value.value.sha256) = $sha | + (.payloads[] | select(.input_id=="input.producer-patch") | .data) = "" | + (.trust_context.verified_payloads[] | select(.input_id=="input.producer-patch") | .content.data) = "" | + (.trust_context.verified_payloads[] | select(.input_id=="input.producer-patch") | .sha256) = $sha' \ + --arg sha "$(printf '' | /usr/bin/shasum -a 256 | /usr/bin/awk '{print $1}')" "$base_input" >"$empty_input" +empty_request="$tmp/empty-request.json" +jq -S -c '.stage_request.content' "$empty_input" >"$empty_request" +empty_request_sha=$(sha_file "$empty_request") +jq -S -c --arg sha "$empty_request_sha" '.stage_request.sha256=$sha' "$empty_input" >"$tmp/empty-final.json" +source_digest=$(printf '%s\n' alpha beta | /usr/bin/shasum -a 256 | /usr/bin/awk '{print $1}') +run_replay no-change "$tmp/empty-final.json" "$source_digest" >"$tmp/no-change.out" +jq -e '.state.phase=="review-wait" and .state.materialization.candidate_tree_id==.state.identity.source_tree_id' "$tmp/no-change.out" >/dev/null || + fail no-change +pass 'empty producer patch records a no-change candidate before review' + +mkdir -m 700 "$tmp/interrupted-state" "$tmp/interrupted-candidate" "$tmp/interrupted-scratch" +python3 "$replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ + --candidate-root "$tmp/interrupted-candidate" --scratch-root "$tmp/interrupted-scratch" --state-dir "$tmp/interrupted-state" \ + --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" --verify-path source.txt --expected-sha256 "$expected_changed" \ + >"$tmp/interrupted.out" & +interrupted_pid=$! +interrupted_wait=0 +while [ ! -f "$tmp/interrupted-state/run.json" ]; do + if ! kill -0 "$interrupted_pid" 2>/dev/null; then + wait "$interrupted_pid" || : + sed -n '1,12p' "$tmp/interrupted.out" >&2 + fail interrupted-start + fi + interrupted_wait=$((interrupted_wait + 1)) + if [ "$interrupted_wait" -gt 100 ]; then + kill -TERM "$interrupted_pid" 2>/dev/null || : + wait "$interrupted_pid" || : + sed -n '1,12p' "$tmp/interrupted.out" >&2 + fail interrupted-start-timeout + fi + sleep 0.1 +done +kill -TERM "$interrupted_pid" +if wait "$interrupted_pid"; then fail interrupted-run; fi +[ "$(jq -r '.phase' "$tmp/interrupted-state/run.json")" = verifying ] || fail interrupted-state +python3 "$replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ + --candidate-root "$tmp/interrupted-candidate" --scratch-root "$tmp/interrupted-scratch" --state-dir "$tmp/interrupted-state" \ + --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" --verify-path source.txt --expected-sha256 "$expected_changed" \ + >"$tmp/interrupted-retry.out" +jq -e '.state.phase=="review-wait"' "$tmp/interrupted-retry.out" >/dev/null || fail interrupted-retry +pass 'interruption after materialization resumes without a duplicate candidate output' + +mkdir -m 700 "$tmp/stale-state" "$tmp/stale-candidate" "$tmp/stale-scratch" +python3 "$replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ + --candidate-root "$tmp/stale-candidate" --scratch-root "$tmp/stale-scratch" --state-dir "$tmp/stale-state" \ + --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" --verify-path source.txt --expected-sha256 "$expected_changed" > /dev/null +if python3 "$replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ + --candidate-root "$tmp/stale-candidate" --scratch-root "$tmp/stale-scratch" --state-dir "$tmp/stale-state" \ + --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" --verify-path source.txt --expected-sha256 "$source_digest" >"$tmp/stale.out"; then fail changed-input-stale; fi +jq -e '.state.phase=="stale"' "$tmp/stale.out" >/dev/null || fail changed-input-stale-state +pass 'changed verifier input cannot reuse the prior run' +changed_input="$tmp/changed-input.json" +jq -S -c '.attempt.attempt_id="attempt.changed"' "$base_input" >"$changed_input" +if python3 "$replay" --input "$changed_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ + --candidate-root "$tmp/stale-candidate" --scratch-root "$tmp/stale-scratch" --state-dir "$tmp/stale-state" \ + --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" --verify-path source.txt --expected-sha256 "$expected_changed" \ + >"$tmp/changed-input-stale.out"; then fail changed-materialization-input-stale; fi +jq -e '.state.phase=="stale"' "$tmp/changed-input-stale.out" >/dev/null || fail changed-materialization-input-stale-state +pass 'changed materialization input cannot reuse the prior run' + +printf 'delivery replay: %s focused checks passed\n' "$passed" diff --git a/work/delivery-loop-first/plan.md b/work/delivery-loop-first/plan.md new file mode 100644 index 0000000..0200d0d --- /dev/null +++ b/work/delivery-loop-first/plan.md @@ -0,0 +1,23 @@ +# Delivery loop first plan + +This PR adds one inactive offline replay slice. It owns only these paths: + +- `delivery/v1/replay.py` +- `scripts/test/delivery-replay.test.sh` and its test-owned fixtures +- `work/delivery-loop-first/plan.md` +- `README.md`, `RESTORE.md`, and `ci/required-files.txt` + +The replay accepts an existing canonical local-materialization input and +caller-owned source, candidate, scratch, and private state directories. It calls +the existing local materializer, verifies one repo-relative candidate blob against +one supplied SHA-256, and journals identities and phase atomically. It never runs +candidate code or a user command string. + +States are `materializing`, `verifying`, `review-wait`, `publish-wait`, `failed`, +and `completed-offline`. Review and publisher records are supplied offline test +observations. They name the exact request and candidate plus an actor, but do not +authenticate anyone or authorize publication. Missing review remains waiting. + +The slice is not profile selection, qualification, model execution, target access, +deployment, release, install, merge, or production publication. It is not the +whole delivery loop. From 0aca16fc7dedd145c7e2ca432364046de251a77c Mon Sep 17 00:00:00 2001 From: ci Date: Fri, 4 Sep 2026 18:15:11 -0400 Subject: [PATCH 02/20] Simplify delivery replay state loading --- delivery/v1/replay.py | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/delivery/v1/replay.py b/delivery/v1/replay.py index 0ff5a28..bda8568 100755 --- a/delivery/v1/replay.py +++ b/delivery/v1/replay.py @@ -100,15 +100,6 @@ def atomic_json(path, value): os.unlink(temporary) -def load_state(path): - if not path.exists(): - return None - value, _ = read_json(path, MAX_OBSERVATION_BYTES) - if not isinstance(value, dict): - raise ReplayError("state journal is malformed") - return value - - def safe_path(value): if not isinstance(value, str) or not value or len(value) > 4096: raise ReplayError("verification path is invalid") @@ -250,7 +241,11 @@ def replay(arguments): lock_descriptor = os.open(lock_path, os.O_RDWR | os.O_CREAT | getattr(os, "O_NOFOLLOW", 0), 0o600) with os.fdopen(lock_descriptor, "a+b") as lock: fcntl.flock(lock, fcntl.LOCK_EX) - state = load_state(state_path) + state = None + if state_path.exists(): + state, _ = read_json(state_path, MAX_OBSERVATION_BYTES) + if not isinstance(state, dict): + raise ReplayError("state journal is malformed") if state is not None and state.get("identity", {}).get("run_key") != identity["run_key"]: result({"phase": "stale", "reason": "run identity changed"}) return 2 From 90ee42f019785b10f06a1f6aff4f1222b38c7c73 Mon Sep 17 00:00:00 2001 From: ci Date: Fri, 4 Sep 2026 18:21:31 -0400 Subject: [PATCH 03/20] Revalidate publish wait replay state --- delivery/v1/replay.py | 7 +++++++ scripts/test/delivery-replay.test.sh | 7 +++++++ 2 files changed, 14 insertions(+) diff --git a/delivery/v1/replay.py b/delivery/v1/replay.py index bda8568..4fe0e4b 100755 --- a/delivery/v1/replay.py +++ b/delivery/v1/replay.py @@ -318,6 +318,13 @@ def replay(arguments): state["phase"] = "publish-wait" atomic_json(state_path, state) if state["phase"] == "publish-wait": + verify_candidate(arguments.candidate_root, state["identity"]["candidate_tree_id"], + identity["verifier"]["path"], identity["verifier"]["expected_sha256"]) + if arguments.review_observation is not None and ( + observation(arguments.review_observation, "delivery_replay_review_observation", + state["identity"], "verdict") != state.get("review") + ): + raise ReplayError("supplied offline review changed after review wait") publisher = observation(arguments.publisher_observation, "delivery_replay_publisher_observation", state["identity"], "disposition") if publisher is None: result(state) diff --git a/scripts/test/delivery-replay.test.sh b/scripts/test/delivery-replay.test.sh index 3f1c147..46de5f8 100755 --- a/scripts/test/delivery-replay.test.sh +++ b/scripts/test/delivery-replay.test.sh @@ -88,6 +88,13 @@ python3 "$replay" --input "$base_input" --source-repository-id fixture.target -- --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" --verify-path source.txt --expected-sha256 "$expected_changed" \ --review-observation "$tmp/review.json" >"$tmp/publish-wait.out" jq -e '.state.phase=="publish-wait"' "$tmp/publish-wait.out" >/dev/null || fail missing-publisher-waits +jq -S -c '.note="changed after review wait"' "$tmp/review.json" >"$tmp/changed-review.json" +if python3 "$replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ + --candidate-root "$tmp/changed-candidate" --scratch-root "$tmp/changed-scratch" --state-dir "$tmp/changed-state" \ + --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" --verify-path source.txt --expected-sha256 "$expected_changed" \ + --review-observation "$tmp/changed-review.json" --publisher-observation "$tmp/publisher.json" >"$tmp/changed-review.out" 2>&1; then fail changed-review-after-wait; fi +grep -Fq 'review changed after review wait' "$tmp/changed-review.out" || fail changed-review-after-wait-error +pass 'a changed supplied review cannot advance publish wait' python3 "$replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ --candidate-root "$tmp/changed-candidate" --scratch-root "$tmp/changed-scratch" --state-dir "$tmp/changed-state" \ --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" --verify-path source.txt --expected-sha256 "$expected_changed" \ From 653771542d68df825c2a21417bb1c9f5599e05f3 Mon Sep 17 00:00:00 2001 From: ci Date: Fri, 4 Sep 2026 18:31:36 -0400 Subject: [PATCH 04/20] Snapshot delivery replay input --- delivery/v1/replay.py | 52 +++++++++++++++++++++++----- scripts/test/delivery-replay.test.sh | 29 ++++++++++++++++ 2 files changed, 72 insertions(+), 9 deletions(-) diff --git a/delivery/v1/replay.py b/delivery/v1/replay.py index 4fe0e4b..a241f82 100755 --- a/delivery/v1/replay.py +++ b/delivery/v1/replay.py @@ -33,7 +33,7 @@ def canonical(value): return json.dumps(value, sort_keys=True, separators=(",", ":")).encode() -def read_json(path, limit): +def read_bytes(path, limit): descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)) try: chunks = [] @@ -49,12 +49,21 @@ def read_json(path, limit): os.close(descriptor) if len(data) > limit: raise ReplayError("input exceeds its size limit") + return data + + +def parse_json(data): try: - return json.loads(data), digest_bytes(data) + return json.loads(data) except json.JSONDecodeError as error: raise ReplayError("input is not JSON") from error +def read_json(path, limit): + data = read_bytes(path, limit) + return parse_json(data), digest_bytes(data) + + def private_directory(path): value = Path(path) stat = value.stat() @@ -82,7 +91,10 @@ def disjoint(*paths): def atomic_json(path, value): - encoded = canonical(value) + b"\n" + atomic_bytes(path, canonical(value) + b"\n") + + +def atomic_bytes(path, encoded): descriptor, temporary = tempfile.mkstemp(prefix=".replay-", dir=path.parent) try: with os.fdopen(descriptor, "wb") as handle: @@ -141,6 +153,7 @@ def input_identity(input_value, input_sha, arguments, materializer): "request_sha256": request_sha, "source_commit_id": source["commit_id"], "source_tree_id": source_tree_id, + "source_hash_algorithm": source.get("hash_algorithm"), "verifier": { "id": "delivery.fixed-content-sha256.v1", "path": safe_path(arguments.verify_path), @@ -155,9 +168,9 @@ def input_identity(input_value, input_sha, arguments, materializer): return identity -def run_materializer(arguments, materializer): +def run_materializer(arguments, materializer, input_path, identity): command = [ - str(materializer), "materialize", str(Path(arguments.input).resolve()), + str(materializer), "materialize", str(input_path), arguments.source_repository_id, str(Path(arguments.source_git_dir).resolve()), str(Path(arguments.candidate_root).resolve()), str(Path(arguments.scratch_root).resolve()), str(Path(arguments.closure_helper).resolve()), str(Path(arguments.jq_bin).resolve()), @@ -172,9 +185,22 @@ def run_materializer(arguments, materializer): receipt_text = response["payloads"][0]["data"] receipt = json.loads(receipt_text) candidate = receipt["candidate"] + receipt_sha = response["payloads"][0]["sha256"] + if ( + receipt_sha != digest_bytes(receipt_text.encode()) or + receipt["request_ref"]["sha256"] != identity["request_sha256"] or + response["stage_result"]["body"]["request_ref"]["sha256"] != identity["request_sha256"] or + receipt["source"] != { + "repository_id": identity["source_repository_id"], + "hash_algorithm": identity["source_hash_algorithm"], + "commit_id": identity["source_commit_id"], + "tree_id": identity["source_tree_id"], + } + ): + raise ReplayError("materializer response does not match the input snapshot") return { "response_sha256": digest_bytes(result.stdout), - "receipt_sha256": response["payloads"][0]["sha256"], + "receipt_sha256": receipt_sha, "candidate_commit_id": candidate["commit_id"], "candidate_tree_id": candidate["tree_id"], } @@ -226,12 +252,15 @@ def result(state): def replay(arguments): repository = Path(__file__).resolve().parents[2] materializer = trusted_file(repository / "adapters/local-git-materializer/v1/materialize.sh") - input_value, input_sha = read_json(arguments.input, MAX_INPUT_BYTES) - identity = input_identity(input_value, input_sha, arguments, materializer) state_dir = private_directory(arguments.state_dir) disjoint(state_dir, arguments.source_git_dir, arguments.candidate_root, arguments.scratch_root) state_path = state_dir / "run.json" + input_snapshot_path = state_dir / "materialization-input.json" lock_path = state_dir / "replay.lock" + input_bytes = read_bytes(arguments.input, MAX_INPUT_BYTES) + input_value = parse_json(input_bytes) + input_sha = digest_bytes(input_bytes) + identity = input_identity(input_value, input_sha, arguments, materializer) interrupted = {"value": False} previous_term = signal.getsignal(signal.SIGTERM) previous_int = signal.getsignal(signal.SIGINT) @@ -252,7 +281,12 @@ def replay(arguments): if state is None: state = {"schema_version": 1, "kind": "delivery_replay_state", "identity": identity, "phase": "materializing", "authority": "none", "qualification": "unavailable"} + atomic_bytes(input_snapshot_path, input_bytes) atomic_json(state_path, state) + elif not input_snapshot_path.is_file() or input_snapshot_path.is_symlink() or ( + digest_bytes(read_bytes(input_snapshot_path, MAX_INPUT_BYTES)) != identity["input_sha256"] + ): + raise ReplayError("saved materialization input snapshot is unavailable") if state["phase"] == "failed": if state.get("recoverable"): state["recovery"] = "start a new replay with fresh empty candidate, scratch, and state directories" @@ -272,7 +306,7 @@ def replay(arguments): return 0 if state["phase"] == "materializing": try: - state["materialization"] = run_materializer(arguments, materializer) + state["materialization"] = run_materializer(arguments, materializer, input_snapshot_path, identity) except ReplayError as error: state.update({"phase": "failed", "recoverable": True, "reason": str(error)}) atomic_json(state_path, state) diff --git a/scripts/test/delivery-replay.test.sh b/scripts/test/delivery-replay.test.sh index 46de5f8..1a9aba9 100755 --- a/scripts/test/delivery-replay.test.sh +++ b/scripts/test/delivery-replay.test.sh @@ -81,6 +81,35 @@ request_sha=$(jq -r '.identity.request_sha256' "$tmp/changed-state/run.json") candidate_tree=$(jq -r '.identity.candidate_tree_id' "$tmp/changed-state/run.json") pass 'changed materialization and fixed read-only verification wait for review' +cp "$base_input" "$tmp/mutable-input.json" +mkdir -m 700 "$tmp/mutation-state" "$tmp/mutation-candidate" "$tmp/mutation-scratch" +python3 "$replay" --input "$tmp/mutable-input.json" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ + --candidate-root "$tmp/mutation-candidate" --scratch-root "$tmp/mutation-scratch" --state-dir "$tmp/mutation-state" \ + --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" --verify-path source.txt --expected-sha256 "$expected_changed" \ + >"$tmp/mutation.out" & +mutation_pid=$! +mutation_wait=0 +while [ ! -f "$tmp/mutation-state/materialization-input.json" ]; do + if ! kill -0 "$mutation_pid" 2>/dev/null; then + wait "$mutation_pid" || : + sed -n '1,12p' "$tmp/mutation.out" >&2 + fail input-snapshot-start + fi + mutation_wait=$((mutation_wait + 1)) + if [ "$mutation_wait" -gt 100 ]; then + kill -TERM "$mutation_pid" 2>/dev/null || : + wait "$mutation_pid" || : + fail input-snapshot-timeout + fi + sleep 0.1 +done +printf '%s\n' '{"replaced":"after snapshot"}' >"$tmp/mutable-input.json" +wait "$mutation_pid" || fail input-snapshot-run +[ "$(sha_file "$tmp/mutation-state/materialization-input.json")" = "$(sha_file "$base_input")" ] || fail input-snapshot-bytes +jq -e '.state.phase=="review-wait" and .state.identity.input_sha256==$sha' --arg sha "$(sha_file "$base_input")" \ + "$tmp/mutation.out" >/dev/null || fail input-snapshot-output +pass 'replacement of the original input after snapshot cannot change materialization' + printf '%s\n' '{"schema_version":1,"kind":"delivery_replay_review_observation","actor_id":"test.reviewer","request_sha256":"'"$request_sha"'","candidate_tree_id":"'"$candidate_tree"'","verdict":"clean"}' >"$tmp/review.json" printf '%s\n' '{"schema_version":1,"kind":"delivery_replay_publisher_observation","actor_id":"test.publisher","request_sha256":"'"$request_sha"'","candidate_tree_id":"'"$candidate_tree"'","disposition":"offline-simulated"}' >"$tmp/publisher.json" python3 "$replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ From 0462ad2a9c62d96d8347988326d3469cd75af27b Mon Sep 17 00:00:00 2001 From: ci Date: Fri, 4 Sep 2026 19:14:36 -0400 Subject: [PATCH 05/20] Reconcile interrupted delivery materialization --- delivery/v1/replay.py | 56 ++++++++++++++++++-- scripts/test/delivery-replay.test.sh | 78 ++++++++++++++++++++++++++++ 2 files changed, 130 insertions(+), 4 deletions(-) diff --git a/delivery/v1/replay.py b/delivery/v1/replay.py index a241f82..431673f 100755 --- a/delivery/v1/replay.py +++ b/delivery/v1/replay.py @@ -9,6 +9,7 @@ from pathlib import Path import re import signal +import shutil import subprocess import sys import tempfile @@ -55,7 +56,7 @@ def read_bytes(path, limit): def parse_json(data): try: return json.loads(data) - except json.JSONDecodeError as error: + except (json.JSONDecodeError, UnicodeDecodeError) as error: raise ReplayError("input is not JSON") from error @@ -168,11 +169,13 @@ def input_identity(input_value, input_sha, arguments, materializer): return identity -def run_materializer(arguments, materializer, input_path, identity): +def run_materializer(arguments, materializer, input_path, identity, candidate_root=None, scratch_root=None): + candidate_root = Path(arguments.candidate_root).resolve() if candidate_root is None else candidate_root + scratch_root = Path(arguments.scratch_root).resolve() if scratch_root is None else scratch_root command = [ str(materializer), "materialize", str(input_path), arguments.source_repository_id, str(Path(arguments.source_git_dir).resolve()), - str(Path(arguments.candidate_root).resolve()), str(Path(arguments.scratch_root).resolve()), + str(candidate_root), str(scratch_root), str(Path(arguments.closure_helper).resolve()), str(Path(arguments.jq_bin).resolve()), ] environment = {"PATH": "/usr/bin:/bin", "LC_ALL": "C"} @@ -203,11 +206,52 @@ def run_materializer(arguments, materializer, input_path, identity): "receipt_sha256": receipt_sha, "candidate_commit_id": candidate["commit_id"], "candidate_tree_id": candidate["tree_id"], + "candidate_parent_commit_id": candidate["parent_commit_id"], } except (KeyError, IndexError, TypeError, json.JSONDecodeError) as error: raise ReplayError("materializer response is malformed") from error +def candidate_identity(candidate_root): + repository = Path(candidate_root).resolve() / "repository.git" + if not repository.is_dir() or repository.is_symlink(): + return None + environment = {"PATH": "/usr/bin:/bin", "LC_ALL": "C", "GIT_CONFIG_NOSYSTEM": "1", + "GIT_CONFIG_GLOBAL": "/dev/null", "GIT_NO_REPLACE_OBJECTS": "1", + "GIT_NO_LAZY_FETCH": "1", "GIT_TERMINAL_PROMPT": "0"} + values = [] + for revision in ("refs/heads/candidate", "refs/heads/candidate^{tree}", "refs/heads/candidate^"): + result = subprocess.run(["/usr/bin/git", f"--git-dir={repository}", "rev-parse", revision], + env=environment, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, check=False) + value = result.stdout.decode().strip() + if result.returncode != 0 or not OID.fullmatch(value): + return None + values.append(value) + return {"candidate_commit_id": values[0], "candidate_tree_id": values[1], + "candidate_parent_commit_id": values[2]} + + +def reconcile_materialization(arguments, materializer, input_path, identity, state_dir): + existing = candidate_identity(arguments.candidate_root) + if existing is None: + return None + recovery_candidate = Path(tempfile.mkdtemp(prefix="reconcile-candidate-", dir=state_dir)) + recovery_scratch = Path(tempfile.mkdtemp(prefix="reconcile-scratch-", dir=state_dir)) + try: + recomputed = run_materializer(arguments, materializer, input_path, identity, + recovery_candidate, recovery_scratch) + finally: + shutil.rmtree(recovery_candidate, ignore_errors=True) + shutil.rmtree(recovery_scratch, ignore_errors=True) + if existing != { + "candidate_commit_id": recomputed["candidate_commit_id"], + "candidate_tree_id": recomputed["candidate_tree_id"], + "candidate_parent_commit_id": recomputed["candidate_parent_commit_id"], + }: + raise ReplayError("existing candidate does not match frozen materialization input") + return recomputed + + def verify_candidate(candidate_root, candidate_tree, path, expected): repository = Path(candidate_root).resolve() / "repository.git" if not repository.is_dir() or repository.is_symlink() or not OID.fullmatch(candidate_tree): @@ -306,7 +350,11 @@ def replay(arguments): return 0 if state["phase"] == "materializing": try: - state["materialization"] = run_materializer(arguments, materializer, input_snapshot_path, identity) + reconciled = reconcile_materialization(arguments, materializer, input_snapshot_path, + identity, state_dir) + state["materialization"] = reconciled or run_materializer( + arguments, materializer, input_snapshot_path, identity + ) except ReplayError as error: state.update({"phase": "failed", "recoverable": True, "reason": str(error)}) atomic_json(state_path, state) diff --git a/scripts/test/delivery-replay.test.sh b/scripts/test/delivery-replay.test.sh index 1a9aba9..0c371b8 100755 --- a/scripts/test/delivery-replay.test.sh +++ b/scripts/test/delivery-replay.test.sh @@ -110,6 +110,84 @@ jq -e '.state.phase=="review-wait" and .state.identity.input_sha256==$sha' --arg "$tmp/mutation.out" >/dev/null || fail input-snapshot-output pass 'replacement of the original input after snapshot cannot change materialization' +kill_wrapper="$tmp/kill-after-materialize.py" +printf '%s\n' \ + 'import importlib.util, os, signal, sys' \ + 'path, arguments = sys.argv[1], sys.argv[2:]' \ + 'spec = importlib.util.spec_from_file_location("replay", path)' \ + 'module = importlib.util.module_from_spec(spec)' \ + 'spec.loader.exec_module(module)' \ + 'original = module.run_materializer' \ + 'def stop_after_materialization(*args, **kwargs):' \ + ' result = original(*args, **kwargs)' \ + ' os.kill(os.getpid(), signal.SIGKILL)' \ + ' return result' \ + 'module.run_materializer = stop_after_materialization' \ + 'sys.argv = [path] + arguments' \ + 'raise SystemExit(module.main())' >"$kill_wrapper" +mkdir -m 700 "$tmp/reconcile-state" "$tmp/reconcile-candidate" "$tmp/reconcile-scratch" +if python3 "$kill_wrapper" "$replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ + --candidate-root "$tmp/reconcile-candidate" --scratch-root "$tmp/reconcile-scratch" --state-dir "$tmp/reconcile-state" \ + --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" --verify-path source.txt --expected-sha256 "$expected_changed" \ + >"$tmp/reconcile-killed.out" 2>&1; then fail reconcile-kill; fi +[ "$(jq -r '.phase' "$tmp/reconcile-state/run.json")" = materializing ] && [ -d "$tmp/reconcile-candidate/repository.git" ] || + fail reconcile-window +python3 "$replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ + --candidate-root "$tmp/reconcile-candidate" --scratch-root "$tmp/reconcile-scratch" --state-dir "$tmp/reconcile-state" \ + --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" --verify-path source.txt --expected-sha256 "$expected_changed" \ + >"$tmp/reconcile-retry.out" +jq -e '.state.phase=="review-wait"' "$tmp/reconcile-retry.out" >/dev/null || fail reconcile-retry +pass 'SIGKILL after materializer output reconciles the existing candidate once' + +mkdir -m 700 "$tmp/reconcile-bad-state" "$tmp/reconcile-bad-candidate" "$tmp/reconcile-bad-scratch" +if python3 "$kill_wrapper" "$replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ + --candidate-root "$tmp/reconcile-bad-candidate" --scratch-root "$tmp/reconcile-bad-scratch" --state-dir "$tmp/reconcile-bad-state" \ + --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" --verify-path source.txt --expected-sha256 "$expected_changed" \ + >"$tmp/reconcile-bad-killed.out" 2>&1; then fail reconcile-bad-kill; fi +bad_repo="$tmp/reconcile-bad-candidate/repository.git" +bad_commit=$(printf '%s\n' mismatch | /usr/bin/env -i HOME="$tmp/home" PATH=/usr/bin:/bin LC_ALL=C \ + GIT_AUTHOR_NAME=fixture GIT_AUTHOR_EMAIL=fixture@example.invalid GIT_COMMITTER_NAME=fixture \ + GIT_COMMITTER_EMAIL=fixture@example.invalid /usr/bin/git --git-dir="$bad_repo" commit-tree "$source_tree" -p "$source_commit") +/usr/bin/git --git-dir="$bad_repo" update-ref refs/heads/candidate "$bad_commit" +if python3 "$replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ + --candidate-root "$tmp/reconcile-bad-candidate" --scratch-root "$tmp/reconcile-bad-scratch" --state-dir "$tmp/reconcile-bad-state" \ + --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" --verify-path source.txt --expected-sha256 "$expected_changed" \ + >"$tmp/reconcile-bad.out" 2>&1; then fail reconcile-mismatch; fi +jq -e '.state.phase=="failed" and (.state.reason|contains("does not match frozen"))' "$tmp/reconcile-bad.out" >/dev/null || + fail reconcile-mismatch-state +pass 'a mismatched interrupted candidate is rejected without cleanup' + +printf '\377' >"$tmp/invalid-input.json" +mkdir -m 700 "$tmp/invalid-input-state" "$tmp/invalid-input-candidate" "$tmp/invalid-input-scratch" +if python3 "$replay" --input "$tmp/invalid-input.json" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ + --candidate-root "$tmp/invalid-input-candidate" --scratch-root "$tmp/invalid-input-scratch" --state-dir "$tmp/invalid-input-state" \ + --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" --verify-path source.txt --expected-sha256 "$expected_changed" \ + >"$tmp/invalid-input.out" 2>&1; then fail invalid-utf8-input; fi +if ! grep -Fq 'delivery replay: input is not JSON' "$tmp/invalid-input.out" || + grep -Fq Traceback "$tmp/invalid-input.out"; then + fail invalid-utf8-input-error +fi +printf '\377' >"$tmp/reconcile-state/invalid-review.json" +if python3 "$replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ + --candidate-root "$tmp/reconcile-candidate" --scratch-root "$tmp/reconcile-scratch" --state-dir "$tmp/reconcile-state" \ + --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" --verify-path source.txt --expected-sha256 "$expected_changed" \ + --review-observation "$tmp/reconcile-state/invalid-review.json" >"$tmp/invalid-review.out" 2>&1; then fail invalid-utf8-review; fi +if ! grep -Fq 'delivery replay: input is not JSON' "$tmp/invalid-review.out" || + grep -Fq Traceback "$tmp/invalid-review.out"; then + fail invalid-utf8-review-error +fi +mkdir -m 700 "$tmp/invalid-journal-state" "$tmp/invalid-journal-candidate" "$tmp/invalid-journal-scratch" +printf '\377' >"$tmp/invalid-journal-state/run.json" +if python3 "$replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ + --candidate-root "$tmp/invalid-journal-candidate" --scratch-root "$tmp/invalid-journal-scratch" --state-dir "$tmp/invalid-journal-state" \ + --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" --verify-path source.txt --expected-sha256 "$expected_changed" \ + >"$tmp/invalid-journal.out" 2>&1; then fail invalid-utf8-journal; fi +if ! grep -Fq 'delivery replay: input is not JSON' "$tmp/invalid-journal.out" || + grep -Fq Traceback "$tmp/invalid-journal.out"; then + fail invalid-utf8-journal-error +fi +pass 'invalid UTF-8 input, review, and journal records fail without a traceback' + printf '%s\n' '{"schema_version":1,"kind":"delivery_replay_review_observation","actor_id":"test.reviewer","request_sha256":"'"$request_sha"'","candidate_tree_id":"'"$candidate_tree"'","verdict":"clean"}' >"$tmp/review.json" printf '%s\n' '{"schema_version":1,"kind":"delivery_replay_publisher_observation","actor_id":"test.publisher","request_sha256":"'"$request_sha"'","candidate_tree_id":"'"$candidate_tree"'","disposition":"offline-simulated"}' >"$tmp/publisher.json" python3 "$replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ From 4c29c1a321aeeff111f7442f1c561da9ab907c3c Mon Sep 17 00:00:00 2001 From: ci Date: Fri, 4 Sep 2026 20:07:26 -0400 Subject: [PATCH 06/20] Validate replay journal recovery states --- delivery/v1/replay.py | 76 +++++++++++++++++++++--- scripts/test/delivery-replay.test.sh | 88 ++++++++++++++++++++++++---- 2 files changed, 147 insertions(+), 17 deletions(-) diff --git a/delivery/v1/replay.py b/delivery/v1/replay.py index 431673f..706eddf 100755 --- a/delivery/v1/replay.py +++ b/delivery/v1/replay.py @@ -212,7 +212,7 @@ def run_materializer(arguments, materializer, input_path, identity, candidate_ro raise ReplayError("materializer response is malformed") from error -def candidate_identity(candidate_root): +def candidate_identity(candidate_root, source_commit): repository = Path(candidate_root).resolve() / "repository.git" if not repository.is_dir() or repository.is_symlink(): return None @@ -220,19 +220,27 @@ def candidate_identity(candidate_root): "GIT_CONFIG_GLOBAL": "/dev/null", "GIT_NO_REPLACE_OBJECTS": "1", "GIT_NO_LAZY_FETCH": "1", "GIT_TERMINAL_PROMPT": "0"} values = [] - for revision in ("refs/heads/candidate", "refs/heads/candidate^{tree}", "refs/heads/candidate^"): + for revision in ("refs/heads/candidate", "refs/heads/candidate^{tree}"): result = subprocess.run(["/usr/bin/git", f"--git-dir={repository}", "rev-parse", revision], env=environment, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, check=False) value = result.stdout.decode().strip() if result.returncode != 0 or not OID.fullmatch(value): return None values.append(value) + if values[0] == source_commit: + return {"candidate_commit_id": values[0], "candidate_tree_id": values[1], + "candidate_parent_commit_id": source_commit} + result = subprocess.run(["/usr/bin/git", f"--git-dir={repository}", "rev-parse", "refs/heads/candidate^"], + env=environment, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, check=False) + parent = result.stdout.decode().strip() + if result.returncode != 0 or not OID.fullmatch(parent): + return None return {"candidate_commit_id": values[0], "candidate_tree_id": values[1], - "candidate_parent_commit_id": values[2]} + "candidate_parent_commit_id": parent} def reconcile_materialization(arguments, materializer, input_path, identity, state_dir): - existing = candidate_identity(arguments.candidate_root) + existing = candidate_identity(arguments.candidate_root, identity["source_commit_id"]) if existing is None: return None recovery_candidate = Path(tempfile.mkdtemp(prefix="reconcile-candidate-", dir=state_dir)) @@ -287,6 +295,61 @@ def observation(path, kind, identity, field): return {"actor_id": value["actor_id"], field: value.get(field), "sha256": source_sha} +def validate_state(state, identity): + if not isinstance(state, dict) or state.get("schema_version") != 1 or \ + state.get("kind") != "delivery_replay_state" or state.get("authority") != "none" or \ + state.get("qualification") != "unavailable": + raise ReplayError("state journal is malformed") + saved = state.get("identity") + if not isinstance(saved, dict) or any( + not isinstance(saved.get(name), str) or not re.fullmatch(r"[0-9a-f]{64}", saved[name]) + for name in ("input_sha256", "request_sha256", "materializer_sha256", "closure_helper_sha256", "jq_sha256", "run_key") + ) or not isinstance(saved.get("source_repository_id"), str) or \ + saved.get("source_hash_algorithm") not in {"sha1", "sha256"} or \ + any(not isinstance(saved.get(name), str) or not OID.fullmatch(saved[name]) + for name in ("source_commit_id", "source_tree_id")) or \ + not isinstance(saved.get("verifier"), dict) or \ + not isinstance(saved["verifier"].get("id"), str) or \ + not isinstance(saved["verifier"].get("path"), str) or \ + not re.fullmatch(r"[0-9a-f]{64}", str(saved["verifier"].get("expected_sha256", ""))): + raise ReplayError("state journal identity is malformed") + for name in ("candidate_commit_id", "candidate_tree_id"): + if name in saved and (not isinstance(saved[name], str) or not OID.fullmatch(saved[name])): + raise ReplayError("state journal candidate identity is malformed") + phase = state.get("phase") + if phase not in {"materializing", "verifying", "review-wait", "publish-wait", "completed-offline", "failed"}: + raise ReplayError("state journal phase is malformed") + needs_materialization = phase in {"verifying", "review-wait", "publish-wait", "completed-offline"} + materialization = state.get("materialization") + if needs_materialization and (not isinstance(materialization, dict) or any( + not isinstance(materialization.get(name), str) or not OID.fullmatch(materialization[name]) + for name in ("candidate_commit_id", "candidate_tree_id", "candidate_parent_commit_id") + ) or any( + not isinstance(materialization.get(name), str) or not re.fullmatch(r"[0-9a-f]{64}", materialization[name]) + for name in ("response_sha256", "receipt_sha256") + )): + raise ReplayError("state journal materialization is malformed") + if phase in {"review-wait", "publish-wait", "completed-offline"}: + verification = state.get("verification") + if not isinstance(verification, dict) or not isinstance(verification.get("id"), str) or \ + not isinstance(verification.get("path"), str) or \ + not re.fullmatch(r"[0-9a-f]{64}", str(verification.get("sha256", ""))): + raise ReplayError("state journal verification is malformed") + if phase in {"publish-wait", "completed-offline"}: + review = state.get("review") + if not isinstance(review, dict) or not ACTOR.fullmatch(str(review.get("actor_id", ""))) or \ + review.get("verdict") != "clean" or not re.fullmatch(r"[0-9a-f]{64}", str(review.get("sha256", ""))): + raise ReplayError("state journal review is malformed") + if phase == "completed-offline": + publisher = state.get("publisher") + if not isinstance(publisher, dict) or not ACTOR.fullmatch(str(publisher.get("actor_id", ""))) or \ + publisher.get("disposition") != "offline-simulated" or \ + not re.fullmatch(r"[0-9a-f]{64}", str(publisher.get("sha256", ""))): + raise ReplayError("state journal publisher is malformed") + if phase == "failed" and not isinstance(state.get("reason"), str): + raise ReplayError("state journal failure is malformed") + + def result(state): print(json.dumps({"kind": "delivery_replay_receipt", "authority": "none", "qualification": "unavailable", "offline_simulation": True, @@ -317,9 +380,8 @@ def replay(arguments): state = None if state_path.exists(): state, _ = read_json(state_path, MAX_OBSERVATION_BYTES) - if not isinstance(state, dict): - raise ReplayError("state journal is malformed") - if state is not None and state.get("identity", {}).get("run_key") != identity["run_key"]: + validate_state(state, identity) + if state is not None and any(state["identity"].get(name) != value for name, value in identity.items()): result({"phase": "stale", "reason": "run identity changed"}) return 2 if state is None: diff --git a/scripts/test/delivery-replay.test.sh b/scripts/test/delivery-replay.test.sh index 0c371b8..fda4949 100755 --- a/scripts/test/delivery-replay.test.sh +++ b/scripts/test/delivery-replay.test.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +# shellcheck disable=SC2016 set -euo pipefail export LC_ALL=C export PYTHONDONTWRITEBYTECODE=1 @@ -59,6 +60,36 @@ make_source() { printf '%s %s\n' "$commit" "$tree" } +make_source_with_ancestor() { + local destination=$1 blob tree base commit + /bin/mkdir -m 700 "$destination" + git_clean init -q --bare "$destination" + blob=$(printf '%s\n' alpha beta | git_clean --git-dir="$destination" hash-object -w --stdin) + tree=$(printf '100644 blob %s\tsource.txt\n' "$blob" | git_clean --git-dir="$destination" mktree) + base=$(printf '%s\n' base | /usr/bin/env -i HOME="$tmp/home" PATH=/usr/bin:/bin LC_ALL=C \ + GIT_CONFIG_NOSYSTEM=1 GIT_CONFIG_GLOBAL=/dev/null GIT_AUTHOR_NAME=fixture \ + GIT_AUTHOR_EMAIL=fixture@example.invalid GIT_COMMITTER_NAME=fixture \ + GIT_COMMITTER_EMAIL=fixture@example.invalid /usr/bin/git --git-dir="$destination" commit-tree "$tree") + commit=$(printf '%s\n' source | /usr/bin/env -i HOME="$tmp/home" PATH=/usr/bin:/bin LC_ALL=C \ + GIT_CONFIG_NOSYSTEM=1 GIT_CONFIG_GLOBAL=/dev/null GIT_AUTHOR_NAME=fixture \ + GIT_AUTHOR_EMAIL=fixture@example.invalid GIT_COMMITTER_NAME=fixture \ + GIT_COMMITTER_EMAIL=fixture@example.invalid /usr/bin/git --git-dir="$destination" commit-tree "$tree" -p "$base") + git_clean --git-dir="$destination" update-ref refs/heads/main "$commit" + printf '%s %s\n' "$commit" "$tree" +} + +make_empty_input() { + local input=$1 output=$2 + local intermediate="$output.intermediate" request="$output.request" + "$jq_bin" -S -c '(.stage_request.content.body.inputs[] | select(.input_id=="input.producer-patch") | .value.value.value.sha256) = $sha | + (.payloads[] | select(.input_id=="input.producer-patch") | .data) = "" | + (.trust_context.verified_payloads[] | select(.input_id=="input.producer-patch") | .content.data) = "" | + (.trust_context.verified_payloads[] | select(.input_id=="input.producer-patch") | .sha256) = $sha' \ + --arg sha "$(printf '' | /usr/bin/shasum -a 256 | /usr/bin/awk '{print $1}')" "$input" >"$intermediate" + "$jq_bin" -S -c '.stage_request.content' "$intermediate" >"$request" + "$jq_bin" -S -c --arg sha "$(sha_file "$request")" '.stage_request.sha256=$sha' "$intermediate" >"$output" +} + read -r source_commit source_tree < <(make_source "$tmp/source.git") "$fixture_builder" build "$tmp/fixture" "$jq_bin" sha1 "$source_commit" "$source_tree" base_input="$tmp/fixture/input.json" @@ -195,6 +226,29 @@ python3 "$replay" --input "$base_input" --source-repository-id fixture.target -- --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" --verify-path source.txt --expected-sha256 "$expected_changed" \ --review-observation "$tmp/review.json" >"$tmp/publish-wait.out" jq -e '.state.phase=="publish-wait"' "$tmp/publish-wait.out" >/dev/null || fail missing-publisher-waits +expect_malformed_state() { + local name=$1 filter=$2 + local state_root="$tmp/malformed-$name-state" + /bin/mkdir -m 700 "$state_root" + cp "$tmp/changed-state/materialization-input.json" "$state_root/materialization-input.json" + jq -S -c "$filter" "$tmp/changed-state/run.json" >"$state_root/run.json" + if python3 "$replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ + --candidate-root "$tmp/changed-candidate" --scratch-root "$tmp/changed-scratch" --state-dir "$state_root" \ + --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" --verify-path source.txt --expected-sha256 "$expected_changed" \ + >"$tmp/malformed-$name.out" 2>&1; then fail "malformed-$name"; fi + if ! grep -Fq 'delivery replay: state journal' "$tmp/malformed-$name.out" || + grep -Fq Traceback "$tmp/malformed-$name.out"; then + fail "malformed-$name-error" + fi +} +expect_malformed_state identity-type '.identity=[]' +expect_malformed_state missing-phase 'del(.phase)' +expect_malformed_state invalid-phase '.phase="unknown"' +expect_malformed_state missing-materialization '(.phase="verifying") | del(.materialization)' +expect_malformed_state missing-verification '(.phase="review-wait") | del(.verification)' +expect_malformed_state missing-review '(.phase="publish-wait") | del(.review)' +expect_malformed_state missing-publisher '(.phase="completed-offline") | del(.publisher)' +pass 'malformed state phases and nested records fail without a traceback' jq -S -c '.note="changed after review wait"' "$tmp/review.json" >"$tmp/changed-review.json" if python3 "$replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ --candidate-root "$tmp/changed-candidate" --scratch-root "$tmp/changed-scratch" --state-dir "$tmp/changed-state" \ @@ -234,22 +288,36 @@ if python3 "$replay" --input "$base_input" --source-repository-id fixture.target grep -Fq 'does not match this candidate' "$tmp/mismatch.out" || fail mismatched-review-error pass 'mismatched supplied review cannot complete the replay' -empty_input="$tmp/empty-input.json" -jq -S -c '(.stage_request.content.body.inputs[] | select(.input_id=="input.producer-patch") | .value.value.value.sha256) = $sha | - (.payloads[] | select(.input_id=="input.producer-patch") | .data) = "" | - (.trust_context.verified_payloads[] | select(.input_id=="input.producer-patch") | .content.data) = "" | - (.trust_context.verified_payloads[] | select(.input_id=="input.producer-patch") | .sha256) = $sha' \ - --arg sha "$(printf '' | /usr/bin/shasum -a 256 | /usr/bin/awk '{print $1}')" "$base_input" >"$empty_input" -empty_request="$tmp/empty-request.json" -jq -S -c '.stage_request.content' "$empty_input" >"$empty_request" -empty_request_sha=$(sha_file "$empty_request") -jq -S -c --arg sha "$empty_request_sha" '.stage_request.sha256=$sha' "$empty_input" >"$tmp/empty-final.json" +make_empty_input "$base_input" "$tmp/empty-final.json" source_digest=$(printf '%s\n' alpha beta | /usr/bin/shasum -a 256 | /usr/bin/awk '{print $1}') run_replay no-change "$tmp/empty-final.json" "$source_digest" >"$tmp/no-change.out" jq -e '.state.phase=="review-wait" and .state.materialization.candidate_tree_id==.state.identity.source_tree_id' "$tmp/no-change.out" >/dev/null || fail no-change pass 'empty producer patch records a no-change candidate before review' +recover_no_change() { + local name=$1 input=$2 source=$3 + local state="$tmp/$name-state" candidate="$tmp/$name-candidate" scratch="$tmp/$name-scratch" + /bin/mkdir -m 700 "$state" "$candidate" "$scratch" + if python3 "$kill_wrapper" "$replay" --input "$input" --source-repository-id fixture.target --source-git-dir "$source" \ + --candidate-root "$candidate" --scratch-root "$scratch" --state-dir "$state" \ + --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" --verify-path source.txt --expected-sha256 "$source_digest" \ + >"$tmp/$name-killed.out" 2>&1; then fail "$name-kill"; fi + [ "$(jq -r '.phase' "$state/run.json")" = materializing ] && [ -d "$candidate/repository.git" ] || fail "$name-window" + python3 "$replay" --input "$input" --source-repository-id fixture.target --source-git-dir "$source" \ + --candidate-root "$candidate" --scratch-root "$scratch" --state-dir "$state" \ + --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" --verify-path source.txt --expected-sha256 "$source_digest" \ + >"$tmp/$name-retry.out" + jq -e '.state.phase=="review-wait" and .state.materialization.candidate_commit_id==.state.identity.source_commit_id' \ + "$tmp/$name-retry.out" >/dev/null || fail "$name-retry" +} +recover_no_change no-change-root "$tmp/empty-final.json" "$tmp/source.git" +read -r ancestor_commit ancestor_tree < <(make_source_with_ancestor "$tmp/ancestor-source.git") +"$fixture_builder" build "$tmp/ancestor-fixture" "$jq_bin" sha1 "$ancestor_commit" "$ancestor_tree" +make_empty_input "$tmp/ancestor-fixture/input.json" "$tmp/ancestor-empty.json" +recover_no_change no-change-ancestor "$tmp/ancestor-empty.json" "$tmp/ancestor-source.git" +pass 'SIGKILL no-change recovery accepts both root and ancestor source commits' + mkdir -m 700 "$tmp/interrupted-state" "$tmp/interrupted-candidate" "$tmp/interrupted-scratch" python3 "$replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ --candidate-root "$tmp/interrupted-candidate" --scratch-root "$tmp/interrupted-scratch" --state-dir "$tmp/interrupted-state" \ From e02c6346a59cd4e6c35f97f9cd358a2df24ce40f Mon Sep 17 00:00:00 2001 From: ci Date: Fri, 4 Sep 2026 20:57:46 -0400 Subject: [PATCH 07/20] fix replay identity and journal validation --- delivery/v1/replay.py | 95 ++++++++++++++++++++++------ scripts/test/delivery-replay.test.sh | 63 ++++++++++++++++++ 2 files changed, 140 insertions(+), 18 deletions(-) diff --git a/delivery/v1/replay.py b/delivery/v1/replay.py index 706eddf..05791fe 100755 --- a/delivery/v1/replay.py +++ b/delivery/v1/replay.py @@ -20,6 +20,21 @@ MAX_VERIFIED_BLOB_BYTES = 1024 * 1024 OID = re.compile(r"[0-9a-f]{40}|[0-9a-f]{64}\Z") ACTOR = re.compile(r"[a-z0-9][a-z0-9._:-]{0,127}\Z") +PACKAGE_FILES = ( + "adapters/local-git-materializer/v1/materialize.sh", + "adapters/local-git-materializer/v1/protocol.jq", + "scripts/core-contract.sh", + "core/v2/generation-registry.json", +) +GENERATION_FILES = ( + "core-ingress.sh", + "contracts.jq", + "modules/schema.jq", + "modules/profile_graph.jq", + "modules/stage_request.jq", + "modules/result_facts.jq", + "modules/result_truth.jq", +) class ReplayError(Exception): @@ -60,11 +75,6 @@ def parse_json(data): raise ReplayError("input is not JSON") from error -def read_json(path, limit): - data = read_bytes(path, limit) - return parse_json(data), digest_bytes(data) - - def private_directory(path): value = Path(path) stat = value.stat() @@ -124,7 +134,30 @@ def safe_path(value): return value -def input_identity(input_value, input_sha, arguments, materializer): +def package_paths(generation): + root = f"core/v2/generations/{generation}" + return PACKAGE_FILES + tuple(f"{root}/{name}" for name in GENERATION_FILES) + + +def materializer_package_identity(repository): + core_path = trusted_file(repository / "scripts/core-contract.sh") + core_bytes = core_path.read_bytes() + match = re.search( + rb"^PORTABLE_CORE_GENERATION='(g-[0-9a-f]{64})'$", core_bytes, re.MULTILINE + ) + if match is None: + raise ReplayError("materializer package generation is unavailable") + generation = match.group(1).decode() + files = { + relative: digest_bytes(trusted_file(repository / relative).read_bytes()) + for relative in package_paths(generation) + } + package = {"generation_id": generation, "files": files} + package["sha256"] = digest_bytes(canonical(package)) + return package + + +def input_identity(input_value, input_sha, arguments, repository): try: request = input_value["stage_request"] request_sha = request["sha256"] @@ -148,7 +181,7 @@ def input_identity(input_value, input_sha, arguments, materializer): raise ReplayError("expected verifier digest is invalid") closure = trusted_file(arguments.closure_helper) jq_bin = trusted_file(arguments.jq_bin) - materializer_sha = digest_bytes(materializer.read_bytes()) + package = materializer_package_identity(repository) identity = { "input_sha256": input_sha, "request_sha256": request_sha, @@ -160,7 +193,9 @@ def input_identity(input_value, input_sha, arguments, materializer): "path": safe_path(arguments.verify_path), "expected_sha256": expected, }, - "materializer_sha256": materializer_sha, + "driver_sha256": digest_bytes(trusted_file(Path(__file__).resolve()).read_bytes()), + "materializer_sha256": package["files"][PACKAGE_FILES[0]], + "materializer_package": package, "closure_helper_sha256": digest_bytes(closure.read_bytes()), "jq_sha256": digest_bytes(jq_bin.read_bytes()), "source_repository_id": arguments.source_repository_id, @@ -285,10 +320,12 @@ def verify_candidate(candidate_root, candidate_tree, path, expected): def observation(path, kind, identity, field): if path is None: return None - value, source_sha = read_json(path, MAX_OBSERVATION_BYTES) + source = read_bytes(path, MAX_OBSERVATION_BYTES) + value = parse_json(source) + source_sha = digest_bytes(source) if not isinstance(value, dict) or value.get("schema_version") != 1 or value.get("kind") != kind: raise ReplayError("offline observation is malformed") - if not ACTOR.fullmatch(str(value.get("actor_id", ""))): + if not isinstance(value.get("actor_id"), str) or not ACTOR.fullmatch(value["actor_id"]): raise ReplayError("offline observation actor is invalid") if value.get("request_sha256") != identity["request_sha256"] or value.get("candidate_tree_id") != identity["candidate_tree_id"]: raise ReplayError("offline observation does not match this candidate") @@ -303,7 +340,8 @@ def validate_state(state, identity): saved = state.get("identity") if not isinstance(saved, dict) or any( not isinstance(saved.get(name), str) or not re.fullmatch(r"[0-9a-f]{64}", saved[name]) - for name in ("input_sha256", "request_sha256", "materializer_sha256", "closure_helper_sha256", "jq_sha256", "run_key") + for name in ("input_sha256", "request_sha256", "driver_sha256", "materializer_sha256", + "closure_helper_sha256", "jq_sha256", "run_key") ) or not isinstance(saved.get("source_repository_id"), str) or \ saved.get("source_hash_algorithm") not in {"sha1", "sha256"} or \ any(not isinstance(saved.get(name), str) or not OID.fullmatch(saved[name]) @@ -313,13 +351,27 @@ def validate_state(state, identity): not isinstance(saved["verifier"].get("path"), str) or \ not re.fullmatch(r"[0-9a-f]{64}", str(saved["verifier"].get("expected_sha256", ""))): raise ReplayError("state journal identity is malformed") - for name in ("candidate_commit_id", "candidate_tree_id"): - if name in saved and (not isinstance(saved[name], str) or not OID.fullmatch(saved[name])): - raise ReplayError("state journal candidate identity is malformed") + package = saved.get("materializer_package") + if not isinstance(package, dict) or not isinstance(package.get("generation_id"), str) or \ + not re.fullmatch(r"g-[0-9a-f]{64}", package["generation_id"]) or \ + not isinstance(package.get("files"), dict) or \ + set(package["files"]) != set(package_paths(package["generation_id"])) or any( + not isinstance(value, str) or not re.fullmatch(r"[0-9a-f]{64}", value) + for value in package["files"].values() + ) or not isinstance(package.get("sha256"), str) or \ + package["sha256"] != digest_bytes(canonical({ + "generation_id": package["generation_id"], "files": package["files"] + })) or saved["materializer_sha256"] != package["files"][PACKAGE_FILES[0]]: + raise ReplayError("state journal materializer package is malformed") phase = state.get("phase") if phase not in {"materializing", "verifying", "review-wait", "publish-wait", "completed-offline", "failed"}: raise ReplayError("state journal phase is malformed") needs_materialization = phase in {"verifying", "review-wait", "publish-wait", "completed-offline"} + for name in ("candidate_commit_id", "candidate_tree_id"): + if (needs_materialization and name not in saved) or ( + name in saved and (not isinstance(saved[name], str) or not OID.fullmatch(saved[name])) + ): + raise ReplayError("state journal candidate identity is malformed") materialization = state.get("materialization") if needs_materialization and (not isinstance(materialization, dict) or any( not isinstance(materialization.get(name), str) or not OID.fullmatch(materialization[name]) @@ -329,6 +381,11 @@ def validate_state(state, identity): for name in ("response_sha256", "receipt_sha256") )): raise ReplayError("state journal materialization is malformed") + if needs_materialization and any( + saved[name] != materialization[name] + for name in ("candidate_commit_id", "candidate_tree_id") + ): + raise ReplayError("state journal candidate identity does not match materialization") if phase in {"review-wait", "publish-wait", "completed-offline"}: verification = state.get("verification") if not isinstance(verification, dict) or not isinstance(verification.get("id"), str) or \ @@ -337,12 +394,14 @@ def validate_state(state, identity): raise ReplayError("state journal verification is malformed") if phase in {"publish-wait", "completed-offline"}: review = state.get("review") - if not isinstance(review, dict) or not ACTOR.fullmatch(str(review.get("actor_id", ""))) or \ + if not isinstance(review, dict) or not isinstance(review.get("actor_id"), str) or \ + not ACTOR.fullmatch(review["actor_id"]) or \ review.get("verdict") != "clean" or not re.fullmatch(r"[0-9a-f]{64}", str(review.get("sha256", ""))): raise ReplayError("state journal review is malformed") if phase == "completed-offline": publisher = state.get("publisher") - if not isinstance(publisher, dict) or not ACTOR.fullmatch(str(publisher.get("actor_id", ""))) or \ + if not isinstance(publisher, dict) or not isinstance(publisher.get("actor_id"), str) or \ + not ACTOR.fullmatch(publisher["actor_id"]) or \ publisher.get("disposition") != "offline-simulated" or \ not re.fullmatch(r"[0-9a-f]{64}", str(publisher.get("sha256", ""))): raise ReplayError("state journal publisher is malformed") @@ -367,7 +426,7 @@ def replay(arguments): input_bytes = read_bytes(arguments.input, MAX_INPUT_BYTES) input_value = parse_json(input_bytes) input_sha = digest_bytes(input_bytes) - identity = input_identity(input_value, input_sha, arguments, materializer) + identity = input_identity(input_value, input_sha, arguments, repository) interrupted = {"value": False} previous_term = signal.getsignal(signal.SIGTERM) previous_int = signal.getsignal(signal.SIGINT) @@ -379,7 +438,7 @@ def replay(arguments): fcntl.flock(lock, fcntl.LOCK_EX) state = None if state_path.exists(): - state, _ = read_json(state_path, MAX_OBSERVATION_BYTES) + state = parse_json(read_bytes(state_path, MAX_OBSERVATION_BYTES)) validate_state(state, identity) if state is not None and any(state["identity"].get(name) != value for name, value in identity.items()): result({"phase": "stale", "reason": "run identity changed"}) diff --git a/scripts/test/delivery-replay.test.sh b/scripts/test/delivery-replay.test.sh index fda4949..1838785 100755 --- a/scripts/test/delivery-replay.test.sh +++ b/scripts/test/delivery-replay.test.sh @@ -37,6 +37,9 @@ else fi /bin/chmod 0555 "$runtime/jq" jq_bin="$runtime/jq" +PATH="$runtime:/usr/bin:/bin" +export PATH +[ "$(command -v jq)" = "$runtime/jq" ] || fail 'private jq is not first on PATH' /usr/bin/cc -std=c11 -Wall -Wextra -Werror -O2 "$closure_source" -o "$runtime/object-closure" /bin/chmod 0555 "$runtime/object-closure" @@ -221,11 +224,24 @@ pass 'invalid UTF-8 input, review, and journal records fail without a traceback' printf '%s\n' '{"schema_version":1,"kind":"delivery_replay_review_observation","actor_id":"test.reviewer","request_sha256":"'"$request_sha"'","candidate_tree_id":"'"$candidate_tree"'","verdict":"clean"}' >"$tmp/review.json" printf '%s\n' '{"schema_version":1,"kind":"delivery_replay_publisher_observation","actor_id":"test.publisher","request_sha256":"'"$request_sha"'","candidate_tree_id":"'"$candidate_tree"'","disposition":"offline-simulated"}' >"$tmp/publisher.json" +printf '%s\n' '{"schema_version":1,"kind":"delivery_replay_review_observation","actor_id":123,"request_sha256":"'"$request_sha"'","candidate_tree_id":"'"$candidate_tree"'","verdict":"clean"}' >"$tmp/numeric-review.json" +if python3 "$replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ + --candidate-root "$tmp/changed-candidate" --scratch-root "$tmp/changed-scratch" --state-dir "$tmp/changed-state" \ + --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" --verify-path source.txt --expected-sha256 "$expected_changed" \ + --review-observation "$tmp/numeric-review.json" >"$tmp/numeric-review.out" 2>&1; then fail numeric-review-actor; fi +grep -Fq 'offline observation actor is invalid' "$tmp/numeric-review.out" || fail numeric-review-actor-error +pass 'offline observations require a string actor identity' python3 "$replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ --candidate-root "$tmp/changed-candidate" --scratch-root "$tmp/changed-scratch" --state-dir "$tmp/changed-state" \ --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" --verify-path source.txt --expected-sha256 "$expected_changed" \ --review-observation "$tmp/review.json" >"$tmp/publish-wait.out" jq -e '.state.phase=="publish-wait"' "$tmp/publish-wait.out" >/dev/null || fail missing-publisher-waits +printf '%s\n' '{"schema_version":1,"kind":"delivery_replay_publisher_observation","actor_id":123,"request_sha256":"'"$request_sha"'","candidate_tree_id":"'"$candidate_tree"'","disposition":"offline-simulated"}' >"$tmp/numeric-publisher.json" +if python3 "$replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ + --candidate-root "$tmp/changed-candidate" --scratch-root "$tmp/changed-scratch" --state-dir "$tmp/changed-state" \ + --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" --verify-path source.txt --expected-sha256 "$expected_changed" \ + --publisher-observation "$tmp/numeric-publisher.json" >"$tmp/numeric-publisher.out" 2>&1; then fail numeric-publisher-actor; fi +grep -Fq 'offline observation actor is invalid' "$tmp/numeric-publisher.out" || fail numeric-publisher-actor-error expect_malformed_state() { local name=$1 filter=$2 local state_root="$tmp/malformed-$name-state" @@ -248,6 +264,19 @@ expect_malformed_state missing-materialization '(.phase="verifying") | del(.mate expect_malformed_state missing-verification '(.phase="review-wait") | del(.verification)' expect_malformed_state missing-review '(.phase="publish-wait") | del(.review)' expect_malformed_state missing-publisher '(.phase="completed-offline") | del(.publisher)' +for candidate_phase in verifying review-wait publish-wait completed-offline; do + expect_malformed_state "missing-candidate-commit-$candidate_phase" \ + "(.phase=\"$candidate_phase\") | del(.identity.candidate_commit_id)" + expect_malformed_state "missing-candidate-tree-$candidate_phase" \ + "(.phase=\"$candidate_phase\") | del(.identity.candidate_tree_id)" +done +expect_malformed_state mismatched-candidate-commit \ + '(.phase="verifying") | .identity.candidate_commit_id="0000000000000000000000000000000000000000"' +expect_malformed_state mismatched-candidate-tree \ + '(.phase="completed-offline") | .identity.candidate_tree_id="0000000000000000000000000000000000000000"' +expect_malformed_state numeric-review-actor '.review.actor_id=123' +expect_malformed_state numeric-publisher-actor \ + '(.phase="completed-offline") | .publisher={"actor_id":123,"disposition":"offline-simulated","sha256":"0000000000000000000000000000000000000000000000000000000000000000"}' pass 'malformed state phases and nested records fail without a traceback' jq -S -c '.note="changed after review wait"' "$tmp/review.json" >"$tmp/changed-review.json" if python3 "$replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ @@ -368,4 +397,38 @@ if python3 "$replay" --input "$changed_input" --source-repository-id fixture.tar jq -e '.state.phase=="stale"' "$tmp/changed-input-stale.out" >/dev/null || fail changed-materialization-input-stale-state pass 'changed materialization input cannot reuse the prior run' +package_root="$tmp/replay-package" +generation=$(/usr/bin/sed -n \ + "s/^PORTABLE_CORE_GENERATION='\(g-[0-9a-f]\{64\}\)'$/\1/p" "$root/scripts/core-contract.sh") +/bin/mkdir -p "$package_root/delivery/v1" "$package_root/adapters/local-git-materializer/v1" \ + "$package_root/scripts" "$package_root/core/v2/generations" +/bin/cp "$replay" "$package_root/delivery/v1/replay.py" +/bin/cp "$root/adapters/local-git-materializer/v1/materialize.sh" \ + "$root/adapters/local-git-materializer/v1/protocol.jq" \ + "$package_root/adapters/local-git-materializer/v1/" +/bin/cp "$root/scripts/core-contract.sh" "$package_root/scripts/core-contract.sh" +/bin/cp "$root/core/v2/generation-registry.json" "$package_root/core/v2/generation-registry.json" +/bin/cp -R "$root/core/v2/generations/$generation" "$package_root/core/v2/generations/" +package_replay="$package_root/delivery/v1/replay.py" +/bin/mkdir -m 700 "$tmp/package-state" "$tmp/package-candidate" "$tmp/package-scratch" +python3 "$package_replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ + --candidate-root "$tmp/package-candidate" --scratch-root "$tmp/package-scratch" --state-dir "$tmp/package-state" \ + --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" --verify-path source.txt --expected-sha256 "$expected_changed" \ + >"$tmp/package-first.out" +printf '\n' >>"$package_root/adapters/local-git-materializer/v1/protocol.jq" +if python3 "$package_replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ + --candidate-root "$tmp/package-candidate" --scratch-root "$tmp/package-scratch" --state-dir "$tmp/package-state" \ + --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" --verify-path source.txt --expected-sha256 "$expected_changed" \ + >"$tmp/package-dependency-stale.out"; then fail changed-package-dependency; fi +jq -e '.state.phase=="stale"' "$tmp/package-dependency-stale.out" >/dev/null || fail changed-package-dependency-state +/bin/cp "$root/adapters/local-git-materializer/v1/protocol.jq" \ + "$package_root/adapters/local-git-materializer/v1/protocol.jq" +printf '\n' >>"$package_replay" +if python3 "$package_replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ + --candidate-root "$tmp/package-candidate" --scratch-root "$tmp/package-scratch" --state-dir "$tmp/package-state" \ + --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" --verify-path source.txt --expected-sha256 "$expected_changed" \ + >"$tmp/package-driver-stale.out"; then fail changed-replay-driver; fi +jq -e '.state.phase=="stale"' "$tmp/package-driver-stale.out" >/dev/null || fail changed-replay-driver-state +pass 'changed executable package or replay driver cannot reuse a prior run' + printf 'delivery replay: %s focused checks passed\n' "$passed" From a7de157f1dc7bb0a31b37306a86ddfe33811f97f Mon Sep 17 00:00:00 2001 From: ci Date: Sat, 5 Sep 2026 06:52:53 -0400 Subject: [PATCH 08/20] Fix replay execution identity validation --- delivery/v1/replay.py | 111 ++++++++++++++++---- scripts/test/delivery-replay.test.sh | 145 ++++++++++++++++++++++++++- 2 files changed, 229 insertions(+), 27 deletions(-) diff --git a/delivery/v1/replay.py b/delivery/v1/replay.py index 05791fe..0b11b63 100755 --- a/delivery/v1/replay.py +++ b/delivery/v1/replay.py @@ -71,7 +71,7 @@ def read_bytes(path, limit): def parse_json(data): try: return json.loads(data) - except (json.JSONDecodeError, UnicodeDecodeError) as error: + except (ValueError, UnicodeDecodeError) as error: raise ReplayError("input is not JSON") from error @@ -139,6 +139,40 @@ def package_paths(generation): return PACKAGE_FILES + tuple(f"{root}/{name}" for name in GENERATION_FILES) +def snapshot_file(source, destination, mode): + data = read_bytes(trusted_file(source), MAX_INPUT_BYTES) + destination.parent.mkdir(parents=True, exist_ok=True) + atomic_bytes(destination, data) + os.chmod(destination, mode) + return data + + +def create_execution_snapshot(repository, arguments, state_dir): + root = Path(tempfile.mkdtemp(prefix="execution-", dir=state_dir)) + try: + driver = root / "delivery/v1/replay.py" + snapshot_file(Path(__file__).resolve(), driver, 0o500) + core_relative = "scripts/core-contract.sh" + core = snapshot_file(repository / core_relative, root / core_relative, 0o500) + match = re.search( + rb"^PORTABLE_CORE_GENERATION='(g-[0-9a-f]{64})'$", core, re.MULTILINE + ) + if match is None: + raise ReplayError("materializer package generation is unavailable") + generation = match.group(1).decode() + for relative in package_paths(generation): + if relative == core_relative: + continue + mode = 0o500 if relative.endswith(".sh") else 0o400 + snapshot_file(repository / relative, root / relative, mode) + snapshot_file(arguments.closure_helper, root / ".dependencies/object-closure", 0o500) + snapshot_file(arguments.jq_bin, root / ".dependencies/jq", 0o500) + return root + except (OSError, ReplayError): + shutil.rmtree(root, ignore_errors=True) + raise + + def materializer_package_identity(repository): core_path = trusted_file(repository / "scripts/core-contract.sh") core_bytes = core_path.read_bytes() @@ -157,7 +191,7 @@ def materializer_package_identity(repository): return package -def input_identity(input_value, input_sha, arguments, repository): +def input_identity(input_value, input_sha, arguments, execution): try: request = input_value["stage_request"] request_sha = request["sha256"] @@ -179,9 +213,7 @@ def input_identity(input_value, input_sha, arguments, repository): expected = arguments.expected_sha256 if not re.fullmatch(r"[0-9a-f]{64}", expected): raise ReplayError("expected verifier digest is invalid") - closure = trusted_file(arguments.closure_helper) - jq_bin = trusted_file(arguments.jq_bin) - package = materializer_package_identity(repository) + package = materializer_package_identity(execution) identity = { "input_sha256": input_sha, "request_sha256": request_sha, @@ -193,25 +225,29 @@ def input_identity(input_value, input_sha, arguments, repository): "path": safe_path(arguments.verify_path), "expected_sha256": expected, }, - "driver_sha256": digest_bytes(trusted_file(Path(__file__).resolve()).read_bytes()), + "driver_sha256": digest_bytes(read_bytes(trusted_file(Path(__file__).resolve()), MAX_INPUT_BYTES)), "materializer_sha256": package["files"][PACKAGE_FILES[0]], "materializer_package": package, - "closure_helper_sha256": digest_bytes(closure.read_bytes()), - "jq_sha256": digest_bytes(jq_bin.read_bytes()), + "closure_helper_sha256": digest_bytes(read_bytes( + trusted_file(execution / ".dependencies/object-closure"), MAX_INPUT_BYTES + )), + "jq_sha256": digest_bytes(read_bytes( + trusted_file(execution / ".dependencies/jq"), MAX_INPUT_BYTES + )), "source_repository_id": arguments.source_repository_id, } identity["run_key"] = digest_bytes(canonical(identity)) return identity -def run_materializer(arguments, materializer, input_path, identity, candidate_root=None, scratch_root=None): +def run_materializer(arguments, execution, input_path, identity, candidate_root=None, scratch_root=None): candidate_root = Path(arguments.candidate_root).resolve() if candidate_root is None else candidate_root scratch_root = Path(arguments.scratch_root).resolve() if scratch_root is None else scratch_root command = [ - str(materializer), "materialize", str(input_path), + str(execution / PACKAGE_FILES[0]), "materialize", str(input_path), arguments.source_repository_id, str(Path(arguments.source_git_dir).resolve()), str(candidate_root), str(scratch_root), - str(Path(arguments.closure_helper).resolve()), str(Path(arguments.jq_bin).resolve()), + str(execution / ".dependencies/object-closure"), str(execution / ".dependencies/jq"), ] environment = {"PATH": "/usr/bin:/bin", "LC_ALL": "C"} result = subprocess.run(command, env=environment, stdout=subprocess.PIPE, @@ -274,14 +310,14 @@ def candidate_identity(candidate_root, source_commit): "candidate_parent_commit_id": parent} -def reconcile_materialization(arguments, materializer, input_path, identity, state_dir): +def reconcile_materialization(arguments, execution, input_path, identity, state_dir): existing = candidate_identity(arguments.candidate_root, identity["source_commit_id"]) if existing is None: return None recovery_candidate = Path(tempfile.mkdtemp(prefix="reconcile-candidate-", dir=state_dir)) recovery_scratch = Path(tempfile.mkdtemp(prefix="reconcile-scratch-", dir=state_dir)) try: - recomputed = run_materializer(arguments, materializer, input_path, identity, + recomputed = run_materializer(arguments, execution, input_path, identity, recovery_candidate, recovery_scratch) finally: shutil.rmtree(recovery_candidate, ignore_errors=True) @@ -388,9 +424,11 @@ def validate_state(state, identity): raise ReplayError("state journal candidate identity does not match materialization") if phase in {"review-wait", "publish-wait", "completed-offline"}: verification = state.get("verification") - if not isinstance(verification, dict) or not isinstance(verification.get("id"), str) or \ - not isinstance(verification.get("path"), str) or \ - not re.fullmatch(r"[0-9a-f]{64}", str(verification.get("sha256", ""))): + if verification != { + "id": saved["verifier"]["id"], + "path": saved["verifier"]["path"], + "sha256": saved["verifier"]["expected_sha256"], + }: raise ReplayError("state journal verification is malformed") if phase in {"publish-wait", "completed-offline"}: review = state.get("review") @@ -416,8 +454,9 @@ def result(state): def replay(arguments): - repository = Path(__file__).resolve().parents[2] - materializer = trusted_file(repository / "adapters/local-git-materializer/v1/materialize.sh") + execution = Path(arguments.execution_root).resolve() + if Path(__file__).resolve() != execution / "delivery/v1/replay.py": + raise ReplayError("execution snapshot is invalid") state_dir = private_directory(arguments.state_dir) disjoint(state_dir, arguments.source_git_dir, arguments.candidate_root, arguments.scratch_root) state_path = state_dir / "run.json" @@ -426,7 +465,7 @@ def replay(arguments): input_bytes = read_bytes(arguments.input, MAX_INPUT_BYTES) input_value = parse_json(input_bytes) input_sha = digest_bytes(input_bytes) - identity = input_identity(input_value, input_sha, arguments, repository) + identity = input_identity(input_value, input_sha, arguments, execution) interrupted = {"value": False} previous_term = signal.getsignal(signal.SIGTERM) previous_int = signal.getsignal(signal.SIGINT) @@ -471,10 +510,10 @@ def replay(arguments): return 0 if state["phase"] == "materializing": try: - reconciled = reconcile_materialization(arguments, materializer, input_snapshot_path, + reconciled = reconcile_materialization(arguments, execution, input_snapshot_path, identity, state_dir) state["materialization"] = reconciled or run_materializer( - arguments, materializer, input_snapshot_path, identity + arguments, execution, input_snapshot_path, identity ) except ReplayError as error: state.update({"phase": "failed", "recoverable": True, "reason": str(error)}) @@ -563,8 +602,36 @@ def main(): parser.add_argument("--expected-sha256", required=True) parser.add_argument("--review-observation") parser.add_argument("--publisher-observation") + parser.add_argument("--execution-root", help=argparse.SUPPRESS) + arguments = parser.parse_args() try: - return replay(parser.parse_args()) + if arguments.execution_root is None: + state_dir = private_directory(arguments.state_dir) + execution = create_execution_snapshot(Path(__file__).resolve().parents[2], arguments, state_dir) + child = None + pending_signal = {"value": None} + + def forward_signal(number, _frame): + if child is None: + pending_signal["value"] = number + else: + child.send_signal(number) + + previous_term = signal.signal(signal.SIGTERM, forward_signal) + previous_int = signal.signal(signal.SIGINT, forward_signal) + try: + child = subprocess.Popen([ + sys.executable, str(execution / "delivery/v1/replay.py"), + *sys.argv[1:], "--execution-root", str(execution), + ]) + if pending_signal["value"] is not None: + child.send_signal(pending_signal["value"]) + return child.wait() + finally: + signal.signal(signal.SIGTERM, previous_term) + signal.signal(signal.SIGINT, previous_int) + shutil.rmtree(execution, ignore_errors=True) + return replay(arguments) except (OSError, ReplayError) as error: print(f"delivery replay: {error}", file=sys.stderr) return 1 diff --git a/scripts/test/delivery-replay.test.sh b/scripts/test/delivery-replay.test.sh index 1838785..1b3f81f 100755 --- a/scripts/test/delivery-replay.test.sh +++ b/scripts/test/delivery-replay.test.sh @@ -9,6 +9,7 @@ root=$(CDPATH='' cd -P -- "${BASH_SOURCE[0]%/*}/../.." && pwd -P) replay="$root/delivery/v1/replay.py" fixture_builder="$root/scripts/test/local-git-materializer-fixtures.sh" closure_source="$root/adapters/local-git-materializer/v1/object-closure.c" +python_with_int_limit=$(command -v python3) tmp=$(/usr/bin/mktemp -d "${TMPDIR:-/tmp}/ystack-delivery-replay.XXXXXX") cleanup() { /bin/rm -rf -- "$tmp"; } trap cleanup EXIT @@ -146,19 +147,26 @@ pass 'replacement of the original input after snapshot cannot change materializa kill_wrapper="$tmp/kill-after-materialize.py" printf '%s\n' \ - 'import importlib.util, os, signal, sys' \ + 'import importlib.util, os, pathlib, signal, sys, types' \ 'path, arguments = sys.argv[1], sys.argv[2:]' \ 'spec = importlib.util.spec_from_file_location("replay", path)' \ 'module = importlib.util.module_from_spec(spec)' \ 'spec.loader.exec_module(module)' \ - 'original = module.run_materializer' \ + 'argument = lambda name: arguments[arguments.index(name) + 1]' \ + 'values = types.SimpleNamespace(closure_helper=argument("--closure-helper"), jq_bin=argument("--jq-bin"))' \ + 'snapshot = module.create_execution_snapshot(pathlib.Path(path).resolve().parents[2], values, pathlib.Path(argument("--state-dir")))' \ + 'snapshot_path = snapshot / "delivery/v1/replay.py"' \ + 'snapshot_spec = importlib.util.spec_from_file_location("snapshot_replay", snapshot_path)' \ + 'snapshot_module = importlib.util.module_from_spec(snapshot_spec)' \ + 'snapshot_spec.loader.exec_module(snapshot_module)' \ + 'original = snapshot_module.run_materializer' \ 'def stop_after_materialization(*args, **kwargs):' \ ' result = original(*args, **kwargs)' \ ' os.kill(os.getpid(), signal.SIGKILL)' \ ' return result' \ - 'module.run_materializer = stop_after_materialization' \ - 'sys.argv = [path] + arguments' \ - 'raise SystemExit(module.main())' >"$kill_wrapper" + 'snapshot_module.run_materializer = stop_after_materialization' \ + 'sys.argv = [str(snapshot_path), *arguments, "--execution-root", str(snapshot)]' \ + 'raise SystemExit(snapshot_module.main())' >"$kill_wrapper" mkdir -m 700 "$tmp/reconcile-state" "$tmp/reconcile-candidate" "$tmp/reconcile-scratch" if python3 "$kill_wrapper" "$replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ --candidate-root "$tmp/reconcile-candidate" --scratch-root "$tmp/reconcile-scratch" --state-dir "$tmp/reconcile-state" \ @@ -191,6 +199,76 @@ jq -e '.state.phase=="failed" and (.state.reason|contains("does not match frozen fail reconcile-mismatch-state pass 'a mismatched interrupted candidate is rejected without cleanup' +for tree_case in numeric list null; do + case "$tree_case" in + numeric) tree_value=123 ;; + list) tree_value='[]' ;; + null) tree_value=null ;; + esac + "$jq_bin" -S -c "(.stage_request.content.body.operation.arguments.source_tree_input_id) as \$id | + (.stage_request.content.body.inputs[] | select(.input_id == \$id) | + .value.value.value.object_id) = $tree_value" "$base_input" >"$tmp/$tree_case-tree.json" + mkdir -m 700 "$tmp/$tree_case-tree-state" "$tmp/$tree_case-tree-candidate" "$tmp/$tree_case-tree-scratch" + if python3 "$replay" --input "$tmp/$tree_case-tree.json" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ + --candidate-root "$tmp/$tree_case-tree-candidate" --scratch-root "$tmp/$tree_case-tree-scratch" \ + --state-dir "$tmp/$tree_case-tree-state" --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" \ + --verify-path source.txt --expected-sha256 "$expected_changed" >"$tmp/$tree_case-tree.out" 2>&1; then + fail "$tree_case-source-tree" + fi + if ! grep -Fq 'delivery replay: materialization input tree identity is invalid' "$tmp/$tree_case-tree.out" || + grep -Fq Traceback "$tmp/$tree_case-tree.out"; then + fail "$tree_case-source-tree-error" + fi +done +pass 'non-string source tree identities fail without a traceback' + +mkdir -m 700 "$tmp/caller-execution-root" "$tmp/caller-execution-state" \ + "$tmp/caller-execution-candidate" "$tmp/caller-execution-scratch" +printf '%s\n' keep >"$tmp/caller-execution-root/sentinel" +if python3 "$replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ + --candidate-root "$tmp/caller-execution-candidate" --scratch-root "$tmp/caller-execution-scratch" \ + --state-dir "$tmp/caller-execution-state" --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" \ + --verify-path source.txt --expected-sha256 "$expected_changed" \ + --execution-root "$tmp/caller-execution-root" >"$tmp/caller-execution.out" 2>&1; then + fail caller-execution-root +fi +[ "$(cat "$tmp/caller-execution-root/sentinel")" = keep ] || fail caller-execution-root-deleted +grep -Fq 'delivery replay: execution snapshot is invalid' "$tmp/caller-execution.out" || + fail caller-execution-root-error +pass 'a caller-supplied execution root is rejected without deleting it' + +huge_integer=$(printf '1%.0s' {1..5000}) +printf '{"huge":%s}\n' "$huge_integer" >"$tmp/huge-input.json" +mkdir -m 700 "$tmp/huge-input-state" "$tmp/huge-input-candidate" "$tmp/huge-input-scratch" +if PYTHONINTMAXSTRDIGITS=4300 "$python_with_int_limit" "$replay" --input "$tmp/huge-input.json" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ + --candidate-root "$tmp/huge-input-candidate" --scratch-root "$tmp/huge-input-scratch" --state-dir "$tmp/huge-input-state" \ + --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" --verify-path source.txt --expected-sha256 "$expected_changed" \ + >"$tmp/huge-input.out" 2>&1; then fail huge-integer-input; fi +if ! grep -Fq 'delivery replay: input is not JSON' "$tmp/huge-input.out" || + grep -Fq Traceback "$tmp/huge-input.out"; then + fail huge-integer-input-error +fi +printf '{"huge":%s}\n' "$huge_integer" >"$tmp/huge-observation.json" +if PYTHONINTMAXSTRDIGITS=4300 "$python_with_int_limit" "$replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ + --candidate-root "$tmp/reconcile-candidate" --scratch-root "$tmp/reconcile-scratch" --state-dir "$tmp/reconcile-state" \ + --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" --verify-path source.txt --expected-sha256 "$expected_changed" \ + --review-observation "$tmp/huge-observation.json" >"$tmp/huge-observation.out" 2>&1; then fail huge-integer-observation; fi +if ! grep -Fq 'delivery replay: input is not JSON' "$tmp/huge-observation.out" || + grep -Fq Traceback "$tmp/huge-observation.out"; then + fail huge-integer-observation-error +fi +mkdir -m 700 "$tmp/huge-journal-state" "$tmp/huge-journal-candidate" "$tmp/huge-journal-scratch" +printf '{"huge":%s}\n' "$huge_integer" >"$tmp/huge-journal-state/run.json" +if PYTHONINTMAXSTRDIGITS=4300 "$python_with_int_limit" "$replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ + --candidate-root "$tmp/huge-journal-candidate" --scratch-root "$tmp/huge-journal-scratch" --state-dir "$tmp/huge-journal-state" \ + --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" --verify-path source.txt --expected-sha256 "$expected_changed" \ + >"$tmp/huge-journal.out" 2>&1; then fail huge-integer-journal; fi +if ! grep -Fq 'delivery replay: input is not JSON' "$tmp/huge-journal.out" || + grep -Fq Traceback "$tmp/huge-journal.out"; then + fail huge-integer-journal-error +fi +pass 'huge JSON integers in input, observation, and journal fail without a traceback' + printf '\377' >"$tmp/invalid-input.json" mkdir -m 700 "$tmp/invalid-input-state" "$tmp/invalid-input-candidate" "$tmp/invalid-input-scratch" if python3 "$replay" --input "$tmp/invalid-input.json" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ @@ -277,6 +355,14 @@ expect_malformed_state mismatched-candidate-tree \ expect_malformed_state numeric-review-actor '.review.actor_id=123' expect_malformed_state numeric-publisher-actor \ '(.phase="completed-offline") | .publisher={"actor_id":123,"disposition":"offline-simulated","sha256":"0000000000000000000000000000000000000000000000000000000000000000"}' +for verification_phase in review-wait publish-wait completed-offline; do + expect_malformed_state "verification-id-$verification_phase" \ + "(.phase=\"$verification_phase\") | .verification.id=\"delivery.other.v1\"" + expect_malformed_state "verification-path-$verification_phase" \ + "(.phase=\"$verification_phase\") | .verification.path=\"other.txt\"" + expect_malformed_state "verification-sha-$verification_phase" \ + "(.phase=\"$verification_phase\") | .verification.sha256=\"0000000000000000000000000000000000000000000000000000000000000000\"" +done pass 'malformed state phases and nested records fail without a traceback' jq -S -c '.note="changed after review wait"' "$tmp/review.json" >"$tmp/changed-review.json" if python3 "$replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ @@ -431,4 +517,53 @@ if python3 "$package_replay" --input "$base_input" --source-repository-id fixtur jq -e '.state.phase=="stale"' "$tmp/package-driver-stale.out" >/dev/null || fail changed-replay-driver-state pass 'changed executable package or replay driver cannot reuse a prior run' +/bin/cp "$replay" "$package_replay" +/bin/cp "$root/adapters/local-git-materializer/v1/protocol.jq" \ + "$package_root/adapters/local-git-materializer/v1/protocol.jq" +/bin/cp "$runtime/object-closure" "$tmp/race-object-closure" +/bin/cp "$jq_bin" "$tmp/race-jq" +/bin/chmod 0555 "$tmp/race-object-closure" "$tmp/race-jq" +race_wrapper="$tmp/snapshot-race.py" +printf '%s\n' \ + 'import argparse, importlib.util, os, pathlib, stat, subprocess, sys' \ + 'driver, repository, state_dir, helper, jq_bin, *arguments = sys.argv[1:]' \ + 'spec = importlib.util.spec_from_file_location("replay", driver)' \ + 'module = importlib.util.module_from_spec(spec)' \ + 'spec.loader.exec_module(module)' \ + 'values = argparse.Namespace(closure_helper=helper, jq_bin=jq_bin)' \ + 'snapshot = module.create_execution_snapshot(pathlib.Path(repository), values, pathlib.Path(state_dir))' \ + 'targets = [pathlib.Path(driver), pathlib.Path(repository) / "adapters/local-git-materializer/v1/protocol.jq", pathlib.Path(helper), pathlib.Path(jq_bin)]' \ + 'saved = [target.read_bytes() for target in targets]' \ + 'modes = [stat.S_IMODE(target.stat().st_mode) for target in targets]' \ + 'try:' \ + ' for target in targets:' \ + ' target.chmod(0o700)' \ + ' target.write_bytes(b"replaced after execution snapshot\n")' \ + ' result = subprocess.run([sys.executable, str(snapshot / "delivery/v1/replay.py"), *arguments, "--execution-root", str(snapshot)], stdout=subprocess.PIPE, stderr=subprocess.PIPE)' \ + 'finally:' \ + ' for target, data, mode in zip(targets, saved, modes):' \ + ' target.write_bytes(data)' \ + ' target.chmod(mode)' \ + 'sys.stdout.buffer.write(result.stdout)' \ + 'sys.stderr.buffer.write(result.stderr)' \ + 'raise SystemExit(result.returncode)' >"$race_wrapper" +/bin/mkdir -m 700 "$tmp/race-state" "$tmp/race-candidate" "$tmp/race-scratch" +driver_sha=$(sha_file "$package_replay") +package_sha=$(sha_file "$package_root/adapters/local-git-materializer/v1/protocol.jq") +helper_sha=$(sha_file "$tmp/race-object-closure") +jq_sha=$(sha_file "$tmp/race-jq") +python3 "$race_wrapper" "$package_replay" "$package_root" "$tmp/race-state" \ + "$tmp/race-object-closure" "$tmp/race-jq" \ + --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ + --candidate-root "$tmp/race-candidate" --scratch-root "$tmp/race-scratch" --state-dir "$tmp/race-state" \ + --closure-helper "$tmp/race-object-closure" --jq-bin "$tmp/race-jq" \ + --verify-path source.txt --expected-sha256 "$expected_changed" >"$tmp/race.out" +jq -e '.state.phase=="review-wait" and + .state.identity.driver_sha256==$driver and + .state.identity.materializer_package.files["adapters/local-git-materializer/v1/protocol.jq"]==$package and + .state.identity.closure_helper_sha256==$helper and .state.identity.jq_sha256==$jq' \ + --arg driver "$driver_sha" --arg package "$package_sha" --arg helper "$helper_sha" --arg jq "$jq_sha" \ + "$tmp/race.out" >/dev/null || fail immutable-execution-snapshot +pass 'replacement after the private execution snapshot cannot change executed or recorded bytes' + printf 'delivery replay: %s focused checks passed\n' "$passed" From 6d53d29a7b78b6647a570faaa4941b2e5fee8923 Mon Sep 17 00:00:00 2001 From: ci Date: Sat, 5 Sep 2026 08:13:01 -0400 Subject: [PATCH 09/20] Harden replay wait invariants --- delivery/v1/replay.py | 132 +++++++++++--------- scripts/test/delivery-replay.test.sh | 177 +++++++++++++++++++++++---- 2 files changed, 233 insertions(+), 76 deletions(-) diff --git a/delivery/v1/replay.py b/delivery/v1/replay.py index 0b11b63..4e50663 100755 --- a/delivery/v1/replay.py +++ b/delivery/v1/replay.py @@ -15,6 +15,7 @@ import tempfile +LOADED_DRIVER_CODE = sys._getframe().f_code MAX_INPUT_BYTES = 8 * 1024 * 1024 MAX_OBSERVATION_BYTES = 64 * 1024 MAX_VERIFIED_BLOB_BYTES = 1024 * 1024 @@ -150,8 +151,6 @@ def snapshot_file(source, destination, mode): def create_execution_snapshot(repository, arguments, state_dir): root = Path(tempfile.mkdtemp(prefix="execution-", dir=state_dir)) try: - driver = root / "delivery/v1/replay.py" - snapshot_file(Path(__file__).resolve(), driver, 0o500) core_relative = "scripts/core-contract.sh" core = snapshot_file(repository / core_relative, root / core_relative, 0o500) match = re.search( @@ -173,6 +172,17 @@ def create_execution_snapshot(repository, arguments, state_dir): raise +def driver_identity(): + source = read_bytes(trusted_file(Path(__file__).resolve()), MAX_INPUT_BYTES) + try: + current = compile(source, LOADED_DRIVER_CODE.co_filename, "exec") + except (SyntaxError, TypeError, ValueError) as error: + raise ReplayError("loaded replay driver identity is unavailable") from error + if current != LOADED_DRIVER_CODE: + raise ReplayError("loaded replay driver changed during startup") + return digest_bytes(source) + + def materializer_package_identity(repository): core_path = trusted_file(repository / "scripts/core-contract.sh") core_bytes = core_path.read_bytes() @@ -225,7 +235,7 @@ def input_identity(input_value, input_sha, arguments, execution): "path": safe_path(arguments.verify_path), "expected_sha256": expected, }, - "driver_sha256": digest_bytes(read_bytes(trusted_file(Path(__file__).resolve()), MAX_INPUT_BYTES)), + "driver_sha256": driver_identity(), "materializer_sha256": package["files"][PACKAGE_FILES[0]], "materializer_package": package, "closure_helper_sha256": digest_bytes(read_bytes( @@ -353,6 +363,18 @@ def verify_candidate(candidate_root, candidate_tree, path, expected): return actual +def revalidate_candidate(arguments, state): + expected = { + name: state["materialization"][name] + for name in ("candidate_commit_id", "candidate_tree_id", "candidate_parent_commit_id") + } + if candidate_identity(arguments.candidate_root, state["identity"]["source_commit_id"]) != expected: + raise ReplayError("candidate repository no longer matches saved materialization") + verify_candidate(arguments.candidate_root, expected["candidate_tree_id"], + state["identity"]["verifier"]["path"], + state["identity"]["verifier"]["expected_sha256"]) + + def observation(path, kind, identity, field): if path is None: return None @@ -453,12 +475,15 @@ def result(state): "state": state}, sort_keys=True, separators=(",", ":"))) -def replay(arguments): - execution = Path(arguments.execution_root).resolve() - if Path(__file__).resolve() != execution / "delivery/v1/replay.py": - raise ReplayError("execution snapshot is invalid") - state_dir = private_directory(arguments.state_dir) - disjoint(state_dir, arguments.source_git_dir, arguments.candidate_root, arguments.scratch_root) +def stop_if_interrupted(state, interrupted): + if not interrupted["value"]: + return False + if state is not None: + result(state) + return True + + +def replay_locked(arguments, execution, state_dir): state_path = state_dir / "run.json" input_snapshot_path = state_dir / "materialization-input.json" lock_path = state_dir / "replay.lock" @@ -482,6 +507,8 @@ def replay(arguments): if state is not None and any(state["identity"].get(name) != value for name, value in identity.items()): result({"phase": "stale", "reason": "run identity changed"}) return 2 + if stop_if_interrupted(state, interrupted): + return 75 if state is None: state = {"schema_version": 1, "kind": "delivery_replay_state", "identity": identity, "phase": "materializing", "authority": "none", "qualification": "unavailable"} @@ -498,14 +525,19 @@ def replay(arguments): result(state) return 1 if state["phase"] == "completed-offline": - verify_candidate(arguments.candidate_root, state["identity"]["candidate_tree_id"], - identity["verifier"]["path"], identity["verifier"]["expected_sha256"]) + revalidate_candidate(arguments, state) + if stop_if_interrupted(state, interrupted): + return 75 for supplied, kind, field, recorded in ( (arguments.review_observation, "delivery_replay_review_observation", "verdict", state.get("review")), (arguments.publisher_observation, "delivery_replay_publisher_observation", "disposition", state.get("publisher")), ): - if supplied is not None and observation(supplied, kind, state["identity"], field) != recorded: - raise ReplayError("supplied offline observation changed after completion") + if supplied is not None: + supplied_observation = observation(supplied, kind, state["identity"], field) + if stop_if_interrupted(state, interrupted): + return 75 + if supplied_observation != recorded: + raise ReplayError("supplied offline observation changed after completion") result(state) return 0 if state["phase"] == "materializing": @@ -526,8 +558,7 @@ def replay(arguments): }) state["phase"] = "verifying" atomic_json(state_path, state) - if interrupted["value"]: - result(state) + if stop_if_interrupted(state, interrupted): return 75 if state["phase"] == "verifying": try: @@ -541,13 +572,15 @@ def replay(arguments): return 1 state["phase"] = "review-wait" atomic_json(state_path, state) - if interrupted["value"]: - result(state) + if stop_if_interrupted(state, interrupted): return 75 if state["phase"] == "review-wait": - verify_candidate(arguments.candidate_root, state["identity"]["candidate_tree_id"], - identity["verifier"]["path"], identity["verifier"]["expected_sha256"]) + revalidate_candidate(arguments, state) + if stop_if_interrupted(state, interrupted): + return 75 review = observation(arguments.review_observation, "delivery_replay_review_observation", state["identity"], "verdict") + if stop_if_interrupted(state, interrupted): + return 75 if review is None: result(state) return 0 @@ -560,14 +593,20 @@ def replay(arguments): state["phase"] = "publish-wait" atomic_json(state_path, state) if state["phase"] == "publish-wait": - verify_candidate(arguments.candidate_root, state["identity"]["candidate_tree_id"], - identity["verifier"]["path"], identity["verifier"]["expected_sha256"]) - if arguments.review_observation is not None and ( - observation(arguments.review_observation, "delivery_replay_review_observation", - state["identity"], "verdict") != state.get("review") - ): - raise ReplayError("supplied offline review changed after review wait") + revalidate_candidate(arguments, state) + if stop_if_interrupted(state, interrupted): + return 75 + if arguments.review_observation is not None: + supplied_review = observation(arguments.review_observation, + "delivery_replay_review_observation", + state["identity"], "verdict") + if stop_if_interrupted(state, interrupted): + return 75 + if supplied_review != state.get("review"): + raise ReplayError("supplied offline review changed after review wait") publisher = observation(arguments.publisher_observation, "delivery_replay_publisher_observation", state["identity"], "disposition") + if stop_if_interrupted(state, interrupted): + return 75 if publisher is None: result(state) return 0 @@ -588,6 +627,17 @@ def replay(arguments): signal.signal(signal.SIGINT, previous_int) +def replay(arguments): + repository = Path(__file__).resolve().parents[2] + state_dir = private_directory(arguments.state_dir) + disjoint(state_dir, arguments.source_git_dir, arguments.candidate_root, arguments.scratch_root) + execution = create_execution_snapshot(repository, arguments, state_dir) + try: + return replay_locked(arguments, execution, state_dir) + finally: + shutil.rmtree(execution, ignore_errors=True) + + def main(): parser = argparse.ArgumentParser() parser.add_argument("--input", required=True) @@ -602,36 +652,8 @@ def main(): parser.add_argument("--expected-sha256", required=True) parser.add_argument("--review-observation") parser.add_argument("--publisher-observation") - parser.add_argument("--execution-root", help=argparse.SUPPRESS) - arguments = parser.parse_args() try: - if arguments.execution_root is None: - state_dir = private_directory(arguments.state_dir) - execution = create_execution_snapshot(Path(__file__).resolve().parents[2], arguments, state_dir) - child = None - pending_signal = {"value": None} - - def forward_signal(number, _frame): - if child is None: - pending_signal["value"] = number - else: - child.send_signal(number) - - previous_term = signal.signal(signal.SIGTERM, forward_signal) - previous_int = signal.signal(signal.SIGINT, forward_signal) - try: - child = subprocess.Popen([ - sys.executable, str(execution / "delivery/v1/replay.py"), - *sys.argv[1:], "--execution-root", str(execution), - ]) - if pending_signal["value"] is not None: - child.send_signal(pending_signal["value"]) - return child.wait() - finally: - signal.signal(signal.SIGTERM, previous_term) - signal.signal(signal.SIGINT, previous_int) - shutil.rmtree(execution, ignore_errors=True) - return replay(arguments) + return replay(parser.parse_args()) except (OSError, ReplayError) as error: print(f"delivery replay: {error}", file=sys.stderr) return 1 diff --git a/scripts/test/delivery-replay.test.sh b/scripts/test/delivery-replay.test.sh index 1b3f81f..8203504 100755 --- a/scripts/test/delivery-replay.test.sh +++ b/scripts/test/delivery-replay.test.sh @@ -114,6 +114,26 @@ jq -e '.state.phase=="review-wait" and .authority=="none" and .offline_simulatio fail missing-review-waits request_sha=$(jq -r '.identity.request_sha256' "$tmp/changed-state/run.json") candidate_tree=$(jq -r '.identity.candidate_tree_id' "$tmp/changed-state/run.json") +candidate_commit=$(jq -r '.identity.candidate_commit_id' "$tmp/changed-state/run.json") +moved_candidate=$(printf '%s\n' moved | /usr/bin/env -i HOME="$tmp/home" PATH=/usr/bin:/bin LC_ALL=C \ + GIT_AUTHOR_NAME=fixture GIT_AUTHOR_EMAIL=fixture@example.invalid GIT_COMMITTER_NAME=fixture \ + GIT_COMMITTER_EMAIL=fixture@example.invalid /usr/bin/git --git-dir="$tmp/changed-candidate/repository.git" \ + commit-tree "$candidate_tree" -p "$candidate_commit") +expect_candidate_move_rejected() { + local phase=$1 + shift + /usr/bin/git --git-dir="$tmp/changed-candidate/repository.git" update-ref refs/heads/candidate "$moved_candidate" + if python3 "$replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ + --candidate-root "$tmp/changed-candidate" --scratch-root "$tmp/changed-scratch" --state-dir "$tmp/changed-state" \ + --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" --verify-path source.txt \ + --expected-sha256 "$expected_changed" "$@" >"$tmp/candidate-moved-$phase.out" 2>&1; then + /usr/bin/git --git-dir="$tmp/changed-candidate/repository.git" update-ref refs/heads/candidate "$candidate_commit" + fail "candidate-moved-$phase" + fi + /usr/bin/git --git-dir="$tmp/changed-candidate/repository.git" update-ref refs/heads/candidate "$candidate_commit" + grep -Fq 'candidate repository no longer matches saved materialization' "$tmp/candidate-moved-$phase.out" || + fail "candidate-moved-$phase-error" +} pass 'changed materialization and fixed read-only verification wait for review' cp "$base_input" "$tmp/mutable-input.json" @@ -147,26 +167,19 @@ pass 'replacement of the original input after snapshot cannot change materializa kill_wrapper="$tmp/kill-after-materialize.py" printf '%s\n' \ - 'import importlib.util, os, pathlib, signal, sys, types' \ + 'import importlib.util, os, signal, sys' \ 'path, arguments = sys.argv[1], sys.argv[2:]' \ 'spec = importlib.util.spec_from_file_location("replay", path)' \ 'module = importlib.util.module_from_spec(spec)' \ 'spec.loader.exec_module(module)' \ - 'argument = lambda name: arguments[arguments.index(name) + 1]' \ - 'values = types.SimpleNamespace(closure_helper=argument("--closure-helper"), jq_bin=argument("--jq-bin"))' \ - 'snapshot = module.create_execution_snapshot(pathlib.Path(path).resolve().parents[2], values, pathlib.Path(argument("--state-dir")))' \ - 'snapshot_path = snapshot / "delivery/v1/replay.py"' \ - 'snapshot_spec = importlib.util.spec_from_file_location("snapshot_replay", snapshot_path)' \ - 'snapshot_module = importlib.util.module_from_spec(snapshot_spec)' \ - 'snapshot_spec.loader.exec_module(snapshot_module)' \ - 'original = snapshot_module.run_materializer' \ + 'original = module.run_materializer' \ 'def stop_after_materialization(*args, **kwargs):' \ ' result = original(*args, **kwargs)' \ ' os.kill(os.getpid(), signal.SIGKILL)' \ ' return result' \ - 'snapshot_module.run_materializer = stop_after_materialization' \ - 'sys.argv = [str(snapshot_path), *arguments, "--execution-root", str(snapshot)]' \ - 'raise SystemExit(snapshot_module.main())' >"$kill_wrapper" + 'module.run_materializer = stop_after_materialization' \ + 'sys.argv = [path] + arguments' \ + 'raise SystemExit(module.main())' >"$kill_wrapper" mkdir -m 700 "$tmp/reconcile-state" "$tmp/reconcile-candidate" "$tmp/reconcile-scratch" if python3 "$kill_wrapper" "$replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ --candidate-root "$tmp/reconcile-candidate" --scratch-root "$tmp/reconcile-scratch" --state-dir "$tmp/reconcile-state" \ @@ -233,9 +246,9 @@ if python3 "$replay" --input "$base_input" --source-repository-id fixture.target fail caller-execution-root fi [ "$(cat "$tmp/caller-execution-root/sentinel")" = keep ] || fail caller-execution-root-deleted -grep -Fq 'delivery replay: execution snapshot is invalid' "$tmp/caller-execution.out" || +grep -Fq 'unrecognized arguments: --execution-root' "$tmp/caller-execution.out" || fail caller-execution-root-error -pass 'a caller-supplied execution root is rejected without deleting it' +pass 'the replay CLI has no caller-selected execution-root path' huge_integer=$(printf '1%.0s' {1..5000}) printf '{"huge":%s}\n' "$huge_integer" >"$tmp/huge-input.json" @@ -302,6 +315,78 @@ pass 'invalid UTF-8 input, review, and journal records fail without a traceback' printf '%s\n' '{"schema_version":1,"kind":"delivery_replay_review_observation","actor_id":"test.reviewer","request_sha256":"'"$request_sha"'","candidate_tree_id":"'"$candidate_tree"'","verdict":"clean"}' >"$tmp/review.json" printf '%s\n' '{"schema_version":1,"kind":"delivery_replay_publisher_observation","actor_id":"test.publisher","request_sha256":"'"$request_sha"'","candidate_tree_id":"'"$candidate_tree"'","disposition":"offline-simulated"}' >"$tmp/publisher.json" +expect_candidate_move_rejected review-wait --review-observation "$tmp/review.json" +lock_holder="$tmp/lock-holder.py" +printf '%s\n' \ + 'import fcntl, pathlib, sys, time' \ + 'lock_path, ready, release = map(pathlib.Path, sys.argv[1:])' \ + 'with lock_path.open("a+b") as lock:' \ + ' fcntl.flock(lock, fcntl.LOCK_EX)' \ + ' ready.write_text("ready")' \ + ' while not release.exists(): time.sleep(0.01)' >"$lock_holder" +lock_wrapper="$tmp/lock-replay.py" +printf '%s\n' \ + 'import importlib.util, pathlib, sys' \ + 'path, marker, arguments = sys.argv[1], pathlib.Path(sys.argv[2]), sys.argv[3:]' \ + 'spec = importlib.util.spec_from_file_location("replay", path)' \ + 'module = importlib.util.module_from_spec(spec)' \ + 'spec.loader.exec_module(module)' \ + 'original = module.fcntl.flock' \ + 'def marked_flock(*args, **kwargs):' \ + ' marker.write_text("waiting")' \ + ' return original(*args, **kwargs)' \ + 'module.fcntl.flock = marked_flock' \ + 'sys.argv = [path] + arguments' \ + 'raise SystemExit(module.main())' >"$lock_wrapper" +mkdir -m 700 "$tmp/lock-state" +cp "$tmp/changed-state/materialization-input.json" "$tmp/changed-state/run.json" "$tmp/lock-state/" +python3 "$lock_holder" "$tmp/lock-state/replay.lock" "$tmp/lock-ready" "$tmp/lock-release" & +lock_holder_pid=$! +while [ ! -f "$tmp/lock-ready" ]; do sleep 0.01; done +python3 "$lock_wrapper" "$replay" "$tmp/lock-waiting" \ + --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ + --candidate-root "$tmp/changed-candidate" --scratch-root "$tmp/changed-scratch" --state-dir "$tmp/lock-state" \ + --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" --verify-path source.txt \ + --expected-sha256 "$expected_changed" --review-observation "$tmp/review.json" >"$tmp/lock-cancel.out" & +lock_replay_pid=$! +while [ ! -f "$tmp/lock-waiting" ]; do sleep 0.01; done +kill -TERM "$lock_replay_pid" +touch "$tmp/lock-release" +wait "$lock_holder_pid" +if wait "$lock_replay_pid"; then fail lock-cancel-status; else lock_status=$?; fi +[ "$lock_status" -eq 75 ] || fail lock-cancel-code +jq -e '(.phase=="review-wait") and (has("review")|not)' "$tmp/lock-state/run.json" >/dev/null || + fail lock-cancel-state + +observation_wrapper="$tmp/observation-interrupt.py" +printf '%s\n' \ + 'import importlib.util, os, signal, sys' \ + 'path, signal_name, arguments = sys.argv[1], sys.argv[2], sys.argv[3:]' \ + 'spec = importlib.util.spec_from_file_location("replay", path)' \ + 'module = importlib.util.module_from_spec(spec)' \ + 'spec.loader.exec_module(module)' \ + 'original = module.observation' \ + 'def interrupt_after_observation(*args, **kwargs):' \ + ' result = original(*args, **kwargs)' \ + ' os.kill(os.getpid(), getattr(signal, signal_name))' \ + ' return result' \ + 'module.observation = interrupt_after_observation' \ + 'sys.argv = [path] + arguments' \ + 'raise SystemExit(module.main())' >"$observation_wrapper" +mkdir -m 700 "$tmp/review-cancel-state" +cp "$tmp/changed-state/materialization-input.json" "$tmp/changed-state/run.json" "$tmp/review-cancel-state/" +if python3 "$observation_wrapper" "$replay" SIGINT \ + --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ + --candidate-root "$tmp/changed-candidate" --scratch-root "$tmp/changed-scratch" --state-dir "$tmp/review-cancel-state" \ + --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" --verify-path source.txt \ + --expected-sha256 "$expected_changed" --review-observation "$tmp/review.json" >"$tmp/review-cancel.out"; then + fail review-cancel-status +else + review_cancel_status=$? +fi +[ "$review_cancel_status" -eq 75 ] || fail review-cancel-code +jq -e '(.phase=="review-wait") and (has("review")|not)' "$tmp/review-cancel-state/run.json" >/dev/null || + fail review-cancel-state printf '%s\n' '{"schema_version":1,"kind":"delivery_replay_review_observation","actor_id":123,"request_sha256":"'"$request_sha"'","candidate_tree_id":"'"$candidate_tree"'","verdict":"clean"}' >"$tmp/numeric-review.json" if python3 "$replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ --candidate-root "$tmp/changed-candidate" --scratch-root "$tmp/changed-scratch" --state-dir "$tmp/changed-state" \ @@ -314,6 +399,22 @@ python3 "$replay" --input "$base_input" --source-repository-id fixture.target -- --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" --verify-path source.txt --expected-sha256 "$expected_changed" \ --review-observation "$tmp/review.json" >"$tmp/publish-wait.out" jq -e '.state.phase=="publish-wait"' "$tmp/publish-wait.out" >/dev/null || fail missing-publisher-waits +expect_candidate_move_rejected publish-wait --publisher-observation "$tmp/publisher.json" +mkdir -m 700 "$tmp/publish-cancel-state" +cp "$tmp/changed-state/materialization-input.json" "$tmp/changed-state/run.json" "$tmp/publish-cancel-state/" +if python3 "$observation_wrapper" "$replay" SIGTERM \ + --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ + --candidate-root "$tmp/changed-candidate" --scratch-root "$tmp/changed-scratch" --state-dir "$tmp/publish-cancel-state" \ + --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" --verify-path source.txt \ + --expected-sha256 "$expected_changed" --publisher-observation "$tmp/publisher.json" >"$tmp/publish-cancel.out"; then + fail publish-cancel-status +else + publish_cancel_status=$? +fi +[ "$publish_cancel_status" -eq 75 ] || fail publish-cancel-code +jq -e '(.phase=="publish-wait") and (has("publisher")|not)' "$tmp/publish-cancel-state/run.json" >/dev/null || + fail publish-cancel-state +pass 'SIGTERM at the lock and SIGINT or SIGTERM after wait observations do not advance state' printf '%s\n' '{"schema_version":1,"kind":"delivery_replay_publisher_observation","actor_id":123,"request_sha256":"'"$request_sha"'","candidate_tree_id":"'"$candidate_tree"'","disposition":"offline-simulated"}' >"$tmp/numeric-publisher.json" if python3 "$replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ --candidate-root "$tmp/changed-candidate" --scratch-root "$tmp/changed-scratch" --state-dir "$tmp/changed-state" \ @@ -377,6 +478,8 @@ python3 "$replay" --input "$base_input" --source-repository-id fixture.target -- --review-observation "$tmp/review.json" --publisher-observation "$tmp/publisher.json" >"$tmp/completed.out" jq -e '.state.phase=="completed-offline" and .state.publisher.disposition=="offline-simulated"' "$tmp/completed.out" >/dev/null || fail completed-offline +expect_candidate_move_rejected completed-offline +pass 'review, publish, and completed waits reject a moved same-tree candidate ref' cp "$tmp/completed.out" "$tmp/completed-first.out" python3 "$replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ --candidate-root "$tmp/changed-candidate" --scratch-root "$tmp/changed-scratch" --state-dir "$tmp/changed-state" \ @@ -520,33 +623,65 @@ pass 'changed executable package or replay driver cannot reuse a prior run' /bin/cp "$replay" "$package_replay" /bin/cp "$root/adapters/local-git-materializer/v1/protocol.jq" \ "$package_root/adapters/local-git-materializer/v1/protocol.jq" +printf '%s\n' keep >"$package_root/sentinel" +if python3 "$package_replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ + --candidate-root "$tmp/package-candidate" --scratch-root "$tmp/package-scratch" --state-dir "$tmp/package-state" \ + --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" --verify-path source.txt \ + --expected-sha256 "$expected_changed" --execution-root "$package_root" >"$tmp/copied-root-bypass.out" 2>&1; then + fail copied-root-bypass +fi +grep -Fq 'unrecognized arguments: --execution-root' "$tmp/copied-root-bypass.out" || fail copied-root-bypass-error +[ "$(cat "$package_root/sentinel")" = keep ] || fail copied-root-bypass-deleted +pass 'a copied expected layout cannot select or delete an execution root' + +driver_wrapper="$tmp/driver-identity.py" +printf '%s\n' \ + 'import hashlib, importlib.util, pathlib, sys' \ + 'driver = pathlib.Path(sys.argv[1])' \ + 'spec = importlib.util.spec_from_file_location("replay", driver)' \ + 'module = importlib.util.module_from_spec(spec)' \ + 'spec.loader.exec_module(module)' \ + 'saved = driver.read_bytes()' \ + 'assert module.driver_identity() == hashlib.sha256(saved).hexdigest()' \ + 'for changed, expected in ((saved + b"\nCHANGED_EXECUTABLE_STATEMENT = True\n", "changed during startup"), (saved + b"\nif\n", "identity is unavailable"), (saved + b"\n\\xff\n", "identity is unavailable")):' \ + ' driver.write_bytes(changed)' \ + ' try:' \ + ' module.driver_identity()' \ + ' except module.ReplayError as error:' \ + ' assert expected in str(error)' \ + ' else:' \ + ' raise AssertionError("changed driver was accepted")' \ + 'driver.write_bytes(saved)' >"$driver_wrapper" +python3 "$driver_wrapper" "$package_replay" || fail driver-loaded-identity +pass 'driver identity binds normal loaded code and rejects changed, invalid, or undecodable source' + /bin/cp "$runtime/object-closure" "$tmp/race-object-closure" /bin/cp "$jq_bin" "$tmp/race-jq" /bin/chmod 0555 "$tmp/race-object-closure" "$tmp/race-jq" race_wrapper="$tmp/snapshot-race.py" printf '%s\n' \ - 'import argparse, importlib.util, os, pathlib, stat, subprocess, sys' \ + 'import argparse, importlib.util, pathlib, shutil, stat, sys' \ 'driver, repository, state_dir, helper, jq_bin, *arguments = sys.argv[1:]' \ 'spec = importlib.util.spec_from_file_location("replay", driver)' \ 'module = importlib.util.module_from_spec(spec)' \ 'spec.loader.exec_module(module)' \ - 'values = argparse.Namespace(closure_helper=helper, jq_bin=jq_bin)' \ + 'value = lambda name: arguments[arguments.index(name) + 1]' \ + 'values = argparse.Namespace(input=value("--input"), source_repository_id=value("--source-repository-id"), source_git_dir=value("--source-git-dir"), candidate_root=value("--candidate-root"), scratch_root=value("--scratch-root"), state_dir=value("--state-dir"), closure_helper=helper, jq_bin=jq_bin, verify_path=value("--verify-path"), expected_sha256=value("--expected-sha256"), review_observation=None, publisher_observation=None)' \ 'snapshot = module.create_execution_snapshot(pathlib.Path(repository), values, pathlib.Path(state_dir))' \ - 'targets = [pathlib.Path(driver), pathlib.Path(repository) / "adapters/local-git-materializer/v1/protocol.jq", pathlib.Path(helper), pathlib.Path(jq_bin)]' \ + 'targets = [pathlib.Path(repository) / "adapters/local-git-materializer/v1/protocol.jq", pathlib.Path(helper), pathlib.Path(jq_bin)]' \ 'saved = [target.read_bytes() for target in targets]' \ 'modes = [stat.S_IMODE(target.stat().st_mode) for target in targets]' \ 'try:' \ ' for target in targets:' \ ' target.chmod(0o700)' \ ' target.write_bytes(b"replaced after execution snapshot\n")' \ - ' result = subprocess.run([sys.executable, str(snapshot / "delivery/v1/replay.py"), *arguments, "--execution-root", str(snapshot)], stdout=subprocess.PIPE, stderr=subprocess.PIPE)' \ + ' status = module.replay_locked(values, snapshot, pathlib.Path(state_dir))' \ 'finally:' \ ' for target, data, mode in zip(targets, saved, modes):' \ ' target.write_bytes(data)' \ ' target.chmod(mode)' \ - 'sys.stdout.buffer.write(result.stdout)' \ - 'sys.stderr.buffer.write(result.stderr)' \ - 'raise SystemExit(result.returncode)' >"$race_wrapper" + ' shutil.rmtree(snapshot, ignore_errors=True)' \ + 'raise SystemExit(status)' >"$race_wrapper" /bin/mkdir -m 700 "$tmp/race-state" "$tmp/race-candidate" "$tmp/race-scratch" driver_sha=$(sha_file "$package_replay") package_sha=$(sha_file "$package_root/adapters/local-git-materializer/v1/protocol.jq") From 1fa6e090eccebaa795e05f121b66da9d1ad09466 Mon Sep 17 00:00:00 2001 From: ci Date: Sat, 5 Sep 2026 09:15:58 -0400 Subject: [PATCH 10/20] Preserve replay cancellation identity --- delivery/v1/replay.py | 21 +++++- scripts/test/delivery-replay.test.sh | 106 +++++++++++++++++++++++++-- 2 files changed, 119 insertions(+), 8 deletions(-) diff --git a/delivery/v1/replay.py b/delivery/v1/replay.py index 4e50663..1578c80 100755 --- a/delivery/v1/replay.py +++ b/delivery/v1/replay.py @@ -148,6 +148,13 @@ def snapshot_file(source, destination, mode): return data +def snapshot_native_executable(source, destination): + data = snapshot_file(source, destination, 0o500) + if not data.startswith((b"\x7fELF", b"\xcf\xfa\xed\xfe", b"\xfe\xed\xfa\xcf", + b"\xca\xfe\xba\xbe", b"\xbe\xba\xfe\xca")): + raise ReplayError("dependency is not a native executable") + + def create_execution_snapshot(repository, arguments, state_dir): root = Path(tempfile.mkdtemp(prefix="execution-", dir=state_dir)) try: @@ -164,8 +171,8 @@ def create_execution_snapshot(repository, arguments, state_dir): continue mode = 0o500 if relative.endswith(".sh") else 0o400 snapshot_file(repository / relative, root / relative, mode) - snapshot_file(arguments.closure_helper, root / ".dependencies/object-closure", 0o500) - snapshot_file(arguments.jq_bin, root / ".dependencies/jq", 0o500) + snapshot_native_executable(arguments.closure_helper, root / ".dependencies/object-closure") + snapshot_native_executable(arguments.jq_bin, root / ".dependencies/jq") return root except (OSError, ReplayError): shutil.rmtree(root, ignore_errors=True) @@ -218,8 +225,12 @@ def input_identity(input_value, input_sha, arguments, execution): raise ReplayError("materialization input request identity is invalid") if not isinstance(source_tree_id, str) or not OID.fullmatch(source_tree_id): raise ReplayError("materialization input tree identity is invalid") - if not isinstance(source, dict) or not OID.fullmatch(str(source.get("commit_id", ""))): + if not isinstance(source, dict) or not isinstance(source.get("commit_id"), str) or \ + not OID.fullmatch(source["commit_id"]): raise ReplayError("materialization input commit identity is invalid") + if not isinstance(source.get("hash_algorithm"), str) or \ + source["hash_algorithm"] not in {"sha1", "sha256"}: + raise ReplayError("materialization input hash algorithm is invalid") expected = arguments.expected_sha256 if not re.fullmatch(r"[0-9a-f]{64}", expected): raise ReplayError("expected verifier digest is invalid") @@ -548,6 +559,8 @@ def replay_locked(arguments, execution, state_dir): arguments, execution, input_snapshot_path, identity ) except ReplayError as error: + if stop_if_interrupted(state, interrupted): + return 75 state.update({"phase": "failed", "recoverable": True, "reason": str(error)}) atomic_json(state_path, state) result(state) @@ -566,6 +579,8 @@ def replay_locked(arguments, execution, state_dir): "sha256": verify_candidate(arguments.candidate_root, state["identity"]["candidate_tree_id"], identity["verifier"]["path"], identity["verifier"]["expected_sha256"])} except ReplayError as error: + if stop_if_interrupted(state, interrupted): + return 75 state.update({"phase": "failed", "recoverable": False, "reason": str(error)}) atomic_json(state_path, state) result(state) diff --git a/scripts/test/delivery-replay.test.sh b/scripts/test/delivery-replay.test.sh index 8203504..adf2b1d 100755 --- a/scripts/test/delivery-replay.test.sh +++ b/scripts/test/delivery-replay.test.sh @@ -31,11 +31,7 @@ jq_bin="${TMPDIR:-/tmp}/ystack-portable-core-jq16/$asset" runtime="$tmp/runtime" /bin/mkdir -m 700 "$runtime" "$tmp/home" -if [ "$platform" = Darwin:arm64 ]; then - printf '%s\n' '#!/bin/sh' "exec /usr/bin/arch -x86_64 '$jq_bin' \"\$@\"" > "$runtime/jq" -else - /bin/cp "$jq_bin" "$runtime/jq" -fi +/bin/cp "$jq_bin" "$runtime/jq" /bin/chmod 0555 "$runtime/jq" jq_bin="$runtime/jq" PATH="$runtime:/usr/bin:/bin" @@ -109,6 +105,20 @@ run_replay() { --verify-path source.txt --expected-sha256 "$expected" } +printf '%s\n' '#!/bin/sh' "exec '$jq_bin' \"\$@\"" >"$tmp/jq-launcher" +/bin/chmod 0555 "$tmp/jq-launcher" +/bin/mkdir -m 700 "$tmp/launcher-state" "$tmp/launcher-candidate" "$tmp/launcher-scratch" +if python3 "$replay" --input "$base_input" --source-repository-id fixture.target \ + --source-git-dir "$tmp/source.git" --candidate-root "$tmp/launcher-candidate" --scratch-root "$tmp/launcher-scratch" \ + --state-dir "$tmp/launcher-state" --closure-helper "$runtime/object-closure" --jq-bin "$tmp/jq-launcher" \ + --verify-path source.txt --expected-sha256 "$expected_changed" >"$tmp/launcher.out" 2>&1; then + fail dependency-launcher +fi +grep -Fq 'delivery replay: dependency is not a native executable' "$tmp/launcher.out" || + fail dependency-launcher-error +[ ! -e "$tmp/launcher-state/run.json" ] || fail dependency-launcher-journal +pass 'dependency launchers are rejected before replay state is created' + run_replay changed "$base_input" "$expected_changed" >"$tmp/changed.out" jq -e '.state.phase=="review-wait" and .authority=="none" and .offline_simulation==true' "$tmp/changed.out" >/dev/null || fail missing-review-waits @@ -212,6 +222,63 @@ jq -e '.state.phase=="failed" and (.state.reason|contains("does not match frozen fail reconcile-mismatch-state pass 'a mismatched interrupted candidate is rejected without cleanup' +group_interrupt_wrapper="$tmp/materialization-group-interrupt.py" +printf '%s\n' \ + 'import importlib.util, os, signal, sys' \ + 'path, point, signal_name, arguments = sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4:]' \ + 'os.setsid()' \ + 'spec = importlib.util.spec_from_file_location("replay", path)' \ + 'module = importlib.util.module_from_spec(spec)' \ + 'spec.loader.exec_module(module)' \ + 'original = module.run_materializer' \ + 'def interrupt_materialization(*args, **kwargs):' \ + ' if point == "before":' \ + ' os.killpg(os.getpgrp(), getattr(signal, signal_name))' \ + ' raise module.ReplayError("materialization did not complete")' \ + ' original(*args, **kwargs)' \ + ' os.killpg(os.getpgrp(), getattr(signal, signal_name))' \ + ' raise module.ReplayError("materialization did not complete")' \ + 'def interrupt_verification(*_args, **_kwargs):' \ + ' os.killpg(os.getpgrp(), getattr(signal, signal_name))' \ + ' raise module.ReplayError("fixed verifier could not read the candidate blob")' \ + 'if point == "verify":' \ + ' module.verify_candidate = interrupt_verification' \ + 'else:' \ + ' module.run_materializer = interrupt_materialization' \ + 'sys.argv = [path] + arguments' \ + 'raise SystemExit(module.main())' >"$group_interrupt_wrapper" +for interrupt_case in before after verify; do + if [ "$interrupt_case" = after ]; then interrupt_signal=SIGTERM; else interrupt_signal=SIGINT; fi + mkdir -m 700 "$tmp/group-$interrupt_case-state" "$tmp/group-$interrupt_case-candidate" \ + "$tmp/group-$interrupt_case-scratch" + if python3 "$group_interrupt_wrapper" "$replay" "$interrupt_case" "$interrupt_signal" \ + --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ + --candidate-root "$tmp/group-$interrupt_case-candidate" --scratch-root "$tmp/group-$interrupt_case-scratch" \ + --state-dir "$tmp/group-$interrupt_case-state" --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" \ + --verify-path source.txt --expected-sha256 "$expected_changed" >"$tmp/group-$interrupt_case.out"; then + fail "group-$interrupt_case-status" + else + interrupt_status=$? + fi + [ "$interrupt_status" -eq 75 ] || fail "group-$interrupt_case-code" + if [ "$interrupt_case" = before ]; then + expected_interrupt_phase=materializing + [ ! -e "$tmp/group-$interrupt_case-candidate/repository.git" ] || fail group-before-effect + else + if [ "$interrupt_case" = after ]; then expected_interrupt_phase=materializing; else expected_interrupt_phase=verifying; fi + [ -d "$tmp/group-$interrupt_case-candidate/repository.git" ] || fail group-after-candidate + fi + jq -e --arg phase "$expected_interrupt_phase" '.phase==$phase' \ + "$tmp/group-$interrupt_case-state/run.json" >/dev/null || fail "group-$interrupt_case-state" + python3 "$replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ + --candidate-root "$tmp/group-$interrupt_case-candidate" --scratch-root "$tmp/group-$interrupt_case-scratch" \ + --state-dir "$tmp/group-$interrupt_case-state" --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" \ + --verify-path source.txt --expected-sha256 "$expected_changed" >"$tmp/group-$interrupt_case-resume.out" + jq -e '.state.phase=="review-wait"' "$tmp/group-$interrupt_case-resume.out" >/dev/null || + fail "group-$interrupt_case-resume" +done +pass 'process-group cancellation during materialization or verification resumes the same attempt' + for tree_case in numeric list null; do case "$tree_case" in numeric) tree_value=123 ;; @@ -235,6 +302,35 @@ for tree_case in numeric list null; do done pass 'non-string source tree identities fail without a traceback' +for identity_field in commit_id hash_algorithm; do + case "$identity_field" in + commit_id) identity_error='materialization input commit identity is invalid' ;; + hash_algorithm) identity_error='materialization input hash algorithm is invalid' ;; + esac + for identity_case in numeric list null; do + case "$identity_case" in + numeric) identity_value=1111111111111111111111111111111111111111 ;; + list) identity_value='[]' ;; + null) identity_value=null ;; + esac + "$jq_bin" -S -c ".stage_request.content.body.target_revision.value.$identity_field = $identity_value" \ + "$base_input" >"$tmp/$identity_field-$identity_case.json" + mkdir -m 700 "$tmp/$identity_field-$identity_case-state" "$tmp/$identity_field-$identity_case-candidate" \ + "$tmp/$identity_field-$identity_case-scratch" + if python3 "$replay" --input "$tmp/$identity_field-$identity_case.json" --source-repository-id fixture.target \ + --source-git-dir "$tmp/source.git" --candidate-root "$tmp/$identity_field-$identity_case-candidate" \ + --scratch-root "$tmp/$identity_field-$identity_case-scratch" --state-dir "$tmp/$identity_field-$identity_case-state" \ + --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" --verify-path source.txt \ + --expected-sha256 "$expected_changed" >"$tmp/$identity_field-$identity_case.out" 2>&1; then + fail "$identity_field-$identity_case" + fi + [ ! -e "$tmp/$identity_field-$identity_case-state/run.json" ] || fail "$identity_field-$identity_case-journal" + grep -Fq "delivery replay: $identity_error" \ + "$tmp/$identity_field-$identity_case.out" || fail "$identity_field-$identity_case-error" + done +done +pass 'non-string commit and hash-algorithm identities are rejected before journaling' + mkdir -m 700 "$tmp/caller-execution-root" "$tmp/caller-execution-state" \ "$tmp/caller-execution-candidate" "$tmp/caller-execution-scratch" printf '%s\n' keep >"$tmp/caller-execution-root/sentinel" From cc2cb987bbdd39ec62a766e6ff632f9f94b90af6 Mon Sep 17 00:00:00 2001 From: ci Date: Sat, 5 Sep 2026 10:29:45 -0400 Subject: [PATCH 11/20] Bind replay completion atomically --- delivery/v1/replay.py | 182 ++++++++++++++++++++------- scripts/test/delivery-replay.test.sh | 150 ++++++++++++++++++---- 2 files changed, 259 insertions(+), 73 deletions(-) diff --git a/delivery/v1/replay.py b/delivery/v1/replay.py index 1578c80..86e7d8a 100755 --- a/delivery/v1/replay.py +++ b/delivery/v1/replay.py @@ -2,6 +2,7 @@ """Run one inactive, offline delivery replay without executing candidate code.""" import argparse +from contextlib import contextmanager import fcntl import hashlib import json @@ -13,9 +14,25 @@ import subprocess import sys import tempfile +import time -LOADED_DRIVER_CODE = sys._getframe().f_code +LOADED_DRIVER_BYTES = globals().get("_REPLAY_DRIVER_BYTES") +if LOADED_DRIVER_BYTES is None and __name__ == "__main__": + try: + driver_descriptor = os.open(__file__, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)) + with os.fdopen(driver_descriptor, "rb") as driver_handle: + driver_source = driver_handle.read(8 * 1024 * 1024 + 1) + if len(driver_source) > 8 * 1024 * 1024: + raise OSError("replay driver exceeds its size limit") + driver_code = compile(driver_source, __file__, "exec") + except (OSError, SyntaxError, TypeError, ValueError) as error: + print(f"delivery replay: loaded replay driver identity is unavailable: {error}", file=sys.stderr) + raise SystemExit(1) from error + globals()["_REPLAY_DRIVER_BYTES"] = driver_source + exec(driver_code, globals()) + raise SystemExit(1) + MAX_INPUT_BYTES = 8 * 1024 * 1024 MAX_OBSERVATION_BYTES = 64 * 1024 MAX_VERIFIED_BLOB_BYTES = 1024 * 1024 @@ -149,14 +166,22 @@ def snapshot_file(source, destination, mode): def snapshot_native_executable(source, destination): - data = snapshot_file(source, destination, 0o500) + data = read_bytes(trusted_file(source), MAX_INPUT_BYTES) if not data.startswith((b"\x7fELF", b"\xcf\xfa\xed\xfe", b"\xfe\xed\xfa\xcf", b"\xca\xfe\xba\xbe", b"\xbe\xba\xfe\xca")): raise ReplayError("dependency is not a native executable") + destination.parent.mkdir(parents=True, exist_ok=True) + atomic_bytes(destination, data) + os.chmod(destination, 0o500) def create_execution_snapshot(repository, arguments, state_dir): - root = Path(tempfile.mkdtemp(prefix="execution-", dir=state_dir)) + root = state_dir / "execution" + if root.exists(): + if root.is_symlink() or not root.is_dir(): + raise ReplayError("execution bundle is unavailable") + return root + os.mkdir(root, 0o700) try: core_relative = "scripts/core-contract.sh" core = snapshot_file(repository / core_relative, root / core_relative, 0o500) @@ -175,19 +200,31 @@ def create_execution_snapshot(repository, arguments, state_dir): snapshot_native_executable(arguments.jq_bin, root / ".dependencies/jq") return root except (OSError, ReplayError): - shutil.rmtree(root, ignore_errors=True) raise -def driver_identity(): - source = read_bytes(trusted_file(Path(__file__).resolve()), MAX_INPUT_BYTES) +def execution_sources_match(repository, arguments, execution): try: - current = compile(source, LOADED_DRIVER_CODE.co_filename, "exec") - except (SyntaxError, TypeError, ValueError) as error: - raise ReplayError("loaded replay driver identity is unavailable") from error - if current != LOADED_DRIVER_CODE: - raise ReplayError("loaded replay driver changed during startup") - return digest_bytes(source) + if materializer_package_identity(repository) != materializer_package_identity(execution): + return False + for source, saved in ( + (arguments.closure_helper, execution / ".dependencies/object-closure"), + (arguments.jq_bin, execution / ".dependencies/jq"), + ): + current = read_bytes(trusted_file(source), MAX_INPUT_BYTES) + if not current.startswith((b"\x7fELF", b"\xcf\xfa\xed\xfe", b"\xfe\xed\xfa\xcf", + b"\xca\xfe\xba\xbe", b"\xbe\xba\xfe\xca")) or \ + digest_bytes(current) != digest_bytes(read_bytes(trusted_file(saved), MAX_INPUT_BYTES)): + return False + return True + except (OSError, ReplayError): + return False + + +def driver_identity(): + if not isinstance(LOADED_DRIVER_BYTES, bytes) or len(LOADED_DRIVER_BYTES) > MAX_INPUT_BYTES: + raise ReplayError("loaded replay driver identity is unavailable") + return digest_bytes(LOADED_DRIVER_BYTES) def materializer_package_identity(repository): @@ -331,6 +368,50 @@ def candidate_identity(candidate_root, source_commit): "candidate_parent_commit_id": parent} +@contextmanager +def hold_candidate_ref(candidate_root, expected_commit): + repository = Path(candidate_root).resolve() / "repository.git" + lock_path = repository / "refs/heads/candidate.lock" + command = ["/usr/bin/git", f"--git-dir={repository}", "-c", "core.hooksPath=/dev/null", + "update-ref", "--stdin"] + process = subprocess.Popen(command, stdin=subprocess.PIPE, stdout=subprocess.DEVNULL, + stderr=subprocess.PIPE) + try: + process.stdin.write( + f"option no-deref\nstart\nverify refs/heads/candidate {expected_commit}\nprepare\n".encode() + ) + process.stdin.flush() + for _ in range(1000): + if lock_path.is_file() and not lock_path.is_symlink(): + break + if process.poll() is not None: + raise ReplayError("candidate repository identity guard failed") + time.sleep(0.001) + else: + raise ReplayError("candidate repository identity guard timed out") + symbolic = subprocess.run( + ["/usr/bin/git", f"--git-dir={repository}", "symbolic-ref", "-q", "refs/heads/candidate"], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=False + ) + if symbolic.returncode != 1: + raise ReplayError("candidate repository identity guard failed") + yield + finally: + if process.poll() is None: + try: + process.stdin.write(b"abort\n") + process.stdin.flush() + except (BrokenPipeError, OSError): + pass + if process.stdin is not None: + process.stdin.close() + try: + process.wait(timeout=5) + except subprocess.TimeoutExpired: + process.terminate() + process.wait(timeout=5) + + def reconcile_materialization(arguments, execution, input_path, identity, state_dir): existing = candidate_identity(arguments.candidate_root, identity["source_commit_id"]) if existing is None: @@ -483,7 +564,7 @@ def validate_state(state, identity): def result(state): print(json.dumps({"kind": "delivery_replay_receipt", "authority": "none", "qualification": "unavailable", "offline_simulation": True, - "state": state}, sort_keys=True, separators=(",", ":"))) + "state": state}, sort_keys=True, separators=(",", ":")), flush=True) def stop_if_interrupted(state, interrupted): @@ -494,14 +575,11 @@ def stop_if_interrupted(state, interrupted): return True -def replay_locked(arguments, execution, state_dir): +def replay_locked(arguments, state_dir): + repository = Path(__file__).resolve().parents[2] state_path = state_dir / "run.json" input_snapshot_path = state_dir / "materialization-input.json" lock_path = state_dir / "replay.lock" - input_bytes = read_bytes(arguments.input, MAX_INPUT_BYTES) - input_value = parse_json(input_bytes) - input_sha = digest_bytes(input_bytes) - identity = input_identity(input_value, input_sha, arguments, execution) interrupted = {"value": False} previous_term = signal.getsignal(signal.SIGTERM) previous_int = signal.getsignal(signal.SIGINT) @@ -511,6 +589,12 @@ def replay_locked(arguments, execution, state_dir): lock_descriptor = os.open(lock_path, os.O_RDWR | os.O_CREAT | getattr(os, "O_NOFOLLOW", 0), 0o600) with os.fdopen(lock_descriptor, "a+b") as lock: fcntl.flock(lock, fcntl.LOCK_EX) + execution = create_execution_snapshot(repository, arguments, state_dir) + sources_match = execution_sources_match(repository, arguments, execution) + input_bytes = read_bytes(arguments.input, MAX_INPUT_BYTES) + input_value = parse_json(input_bytes) + input_sha = digest_bytes(input_bytes) + identity = input_identity(input_value, input_sha, arguments, execution) state = None if state_path.exists(): state = parse_json(read_bytes(state_path, MAX_OBSERVATION_BYTES)) @@ -518,6 +602,11 @@ def replay_locked(arguments, execution, state_dir): if state is not None and any(state["identity"].get(name) != value for name, value in identity.items()): result({"phase": "stale", "reason": "run identity changed"}) return 2 + if not sources_match: + if state is not None: + result({"phase": "stale", "reason": "execution dependencies changed"}) + return 2 + raise ReplayError("execution bundle does not match current dependencies") if stop_if_interrupted(state, interrupted): return 75 if state is None: @@ -608,33 +697,37 @@ def replay_locked(arguments, execution, state_dir): state["phase"] = "publish-wait" atomic_json(state_path, state) if state["phase"] == "publish-wait": - revalidate_candidate(arguments, state) - if stop_if_interrupted(state, interrupted): - return 75 - if arguments.review_observation is not None: - supplied_review = observation(arguments.review_observation, - "delivery_replay_review_observation", - state["identity"], "verdict") + with hold_candidate_ref(arguments.candidate_root, + state["materialization"]["candidate_commit_id"]): + revalidate_candidate(arguments, state) if stop_if_interrupted(state, interrupted): return 75 - if supplied_review != state.get("review"): - raise ReplayError("supplied offline review changed after review wait") - publisher = observation(arguments.publisher_observation, "delivery_replay_publisher_observation", state["identity"], "disposition") - if stop_if_interrupted(state, interrupted): - return 75 - if publisher is None: - result(state) - return 0 - if publisher["disposition"] != "offline-simulated": - state.update({"phase": "failed", "recoverable": False, "reason": "offline publisher disposition is invalid"}) + if arguments.review_observation is not None: + supplied_review = observation(arguments.review_observation, + "delivery_replay_review_observation", + state["identity"], "verdict") + if stop_if_interrupted(state, interrupted): + return 75 + if supplied_review != state.get("review"): + raise ReplayError("supplied offline review changed after review wait") + publisher = observation(arguments.publisher_observation, + "delivery_replay_publisher_observation", + state["identity"], "disposition") + if stop_if_interrupted(state, interrupted): + return 75 + if publisher is None: + result(state) + return 0 + if publisher["disposition"] != "offline-simulated": + state.update({"phase": "failed", "recoverable": False, "reason": "offline publisher disposition is invalid"}) + atomic_json(state_path, state) + result(state) + return 1 + state["publisher"] = publisher + state["phase"] = "completed-offline" atomic_json(state_path, state) result(state) - return 1 - state["publisher"] = publisher - state["phase"] = "completed-offline" - atomic_json(state_path, state) - result(state) - return 0 + return 0 result(state) return 1 finally: @@ -643,14 +736,9 @@ def replay_locked(arguments, execution, state_dir): def replay(arguments): - repository = Path(__file__).resolve().parents[2] state_dir = private_directory(arguments.state_dir) disjoint(state_dir, arguments.source_git_dir, arguments.candidate_root, arguments.scratch_root) - execution = create_execution_snapshot(repository, arguments, state_dir) - try: - return replay_locked(arguments, execution, state_dir) - finally: - shutil.rmtree(execution, ignore_errors=True) + return replay_locked(arguments, state_dir) def main(): diff --git a/scripts/test/delivery-replay.test.sh b/scripts/test/delivery-replay.test.sh index adf2b1d..f67d178 100755 --- a/scripts/test/delivery-replay.test.sh +++ b/scripts/test/delivery-replay.test.sh @@ -141,7 +141,8 @@ expect_candidate_move_rejected() { fail "candidate-moved-$phase" fi /usr/bin/git --git-dir="$tmp/changed-candidate/repository.git" update-ref refs/heads/candidate "$candidate_commit" - grep -Fq 'candidate repository no longer matches saved materialization' "$tmp/candidate-moved-$phase.out" || + grep -Eq 'candidate repository no longer matches saved materialization|candidate repository identity guard failed' \ + "$tmp/candidate-moved-$phase.out" || fail "candidate-moved-$phase-error" } pass 'changed materialization and fixed read-only verification wait for review' @@ -177,11 +178,12 @@ pass 'replacement of the original input after snapshot cannot change materializa kill_wrapper="$tmp/kill-after-materialize.py" printf '%s\n' \ - 'import importlib.util, os, signal, sys' \ + 'import importlib.util, os, pathlib, signal, sys' \ 'path, arguments = sys.argv[1], sys.argv[2:]' \ 'spec = importlib.util.spec_from_file_location("replay", path)' \ 'module = importlib.util.module_from_spec(spec)' \ - 'spec.loader.exec_module(module)' \ + 'module._REPLAY_DRIVER_BYTES = pathlib.Path(path).read_bytes()' \ + 'exec(compile(module._REPLAY_DRIVER_BYTES, path, "exec"), module.__dict__)' \ 'original = module.run_materializer' \ 'def stop_after_materialization(*args, **kwargs):' \ ' result = original(*args, **kwargs)' \ @@ -204,6 +206,25 @@ python3 "$replay" --input "$base_input" --source-repository-id fixture.target -- jq -e '.state.phase=="review-wait"' "$tmp/reconcile-retry.out" >/dev/null || fail reconcile-retry pass 'SIGKILL after materializer output reconciles the existing candidate once' +mkdir -m 700 "$tmp/repeated-kill-state" "$tmp/repeated-kill-candidate" "$tmp/repeated-kill-scratch" +for kill_round in 1 2 3; do + if python3 "$kill_wrapper" "$replay" --input "$base_input" --source-repository-id fixture.target \ + --source-git-dir "$tmp/source.git" --candidate-root "$tmp/repeated-kill-candidate" \ + --scratch-root "$tmp/repeated-kill-scratch" --state-dir "$tmp/repeated-kill-state" \ + --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" --verify-path source.txt \ + --expected-sha256 "$expected_changed" >"$tmp/repeated-kill-$kill_round.out" 2>&1; then + fail "repeated-kill-$kill_round" + fi + [ "$(find "$tmp/repeated-kill-state" -maxdepth 1 -type d -name 'execution*' | wc -l | tr -d ' ')" = 1 ] || + fail "repeated-kill-snapshot-count-$kill_round" +done +python3 "$replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ + --candidate-root "$tmp/repeated-kill-candidate" --scratch-root "$tmp/repeated-kill-scratch" \ + --state-dir "$tmp/repeated-kill-state" --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" \ + --verify-path source.txt --expected-sha256 "$expected_changed" >"$tmp/repeated-kill-resume.out" +jq -e '.state.phase=="review-wait"' "$tmp/repeated-kill-resume.out" >/dev/null || fail repeated-kill-resume +pass 'repeated SIGKILL recovery reuses one bounded execution bundle' + mkdir -m 700 "$tmp/reconcile-bad-state" "$tmp/reconcile-bad-candidate" "$tmp/reconcile-bad-scratch" if python3 "$kill_wrapper" "$replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ --candidate-root "$tmp/reconcile-bad-candidate" --scratch-root "$tmp/reconcile-bad-scratch" --state-dir "$tmp/reconcile-bad-state" \ @@ -224,12 +245,13 @@ pass 'a mismatched interrupted candidate is rejected without cleanup' group_interrupt_wrapper="$tmp/materialization-group-interrupt.py" printf '%s\n' \ - 'import importlib.util, os, signal, sys' \ + 'import importlib.util, os, pathlib, signal, sys' \ 'path, point, signal_name, arguments = sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4:]' \ 'os.setsid()' \ 'spec = importlib.util.spec_from_file_location("replay", path)' \ 'module = importlib.util.module_from_spec(spec)' \ - 'spec.loader.exec_module(module)' \ + 'module._REPLAY_DRIVER_BYTES = pathlib.Path(path).read_bytes()' \ + 'exec(compile(module._REPLAY_DRIVER_BYTES, path, "exec"), module.__dict__)' \ 'original = module.run_materializer' \ 'def interrupt_materialization(*args, **kwargs):' \ ' if point == "before":' \ @@ -426,7 +448,8 @@ printf '%s\n' \ 'path, marker, arguments = sys.argv[1], pathlib.Path(sys.argv[2]), sys.argv[3:]' \ 'spec = importlib.util.spec_from_file_location("replay", path)' \ 'module = importlib.util.module_from_spec(spec)' \ - 'spec.loader.exec_module(module)' \ + 'module._REPLAY_DRIVER_BYTES = pathlib.Path(path).read_bytes()' \ + 'exec(compile(module._REPLAY_DRIVER_BYTES, path, "exec"), module.__dict__)' \ 'original = module.fcntl.flock' \ 'def marked_flock(*args, **kwargs):' \ ' marker.write_text("waiting")' \ @@ -456,11 +479,12 @@ jq -e '(.phase=="review-wait") and (has("review")|not)' "$tmp/lock-state/run.jso observation_wrapper="$tmp/observation-interrupt.py" printf '%s\n' \ - 'import importlib.util, os, signal, sys' \ + 'import importlib.util, os, pathlib, signal, sys' \ 'path, signal_name, arguments = sys.argv[1], sys.argv[2], sys.argv[3:]' \ 'spec = importlib.util.spec_from_file_location("replay", path)' \ 'module = importlib.util.module_from_spec(spec)' \ - 'spec.loader.exec_module(module)' \ + 'module._REPLAY_DRIVER_BYTES = pathlib.Path(path).read_bytes()' \ + 'exec(compile(module._REPLAY_DRIVER_BYTES, path, "exec"), module.__dict__)' \ 'original = module.observation' \ 'def interrupt_after_observation(*args, **kwargs):' \ ' result = original(*args, **kwargs)' \ @@ -496,6 +520,19 @@ python3 "$replay" --input "$base_input" --source-repository-id fixture.target -- --review-observation "$tmp/review.json" >"$tmp/publish-wait.out" jq -e '.state.phase=="publish-wait"' "$tmp/publish-wait.out" >/dev/null || fail missing-publisher-waits expect_candidate_move_rejected publish-wait --publisher-observation "$tmp/publisher.json" +git_clean --git-dir="$tmp/changed-candidate/repository.git" update-ref refs/heads/alternate "$candidate_commit" +git_clean --git-dir="$tmp/changed-candidate/repository.git" symbolic-ref refs/heads/candidate refs/heads/alternate +if python3 "$replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ + --candidate-root "$tmp/changed-candidate" --scratch-root "$tmp/changed-scratch" --state-dir "$tmp/changed-state" \ + --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" --verify-path source.txt \ + --expected-sha256 "$expected_changed" --publisher-observation "$tmp/publisher.json" >"$tmp/candidate-symref.out" 2>&1; then + fail candidate-symref +fi +git_clean --git-dir="$tmp/changed-candidate/repository.git" symbolic-ref --delete refs/heads/candidate +git_clean --git-dir="$tmp/changed-candidate/repository.git" update-ref refs/heads/candidate "$candidate_commit" +git_clean --git-dir="$tmp/changed-candidate/repository.git" update-ref -d refs/heads/alternate +grep -Fq 'candidate repository identity guard failed' "$tmp/candidate-symref.out" || fail candidate-symref-error +pass 'candidate completion guard rejects a symbolic candidate ref' mkdir -m 700 "$tmp/publish-cancel-state" cp "$tmp/changed-state/materialization-input.json" "$tmp/changed-state/run.json" "$tmp/publish-cancel-state/" if python3 "$observation_wrapper" "$replay" SIGTERM \ @@ -511,6 +548,63 @@ fi jq -e '(.phase=="publish-wait") and (has("publisher")|not)' "$tmp/publish-cancel-state/run.json" >/dev/null || fail publish-cancel-state pass 'SIGTERM at the lock and SIGINT or SIGTERM after wait observations do not advance state' + +atomic_wrapper="$tmp/atomic-candidate-observation.py" +printf '%s\n' \ + 'import importlib.util, os, pathlib, signal, subprocess, sys' \ + 'path, mode, marker, moved, arguments = sys.argv[1], sys.argv[2], pathlib.Path(sys.argv[3]), sys.argv[4], sys.argv[5:]' \ + 'spec = importlib.util.spec_from_file_location("replay", path)' \ + 'module = importlib.util.module_from_spec(spec)' \ + 'module._REPLAY_DRIVER_BYTES = pathlib.Path(path).read_bytes()' \ + 'exec(compile(module._REPLAY_DRIVER_BYTES, path, "exec"), module.__dict__)' \ + 'original = module.observation' \ + 'def act_during_publisher(*args, **kwargs):' \ + ' result = original(*args, **kwargs)' \ + ' if args[1] == "delivery_replay_publisher_observation":' \ + ' if mode == "move":' \ + ' candidate = pathlib.Path(arguments[arguments.index("--candidate-root") + 1]) / "repository.git"' \ + ' attempt = subprocess.run(["/usr/bin/git", f"--git-dir={candidate}", "-c", "core.hooksPath=/dev/null", "update-ref", "refs/heads/candidate", moved], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)' \ + ' marker.write_text(str(attempt.returncode))' \ + ' else:' \ + ' os.kill(os.getpid(), signal.SIGKILL)' \ + ' return result' \ + 'module.observation = act_during_publisher' \ + 'sys.argv = [path] + arguments' \ + 'raise SystemExit(module.main())' >"$atomic_wrapper" +mkdir -m 700 "$tmp/atomic-move-state" +cp "$tmp/changed-state/materialization-input.json" "$tmp/changed-state/run.json" "$tmp/atomic-move-state/" +python3 "$atomic_wrapper" "$replay" move "$tmp/atomic-move-status" "$moved_candidate" \ + --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ + --candidate-root "$tmp/changed-candidate" --scratch-root "$tmp/changed-scratch" --state-dir "$tmp/atomic-move-state" \ + --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" --verify-path source.txt \ + --expected-sha256 "$expected_changed" --publisher-observation "$tmp/publisher.json" >"$tmp/atomic-move.out" +jq -e '.state.phase=="completed-offline"' "$tmp/atomic-move.out" >/dev/null || fail atomic-move-completion +[ "$(cat "$tmp/atomic-move-status")" != 0 ] || fail atomic-move-lock +[ "$(git_clean --git-dir="$tmp/changed-candidate/repository.git" rev-parse refs/heads/candidate)" = "$candidate_commit" ] || + fail atomic-move-ref + +mkdir -m 700 "$tmp/atomic-kill-state" +cp "$tmp/changed-state/materialization-input.json" "$tmp/changed-state/run.json" "$tmp/atomic-kill-state/" +if python3 "$atomic_wrapper" "$replay" kill "$tmp/unused-kill-status" "$moved_candidate" \ + --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ + --candidate-root "$tmp/changed-candidate" --scratch-root "$tmp/changed-scratch" --state-dir "$tmp/atomic-kill-state" \ + --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" --verify-path source.txt \ + --expected-sha256 "$expected_changed" --publisher-observation "$tmp/publisher.json" >"$tmp/atomic-kill.out" 2>&1; then + fail atomic-kill-status +fi +atomic_lock="$tmp/changed-candidate/repository.git/refs/heads/candidate.lock" +atomic_wait=0 +while [ -e "$atomic_lock" ]; do + atomic_wait=$((atomic_wait + 1)) + [ "$atomic_wait" -le 100 ] || fail atomic-kill-lock-release + sleep 0.01 +done +python3 "$replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ + --candidate-root "$tmp/changed-candidate" --scratch-root "$tmp/changed-scratch" --state-dir "$tmp/atomic-kill-state" \ + --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" --verify-path source.txt \ + --expected-sha256 "$expected_changed" --publisher-observation "$tmp/publisher.json" >"$tmp/atomic-kill-resume.out" +jq -e '.state.phase=="completed-offline"' "$tmp/atomic-kill-resume.out" >/dev/null || fail atomic-kill-resume +pass 'candidate ref guard blocks publisher-time moves and releases after SIGKILL' printf '%s\n' '{"schema_version":1,"kind":"delivery_replay_publisher_observation","actor_id":123,"request_sha256":"'"$request_sha"'","candidate_tree_id":"'"$candidate_tree"'","disposition":"offline-simulated"}' >"$tmp/numeric-publisher.json" if python3 "$replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ --candidate-root "$tmp/changed-candidate" --scratch-root "$tmp/changed-scratch" --state-dir "$tmp/changed-state" \ @@ -734,36 +828,33 @@ driver_wrapper="$tmp/driver-identity.py" printf '%s\n' \ 'import hashlib, importlib.util, pathlib, sys' \ 'driver = pathlib.Path(sys.argv[1])' \ + 'saved = driver.read_bytes()' \ 'spec = importlib.util.spec_from_file_location("replay", driver)' \ 'module = importlib.util.module_from_spec(spec)' \ - 'spec.loader.exec_module(module)' \ - 'saved = driver.read_bytes()' \ + 'module._REPLAY_DRIVER_BYTES = saved' \ + 'exec(compile(saved, str(driver), "exec"), module.__dict__)' \ 'assert module.driver_identity() == hashlib.sha256(saved).hexdigest()' \ - 'for changed, expected in ((saved + b"\nCHANGED_EXECUTABLE_STATEMENT = True\n", "changed during startup"), (saved + b"\nif\n", "identity is unavailable"), (saved + b"\n\\xff\n", "identity is unavailable")):' \ + 'for changed in (saved + b"\n# comment after load\n", saved + b"\n\n", saved + b"\nCHANGED_EXECUTABLE_STATEMENT = True\n", saved + b"\n\\xff\n"):' \ ' driver.write_bytes(changed)' \ - ' try:' \ - ' module.driver_identity()' \ - ' except module.ReplayError as error:' \ - ' assert expected in str(error)' \ - ' else:' \ - ' raise AssertionError("changed driver was accepted")' \ + ' assert module.driver_identity() == hashlib.sha256(saved).hexdigest()' \ 'driver.write_bytes(saved)' >"$driver_wrapper" python3 "$driver_wrapper" "$package_replay" || fail driver-loaded-identity -pass 'driver identity binds normal loaded code and rejects changed, invalid, or undecodable source' +pass 'driver identity remains bound to the exact source buffer loaded once' /bin/cp "$runtime/object-closure" "$tmp/race-object-closure" /bin/cp "$jq_bin" "$tmp/race-jq" /bin/chmod 0555 "$tmp/race-object-closure" "$tmp/race-jq" race_wrapper="$tmp/snapshot-race.py" printf '%s\n' \ - 'import argparse, importlib.util, pathlib, shutil, stat, sys' \ + 'import argparse, importlib.util, pathlib, stat, sys' \ 'driver, repository, state_dir, helper, jq_bin, *arguments = sys.argv[1:]' \ 'spec = importlib.util.spec_from_file_location("replay", driver)' \ 'module = importlib.util.module_from_spec(spec)' \ - 'spec.loader.exec_module(module)' \ + 'module._REPLAY_DRIVER_BYTES = pathlib.Path(driver).read_bytes()' \ + 'exec(compile(module._REPLAY_DRIVER_BYTES, driver, "exec"), module.__dict__)' \ 'value = lambda name: arguments[arguments.index(name) + 1]' \ 'values = argparse.Namespace(input=value("--input"), source_repository_id=value("--source-repository-id"), source_git_dir=value("--source-git-dir"), candidate_root=value("--candidate-root"), scratch_root=value("--scratch-root"), state_dir=value("--state-dir"), closure_helper=helper, jq_bin=jq_bin, verify_path=value("--verify-path"), expected_sha256=value("--expected-sha256"), review_observation=None, publisher_observation=None)' \ - 'snapshot = module.create_execution_snapshot(pathlib.Path(repository), values, pathlib.Path(state_dir))' \ + 'module.create_execution_snapshot(pathlib.Path(repository), values, pathlib.Path(state_dir))' \ 'targets = [pathlib.Path(repository) / "adapters/local-git-materializer/v1/protocol.jq", pathlib.Path(helper), pathlib.Path(jq_bin)]' \ 'saved = [target.read_bytes() for target in targets]' \ 'modes = [stat.S_IMODE(target.stat().st_mode) for target in targets]' \ @@ -771,13 +862,16 @@ printf '%s\n' \ ' for target in targets:' \ ' target.chmod(0o700)' \ ' target.write_bytes(b"replaced after execution snapshot\n")' \ - ' status = module.replay_locked(values, snapshot, pathlib.Path(state_dir))' \ + ' try:' \ + ' module.replay_locked(values, pathlib.Path(state_dir))' \ + ' except module.ReplayError as error:' \ + ' assert "execution bundle does not match current dependencies" in str(error)' \ + ' else:' \ + ' raise AssertionError("changed execution sources were accepted")' \ 'finally:' \ ' for target, data, mode in zip(targets, saved, modes):' \ ' target.write_bytes(data)' \ - ' target.chmod(mode)' \ - ' shutil.rmtree(snapshot, ignore_errors=True)' \ - 'raise SystemExit(status)' >"$race_wrapper" + ' target.chmod(mode)' >"$race_wrapper" /bin/mkdir -m 700 "$tmp/race-state" "$tmp/race-candidate" "$tmp/race-scratch" driver_sha=$(sha_file "$package_replay") package_sha=$(sha_file "$package_root/adapters/local-git-materializer/v1/protocol.jq") @@ -786,6 +880,10 @@ jq_sha=$(sha_file "$tmp/race-jq") python3 "$race_wrapper" "$package_replay" "$package_root" "$tmp/race-state" \ "$tmp/race-object-closure" "$tmp/race-jq" \ --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ + --candidate-root "$tmp/race-candidate" --scratch-root "$tmp/race-scratch" --state-dir "$tmp/race-state" \ + --closure-helper "$tmp/race-object-closure" --jq-bin "$tmp/race-jq" \ + --verify-path source.txt --expected-sha256 "$expected_changed" +python3 "$package_replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ --candidate-root "$tmp/race-candidate" --scratch-root "$tmp/race-scratch" --state-dir "$tmp/race-state" \ --closure-helper "$tmp/race-object-closure" --jq-bin "$tmp/race-jq" \ --verify-path source.txt --expected-sha256 "$expected_changed" >"$tmp/race.out" @@ -795,6 +893,6 @@ jq -e '.state.phase=="review-wait" and .state.identity.closure_helper_sha256==$helper and .state.identity.jq_sha256==$jq' \ --arg driver "$driver_sha" --arg package "$package_sha" --arg helper "$helper_sha" --arg jq "$jq_sha" \ "$tmp/race.out" >/dev/null || fail immutable-execution-snapshot -pass 'replacement after the private execution snapshot cannot change executed or recorded bytes' +pass 'the one state-owned execution bundle rejects drift and records its exact bytes' printf 'delivery replay: %s focused checks passed\n' "$passed" From 5f9c0d3fdcf57c1510a857fb7c49067fc55824d3 Mon Sep 17 00:00:00 2001 From: ci Date: Sat, 5 Sep 2026 12:48:06 -0400 Subject: [PATCH 12/20] Rebuild incomplete execution bundles and isolate the ref guard Completes the in-progress fix for the two review findings on this PR. - The execution bundle is built in a program-owned staging directory and published by one rename, so an interrupted or rejected first attempt never leaves a usable-looking bundle. Foreign staging data and links are preserved, never deleted. A corrected native dependency can reuse the same state directory after a launcher rejection. - Every candidate Git operation, including the update-ref guard and the symbolic-ref check, runs under one sanitized environment, so ambient GIT_NAMESPACE, GIT_COMMON_DIR, config, work-tree, and hook settings cannot redirect or block the ref guard. - An existing bundle is judged by execution_sources_match, so changed or invalid dependencies read as a mismatch rather than a build failure. Three new focused checks cover each behavior; 33/33 pass. Co-Authored-By: Claude Fable 5.1 --- delivery/v1/replay.py | 152 ++++++++++++++++----------- scripts/test/delivery-replay.test.sh | 95 ++++++++++++++++- 2 files changed, 187 insertions(+), 60 deletions(-) diff --git a/delivery/v1/replay.py b/delivery/v1/replay.py index 86e7d8a..22f246c 100755 --- a/delivery/v1/replay.py +++ b/delivery/v1/replay.py @@ -38,6 +38,15 @@ MAX_VERIFIED_BLOB_BYTES = 1024 * 1024 OID = re.compile(r"[0-9a-f]{40}|[0-9a-f]{64}\Z") ACTOR = re.compile(r"[a-z0-9][a-z0-9._:-]{0,127}\Z") +GIT_ENVIRONMENT = { + "PATH": "/usr/bin:/bin", "LC_ALL": "C", "GIT_CONFIG_NOSYSTEM": "1", + "GIT_CONFIG_GLOBAL": "/dev/null", "GIT_NO_REPLACE_OBJECTS": "1", + "GIT_NO_LAZY_FETCH": "1", "GIT_TERMINAL_PROMPT": "0", +} +NATIVE_EXECUTABLE_MAGICS = ( + b"\x7fELF", b"\xcf\xfa\xed\xfe", b"\xfe\xed\xfa\xcf", + b"\xca\xfe\xba\xbe", b"\xbe\xba\xfe\xca", +) PACKAGE_FILES = ( "adapters/local-git-materializer/v1/materialize.sh", "adapters/local-git-materializer/v1/protocol.jq", @@ -157,66 +166,97 @@ def package_paths(generation): return PACKAGE_FILES + tuple(f"{root}/{name}" for name in GENERATION_FILES) -def snapshot_file(source, destination, mode): - data = read_bytes(trusted_file(source), MAX_INPUT_BYTES) - destination.parent.mkdir(parents=True, exist_ok=True) - atomic_bytes(destination, data) - os.chmod(destination, mode) - return data +def execution_source_bytes(repository, arguments): + core_relative = "scripts/core-contract.sh" + core = read_bytes(trusted_file(repository / core_relative), MAX_INPUT_BYTES) + match = re.search( + rb"^PORTABLE_CORE_GENERATION='(g-[0-9a-f]{64})'$", core, re.MULTILINE + ) + if match is None: + raise ReplayError("materializer package generation is unavailable") + generation = match.group(1).decode() + package = { + relative: core if relative == core_relative else + read_bytes(trusted_file(repository / relative), MAX_INPUT_BYTES) + for relative in package_paths(generation) + } + dependencies = { + ".dependencies/object-closure": read_bytes( + trusted_file(arguments.closure_helper), MAX_INPUT_BYTES + ), + ".dependencies/jq": read_bytes(trusted_file(arguments.jq_bin), MAX_INPUT_BYTES), + } + if any(not data.startswith(NATIVE_EXECUTABLE_MAGICS) for data in dependencies.values()): + raise ReplayError("dependency is not a native executable") + return package | dependencies -def snapshot_native_executable(source, destination): - data = read_bytes(trusted_file(source), MAX_INPUT_BYTES) - if not data.startswith((b"\x7fELF", b"\xcf\xfa\xed\xfe", b"\xfe\xed\xfa\xcf", - b"\xca\xfe\xba\xbe", b"\xbe\xba\xfe\xca")): - raise ReplayError("dependency is not a native executable") - destination.parent.mkdir(parents=True, exist_ok=True) - atomic_bytes(destination, data) - os.chmod(destination, 0o500) +def owned_staging_token(owner_path): + if not owner_path.exists() or owner_path.is_symlink() or not owner_path.is_file(): + return None + value = read_bytes(owner_path, 128) + match = re.fullmatch(rb"ystack-delivery-execution-v1:([0-9a-f]{64})\n", value) + return match.group(1) if match is not None else None def create_execution_snapshot(repository, arguments, state_dir): root = state_dir / "execution" - if root.exists(): + staging = state_dir / ".execution-building" + owner_path = state_dir / ".execution-building.owner" + if root.is_symlink() or root.exists(): if root.is_symlink() or not root.is_dir(): raise ReplayError("execution bundle is unavailable") + # An existing bundle is judged by execution_sources_match, so changed or + # invalid dependencies read as a mismatch there, not as a build failure here. return root - os.mkdir(root, 0o700) + source_bytes = execution_source_bytes(repository, arguments) + token = owned_staging_token(owner_path) + if staging.is_symlink() or staging.exists(): + if staging.is_symlink() or not staging.is_dir() or token is None: + raise ReplayError("execution bundle staging is not program-owned") + marker = staging / ".owner" + entries = list(staging.iterdir()) + if entries and ( + marker not in entries or marker.is_symlink() or not marker.is_file() or + read_bytes(marker, 128) != token + b"\n" + ): + raise ReplayError("execution bundle staging is not program-owned") + shutil.rmtree(staging) + if token is None: + if owner_path.exists(): + raise ReplayError("execution bundle staging owner is invalid") + token = os.urandom(32).hex().encode() + descriptor = os.open(owner_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | + getattr(os, "O_NOFOLLOW", 0), 0o600) + with os.fdopen(descriptor, "wb") as owner: + owner.write(b"ystack-delivery-execution-v1:" + token + b"\n") + owner.flush() + os.fsync(owner.fileno()) + os.mkdir(staging, 0o700) + atomic_bytes(staging / ".owner", token + b"\n") + os.chmod(staging / ".owner", 0o400) + for relative, data in source_bytes.items(): + destination = staging / relative + destination.parent.mkdir(parents=True, exist_ok=True) + atomic_bytes(destination, data) + mode = 0o500 if relative.endswith(".sh") or relative.startswith(".dependencies/") else 0o400 + os.chmod(destination, mode) + os.replace(staging, root) + directory = os.open(state_dir, os.O_DIRECTORY) try: - core_relative = "scripts/core-contract.sh" - core = snapshot_file(repository / core_relative, root / core_relative, 0o500) - match = re.search( - rb"^PORTABLE_CORE_GENERATION='(g-[0-9a-f]{64})'$", core, re.MULTILINE - ) - if match is None: - raise ReplayError("materializer package generation is unavailable") - generation = match.group(1).decode() - for relative in package_paths(generation): - if relative == core_relative: - continue - mode = 0o500 if relative.endswith(".sh") else 0o400 - snapshot_file(repository / relative, root / relative, mode) - snapshot_native_executable(arguments.closure_helper, root / ".dependencies/object-closure") - snapshot_native_executable(arguments.jq_bin, root / ".dependencies/jq") - return root - except (OSError, ReplayError): - raise + os.fsync(directory) + finally: + os.close(directory) + os.unlink(owner_path) + return root def execution_sources_match(repository, arguments, execution): try: - if materializer_package_identity(repository) != materializer_package_identity(execution): - return False - for source, saved in ( - (arguments.closure_helper, execution / ".dependencies/object-closure"), - (arguments.jq_bin, execution / ".dependencies/jq"), - ): - current = read_bytes(trusted_file(source), MAX_INPUT_BYTES) - if not current.startswith((b"\x7fELF", b"\xcf\xfa\xed\xfe", b"\xfe\xed\xfa\xcf", - b"\xca\xfe\xba\xbe", b"\xbe\xba\xfe\xca")) or \ - digest_bytes(current) != digest_bytes(read_bytes(trusted_file(saved), MAX_INPUT_BYTES)): - return False - return True + return all( + digest_bytes(data) == digest_bytes(read_bytes(trusted_file(execution / relative), MAX_INPUT_BYTES)) + for relative, data in execution_source_bytes(repository, arguments).items() + ) except (OSError, ReplayError): return False @@ -345,13 +385,10 @@ def candidate_identity(candidate_root, source_commit): repository = Path(candidate_root).resolve() / "repository.git" if not repository.is_dir() or repository.is_symlink(): return None - environment = {"PATH": "/usr/bin:/bin", "LC_ALL": "C", "GIT_CONFIG_NOSYSTEM": "1", - "GIT_CONFIG_GLOBAL": "/dev/null", "GIT_NO_REPLACE_OBJECTS": "1", - "GIT_NO_LAZY_FETCH": "1", "GIT_TERMINAL_PROMPT": "0"} values = [] for revision in ("refs/heads/candidate", "refs/heads/candidate^{tree}"): result = subprocess.run(["/usr/bin/git", f"--git-dir={repository}", "rev-parse", revision], - env=environment, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, check=False) + env=GIT_ENVIRONMENT, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, check=False) value = result.stdout.decode().strip() if result.returncode != 0 or not OID.fullmatch(value): return None @@ -360,7 +397,7 @@ def candidate_identity(candidate_root, source_commit): return {"candidate_commit_id": values[0], "candidate_tree_id": values[1], "candidate_parent_commit_id": source_commit} result = subprocess.run(["/usr/bin/git", f"--git-dir={repository}", "rev-parse", "refs/heads/candidate^"], - env=environment, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, check=False) + env=GIT_ENVIRONMENT, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, check=False) parent = result.stdout.decode().strip() if result.returncode != 0 or not OID.fullmatch(parent): return None @@ -374,8 +411,8 @@ def hold_candidate_ref(candidate_root, expected_commit): lock_path = repository / "refs/heads/candidate.lock" command = ["/usr/bin/git", f"--git-dir={repository}", "-c", "core.hooksPath=/dev/null", "update-ref", "--stdin"] - process = subprocess.Popen(command, stdin=subprocess.PIPE, stdout=subprocess.DEVNULL, - stderr=subprocess.PIPE) + process = subprocess.Popen(command, env=GIT_ENVIRONMENT, stdin=subprocess.PIPE, + stdout=subprocess.DEVNULL, stderr=subprocess.PIPE) try: process.stdin.write( f"option no-deref\nstart\nverify refs/heads/candidate {expected_commit}\nprepare\n".encode() @@ -391,7 +428,7 @@ def hold_candidate_ref(candidate_root, expected_commit): raise ReplayError("candidate repository identity guard timed out") symbolic = subprocess.run( ["/usr/bin/git", f"--git-dir={repository}", "symbolic-ref", "-q", "refs/heads/candidate"], - stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=False + env=GIT_ENVIRONMENT, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=False ) if symbolic.returncode != 1: raise ReplayError("candidate repository identity guard failed") @@ -437,16 +474,13 @@ def verify_candidate(candidate_root, candidate_tree, path, expected): repository = Path(candidate_root).resolve() / "repository.git" if not repository.is_dir() or repository.is_symlink() or not OID.fullmatch(candidate_tree): raise ReplayError("candidate repository identity is unavailable") - environment = {"PATH": "/usr/bin:/bin", "LC_ALL": "C", "GIT_CONFIG_NOSYSTEM": "1", - "GIT_CONFIG_GLOBAL": "/dev/null", "GIT_NO_REPLACE_OBJECTS": "1", - "GIT_NO_LAZY_FETCH": "1", "GIT_TERMINAL_PROMPT": "0"} object_name = f"{candidate_tree}:{path}" size = subprocess.run(["/usr/bin/git", f"--git-dir={repository}", "cat-file", "-s", object_name], - env=environment, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, check=False) + env=GIT_ENVIRONMENT, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, check=False) if size.returncode != 0 or not size.stdout.strip().isdigit() or int(size.stdout) > MAX_VERIFIED_BLOB_BYTES: raise ReplayError("fixed verifier cannot read the candidate blob") blob = subprocess.run(["/usr/bin/git", f"--git-dir={repository}", "cat-file", "blob", object_name], - env=environment, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, check=False) + env=GIT_ENVIRONMENT, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, check=False) if blob.returncode != 0 or len(blob.stdout) != int(size.stdout): raise ReplayError("fixed verifier could not read the candidate blob") actual = digest_bytes(blob.stdout) diff --git a/scripts/test/delivery-replay.test.sh b/scripts/test/delivery-replay.test.sh index f67d178..4ae7db5 100755 --- a/scripts/test/delivery-replay.test.sh +++ b/scripts/test/delivery-replay.test.sh @@ -117,7 +117,81 @@ fi grep -Fq 'delivery replay: dependency is not a native executable' "$tmp/launcher.out" || fail dependency-launcher-error [ ! -e "$tmp/launcher-state/run.json" ] || fail dependency-launcher-journal -pass 'dependency launchers are rejected before replay state is created' +python3 "$replay" --input "$base_input" --source-repository-id fixture.target \ + --source-git-dir "$tmp/source.git" --candidate-root "$tmp/launcher-candidate" --scratch-root "$tmp/launcher-scratch" \ + --state-dir "$tmp/launcher-state" --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" \ + --verify-path source.txt --expected-sha256 "$expected_changed" >"$tmp/launcher-retry.out" +jq -e '.state.phase=="review-wait"' "$tmp/launcher-retry.out" >/dev/null || fail dependency-launcher-retry +pass 'a corrected native dependency can reuse state after launcher rejection' + +snapshot_interrupt_wrapper="$tmp/snapshot-interrupt.py" +printf '%s\n' \ + 'import importlib.util, pathlib, sys' \ + 'path, point, arguments = sys.argv[1], sys.argv[2], sys.argv[3:]' \ + 'spec = importlib.util.spec_from_file_location("replay", path)' \ + 'module = importlib.util.module_from_spec(spec)' \ + 'module._REPLAY_DRIVER_BYTES = pathlib.Path(path).read_bytes()' \ + 'exec(compile(module._REPLAY_DRIVER_BYTES, path, "exec"), module.__dict__)' \ + 'if point == "directory":' \ + ' original_mkdir = module.os.mkdir' \ + ' def interrupted_mkdir(target, *args, **kwargs):' \ + ' result = original_mkdir(target, *args, **kwargs)' \ + ' if pathlib.Path(target).name == ".execution-building": raise module.ReplayError("snapshot interrupted after directory creation")' \ + ' return result' \ + ' module.os.mkdir = interrupted_mkdir' \ + 'else:' \ + ' original_atomic = module.atomic_bytes' \ + ' writes = {"count": 0}' \ + ' def interrupted_atomic(target, data):' \ + ' original_atomic(target, data)' \ + ' if ".execution-building" in pathlib.Path(target).parts:' \ + ' writes["count"] += 1' \ + ' if writes["count"] == 2: raise module.ReplayError("snapshot interrupted during copy")' \ + ' module.atomic_bytes = interrupted_atomic' \ + 'sys.argv = [path] + arguments' \ + 'raise SystemExit(module.main())' >"$snapshot_interrupt_wrapper" +for snapshot_point in directory copy; do + mkdir -m 700 "$tmp/snapshot-$snapshot_point-state" "$tmp/snapshot-$snapshot_point-candidate" \ + "$tmp/snapshot-$snapshot_point-scratch" + if python3 "$snapshot_interrupt_wrapper" "$replay" "$snapshot_point" \ + --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ + --candidate-root "$tmp/snapshot-$snapshot_point-candidate" --scratch-root "$tmp/snapshot-$snapshot_point-scratch" \ + --state-dir "$tmp/snapshot-$snapshot_point-state" --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" \ + --verify-path source.txt --expected-sha256 "$expected_changed" >"$tmp/snapshot-$snapshot_point.out" 2>&1; then + fail "snapshot-$snapshot_point-interrupt" + fi + [ ! -e "$tmp/snapshot-$snapshot_point-state/run.json" ] || fail "snapshot-$snapshot_point-journal" + [ -d "$tmp/snapshot-$snapshot_point-state/.execution-building" ] || fail "snapshot-$snapshot_point-staging" + python3 "$replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ + --candidate-root "$tmp/snapshot-$snapshot_point-candidate" --scratch-root "$tmp/snapshot-$snapshot_point-scratch" \ + --state-dir "$tmp/snapshot-$snapshot_point-state" --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" \ + --verify-path source.txt --expected-sha256 "$expected_changed" >"$tmp/snapshot-$snapshot_point-retry.out" + jq -e '.state.phase=="review-wait"' "$tmp/snapshot-$snapshot_point-retry.out" >/dev/null || + fail "snapshot-$snapshot_point-retry" +done + +mkdir -m 700 "$tmp/foreign-staging-state" "$tmp/foreign-staging-state/.execution-building" \ + "$tmp/foreign-staging-candidate" "$tmp/foreign-staging-scratch" +printf '%s\n' preserve >"$tmp/foreign-staging-state/.execution-building/sentinel" +if python3 "$replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ + --candidate-root "$tmp/foreign-staging-candidate" --scratch-root "$tmp/foreign-staging-scratch" \ + --state-dir "$tmp/foreign-staging-state" --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" \ + --verify-path source.txt --expected-sha256 "$expected_changed" >"$tmp/foreign-staging.out" 2>&1; then + fail foreign-staging +fi +[ "$(cat "$tmp/foreign-staging-state/.execution-building/sentinel")" = preserve ] || fail foreign-staging-preserved +mkdir -m 700 "$tmp/foreign-link-state" "$tmp/foreign-link-target" "$tmp/foreign-link-candidate" "$tmp/foreign-link-scratch" +printf '%s\n' preserve >"$tmp/foreign-link-target/sentinel" +ln -s "$tmp/foreign-link-target" "$tmp/foreign-link-state/.execution-building" +if python3 "$replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ + --candidate-root "$tmp/foreign-link-candidate" --scratch-root "$tmp/foreign-link-scratch" \ + --state-dir "$tmp/foreign-link-state" --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" \ + --verify-path source.txt --expected-sha256 "$expected_changed" >"$tmp/foreign-link.out" 2>&1; then + fail foreign-staging-link +fi +[ -L "$tmp/foreign-link-state/.execution-building" ] && \ + [ "$(cat "$tmp/foreign-link-target/sentinel")" = preserve ] || fail foreign-staging-link-preserved +pass 'transactional bundle recovery preserves foreign staging data and links' run_replay changed "$base_input" "$expected_changed" >"$tmp/changed.out" jq -e '.state.phase=="review-wait" and .authority=="none" and .offline_simulation==true' "$tmp/changed.out" >/dev/null || @@ -605,6 +679,25 @@ python3 "$replay" --input "$base_input" --source-repository-id fixture.target -- --expected-sha256 "$expected_changed" --publisher-observation "$tmp/publisher.json" >"$tmp/atomic-kill-resume.out" jq -e '.state.phase=="completed-offline"' "$tmp/atomic-kill-resume.out" >/dev/null || fail atomic-kill-resume pass 'candidate ref guard blocks publisher-time moves and releases after SIGKILL' + +mkdir -m 700 "$tmp/git-env-state" "$tmp/git-env-hooks" "$tmp/git-env-work-tree" +cp "$tmp/changed-state/materialization-input.json" "$tmp/changed-state/run.json" "$tmp/git-env-state/" +printf '%s\n' '#!/bin/sh' "printf hook >'$tmp/git-env-hook-ran'" >"$tmp/git-env-hooks/reference-transaction" +/bin/chmod 0555 "$tmp/git-env-hooks/reference-transaction" +printf '%s\n' '[core]' "hooksPath = $tmp/git-env-hooks" >"$tmp/git-env-global" +git_clean init -q --bare "$tmp/git-env-other.git" +/usr/bin/env GIT_NAMESPACE=poison GIT_COMMON_DIR="$tmp/git-env-other.git" GIT_DIR="$tmp/git-env-other.git" \ + GIT_WORK_TREE="$tmp/git-env-work-tree" GIT_CONFIG_NOSYSTEM=0 GIT_CONFIG_GLOBAL="$tmp/git-env-global" \ + GIT_CONFIG_COUNT=1 GIT_CONFIG_KEY_0=core.hooksPath GIT_CONFIG_VALUE_0="$tmp/git-env-hooks" \ + python3 "$replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ + --candidate-root "$tmp/changed-candidate" --scratch-root "$tmp/changed-scratch" --state-dir "$tmp/git-env-state" \ + --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" --verify-path source.txt \ + --expected-sha256 "$expected_changed" --publisher-observation "$tmp/publisher.json" >"$tmp/git-env.out" +jq -e '.state.phase=="completed-offline"' "$tmp/git-env.out" >/dev/null || fail git-env-completion +[ ! -e "$tmp/git-env-hook-ran" ] || fail git-env-hook +[ "$(git_clean --git-dir="$tmp/changed-candidate/repository.git" rev-parse refs/heads/candidate)" = "$candidate_commit" ] || + fail git-env-candidate +pass 'all candidate Git operations ignore ambient repository, namespace, config, work-tree, and hook settings' printf '%s\n' '{"schema_version":1,"kind":"delivery_replay_publisher_observation","actor_id":123,"request_sha256":"'"$request_sha"'","candidate_tree_id":"'"$candidate_tree"'","disposition":"offline-simulated"}' >"$tmp/numeric-publisher.json" if python3 "$replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ --candidate-root "$tmp/changed-candidate" --scratch-root "$tmp/changed-scratch" --state-dir "$tmp/changed-state" \ From 88ef9dece0fa2418fad9a04498a36ab594e1682e Mon Sep 17 00:00:00 2001 From: ci Date: Sat, 5 Sep 2026 13:05:39 -0400 Subject: [PATCH 13/20] Require execute permission on dependencies before snapshotting them Review P2: the execution snapshot copies caller-supplied dependencies with execute permission, so bytes that merely look like a native binary could gain execution through the copy. The caller must now hold an executable file for the jq binary and the closure helper; a non-executable dependency is rejected before any state or snapshot exists. 34/34 focused checks pass. Co-Authored-By: Claude Fable 5.1 --- delivery/v1/replay.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/delivery/v1/replay.py b/delivery/v1/replay.py index 22f246c..d76ec25 100755 --- a/delivery/v1/replay.py +++ b/delivery/v1/replay.py @@ -180,11 +180,18 @@ def execution_source_bytes(repository, arguments): read_bytes(trusted_file(repository / relative), MAX_INPUT_BYTES) for relative in package_paths(generation) } + dependency_sources = { + ".dependencies/object-closure": trusted_file(arguments.closure_helper), + ".dependencies/jq": trusted_file(arguments.jq_bin), + } + for source in dependency_sources.values(): + # The snapshot copies dependencies with execute permission, so the caller + # must already hold an executable file; bytes alone never confer that. + if not os.access(source, os.X_OK) or not (os.stat(source).st_mode & 0o111): + raise ReplayError("dependency is not executable") dependencies = { - ".dependencies/object-closure": read_bytes( - trusted_file(arguments.closure_helper), MAX_INPUT_BYTES - ), - ".dependencies/jq": read_bytes(trusted_file(arguments.jq_bin), MAX_INPUT_BYTES), + relative: read_bytes(source, MAX_INPUT_BYTES) + for relative, source in dependency_sources.items() } if any(not data.startswith(NATIVE_EXECUTABLE_MAGICS) for data in dependencies.values()): raise ReplayError("dependency is not a native executable") From c15a42f18dfd5deceb5dc18e369411872dbe009e Mon Sep 17 00:00:00 2001 From: ci Date: Sat, 5 Sep 2026 13:13:41 -0400 Subject: [PATCH 14/20] Add the regression for non-executable dependencies The previous commit's message counted this check before it landed; the test insertion had failed silently while the fix itself went in. A read-only copy of the jq binary is now rejected before any state or snapshot exists. 34/34 focused checks pass. Co-Authored-By: Claude Fable 5.1 --- scripts/test/delivery-replay.test.sh | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/scripts/test/delivery-replay.test.sh b/scripts/test/delivery-replay.test.sh index 4ae7db5..e41d446 100755 --- a/scripts/test/delivery-replay.test.sh +++ b/scripts/test/delivery-replay.test.sh @@ -124,6 +124,20 @@ python3 "$replay" --input "$base_input" --source-repository-id fixture.target \ jq -e '.state.phase=="review-wait"' "$tmp/launcher-retry.out" >/dev/null || fail dependency-launcher-retry pass 'a corrected native dependency can reuse state after launcher rejection' +/bin/cp "$jq_bin" "$tmp/jq-noexec" +/bin/chmod 0444 "$tmp/jq-noexec" +/bin/mkdir -m 700 "$tmp/noexec-state" "$tmp/noexec-candidate" "$tmp/noexec-scratch" +if python3 "$replay" --input "$base_input" --source-repository-id fixture.target \ + --source-git-dir "$tmp/source.git" --candidate-root "$tmp/noexec-candidate" --scratch-root "$tmp/noexec-scratch" \ + --state-dir "$tmp/noexec-state" --closure-helper "$runtime/object-closure" --jq-bin "$tmp/jq-noexec" \ + --verify-path source.txt --expected-sha256 "$expected_changed" >"$tmp/noexec.out" 2>&1; then + fail dependency-noexec +fi +grep -Fq 'delivery replay: dependency is not executable' "$tmp/noexec.out" || fail dependency-noexec-error +[ ! -e "$tmp/noexec-state/run.json" ] && [ ! -e "$tmp/noexec-state/execution" ] || fail dependency-noexec-state +pass 'a non-executable native dependency is rejected before any snapshot grants it execute permission' + + snapshot_interrupt_wrapper="$tmp/snapshot-interrupt.py" printf '%s\n' \ 'import importlib.util, pathlib, sys' \ From 3dd8bacbd3470e0814d6d20a3e614764f80d7beb Mon Sep 17 00:00:00 2001 From: ci Date: Sat, 5 Sep 2026 13:18:20 -0400 Subject: [PATCH 15/20] Treat deep JSON as bad input and bound reconcile staging Two review findings: - json.loads raises RecursionError on deeply nested input within the byte limit; it is now wrapped as the same "input is not JSON" replay error instead of escaping as a traceback. - Reconcile staging uses one fixed pair of directories under the state directory, cleared on entry and on exit, so a SIGKILL mid-reconcile cannot accumulate materialized repositories across retries. Two focused checks cover them; 36/36 pass. Co-Authored-By: Claude Fable 5.1 --- delivery/v1/replay.py | 17 ++++++++++++++--- scripts/test/delivery-replay.test.sh | 18 ++++++++++++++++++ 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/delivery/v1/replay.py b/delivery/v1/replay.py index d76ec25..86c0c3b 100755 --- a/delivery/v1/replay.py +++ b/delivery/v1/replay.py @@ -98,7 +98,9 @@ def read_bytes(path, limit): def parse_json(data): try: return json.loads(data) - except (ValueError, UnicodeDecodeError) as error: + except (ValueError, UnicodeDecodeError, RecursionError) as error: + # Deeply nested input within the byte limit raises RecursionError; it is + # still just input this program cannot accept, never a crash. raise ReplayError("input is not JSON") from error @@ -460,8 +462,17 @@ def reconcile_materialization(arguments, execution, input_path, identity, state_ existing = candidate_identity(arguments.candidate_root, identity["source_commit_id"]) if existing is None: return None - recovery_candidate = Path(tempfile.mkdtemp(prefix="reconcile-candidate-", dir=state_dir)) - recovery_scratch = Path(tempfile.mkdtemp(prefix="reconcile-scratch-", dir=state_dir)) + # One fixed pair of staging directories, cleared on entry and on exit, so a + # crash mid-reconcile can never accumulate materialized repositories. + recovery_candidate = state_dir / "reconcile-candidate" + recovery_scratch = state_dir / "reconcile-scratch" + for stale in (recovery_candidate, recovery_scratch): + if stale.is_symlink() or stale.exists(): + if stale.is_symlink() or not stale.is_dir(): + raise ReplayError("reconcile staging is unavailable") + shutil.rmtree(stale) + os.mkdir(recovery_candidate, 0o700) + os.mkdir(recovery_scratch, 0o700) try: recomputed = run_materializer(arguments, execution, input_path, identity, recovery_candidate, recovery_scratch) diff --git a/scripts/test/delivery-replay.test.sh b/scripts/test/delivery-replay.test.sh index e41d446..843034d 100755 --- a/scripts/test/delivery-replay.test.sh +++ b/scripts/test/delivery-replay.test.sh @@ -137,6 +137,19 @@ grep -Fq 'delivery replay: dependency is not executable' "$tmp/noexec.out" || fa [ ! -e "$tmp/noexec-state/run.json" ] && [ ! -e "$tmp/noexec-state/execution" ] || fail dependency-noexec-state pass 'a non-executable native dependency is rejected before any snapshot grants it execute permission' +python3 -c 'import sys; sys.stdout.write("[" * 100000 + "]" * 100000)' >"$tmp/deep.json" +/bin/mkdir -m 700 "$tmp/deep-state" "$tmp/deep-candidate" "$tmp/deep-scratch" +if python3 "$replay" --input "$tmp/deep.json" --source-repository-id fixture.target \ + --source-git-dir "$tmp/source.git" --candidate-root "$tmp/deep-candidate" --scratch-root "$tmp/deep-scratch" \ + --state-dir "$tmp/deep-state" --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" \ + --verify-path source.txt --expected-sha256 "$expected_changed" >"$tmp/deep.out" 2>&1; then + fail deep-json-accepted +fi +grep -Fq 'delivery replay: input is not JSON' "$tmp/deep.out" || fail deep-json-error +! grep -Fq 'Traceback' "$tmp/deep.out" || fail deep-json-traceback +pass 'deeply nested JSON is rejected as input, never as a crash' + + snapshot_interrupt_wrapper="$tmp/snapshot-interrupt.py" printf '%s\n' \ @@ -313,6 +326,11 @@ python3 "$replay" --input "$base_input" --source-repository-id fixture.target -- jq -e '.state.phase=="review-wait"' "$tmp/repeated-kill-resume.out" >/dev/null || fail repeated-kill-resume pass 'repeated SIGKILL recovery reuses one bounded execution bundle' +[ "$(find "$tmp/repeated-kill-state" -maxdepth 1 -name 'reconcile-*' | wc -l | tr -d ' ')" = 0 ] || + fail repeated-kill-reconcile-leftovers +pass 'crash recovery leaves no reconcile staging directories behind' + + mkdir -m 700 "$tmp/reconcile-bad-state" "$tmp/reconcile-bad-candidate" "$tmp/reconcile-bad-scratch" if python3 "$kill_wrapper" "$replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ --candidate-root "$tmp/reconcile-bad-candidate" --scratch-root "$tmp/reconcile-bad-scratch" --state-dir "$tmp/reconcile-bad-state" \ From 72bc74c4f0287feb71727d6148c9223effd79d09 Mon Sep 17 00:00:00 2001 From: ci Date: Sun, 6 Sep 2026 00:48:35 -0400 Subject: [PATCH 16/20] Refuse non-regular replay inputs before the first read The replay opened its input, observation, and run.json paths and read them before checking what they were, so a FIFO with no writer or a blocking device could stall it forever. It now opens without blocking, checks for a regular file on the open descriptor, and fails closed with a plain error otherwise. The test adds a FIFO input case under a watchdog. Proof: delivery-replay 36/36, shellcheck clean. Co-Authored-By: Claude Fable 5.1 --- delivery/v1/__pycache__/replay.cpython-314.pyc | Bin 0 -> 62704 bytes delivery/v1/replay.py | 11 ++++++++++- scripts/test/delivery-replay.test.sh | 17 +++++++++++++++++ 3 files changed, 27 insertions(+), 1 deletion(-) create mode 100644 delivery/v1/__pycache__/replay.cpython-314.pyc diff --git a/delivery/v1/__pycache__/replay.cpython-314.pyc b/delivery/v1/__pycache__/replay.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..615ba48f5a797dca5e01b2220f38e41c63276aa9 GIT binary patch literal 62704 zcmeFa33OZMeJ^;ig9J!$-xqL|Kyi^MN}{NRqC`sKPG3@_B~fB25?myTgfBqLV(gf) zn}%}IN>2Qq>DE)zsZ!G&wU!f4t+eh$PMth0^&B_P z@tlV{%ki34jYorDZL9XU&Z8?7!}K0K`)2SM*ssxJWWOel3BS5l^YJ)OTsUsLC!T$? zcr5JK>anul1Wy9{P4pzPUz^9qev>>&>^Iqyd?uxcs|#1FXNy!(<(JpjX~Z{A+L^Sm zH#shkJCn{E>L2I0(XY~5E_cS%Q>mnoo{G3LnfS6@dRKnWWaFEOM|R&?Z;#gblMI3LIzD#J&6j97GnS7j-+Ea{oRWI`Tauj-|0`ZE~;ZluJnxlw2 zt5Ndg_j$JQ2RxPhK~L40_@2W`ouwz#!79b~R6j6If*hxYPvmWU5}(Yc@Tq(npU!9C zHxqwZe0Gh=vz>4B?BEZ1YWXJ5&U6m>=VWqx?wLBdRlE4SOwO~L&xgNlW^EG&HJ)`5dXDTDw#_vFAJ6{XGgWn0ildpr{#qWaO&F_Za!|#Fr z7+(*c=l87=NbR%GGMm*m`=T(NlhT%bxJ; zj-CZ)$|tnE##7!@KB<}H#8Mnb=Gyt$Y5UBy(>^&pJmQ&r(phPr85^6LM98Rf3h%;Y zyWn(94PUljn)FP}%zEt3r<^0Rp2_KP`^fP0=;Y|I$7vs#8Ff~7IW(cfk(p_a^C{28 z;pyRVr+^TB&#-4Als3e-^mH_x9BSrU`&;;-!zcS%dPC-+p^L+l(?dg{`0k;u?xWou z9o+*Vt#EFxG+gMYKy^TwBTnb2+dk=W+uf6MPW#m4#Ys;{Pj#7VjH+paeQIWS)Hy0u zJeA7s5blYdAho2mtoEkni=hMD|giH!DlqHsqgs52{p5Bq2}D=bOU)IW5g?keJy-vYgbdpP!HeT+0z%&Y#&i3 z2gikXTJBy3Lg%?NoJWH*LVLE^qr=?L^V$;5p$l1fHsCFSFeALlbvX<)PSe8|oit7u zv6<1?DSEaJ4LvbCJQe<88yXs$6x<$~JJU1xoIW(<_6&O_M}~$y9%1s_tjFmd8WOTl z^gXInC|w%2nkRSEh~r)Dx-4Yk8}pj+;5NYf1MVk!vrr&~k4Vix07n}<{oQMWH_r{i z7hO+yAxQ7^Tg3Hq(t_-voTHp{P?d0RYP!%9rio$q#MI=ukkLIcT)S&`C~m^})ac~6 z)9t~eF+4duHS2U!ZOVm0_#vzKZRp%(v{j&mejI=99C-8Gx+Uf6!50oLmia99prvBP zQsJ{ydi9m8VG(UIB0g&qt?)-_6OY54*X*G*aGSTxq3e578;Wz!2%ez}&dcslyxZv- z7KS}Df*T(+l_5<7nyx?Zo|z6AMrSX&+##cLnpU__hTPTdjl&KwGd(#nJSEiPQwM#f zVWP&`t~R{TaLu{oxfSoX90^+5RxE8kONUqAAvRb&JaU6SLg>R9tQulH%8GL8lYUS> z^_4U{pdaZBQpa^^bJdf_9F;9zi?{);bjG@Lh%G@|Tu)N0*h-l_DY3ry>Ex-Hw(-04 zO+GDB=?oZ5kYe(>KCY7bE$pQD6PjwyW$4qK)u=%xm$A<%?dK9iJIJhM8>`V2kUQAzOHXOe1B$n_ZqbEOzcIF4l5xs2_YFucAeUk;KV`V3M| zF0-m1X>LWiB2MYYLQYkt3{&-K7_?=L>OZ-J2~D4wPUb-dS5x*lanZUw&ZSrNwhN1FS(XI53RF&8XE-~&$;Z$X%~)JoKn*>9-LvrqvA1D=I)v+jQBQ; zuj8{*!-AcTuLgUWn=P%mJTX@6L(`EI($2U;W*jQb-CZ3g#iO_Da7Xu%_K<1Zi7neB zg!D5m=XA)_-OEk^wjGD`w4H}cy+f_Ne9N(rVN85?d+11ThV)d`ka5_Bl%pZt6w>R@ z&)^g^j7-hAosPJW-bJ8*K|H$+V^hQ9*nLKw?h#?qh3zL~aMF~u&ce<-&)qk23AU^4SK41Key!@2 zs(^LNjV+aAoTU&*Tv+V-y6_O7SodkY$Gmj()s`cjU0t;g1qQx}YDY1xZS z|E_f*?x%^F>uDJ;AN%sL#Wr7B?Lx~>Q?u5R($}mhS39qCE|&SN_BCtztEr18{v;=0 zEqS4J&6aZYsVh${8hy6H1^v&>@mI|+n6Ksd&3Wsl_=URXFJIf`H)X9SW_xq$eTjR$ z=Dlkc+kE>+_sckI>U~Z_ZJ>|4Zs`L#`+bQAZtwD%kN@Vbj=ucnegPL}5pxWf^FDTO zM-Tm7M)N-I{j|L;MclU&OUd8al#ZVd_E}mAbl)+iwq)zRlWm~qqO6v^x`;!Hj!$+- z{XQP%fu*VDyn1iPS90p3m#sE=?;ezQu|av)-KoPG#IA`os2Q|~Ibg4l5AQw=L9;GR zqefYE4((iW#PVqC?e6Mw7=;?7$E*WPi0Rf(XEvMEp_CIix-fH_o#cyQi9CQ?!96)M ztxSA}R%k~S!ZCPmYFfEHJnL|7x}8IGQVKl6FHnSo@aDO@T7#j1=Fi*rDt)A8I8KXvNq`c z#?Tnq_tfvK@B7&lA0!ZjIR(6NY5FxiVSqsTO1~0_{$CLoZ!;{(&p;}F-R&P9Bd z@{;0?0zN=oZI3*TN*ZN%?ZHkQ?G${e{t_@_aj&*Xv6OM2&>;61r|sFsO1Td>8N=F; zj|wM7i*TYwJE=v0gSB%tw7&rlniQNP*q#{TKwI)K5TB_jr%*L>X&P9D{C%$A0SR0{ zVvv&~UZ4RH8sLTEC*4Et%NGGHUkDlTj9dU^dWL2vM+K@y$cU|&UKluhfI>`y(>*iw zr1MRkaD?(HCyz37Q?}R;#aWpgE{?JkD=wVEcMnB$!vk<;O}du$#kTpDHA~XfhARzU z+&|yEX0irNnJcEupecLRl)aSxM&9drZxnpB;AY*s``+I7LGgDT-*)`{Hvi5}w#l^n z5<9%+jy03z3r~OU>1)oQDc5hxWuF>+iTk|feU$fsD+j*V=+)9I*0Q7hopz6ViEOv(=aRS+ps7`J zv)ZG19PRM(+U=KD=w^FYm$n^HRUB8qxwJ?v?_&L032IE@jso+#gw0y08z_l{g?Wtz zH8_gla2W<#h|V}(LZd#=(j%Wla^1PTcpPW2_dwqRzhfrnaqZhUZjfLT=6Ja-6B?>% zJMdHbB`VM?Xg;&3`6RIIUr^ft*hOq9=VoT7M~8(;r`tXQ^2gM$%V7}b1dVVg&N)4M z(Fv3d69PvN$Qo`o{tjcvKp>Wz?GGVyc)u78878Mkolk}IQ_e9@$RJFPPk7wa)(Qqg ziQ9&Gbka@W8c=n7`2+m9XW-3qKi;w1XWR33`f}P!`fGh(i4P_fuOt)KY|~&w~A+d7`3;Aqq4*P zBBJ~e{@fxRWnGW`^rg#7>AuV@%VmMgnh*8cHzK*T1Xvp;;6Dt>1)&eLiXJ`EL^CHk8F=WJ3i6L!hK)aM0$Vjmg z4>i#DBrEUIlayXsv=|>9{gcr1hjGgphW#VRa# zqf2m(O+FRU5xAwjbgm27u-W6h=n_xYix+SvUvwcF+f~DW*w`pTh7Fg5N$j}5TW8(C zV#nNh;)pwxErCxycZ5jSL`^QlR<2jsNbX@dKJ zn$TVoGKti+QQ;`P#Ze7rgkj-w$T$H4J5K#L8IohWwut+(U0ax<%*p#Q{@gltz^><& z2J67fxQATdE5tZ1pE>T|P=UX~&h0 z#Z14oDE$1Tu61kT)#EQ5zt;cqnHSG|Wyo(W`<01Hblf)+9vS~a{I$J)bBToEHow;X zO8bqJw+h}Y@TJu+!~vy_vr9Rg^jiym9?tD#0MC|$t9!2Oxq9%*!9_>VQo3p>rKqhh zv|gM1&}#psjzd|QC=2!b&HWTY->D*$^f#NO4Os1WDecE@0+;{1uw^IrzM;w3VlaH$ z(v*O=zt7HS$>RRLCIipkF<4sCu!;;X@|+5uq(g!^a1ZI2f&)Cs^~i9A7)HJVVE_h_ zX;=F+kLlaEQNUGs;3@ICOyF)wGS+w0s$fUsXcM3oZ2f>r^qt8JIx%!6;OL1;CDlZ3 z8*)KgV%GJ6H8K8HHHk{~Novy10gE$D0@mZ^@&H9mj)73^0cLL}7>Qu+$!SpVCr3k? znh44UjrUIc-6M)dF?w;H>*J0VbKDcjyq4g>eHy7J#&IlBKLg?qj}|`54o6uy^eJxM zP{_5HDlp(K-K55$4{==wsW{>D@IspEkG0j+AM2~fCp`c}{(yzq!9z2HqUI+48wm8; z>g_~G6Fd>5h`9aZ&S~dUF5zG(&pkZm9HKT>@1L3hT*AXiZ?*ZVkNGXfe0?W>?WPJoS5c|yI~vhVG?iI5UB`pm zP1wU+U@G-%qB?4q*ij0RC~Bf=9&vhT!xL4O=; z^3BFb8LhuSTsNJzY_tj*zgc>-^i}s#-Je`uuKm-`{%vV^Y;yB4Hmu~~>C3_wDN%HZ z!c~N_?1*3>4nF_AmfOt>b=T^y>q8h_C8 zYd67G&m}b_X})5C_gXT%Z{))R?g5g<F<=>bbh1amd9UvgrT7l zJ=B^^gx*D+&zd*i@>5g%^X?Tk#eMOozTp0X`#I$`XgalOI;Bn=j*d?8#q&~Z@hxcL zS4|?qE0ttPnh*1_)a#y>1Uo6H>cbk6!0G3RTR)>wCjKCe4RcrmTI|(D;?k(bqS*;4 z0mvzs2We(9r`o%bGcaG2L6dryu~LeH-x8v)Zkk$J8j-0$X zr27H;At6ZnHS#*aDd5udYOT~t4O*!SuMB|45~jvSa@}%yplFBh zV3)W&o=?lvfT5Yt({!Z0=~&Cq(bf)#9vD2cE`U}^$6C5t_@=(r?k@2IfW&yECxj&` zh%Kf=BAT2KQ8nSu@%=4|Ljpk`&2wuBMN1V+d0tEPeDk^`>$P34?(*Ba0|&c&_U_kq z1uWfOefOdgT|K%0?w zl*6g7=~u_0@}r!Dm=Vf2hS87UN2=%@K=)$MzZ8k2&5%KtVG#ST=oB;w6eHuTwuUH1 z+c4{lC&&7=OhODjlm`xAXF<%qF+N(~oYS^fiF4Lvav6D@(z`gnbX}l!}Wpq0mCnZ7jVpSO5i}!J9;4%gxiCmkA?~LTyZWuJK{gi5?Q6#pRs3Gk*JjbsjX`F z#68Sk?rcvt-`8}wqXlwCe9MrsvmMy7b-YR~&g9rq`f>F|&<}-40Kjt)unO;Lc4g~} zB1(UpS3T+^&MuHu&?ReT&YcHyp^8|$vx4*Eq{y%5pP0*&qsxpFQF4s9*|DLpJ~To; zT`;$wNR9NWcjTxC4?d9)dXtkPc;aVbkw#8ESdF<;V%K z*E*YywI1mWLvD=ar-n{Mz|9esew&#da?i{PBTkWa7~_cg5z2IuJPIh8JOivuQp$X@d%L&fY`6q^$ zz?vx16efHOkBm6o?vVcRq3(9ZXc8D7$^bIRMYjmdSwdjFyJp;z#2^dB4GZI-OoJXB zN}wq^Bsiy7A`s|5+T;idvDc! zkn-KU_w(-Qv+uF5e&Uv+CIXmdsyW-gE%WJr6 z(B&At`i#3~F10L}vTY@0+lMJt3r291lTxl;e&Od9a`~H#bW>UAz#X_Rr9VN+ffb9ko-#yv3_Od%oYN-Pv0I4wmiPl^0>d{ zlrR3Y1V~2(2Ti9}O{ZChHu(|{d(DT}EGYp?kyl^z>wk8j3x0hs54pOrX>#Az?Krka z|DBq=WBZL8_6-Ih{a=VRPvr9OX^U6cNjto}4`ZhUhmY%_lu<4^?l^d_EG}T~XtyCQ zI}Lg4I0{{)pF2A*om#*sWvHts25iOPFO3{q+pl#p+5-69^69G7L4l<}#3ee1w#!XN zz-8=kBNwV9!OALmMN~bxqhV{r5imIIFbPj14P!A1grA7J3c=Q)xG{*IimR2D4&di_ z;&uxRB4-Gka2w%~e8yD95GSe6aR}`Ud1fv+r-kn$nxCRN;bHYz(u0<~6-!>wQm|?% zSgHsX?^!9{6D;1lTDWfVPwxTY{>qO@KGllJTR2%G@Y zh4Si(jkQSYT!J-Ras6@fUeZq`4qVxb9vW|9rJEnAw`#(1K@*O4Qh9P2B4bCviOkBL z(r@BTm8uL>vNJJTKbR&(jxF=qFYDrfqG@W;j4`5Jzu}QaHG0m;@|P2uK_X!>*CXR0 zQV6l;&f2Bkjm7MedZdR$tfQRt&1F)RVULQ0S70~>Rg4%oj_Paa#`J3VToqHsJd_j> zF&$F4l7^3qEq~0%%h+|i61xYuOteE=;(3d7Qp8~BTKX+LbQnZA-r65ATO#LSI;L%tw!uU(#PmE z9la`4X_T7ROR1G#Rzk$?z$Yqc`;D=5zNCr*dw(X3zy9MuN5r|2*q_+5Unxa;>cQHI zcF?GTPL;&oSGRHCe^b@O+S3@F65OCu3?79f6taL=9Vj5+tt-GU(9H?x zwoQeM!bOkZbc%?uX;uIue&%9G2lXFe2jxPmf@J6icqT({>K;16w>0&IjNPCp9O)2f zcM@jFdy>3M0SLbdNIJ=vi82QUHcrC( zNXFhd1jU`m8!>xwBGlB%nw+{H_;efE{l1`BIe3u}OS1}*t3mi(ZlaMe<{bm@)H zzW&+ZmOZOm_T1X_HwRb&oxYSVueEC}IcpJ>ELkwFr(|5a^zySWKI_Ze>PvAfn0{(W zf$Rh%D3e8^4?Qeq!)Y&+4Hb!e*Xabo;C&OS8+fUwg)Ht@}AC z5ri{eHz!_Acp>51xZhj=aop#hiNcjwMOu9+ZC-2JdQ!^E@n4Q#to=%&D39k$Dh(!W zTS?mHOR8EhNXiVSAd`{%X6`$}yK`^P1sqKu9R6g>-=X!qWb$&VT<`cgBz5k-q zR5YyZJ9JN{$vCW8i2wNp36YjHk+bH_s=u`>kag%byyJdrYdH8zosy`>iI)do9Q?{z zzqM2n7BQHrT7frP@PbiaPfT26T| zr*b8y^240!D~Su{wT%2g#?}SPnmILS&RH?%1kLspvwb=3mfmZ&`^^W|&1u(8`^{U{ zvh2Yu$4ZuCxh$BqeKl+Q&EvtkV=HyXyvH98)}36fJ9%Z+J2bkWUk@u+#M+7>SP_4(AfrZ1Zo`a9NLK z=|%`ml^g-_^nTBXJ#APUsbm3Sw+_t1-E_#~0MdeW$T8NToP&bgZH7Ln7>oenCE!5> z2g1T3ni7$r2lY@Kk-vkIqyw1_O@%F`5J+}vq5i?pIZ}=$ky-f|j1R5n zGHH(XLHSn)Wd}w>70)P=>ky@lb;?mBR-R^FdHI;gD9wC8wT1tTSV>e-IuAc0RT-lp z{in8+FS?-=lN$f?;`&=TzT)m+@rjk<6aM0Xdj^DFb$`i?_Qb`%ApR_D7Wp2hONwW) zKLSNIA?>BpUP}>aNTr8tX`~O8{;|6pg+CwPw2k|tZSdaH!uysE-ut>OOxfx2#FpeA;{igvXou(&uEa);mAt4#T%MLSb4nU%Dk)1qB@-r7s?>Tg3f*F3 zE}#c$<|;pl5Oj4wWGVKxurQ$OWz=hM;}52#%&ORDQXmB=`(VhIOaA{)qY4) zwHN+(M9QJD#x{lS0@dB0;kdCYXxXu9+3`-Ve{Y9xSEs*c$iHid5rdEW5?j6I)^$s! zKWp2A)T3$9voW}ZoYiB{4_c@i_mX<`;kr=DnDrQudz53eS*mmBv=JecHoO21wggz<(V9Y7B8t}#qVcvKpILHQ2I%H~*GCv0lv3h1R%8t}~ z;I~$X`K?Bmf!6`w*7qC1(Ke~dNtf#->i|pXfsY$#we_21&ZYs}Orx-jKo}T+_ZRR& z2@3BK69@vfkgRnJCHUm9h^(#-iu*QsUnlPyn`c^uZm zRsvAi8dOpFNLB>2b#;gVAzF0^uT#IxSbMF!ULC?4)Nd}ee1H{T54iGEW~4ExV<}*b zG!HRfzUVsi4O!uXia#t!4kfa%EdS5#<^c`fqg2M~|~|l{y-GXkkS;(BH!C z*`x(nu;_!$TVUi9)MF*(`oQ)iN--ka6SIysDefk9gme<5hK#&TT?@XP==fosQDqEe zS`6$2XnV4vx&*I~KzaCFiHq*>4Rb}N-=U4*dWcR@YH-E#DSa9~wa>t(q2>|&$EWw1 z_>8{TV}28*RQ zXE}Aa`k608D*IrW6mw7-Zf?*ah4tz9qFCqDV1*Pz>PNYj#ShJ~_Tgzr2|*&vE_Q-lf={|WwyJsmb30;Z5}r+RG8N%=P9-j;E|)U_>vDg?ySpkFO2 zen7%aG|i-n6tap#8#e8f7;gPg_&q5+k>qxS5+kP8Xed4W9$DFtSt>H5MdM{NYSqv? zbB;2hJxrx4v6Sj3cL+x?&>@>tVX63pFe(+UO{&rZ5W)}x8NeD!i})gmQl&={V}>lM;TAri ze9e^6F+356L7Y9px7n+-jKn>H1O`9@m+%97STH}@&!~L|roZY}E%j@r1aD$Nz*Oknve$3g%Th>2By3qh+twA^*7@V>Pz~Jw!v00u za#qk%yK1Rr@fv-JhrH%P>xtWzTYZVU=a2o&oUs_UX3kpN2En++qf7e&c{TJ}y=E?4 z%6KE^^&D^IeqYgnH8W`PN#2w?Q08B1U7QRSK%-@+FAeYci%;D(>C@xy#&dC5uNL~U zE7r{Uiv=cMuUoSgv%O_U0@+7=*5(gpfV9qt*B&L6*D$>naj ze)(pWZ%czOXW#tsg_A4#%r#5ueD_E9yER-w`b!tDpY&R4h;(&EH15;mOMJ{bFz7W8 zGQ&P+G#@klpy!(s+brDoEg5aKy6+cR+p2WmuQK3y#E8i%Il_1tw)PINHAq3}$4VKj zj!7yyNX22UMmq_r*^G$Xsp-=L(=)`#QC)*@){mH@0lEM(9Nw?eag2g7Wvm6(ZNhEg zHL8>{^#$#tK8SKkOodv4m@-VSM3-{ktS6*6O0Ei}MinOWj#XirNQtu;Icb0?$7pRe z@bJQc7U^a3_ z@mBRWSNuaulk4V!>`f$Y=CVGdti*`268aMq90S@5GL9s*SFzwE^#|+e%hb~-I})Rf z$h%+1C#lO~>CUipaeT7+3!f74_DS;uO2O)q8j(vH%1@67$v{YEL`W7wvLixr5R$77 zsnM&X+?3goC-nyG?B=U_aMO9f7xX9ah5c4OOI-qQM@UXYND)Gc)gd(ohS>qR5uJjc z4$7FLYSvUs@swX?*B4Y0SU5`@RN7G@wWBW)cpttsCaUnh(yow24ek**ClCxL(5@iR z<|Yt$BGBe_^i6$8=bV0$%u0>Ia4-9H3NW)bGT|J#FlSeC5+Qon@@0}9fOw%A2U7=T z*YIU#Hcb=?AZwqZyBGxv_V6nM9N(BMG!#*$yiZmF!_^4oC_$n{2vzi?=#aVM*!6|{%*ElUu^Hi_bJUT=%=ke6 zLh8yPQh^j6$9SRlMgT#NOo%}YVE!r%CPBC|@&VOwW-Bb-J$nu!IZQ!A2T&abh+BJF zNE^d5D~Q%w4^iT+)VXKriBZalY9Vxy$6)Cl^prvQ$G1YY=PA+X9T`px*~omJ$~2zC zrUIkmMPQdG>OwJ3L zm$MNPNYjjfb9uZ(vwNV}F`!!kyX_*&DT&Y&b;k?Gk;6Ft+|vMNAZ^mrz1lSJQXfoB zJFyFOypmp`YV=vV$#s#ygSF);osWrjWJu9hu{Hc4r z10(YtKQ~(zT3=0A8V+P_4OqAO%#P&)@0Hv-|F?iSlY{2+6?6Fw-Hqm(*})yHD?3_! zm2JUFex;K4RrdL|o>)ifE}uE~)!wB%Uw$>cwtBO62duk&<~{GF-a6@RJsmi3+P8P` zZk)kpoIiHg%BAHjHZC8(IpR%em~UTC$h>;?g|myt-aGZJp>GbYmsGvc_AWI zZuoM;;@E2!U%BW@t%QI@CU^mX%x%j}fz0aVCj*(g7p#zPdAZ`t6^nzulnNj#X}OE- zfwZj)acda`i*tdDisc=FjH+cIFnbm(KeJ}Ox_{Xk+`4aN>ptJs1HRmYe(S+CYwER5 zB5l+Mtd%#--o34X?QMQ*+nP1)+S!1$bg3WV%Vz@CeYcJWtSxu4y!^R9>xkbvLMaiH zzXl#fmbcdPT_ATCIYhkfbjFz9XI-dRZS~ZO;saWNGuDFR}6V39q@0 z!Q$f1D+8TiamE5{dt@f0lu7}XVzCM&a<mmWsehpJ{W~0WNqQCNH zghbM09wV7CEsv2r!ymD4P1Lsz>&E~mB$Y=p)dK3O zWcI%XuOG)(RpF-!Vwnf#V8i&yDnJTu=TV^?hmx?9Jen=SE_ie4fSuem6R=(krCtdp zIP1`UoEGcDTyCt;v!5Y&`{0FabeTX)SAQ#HRy$j|`h@)y zYHDuj?>f=ZAwmkoL=PFjQJ8cI2PkDWc|@`k8sUMVPA5Is52Es`2tA2%K({CZ{^(EA+8500$+<5Ve7Rt$;AX?E^S1`QJKFBFdruB~?dR4@ zcfQy5t?qAj`}@uY`^HxK#=O&R9I7d^ng!#}k}_UScrhWERJxK>>PspQCRMH^Rr->u zVdKz}ENZYVY5bO=8>w&QznOn?Ht0C8>NpTe&0HfX&MOJmoFAI=ewo0fcW9tgX--@i z{`}=@oqki%T2k54lgquPx&7pw;qP2tOR8MH2(hH319v%1+<~8m&-=5>bGIsP z<$0@)-m!R3OnLJzuG?$gi~pAGn>K&XsbJ5!m7a6n^D_hnJV9VULh98sSIz_zidGVe zdEFs;{xgLP|L?((-2IG=k8BAr`qP7W1CR4h@4T@wK|m%nwWb2f5DJ zId5{(LCJw2edZJa1f1TZIK7QHi)3<)6iYcJrJYeUe~-8_Bl^K{MKfZBT<2lbHD8G* zJxQmm6MYO~kbDv3AE2gzjAP$?T}HXF5Wanuki2J{FSQ_c8R96%{uWEkQ0c>5>g9*_;w1M7qlVS=yBjsd zI;g0S#f(&C3T>EpbJ?)~tZJCiyCAgVyU;StLhTJux$L!QBHY$blN^&ZfEe zSr^@jRfU}9OjT7NzD-U;+ItI95@dxp5!7-}R<$!OW|DAv21@%#A(|+O7ulbnd2Iv^ z?Q?PQuwqKor})YL6(DTaT!jotA%z5)Dvb5<*H7oU_7M{kfT`$r08rZPgP(C+aT(wQc|b-0(TvC=?I#cP2`NB#s;vJW5b# z$k0QERD~`)GuTrAlZz8hkoP2V5H+hV;R!aXho^CU*dT@WQGnhx11WSm*g|^BA3Ztc zbh;Q|%K+7o?XW-^PCb(n?|CmJKFm^F8pdT|iG58zqhddR;l;UsWn^f;z@vK;Fuo!d16*Dv0yc^VDZw8?hhKi z+xULtdinNXdHqUx{RcVUEquQa0HTf8fUcx3GL){FZ3LYL&ABV)+_l8Rw;R2Od%eA9 zz1E?ib!^2t=Ce-R*JeE-A{^Dn0d4$>e?($`Hdy~3x?ZDNx7uv4PerU~y)=f%& zm{KFDKc(iseDKQ$*Aj9U=aws$^St@HZzbN*d%H%xR_A(B;f?s<)`pd>4IeZFkDORJ za>9G&92qEyUobi%o2tLK^y zcG@-HhK@v=j{87+cng01!I6Tnf7rk4cs2Lk9b55yCnu%Vz}>0T(sOlkt5$cXk)FS& z(ct-eS{*$b_Ox-j@9k&LjT$_EpQGpR>ry-Ob>Gk2)t;dLfmMU&A0#BV7ny!gprvQK zhMtSI;rWLNDIIqG4+|;I5AAz8b4>r3Npb!$TSLz|I(p8hJV(N;OEiGv2#>)QiVN}J zM^fQpvBJTOijawugb_*TehfK?^fD&itYXsw>efdxZe6B<@ts=0mkDNa{H7ETmgN{kZ~*+huE?T z9NIn~hT~;wUo4s)(imY>4!b2e;5HA74P^r*HWa>9qoWEALc0&EtC860 zDrNkK#m9xS>iGQz(C6ZKvzl!P>@SXwiwKEFNW40vMyJ9OlwOvFOrb&&$Kc|J)CWSs z5%qakdm%y`5nqA|9ahGh>zYdiQ)ULn-=<)BPNG!JB`bkp?$sP%kZ?SESRa{$?o>B! z3jYfRpbwBj!sPfgSqTGE5|imR!;a( zk{lN93xfBCVPzCtIY1QPQQ{6$A|{adm`Do~NnwCG!(FCgWih6#a1j;w5Pxohl|T`L zn3yA2+^|yIaO=3YWgxKsai8tv{P8uDZPk&)w`jYn${VFMF$z8GJLg{(ck}uu#bG=~AmKL-XuGk8LwxU&A(Q^7*d2i;uRqz)D z>=1AEWwdycV1)_ym%V`d%g9oD9G6mmKb}-)q2ucftM7(2dc6txi?e=x8Qp}ne?7fy zY4TRvT6sMw8FxU*7?#T70E)bj;LX^I3(H7}muWwHGsQE}i%8=nRy1L4YgG2-!HZ5tX2uv`+aFPkYU$K^^q!(?7bOyh#xs zn)iLo%!~bfMM6sk_nnN|qh?)%a*KR6e0{^&AnjtGfWAeyY&^nFJX*_kV*MlS@*-z~ z>d9P>yjw^iBzrB@RX#_sZ-5R%Vgk%X7@F6BHU(Z18S{nkm5ir8Km|re93IB$0e7a~ z09uNy_Mu9fCY7Z8s=me$9^y?Z>Y5RTS02op>14fSj3rjI87>}+A48h}IjNsUL8(GN z6^Wtf$(W@`y{jldsDzi4K2{m~RJC0`%cL^MP!VoGQF=B0U#VA#LiONYt!6DRlWK*y zGWSag1AK!0h}@(0km^)Z`bBlBDe54bm*+AMJRdVJmC{o`F7$LHh!}mPd|1Uqj-!qy zG#w@|&@>U#H;)W^sz?zLIKNv&cIWU8L2gB`9QjKZbuwWTrMz=fGv}a;0e^0LbfTHr zDN)*nZmx`xZmu9IMOc3Q;-s6&y}$-20j^APM8)k<*fs2olbJu!NH^o{Fxe0?SVk9& zA%7srI$Qu1%B0K1a7%aV(N^dJ<7Uvq;Y-F2QHiub9C0EaNF~QYJSgD^MJS@~dk;?` zOIYJVH1Q?;Cr@h;nZ=d(_{ocn% zyg8%m1y%2ye|P5XnIE1HcKzN;*Y9~JE&{=}&S>VFe3SuDa$cuRF}?0bD*u%vFK zq|R5eJ6LjXrQ~3+no%4Ri@@9#^V z@7&1wF=1S3;@u-q31-RTp?<6ls~@KVuGS6|MQR51N2q#2nHUf`Gv2II8#IqCdJN5> z2-VO?Jt7It?^i?2@)@f(MXff;`2f5wf{(tTT%#I7k5n{$NZbhczfO;yg9B|FFnstL zI4zW+Eeh}(vMJI`A_&L?tW4lOOaRHX6rM%~ zCLnL_uOZny2ZOljLmC)#NHYFY+S!HXyRa#n;7uzFB$h9E0*RHY=E@(ZoxKY+X*UzbJF@#Z8Ff8RMsYrqYxT1x`oGR3)k_I;@H|a*fMAAxQb-Fnl)vUYNoaus;*hga%J0*3W~m=-X#pMkb%Tl z0y{vPBNPH9Fhgp4SZ`LTqbx}e#5i^P&bikl(*1>D~_XP+4x!?j^f z`^-7FQ^19?bd`}>xHM$Ou{tA=%3W!!xa_M+0glP+GFg?RjJZ!Z|f6YG;811R>S1*a?);+_ZkeXtcNk0oaiHMfg0@r_#ow!vlmsO_t}L zCVRpOnTvWqRDcrd=8rOST(F1()56L5i(`;8fo%qM(M(^^QoL#@UUI%M`TC@H`ypR> z)9uInhqv#PukEyVXvE((!nU8VcqL2w^5lz?Fgcd7?FMeEAuA{ErGKmN zn}v7M@5}{SPX+g$UfnC&y*}hiZ1S3$VCd@$mp^yen_T2G6)#PAYZ?RPhy12Pw_C_k zm$z%!*L==9HhmXItLBLIUr4mIS$m)3TJXaP59@cWSu++5i{}F{ezoge$J>ruExy{r zw~yaB@3oxt>Q8Pw$cYZYcC>!X;NiI8_(<~S#+^)Bfukqc?!(1GXB7)I;oS!#3c_OS zxH4+WYuQQ^ceU7Y<(ss4CjTGa)ndn$cLd(3TBkH$nB(PDjU6L)4)RK#iC1>_-uMUi zNsDImDYm!C$awfJ3Wh|2p&t)B6>(&zg5RT9rZ@^zTwOh8Y@ZeD#{rAy1Me4=G>`&Q}>Vx}p^!#s;Z>rSJP4Yd+^2IC?U2LK|pQhLz z>5YGc7NzqU(#b8=L3)>Gt4Ksr)>Nhxb3*fgS(7EjhyeSw4;~H7!rzbVx7b3?e72PH zgL?{jK?efloQL&Q4o>;f9>35@W$+5^3y_)^B-u=PCv% z>$lEdYJ=2MW5t&o875VJw9_DkSd#grfb8%Wfe_efn>+lKEEnz0cqV5N5}q&l!VA7XX>hfV6dO^Of8XYzdRH*GSX zUt%MriW?K+sg&~FWM$8=oPb-4F4oG1UBF89J|X2Qk69;cn<-`;s-)O_HQRAT*!N+l zy9yEllR>7-NVS9&ecNvV`_xEvVdGNBRCYESm+ew~+&-f8wfPf_OH82-{&zJl=0_Tr z*mc+;wT-X+EypExzQ4`5?3C)l#>LLY1wy|Ouv_etp-xKq;29pKLn*6smy`z7pWV2X z34e|z;jX1bx zI5C&Q(%8>K`DS`}DuVdKC?+HFbi$M-k|mnU-TO8?m31#{bL#7jd$m1828ePB-&m1~ zj|N{W)E8lwQpZ9*he~CxM?My%5XRl)dbO~xtl}qb%r$KFt6$_3ko9Mh(0x-YXwkf` zHx0~Gtw>0E1$ir0g+S5%5(N;A8uV%+g2%GxCDW-{Z^p3bwH0u>y9b#_7QHHrztn5e zk7^d0$&Pa7wb_?1zj%2m?JLjTz*WPQesg8eeAa6|`_AEaJHFm=yU@!Icpo1O@MnB= zXZ_pH`pswWX({=y?-9}E`I@Fe%{MCH35jS>-%(K?K8|Ke3BJc3#QzyzRD;O>s|;fC zhCz%W5Vm0e!;?+n`WXxs?Xzft|V&Fg>}^;SD%eigGk5CB}QJg z$&7SG3@IHuY+d5CRj#`PDz9FZFn5C{i*DlzQC&a}jRUss^iu5DT2XXG^TvF_EA3F^ zN)!GNjRD(-5p5$^m~=U2IZE4bIVKyay);t$@R_8W!Nlp)#F2U0+3-kRPxaZ#{FX9t zgl_O}4gSU8t(rT#d{vM69c+BEc8Ip`p0hWZHGftHZ>}bC1QfoO>f+J}Lb$;N0d}OBSP5VfuE+D2l<)k!YI7Det5#e#D5*L2zFkLneGHa`oyWh0Ex(h+&qu-+v=3 zS9l36RPV5?$q!hs?KBI3DM~x2u3IB={KWn9*x6h1moZwA%O;F=jR~pl)@``4aKo8x=ix52};-E^0_%!vUz=p&hCJV=0VR06T0O8O-67>wdwzRapj-rMBeffv%b zVaM{%>Gc+Q1P(E~l%nZM2If3R6|e}Op$agm4%)bcm+=}(pLPOT-5~zUCj6gL{MX3) zDtUC&i1VKWK4ptIha^Lkk?^f>1JR3VMFatP{2BsOo3c2={K#t=K2w%AZ>Qh1^XBZk zm*2h&8<2ZCy^js}_Y6o;3Vo)cr89n$XrytsFL968yoas?YQ55W?Yz%g1jeddmd{fB z#>k(IKw4J8zW!c5c(@lzr_d}ms>~zBr$8z@LJp}ao*y3U)J9Db_Np~ zeF=>+-*l_bTni$?hLAJ1G>W$mM+E(f)0i9onY^D<-4u2$c)rZw zB!w=v!7yhqW7|r`wq?jY?|26WHutPr_ORxtKOXgE?0g4|-o2}qy|V4f8>!1DZZ-s} z4hE|ZuT&lO9_jP;KklnK>2sV4T28N8ME-BssO4H>>eb0BlU~>zqHD0Igz;BBK})I6 zQhKB9<|Pu|y4~p8)9XL+c(C%MukvKL94|P%du}!cw!y$=v(M7ZQoelQ#S2SYd>Iut zn!$bcS$3?O;urQ@+Y_+l`b>G(b=Ny@Zn;$xsBQC=w)^ZIep5%Vr`Olhi#wbFsQoOf zaLEKCN5QO`m8=@N#o9o@h=TvC4~g|FPWPB5?w^?t;qK(z#XYa>du89!Q@&iB@xi3s zt4X_Y3D?3?0aMni9XD`KQqJPBrS3pp-Oce^mu_DQG(eT6@XB>S@4{w^wZ*r*_^lN!V z-m@+R5bUeD_b$*XM5*eoz`LKek$a z>_Q^I!ki3|Z7fZ=c`x42x zD0N>_v)|ObhK3QYTL4IH!MbM2q>d_Ej1Q!gEHwvG95AeUt!1$ykXE)d7D%fi+fL%@ z@owE4$Y_9seqKD*yfq#Y`ec>qm@nnH*9vt7r+C}%MXTJz5PQK()n7XxPFP2{~ z4cfM>+P2)7yH)ljKILoZ@$)BwHT|nK{p+^0tIu9}cJaxj^MS%0L0j#rtriwJ zt~Ok0xHjpt*q4s^E!*CyeRu!Y_usbpAM5j;I2}AOv~ps|`+F1K^HaVP7k!US2Y1e_ z?wnz2f=#}Z!(QuQR-qEVsbnoJ`{mXbTNf|*(kd3>s6vkGeL-8rs;%Okid!eYb>^FA z?lkxg^!pz_9jqN(tsRsqba^=^P*@kV?OL_%!VP!KMsB{(vc-v?p&8}C zw1dqjvXUSf6onQOS7U%oz7=LaDMvWG#VeXoJj~of+M48cLYdK2E@-xpTsac~ z5{^*T-F|$)ai@>_Qt8!-7b>oec?F?TAS?EaGxP+-B&v6#m$jhUtWh6ro?UF#Dy1A6sl2+a*sHPcdYHa)xtbPZ){9 z8H;jW$6ALQO9+~aa%Uq%fMKjm6K(1RA(6MO>zP=sN}&Ze55dB1jbeHVi-C!e;XFx=i^NCGFReg_9 zGnS)?_-RUxP=rj9>hu6LOM@y#niN<0Reg=2vJ-LZb+(jBNyD2TaO-tkU3g0YFGQ)K z3R{%&0Hu=OWRK>6>U!*4~Z}fkO6gFT7*-WZoykb`t z;mUp87M1D>*RxehtNbd~VWl!$)_M zy;N{m-VG%%u5K=7wUy>zzisn-v!%ix$nQ<)EM_zTLuUcT@Dt}NwMFg;C#&}Xsos1d z-MP&tE0P>!o`$#3KWXGY(Ta?<#*IqPsn)K%%B8x{ELDAvwYE)4PG@m?^joBMjeDqF z7?;@XOZjxRZ?IA78(x-&gPM7h6xy5Hn&;cdI|rd`dQAGJ{7P$0?kPY> zka&RQU?q<96diY5+(G31NwtIMH|}rJszXci)wDmAJd-lAva}Lp2Xpxdo8`R7O}`?} z?H-~iM>*aul_j?d`UU9CBIH$!N51RMZ+$>oSq5mQo*D0W z6)-nYC|8mz*_Gl-?N1VQg4(ff5B5s=^(XZwW27SN-1n!*XL#D9j0Z4VA6SxmxYjam+Pn+#1bHxS>iANXY*h_B>a z>3!)c_H6>i#!fyUe&x7<_b~M@BHngv z_`W0Jd+p#EX`HaBSvQP+&6sPsV-ZZNi`TfzK%we-}T@4*zRJdq*lCM z{=bb=Qgs?V;Jn;01IwhD)lA_*C~0(Oxq#tIOG*SHE%3i^Q1ddqi*6 zvt2?rdWp>y%ox!u)W{*v>8Tg2FfltM1 z1D?{K&2Np^+Zxy``4YADp(n+@-{nd1spv5}IkK?|9PH#Mp_3sJ6N)@J9`%&`RJ5I} znLOg`SJ9~-M#uhtk0Oq-vvY$7rPCVb-v2Aja#zMfb}m3mc`zi;TU;598Nb_+`q#u- zH-YjJ_@u$pQa|)%NGCw-)f|P}$D9G5s@}mo5XVxvJMq8I3kmxF@AINDLp?9HDK<`m{DKx8-#&LgHF7bfez=RBhpq!zo5x;nfSZ672bZ$ zJlCt~(cm%4k)V@=o0V{HYMC5P-bYXY33Ds{4OgfF0FZt$e&68PoQ!K&t!s^;4RcSikHy`m(Jx?aCk z-Gz6linqx7o2YsUe@mh7!CO#Qqdi!8WTo=R?PGWP{FOYbhG0ZJgnxuLryJgOu8W#2 zJF3=B{M{q!rApJHSQI-FQhIffqEN9qT+u1)*Q$j6OCz<9AOJ<8(Or;-%Ed4AkN~NQ zI(=+}n0QoJ^53qK{11^yJkqP&Fb?C+Y3EZe;o!^E1C_{(sRekq3M<1;tng=G+i^L4 zc`o2+2s#d~I1b+O_#Dl53?Di=eZx;iOG<{4?TeyHa315RZmxj5jCVw_QM6Y-AE9`K0f4%J*}#)1y-x?z0`46$IRdW>=?H)JTOdQ29)D z{0(}FAy&tv@#l8Nv;i1(Q!WYetMp4nA$+=gl`LBipM%-cPAC8f9{aF;+IcD5!S+j& zo(VhdvT#q>oflo6%ZgF3N_*Jwb)_8^!oxaNxChTkcFE8SbUmDCHJb_Nhl)2!3zC%o zklK>h5;c>A-$P~5v1R3jhy;H|2}n?WZkK!3<(itL8%$!DP79BSeRvFVboQ{S7(@_7 z`FcfbfN2#l?LftVVhK%2vZ5&bDS1ri;B_jpL@Ft)rXk&_jJb*HQS3~kgJ~zs?buMc zaFWP+g#$_NLAXvOY?VroTW?pXe9(waPBRSv#|~y8He?m!f$24-H8EU`yVnozpjIT2Ii^OT4zsxw3HA)|OxkMIg*NVYCT zWARQrIr2m!@ay#HJ$SG^NBV&59+*%nYtxW);k0-;%rKIL5^y!olan*E?jcO7iy^E0 zN_KT^>?WB}Cl(1rVrPVqo&uPmdFe()rU5cCLE@h<7e93Fvd8Hb@~LVCYO`7yo-Ez?aJu}`v$}UtESCmbTEB2wNUqWY{xXb3D;j(^-J=#eM zCv>~FvhfQ40m+}Gt@f|2qP|IT`b!gEDF`NSSxMgFOLhd4cdR7uxLFiPuERx+$vKOs z0?AwO1he(!hXXl>z4}bL*S9l}&)(6JwOr?xPJKODU zT1BQIu{Und_sKjQI}K>;?|$ce=X>1maqi`ud(QdM-dn9dXyvr-8wa8i#dlj@YQ4Gr zRp*S8&u!x+?dvGVjDxovukYdH<~B^?n>r znO7D^O{Kx2`E<_IHq{a}mj(CDS99iLQ?1dAe1GSyt{-&$c$CZ7JGCFfr?`w#$eC~s ztYllF#sdHO+e4hOZK_$&nWh>A9F}E0)=t!v3nmJ~e9a*)?=WXNGA)fV*|X#RK^((I zty@s%+-m#lHr~pj?x9HcXsCOX@4n1g*;{S5+J5*Pe58O~C}5+EVK&)s3|LY5K4jd_ zGY15tX(fwFbuFKjJn^A_^W3W!XD<3jcv<-qPyK)ynHllF@V?Bw(kP*H7C~!T(B{KB zfn7&OU=UTEK$)Gq?xc`uUMbjoW6P6b%9x9CPjH4##B@g0x&DfPJ)-t3s6B!)eZ{`{ zreasYK3xCuq>ye4lm^ZrPwV^yI@K$r=lEU09e0xE)u{6vMCwT&q^G4HGZ1p9glB4_ z5MX*=KpJM8g5Da@mxS~s5xsjs??#@Mh^IZ|X^(i0FL;he8T0EnD8bLVxU5!=If#ye znaEK1=rGR=e{8Uz+z!rg0x>6GVWvC&;!pi1UTuRpdbN6{diHcgYhTdX1?yF8J`vIt zM0F5@JJ87M+^`zc<2O}!30mka+-oPkSKJY@l!w*jQ3keKP|7o&sCK)5Ja9guty$34 z2$@+cN4C=JAu(mx7SWf6^rfh*19jrV5a}BU^^Ks5SGc~bNOw)pXGiqqA$>XWoJL(2 zAdG6{;?>Z_t1*hY4sI$RP}C^hC;~{oR|J+QRcRqz*>v-4pT99s1+kkE&E~$W$E7bOy$Q=OdZ*3z_voUcm~m*-h`um{&}hOM+GJXYPpR zI*_xU%RPt8=RVcK7?yRE20S?=CVFTqjb0Z$nP~quUR@FsDK%M9OU>JxWgwnD)`E# zd3mJyY^eDx>KQkH{9F@-b~Tx_3*g~pn8#R zP53lDD1v#bIwS%UFvnpLoC#-0XM_34b$#PKj&J22;o+Y+B>t?h+WG&Sk;yHzx(sg& z83q2%NP#z0;6)7sXo%e65WdAp8a6#eWi*MVlb%_h2Jsfwr;Da^8Gl)e8EnPPR?Jmf zwo<(lSD&?owfTW$UhBe!P6uWV_zMG7!P9pd=c~}M31s;bUiab#R1y}iDc3AC->I(z%h_xZ0%TqKrx;4|4=ynSZ58S z2Z1@(M8Ru7MbI8G)-D)pg>9AEu`FaPTQKg97F7g?BA%lm&r!j1 zQt<46-R^0cPk$g4+1u$^HJCD*T~Bi${nooIrOfs~|64h+1eOn%?Ty$?ar$0S z)C4j1;6Y^aZ0Ndd>s@Q9qB39%z&N)H1yztMoH_X1t-5Q=Z7<6ym3F$~&sznpZsoLO z^Jnt&pM#T(MAWITf1M4>+z)-7@b}9v?cNe^ zgPnROvq9>uPm#`Ba`EM#lI-N-X7J^%Nah8aUm21y^s8(gxwr}P*GbtJ`gJ~m{-tU+ zhTf~v;|ogl64Fp!76un6Lc{5t7~&I+h7DfQ>ifA?jEyJ=jgT$DkWiS8FZU7^7j>x^ zTFl#ieVb%S?log5T0km%P)*AIRpu?6v`hZYm51@3HHBL zUM1QG8h?G5@cLQkPoc8Fmf5j53Mdc2=X&J7b8E*BgoI}1i7+XiAg?B zAWl8K>&$t}NfUG4$`o(COf^?8lH(@l z>QeyDl+V>0NHcQ{cCXUAr&9csWED$Y#qj^ZEF?EJxKV{qjbIag^|z|X17A^3J}8Dy zhMDIXT|)gef&Y3dfW~I6n1ffg#_$%k;qa`X)PIX31 zB_UHu&=zs;3%U2rw?q!~gbwuZlHO>FX*QipaUzNHS<(60Ouh#51~Uo6Nc^5RkCrt@ zqnEzUDXNrB zQLULof`i!S&-e3FhL@Ic^t>1 zo-jbHaB%F{izA1x2Jv=4EDvT44*|LyAWg@ZHP=KAc64-XA{A(Q+3vC4!33OweQ2yVv) zoRgQhHh`tP#0{aUrJAT(KYcD@C=40EF^r8<7a>{E=TW8h<|&k70Us+sAyXmX!v%gL zO11r1{CA4KRX_yJ57f*f@QonVJoK4h0e)F&+QU$9Gnz()_-#veqg4EkR0?@`65OCb zegX+a(Kul!o0KT%zSVeOFUh?3@RP?R^Z1j8?~%;i$3Oi!$^87MkMG(f6Q$jK-Qeng z^|?SK-hIJ=xB9vUxSw@hDRpgtoxZ^d=O(H^sP?hZ-q8!r;md3QZejxTJZ!(~ira;i zf^c9!-virwV8}PdUMI?BTk?b&)4&NG02P-H_oRg2tYa*M&+$P!6Pl6HzVQqFK35`2 z4@Q|*QP3aJwG6P`mj<6If#W#P53XM?kN5k=aEWoX4)ij9VXW)iYWrU;7-PXhEYz@x z4j!}}TTJfi>Kp9^ZO%uqr56==X#Lm5E)4as_;rE33s^Q6=ZK>W%=sW z=~~xqBveGICwNeff~u3nUuG??;u6e`t;3Dr6TX;?b#S1m0v~3H)_O<#0BE~@ajg4V zBaRWwnsB#=a1CtU%3^jvIKiT#?Ya*X_Knv-O@?2468ffZ<8l@!={R`_CkJuz7AE;2 zPJRT*qP+21Z~rB{84_%UsHohy7^|=nag4~jPZtJZi$76E5yjN&82lTY{3A~8;$&S1 z_Fou;KZKvCyqaZAM`&aeYfT0v}B5NG?V0uy{*bxb6e77FIbNv7QhM)!YnG9~iQpB^P&<_Z(oUP_+fCnTL8l5wF zEJBniCX;GJH&ijDOa~ydY@liRv<"$tmp/fifo-input.out" 2>&1 & +fifo_pid=$! +( sleep 20; kill -9 "$fifo_pid" 2>/dev/null ) & +fifo_watchdog=$! +if wait "$fifo_pid"; then fail fifo-input; fi +kill "$fifo_watchdog" 2>/dev/null || true +if ! grep -Fq 'delivery replay: input is not a regular file' "$tmp/fifo-input.out" || + grep -Fq Traceback "$tmp/fifo-input.out"; then + fail fifo-input-error +fi printf '\377' >"$tmp/reconcile-state/invalid-review.json" if python3 "$replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ --candidate-root "$tmp/reconcile-candidate" --scratch-root "$tmp/reconcile-scratch" --state-dir "$tmp/reconcile-state" \ From 75d68caa752ec61bd53b507b79807f797f00b437 Mon Sep 17 00:00:00 2001 From: ci Date: Sun, 6 Sep 2026 04:59:07 -0400 Subject: [PATCH 17/20] Remove generated bytecode from the replay slice A local compile check left a __pycache__ file that the previous commit swept in. Generated bytecode has no place in the tracked payload. Co-Authored-By: Claude Fable 5.1 --- delivery/v1/__pycache__/replay.cpython-314.pyc | Bin 62704 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 delivery/v1/__pycache__/replay.cpython-314.pyc diff --git a/delivery/v1/__pycache__/replay.cpython-314.pyc b/delivery/v1/__pycache__/replay.cpython-314.pyc deleted file mode 100644 index 615ba48f5a797dca5e01b2220f38e41c63276aa9..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 62704 zcmeFa33OZMeJ^;ig9J!$-xqL|Kyi^MN}{NRqC`sKPG3@_B~fB25?myTgfBqLV(gf) zn}%}IN>2Qq>DE)zsZ!G&wU!f4t+eh$PMth0^&B_P z@tlV{%ki34jYorDZL9XU&Z8?7!}K0K`)2SM*ssxJWWOel3BS5l^YJ)OTsUsLC!T$? zcr5JK>anul1Wy9{P4pzPUz^9qev>>&>^Iqyd?uxcs|#1FXNy!(<(JpjX~Z{A+L^Sm zH#shkJCn{E>L2I0(XY~5E_cS%Q>mnoo{G3LnfS6@dRKnWWaFEOM|R&?Z;#gblMI3LIzD#J&6j97GnS7j-+Ea{oRWI`Tauj-|0`ZE~;ZluJnxlw2 zt5Ndg_j$JQ2RxPhK~L40_@2W`ouwz#!79b~R6j6If*hxYPvmWU5}(Yc@Tq(npU!9C zHxqwZe0Gh=vz>4B?BEZ1YWXJ5&U6m>=VWqx?wLBdRlE4SOwO~L&xgNlW^EG&HJ)`5dXDTDw#_vFAJ6{XGgWn0ildpr{#qWaO&F_Za!|#Fr z7+(*c=l87=NbR%GGMm*m`=T(NlhT%bxJ; zj-CZ)$|tnE##7!@KB<}H#8Mnb=Gyt$Y5UBy(>^&pJmQ&r(phPr85^6LM98Rf3h%;Y zyWn(94PUljn)FP}%zEt3r<^0Rp2_KP`^fP0=;Y|I$7vs#8Ff~7IW(cfk(p_a^C{28 z;pyRVr+^TB&#-4Als3e-^mH_x9BSrU`&;;-!zcS%dPC-+p^L+l(?dg{`0k;u?xWou z9o+*Vt#EFxG+gMYKy^TwBTnb2+dk=W+uf6MPW#m4#Ys;{Pj#7VjH+paeQIWS)Hy0u zJeA7s5blYdAho2mtoEkni=hMD|giH!DlqHsqgs52{p5Bq2}D=bOU)IW5g?keJy-vYgbdpP!HeT+0z%&Y#&i3 z2gikXTJBy3Lg%?NoJWH*LVLE^qr=?L^V$;5p$l1fHsCFSFeALlbvX<)PSe8|oit7u zv6<1?DSEaJ4LvbCJQe<88yXs$6x<$~JJU1xoIW(<_6&O_M}~$y9%1s_tjFmd8WOTl z^gXInC|w%2nkRSEh~r)Dx-4Yk8}pj+;5NYf1MVk!vrr&~k4Vix07n}<{oQMWH_r{i z7hO+yAxQ7^Tg3Hq(t_-voTHp{P?d0RYP!%9rio$q#MI=ukkLIcT)S&`C~m^})ac~6 z)9t~eF+4duHS2U!ZOVm0_#vzKZRp%(v{j&mejI=99C-8Gx+Uf6!50oLmia99prvBP zQsJ{ydi9m8VG(UIB0g&qt?)-_6OY54*X*G*aGSTxq3e578;Wz!2%ez}&dcslyxZv- z7KS}Df*T(+l_5<7nyx?Zo|z6AMrSX&+##cLnpU__hTPTdjl&KwGd(#nJSEiPQwM#f zVWP&`t~R{TaLu{oxfSoX90^+5RxE8kONUqAAvRb&JaU6SLg>R9tQulH%8GL8lYUS> z^_4U{pdaZBQpa^^bJdf_9F;9zi?{);bjG@Lh%G@|Tu)N0*h-l_DY3ry>Ex-Hw(-04 zO+GDB=?oZ5kYe(>KCY7bE$pQD6PjwyW$4qK)u=%xm$A<%?dK9iJIJhM8>`V2kUQAzOHXOe1B$n_ZqbEOzcIF4l5xs2_YFucAeUk;KV`V3M| zF0-m1X>LWiB2MYYLQYkt3{&-K7_?=L>OZ-J2~D4wPUb-dS5x*lanZUw&ZSrNwhN1FS(XI53RF&8XE-~&$;Z$X%~)JoKn*>9-LvrqvA1D=I)v+jQBQ; zuj8{*!-AcTuLgUWn=P%mJTX@6L(`EI($2U;W*jQb-CZ3g#iO_Da7Xu%_K<1Zi7neB zg!D5m=XA)_-OEk^wjGD`w4H}cy+f_Ne9N(rVN85?d+11ThV)d`ka5_Bl%pZt6w>R@ z&)^g^j7-hAosPJW-bJ8*K|H$+V^hQ9*nLKw?h#?qh3zL~aMF~u&ce<-&)qk23AU^4SK41Key!@2 zs(^LNjV+aAoTU&*Tv+V-y6_O7SodkY$Gmj()s`cjU0t;g1qQx}YDY1xZS z|E_f*?x%^F>uDJ;AN%sL#Wr7B?Lx~>Q?u5R($}mhS39qCE|&SN_BCtztEr18{v;=0 zEqS4J&6aZYsVh${8hy6H1^v&>@mI|+n6Ksd&3Wsl_=URXFJIf`H)X9SW_xq$eTjR$ z=Dlkc+kE>+_sckI>U~Z_ZJ>|4Zs`L#`+bQAZtwD%kN@Vbj=ucnegPL}5pxWf^FDTO zM-Tm7M)N-I{j|L;MclU&OUd8al#ZVd_E}mAbl)+iwq)zRlWm~qqO6v^x`;!Hj!$+- z{XQP%fu*VDyn1iPS90p3m#sE=?;ezQu|av)-KoPG#IA`os2Q|~Ibg4l5AQw=L9;GR zqefYE4((iW#PVqC?e6Mw7=;?7$E*WPi0Rf(XEvMEp_CIix-fH_o#cyQi9CQ?!96)M ztxSA}R%k~S!ZCPmYFfEHJnL|7x}8IGQVKl6FHnSo@aDO@T7#j1=Fi*rDt)A8I8KXvNq`c z#?Tnq_tfvK@B7&lA0!ZjIR(6NY5FxiVSqsTO1~0_{$CLoZ!;{(&p;}F-R&P9Bd z@{;0?0zN=oZI3*TN*ZN%?ZHkQ?G${e{t_@_aj&*Xv6OM2&>;61r|sFsO1Td>8N=F; zj|wM7i*TYwJE=v0gSB%tw7&rlniQNP*q#{TKwI)K5TB_jr%*L>X&P9D{C%$A0SR0{ zVvv&~UZ4RH8sLTEC*4Et%NGGHUkDlTj9dU^dWL2vM+K@y$cU|&UKluhfI>`y(>*iw zr1MRkaD?(HCyz37Q?}R;#aWpgE{?JkD=wVEcMnB$!vk<;O}du$#kTpDHA~XfhARzU z+&|yEX0irNnJcEupecLRl)aSxM&9drZxnpB;AY*s``+I7LGgDT-*)`{Hvi5}w#l^n z5<9%+jy03z3r~OU>1)oQDc5hxWuF>+iTk|feU$fsD+j*V=+)9I*0Q7hopz6ViEOv(=aRS+ps7`J zv)ZG19PRM(+U=KD=w^FYm$n^HRUB8qxwJ?v?_&L032IE@jso+#gw0y08z_l{g?Wtz zH8_gla2W<#h|V}(LZd#=(j%Wla^1PTcpPW2_dwqRzhfrnaqZhUZjfLT=6Ja-6B?>% zJMdHbB`VM?Xg;&3`6RIIUr^ft*hOq9=VoT7M~8(;r`tXQ^2gM$%V7}b1dVVg&N)4M z(Fv3d69PvN$Qo`o{tjcvKp>Wz?GGVyc)u78878Mkolk}IQ_e9@$RJFPPk7wa)(Qqg ziQ9&Gbka@W8c=n7`2+m9XW-3qKi;w1XWR33`f}P!`fGh(i4P_fuOt)KY|~&w~A+d7`3;Aqq4*P zBBJ~e{@fxRWnGW`^rg#7>AuV@%VmMgnh*8cHzK*T1Xvp;;6Dt>1)&eLiXJ`EL^CHk8F=WJ3i6L!hK)aM0$Vjmg z4>i#DBrEUIlayXsv=|>9{gcr1hjGgphW#VRa# zqf2m(O+FRU5xAwjbgm27u-W6h=n_xYix+SvUvwcF+f~DW*w`pTh7Fg5N$j}5TW8(C zV#nNh;)pwxErCxycZ5jSL`^QlR<2jsNbX@dKJ zn$TVoGKti+QQ;`P#Ze7rgkj-w$T$H4J5K#L8IohWwut+(U0ax<%*p#Q{@gltz^><& z2J67fxQATdE5tZ1pE>T|P=UX~&h0 z#Z14oDE$1Tu61kT)#EQ5zt;cqnHSG|Wyo(W`<01Hblf)+9vS~a{I$J)bBToEHow;X zO8bqJw+h}Y@TJu+!~vy_vr9Rg^jiym9?tD#0MC|$t9!2Oxq9%*!9_>VQo3p>rKqhh zv|gM1&}#psjzd|QC=2!b&HWTY->D*$^f#NO4Os1WDecE@0+;{1uw^IrzM;w3VlaH$ z(v*O=zt7HS$>RRLCIipkF<4sCu!;;X@|+5uq(g!^a1ZI2f&)Cs^~i9A7)HJVVE_h_ zX;=F+kLlaEQNUGs;3@ICOyF)wGS+w0s$fUsXcM3oZ2f>r^qt8JIx%!6;OL1;CDlZ3 z8*)KgV%GJ6H8K8HHHk{~Novy10gE$D0@mZ^@&H9mj)73^0cLL}7>Qu+$!SpVCr3k? znh44UjrUIc-6M)dF?w;H>*J0VbKDcjyq4g>eHy7J#&IlBKLg?qj}|`54o6uy^eJxM zP{_5HDlp(K-K55$4{==wsW{>D@IspEkG0j+AM2~fCp`c}{(yzq!9z2HqUI+48wm8; z>g_~G6Fd>5h`9aZ&S~dUF5zG(&pkZm9HKT>@1L3hT*AXiZ?*ZVkNGXfe0?W>?WPJoS5c|yI~vhVG?iI5UB`pm zP1wU+U@G-%qB?4q*ij0RC~Bf=9&vhT!xL4O=; z^3BFb8LhuSTsNJzY_tj*zgc>-^i}s#-Je`uuKm-`{%vV^Y;yB4Hmu~~>C3_wDN%HZ z!c~N_?1*3>4nF_AmfOt>b=T^y>q8h_C8 zYd67G&m}b_X})5C_gXT%Z{))R?g5g<F<=>bbh1amd9UvgrT7l zJ=B^^gx*D+&zd*i@>5g%^X?Tk#eMOozTp0X`#I$`XgalOI;Bn=j*d?8#q&~Z@hxcL zS4|?qE0ttPnh*1_)a#y>1Uo6H>cbk6!0G3RTR)>wCjKCe4RcrmTI|(D;?k(bqS*;4 z0mvzs2We(9r`o%bGcaG2L6dryu~LeH-x8v)Zkk$J8j-0$X zr27H;At6ZnHS#*aDd5udYOT~t4O*!SuMB|45~jvSa@}%yplFBh zV3)W&o=?lvfT5Yt({!Z0=~&Cq(bf)#9vD2cE`U}^$6C5t_@=(r?k@2IfW&yECxj&` zh%Kf=BAT2KQ8nSu@%=4|Ljpk`&2wuBMN1V+d0tEPeDk^`>$P34?(*Ba0|&c&_U_kq z1uWfOefOdgT|K%0?w zl*6g7=~u_0@}r!Dm=Vf2hS87UN2=%@K=)$MzZ8k2&5%KtVG#ST=oB;w6eHuTwuUH1 z+c4{lC&&7=OhODjlm`xAXF<%qF+N(~oYS^fiF4Lvav6D@(z`gnbX}l!}Wpq0mCnZ7jVpSO5i}!J9;4%gxiCmkA?~LTyZWuJK{gi5?Q6#pRs3Gk*JjbsjX`F z#68Sk?rcvt-`8}wqXlwCe9MrsvmMy7b-YR~&g9rq`f>F|&<}-40Kjt)unO;Lc4g~} zB1(UpS3T+^&MuHu&?ReT&YcHyp^8|$vx4*Eq{y%5pP0*&qsxpFQF4s9*|DLpJ~To; zT`;$wNR9NWcjTxC4?d9)dXtkPc;aVbkw#8ESdF<;V%K z*E*YywI1mWLvD=ar-n{Mz|9esew&#da?i{PBTkWa7~_cg5z2IuJPIh8JOivuQp$X@d%L&fY`6q^$ zz?vx16efHOkBm6o?vVcRq3(9ZXc8D7$^bIRMYjmdSwdjFyJp;z#2^dB4GZI-OoJXB zN}wq^Bsiy7A`s|5+T;idvDc! zkn-KU_w(-Qv+uF5e&Uv+CIXmdsyW-gE%WJr6 z(B&At`i#3~F10L}vTY@0+lMJt3r291lTxl;e&Od9a`~H#bW>UAz#X_Rr9VN+ffb9ko-#yv3_Od%oYN-Pv0I4wmiPl^0>d{ zlrR3Y1V~2(2Ti9}O{ZChHu(|{d(DT}EGYp?kyl^z>wk8j3x0hs54pOrX>#Az?Krka z|DBq=WBZL8_6-Ih{a=VRPvr9OX^U6cNjto}4`ZhUhmY%_lu<4^?l^d_EG}T~XtyCQ zI}Lg4I0{{)pF2A*om#*sWvHts25iOPFO3{q+pl#p+5-69^69G7L4l<}#3ee1w#!XN zz-8=kBNwV9!OALmMN~bxqhV{r5imIIFbPj14P!A1grA7J3c=Q)xG{*IimR2D4&di_ z;&uxRB4-Gka2w%~e8yD95GSe6aR}`Ud1fv+r-kn$nxCRN;bHYz(u0<~6-!>wQm|?% zSgHsX?^!9{6D;1lTDWfVPwxTY{>qO@KGllJTR2%G@Y zh4Si(jkQSYT!J-Ras6@fUeZq`4qVxb9vW|9rJEnAw`#(1K@*O4Qh9P2B4bCviOkBL z(r@BTm8uL>vNJJTKbR&(jxF=qFYDrfqG@W;j4`5Jzu}QaHG0m;@|P2uK_X!>*CXR0 zQV6l;&f2Bkjm7MedZdR$tfQRt&1F)RVULQ0S70~>Rg4%oj_Paa#`J3VToqHsJd_j> zF&$F4l7^3qEq~0%%h+|i61xYuOteE=;(3d7Qp8~BTKX+LbQnZA-r65ATO#LSI;L%tw!uU(#PmE z9la`4X_T7ROR1G#Rzk$?z$Yqc`;D=5zNCr*dw(X3zy9MuN5r|2*q_+5Unxa;>cQHI zcF?GTPL;&oSGRHCe^b@O+S3@F65OCu3?79f6taL=9Vj5+tt-GU(9H?x zwoQeM!bOkZbc%?uX;uIue&%9G2lXFe2jxPmf@J6icqT({>K;16w>0&IjNPCp9O)2f zcM@jFdy>3M0SLbdNIJ=vi82QUHcrC( zNXFhd1jU`m8!>xwBGlB%nw+{H_;efE{l1`BIe3u}OS1}*t3mi(ZlaMe<{bm@)H zzW&+ZmOZOm_T1X_HwRb&oxYSVueEC}IcpJ>ELkwFr(|5a^zySWKI_Ze>PvAfn0{(W zf$Rh%D3e8^4?Qeq!)Y&+4Hb!e*Xabo;C&OS8+fUwg)Ht@}AC z5ri{eHz!_Acp>51xZhj=aop#hiNcjwMOu9+ZC-2JdQ!^E@n4Q#to=%&D39k$Dh(!W zTS?mHOR8EhNXiVSAd`{%X6`$}yK`^P1sqKu9R6g>-=X!qWb$&VT<`cgBz5k-q zR5YyZJ9JN{$vCW8i2wNp36YjHk+bH_s=u`>kag%byyJdrYdH8zosy`>iI)do9Q?{z zzqM2n7BQHrT7frP@PbiaPfT26T| zr*b8y^240!D~Su{wT%2g#?}SPnmILS&RH?%1kLspvwb=3mfmZ&`^^W|&1u(8`^{U{ zvh2Yu$4ZuCxh$BqeKl+Q&EvtkV=HyXyvH98)}36fJ9%Z+J2bkWUk@u+#M+7>SP_4(AfrZ1Zo`a9NLK z=|%`ml^g-_^nTBXJ#APUsbm3Sw+_t1-E_#~0MdeW$T8NToP&bgZH7Ln7>oenCE!5> z2g1T3ni7$r2lY@Kk-vkIqyw1_O@%F`5J+}vq5i?pIZ}=$ky-f|j1R5n zGHH(XLHSn)Wd}w>70)P=>ky@lb;?mBR-R^FdHI;gD9wC8wT1tTSV>e-IuAc0RT-lp z{in8+FS?-=lN$f?;`&=TzT)m+@rjk<6aM0Xdj^DFb$`i?_Qb`%ApR_D7Wp2hONwW) zKLSNIA?>BpUP}>aNTr8tX`~O8{;|6pg+CwPw2k|tZSdaH!uysE-ut>OOxfx2#FpeA;{igvXou(&uEa);mAt4#T%MLSb4nU%Dk)1qB@-r7s?>Tg3f*F3 zE}#c$<|;pl5Oj4wWGVKxurQ$OWz=hM;}52#%&ORDQXmB=`(VhIOaA{)qY4) zwHN+(M9QJD#x{lS0@dB0;kdCYXxXu9+3`-Ve{Y9xSEs*c$iHid5rdEW5?j6I)^$s! zKWp2A)T3$9voW}ZoYiB{4_c@i_mX<`;kr=DnDrQudz53eS*mmBv=JecHoO21wggz<(V9Y7B8t}#qVcvKpILHQ2I%H~*GCv0lv3h1R%8t}~ z;I~$X`K?Bmf!6`w*7qC1(Ke~dNtf#->i|pXfsY$#we_21&ZYs}Orx-jKo}T+_ZRR& z2@3BK69@vfkgRnJCHUm9h^(#-iu*QsUnlPyn`c^uZm zRsvAi8dOpFNLB>2b#;gVAzF0^uT#IxSbMF!ULC?4)Nd}ee1H{T54iGEW~4ExV<}*b zG!HRfzUVsi4O!uXia#t!4kfa%EdS5#<^c`fqg2M~|~|l{y-GXkkS;(BH!C z*`x(nu;_!$TVUi9)MF*(`oQ)iN--ka6SIysDefk9gme<5hK#&TT?@XP==fosQDqEe zS`6$2XnV4vx&*I~KzaCFiHq*>4Rb}N-=U4*dWcR@YH-E#DSa9~wa>t(q2>|&$EWw1 z_>8{TV}28*RQ zXE}Aa`k608D*IrW6mw7-Zf?*ah4tz9qFCqDV1*Pz>PNYj#ShJ~_Tgzr2|*&vE_Q-lf={|WwyJsmb30;Z5}r+RG8N%=P9-j;E|)U_>vDg?ySpkFO2 zen7%aG|i-n6tap#8#e8f7;gPg_&q5+k>qxS5+kP8Xed4W9$DFtSt>H5MdM{NYSqv? zbB;2hJxrx4v6Sj3cL+x?&>@>tVX63pFe(+UO{&rZ5W)}x8NeD!i})gmQl&={V}>lM;TAri ze9e^6F+356L7Y9px7n+-jKn>H1O`9@m+%97STH}@&!~L|roZY}E%j@r1aD$Nz*Oknve$3g%Th>2By3qh+twA^*7@V>Pz~Jw!v00u za#qk%yK1Rr@fv-JhrH%P>xtWzTYZVU=a2o&oUs_UX3kpN2En++qf7e&c{TJ}y=E?4 z%6KE^^&D^IeqYgnH8W`PN#2w?Q08B1U7QRSK%-@+FAeYci%;D(>C@xy#&dC5uNL~U zE7r{Uiv=cMuUoSgv%O_U0@+7=*5(gpfV9qt*B&L6*D$>naj ze)(pWZ%czOXW#tsg_A4#%r#5ueD_E9yER-w`b!tDpY&R4h;(&EH15;mOMJ{bFz7W8 zGQ&P+G#@klpy!(s+brDoEg5aKy6+cR+p2WmuQK3y#E8i%Il_1tw)PINHAq3}$4VKj zj!7yyNX22UMmq_r*^G$Xsp-=L(=)`#QC)*@){mH@0lEM(9Nw?eag2g7Wvm6(ZNhEg zHL8>{^#$#tK8SKkOodv4m@-VSM3-{ktS6*6O0Ei}MinOWj#XirNQtu;Icb0?$7pRe z@bJQc7U^a3_ z@mBRWSNuaulk4V!>`f$Y=CVGdti*`268aMq90S@5GL9s*SFzwE^#|+e%hb~-I})Rf z$h%+1C#lO~>CUipaeT7+3!f74_DS;uO2O)q8j(vH%1@67$v{YEL`W7wvLixr5R$77 zsnM&X+?3goC-nyG?B=U_aMO9f7xX9ah5c4OOI-qQM@UXYND)Gc)gd(ohS>qR5uJjc z4$7FLYSvUs@swX?*B4Y0SU5`@RN7G@wWBW)cpttsCaUnh(yow24ek**ClCxL(5@iR z<|Yt$BGBe_^i6$8=bV0$%u0>Ia4-9H3NW)bGT|J#FlSeC5+Qon@@0}9fOw%A2U7=T z*YIU#Hcb=?AZwqZyBGxv_V6nM9N(BMG!#*$yiZmF!_^4oC_$n{2vzi?=#aVM*!6|{%*ElUu^Hi_bJUT=%=ke6 zLh8yPQh^j6$9SRlMgT#NOo%}YVE!r%CPBC|@&VOwW-Bb-J$nu!IZQ!A2T&abh+BJF zNE^d5D~Q%w4^iT+)VXKriBZalY9Vxy$6)Cl^prvQ$G1YY=PA+X9T`px*~omJ$~2zC zrUIkmMPQdG>OwJ3L zm$MNPNYjjfb9uZ(vwNV}F`!!kyX_*&DT&Y&b;k?Gk;6Ft+|vMNAZ^mrz1lSJQXfoB zJFyFOypmp`YV=vV$#s#ygSF);osWrjWJu9hu{Hc4r z10(YtKQ~(zT3=0A8V+P_4OqAO%#P&)@0Hv-|F?iSlY{2+6?6Fw-Hqm(*})yHD?3_! zm2JUFex;K4RrdL|o>)ifE}uE~)!wB%Uw$>cwtBO62duk&<~{GF-a6@RJsmi3+P8P` zZk)kpoIiHg%BAHjHZC8(IpR%em~UTC$h>;?g|myt-aGZJp>GbYmsGvc_AWI zZuoM;;@E2!U%BW@t%QI@CU^mX%x%j}fz0aVCj*(g7p#zPdAZ`t6^nzulnNj#X}OE- zfwZj)acda`i*tdDisc=FjH+cIFnbm(KeJ}Ox_{Xk+`4aN>ptJs1HRmYe(S+CYwER5 zB5l+Mtd%#--o34X?QMQ*+nP1)+S!1$bg3WV%Vz@CeYcJWtSxu4y!^R9>xkbvLMaiH zzXl#fmbcdPT_ATCIYhkfbjFz9XI-dRZS~ZO;saWNGuDFR}6V39q@0 z!Q$f1D+8TiamE5{dt@f0lu7}XVzCM&a<mmWsehpJ{W~0WNqQCNH zghbM09wV7CEsv2r!ymD4P1Lsz>&E~mB$Y=p)dK3O zWcI%XuOG)(RpF-!Vwnf#V8i&yDnJTu=TV^?hmx?9Jen=SE_ie4fSuem6R=(krCtdp zIP1`UoEGcDTyCt;v!5Y&`{0FabeTX)SAQ#HRy$j|`h@)y zYHDuj?>f=ZAwmkoL=PFjQJ8cI2PkDWc|@`k8sUMVPA5Is52Es`2tA2%K({CZ{^(EA+8500$+<5Ve7Rt$;AX?E^S1`QJKFBFdruB~?dR4@ zcfQy5t?qAj`}@uY`^HxK#=O&R9I7d^ng!#}k}_UScrhWERJxK>>PspQCRMH^Rr->u zVdKz}ENZYVY5bO=8>w&QznOn?Ht0C8>NpTe&0HfX&MOJmoFAI=ewo0fcW9tgX--@i z{`}=@oqki%T2k54lgquPx&7pw;qP2tOR8MH2(hH319v%1+<~8m&-=5>bGIsP z<$0@)-m!R3OnLJzuG?$gi~pAGn>K&XsbJ5!m7a6n^D_hnJV9VULh98sSIz_zidGVe zdEFs;{xgLP|L?((-2IG=k8BAr`qP7W1CR4h@4T@wK|m%nwWb2f5DJ zId5{(LCJw2edZJa1f1TZIK7QHi)3<)6iYcJrJYeUe~-8_Bl^K{MKfZBT<2lbHD8G* zJxQmm6MYO~kbDv3AE2gzjAP$?T}HXF5Wanuki2J{FSQ_c8R96%{uWEkQ0c>5>g9*_;w1M7qlVS=yBjsd zI;g0S#f(&C3T>EpbJ?)~tZJCiyCAgVyU;StLhTJux$L!QBHY$blN^&ZfEe zSr^@jRfU}9OjT7NzD-U;+ItI95@dxp5!7-}R<$!OW|DAv21@%#A(|+O7ulbnd2Iv^ z?Q?PQuwqKor})YL6(DTaT!jotA%z5)Dvb5<*H7oU_7M{kfT`$r08rZPgP(C+aT(wQc|b-0(TvC=?I#cP2`NB#s;vJW5b# z$k0QERD~`)GuTrAlZz8hkoP2V5H+hV;R!aXho^CU*dT@WQGnhx11WSm*g|^BA3Ztc zbh;Q|%K+7o?XW-^PCb(n?|CmJKFm^F8pdT|iG58zqhddR;l;UsWn^f;z@vK;Fuo!d16*Dv0yc^VDZw8?hhKi z+xULtdinNXdHqUx{RcVUEquQa0HTf8fUcx3GL){FZ3LYL&ABV)+_l8Rw;R2Od%eA9 zz1E?ib!^2t=Ce-R*JeE-A{^Dn0d4$>e?($`Hdy~3x?ZDNx7uv4PerU~y)=f%& zm{KFDKc(iseDKQ$*Aj9U=aws$^St@HZzbN*d%H%xR_A(B;f?s<)`pd>4IeZFkDORJ za>9G&92qEyUobi%o2tLK^y zcG@-HhK@v=j{87+cng01!I6Tnf7rk4cs2Lk9b55yCnu%Vz}>0T(sOlkt5$cXk)FS& z(ct-eS{*$b_Ox-j@9k&LjT$_EpQGpR>ry-Ob>Gk2)t;dLfmMU&A0#BV7ny!gprvQK zhMtSI;rWLNDIIqG4+|;I5AAz8b4>r3Npb!$TSLz|I(p8hJV(N;OEiGv2#>)QiVN}J zM^fQpvBJTOijawugb_*TehfK?^fD&itYXsw>efdxZe6B<@ts=0mkDNa{H7ETmgN{kZ~*+huE?T z9NIn~hT~;wUo4s)(imY>4!b2e;5HA74P^r*HWa>9qoWEALc0&EtC860 zDrNkK#m9xS>iGQz(C6ZKvzl!P>@SXwiwKEFNW40vMyJ9OlwOvFOrb&&$Kc|J)CWSs z5%qakdm%y`5nqA|9ahGh>zYdiQ)ULn-=<)BPNG!JB`bkp?$sP%kZ?SESRa{$?o>B! z3jYfRpbwBj!sPfgSqTGE5|imR!;a( zk{lN93xfBCVPzCtIY1QPQQ{6$A|{adm`Do~NnwCG!(FCgWih6#a1j;w5Pxohl|T`L zn3yA2+^|yIaO=3YWgxKsai8tv{P8uDZPk&)w`jYn${VFMF$z8GJLg{(ck}uu#bG=~AmKL-XuGk8LwxU&A(Q^7*d2i;uRqz)D z>=1AEWwdycV1)_ym%V`d%g9oD9G6mmKb}-)q2ucftM7(2dc6txi?e=x8Qp}ne?7fy zY4TRvT6sMw8FxU*7?#T70E)bj;LX^I3(H7}muWwHGsQE}i%8=nRy1L4YgG2-!HZ5tX2uv`+aFPkYU$K^^q!(?7bOyh#xs zn)iLo%!~bfMM6sk_nnN|qh?)%a*KR6e0{^&AnjtGfWAeyY&^nFJX*_kV*MlS@*-z~ z>d9P>yjw^iBzrB@RX#_sZ-5R%Vgk%X7@F6BHU(Z18S{nkm5ir8Km|re93IB$0e7a~ z09uNy_Mu9fCY7Z8s=me$9^y?Z>Y5RTS02op>14fSj3rjI87>}+A48h}IjNsUL8(GN z6^Wtf$(W@`y{jldsDzi4K2{m~RJC0`%cL^MP!VoGQF=B0U#VA#LiONYt!6DRlWK*y zGWSag1AK!0h}@(0km^)Z`bBlBDe54bm*+AMJRdVJmC{o`F7$LHh!}mPd|1Uqj-!qy zG#w@|&@>U#H;)W^sz?zLIKNv&cIWU8L2gB`9QjKZbuwWTrMz=fGv}a;0e^0LbfTHr zDN)*nZmx`xZmu9IMOc3Q;-s6&y}$-20j^APM8)k<*fs2olbJu!NH^o{Fxe0?SVk9& zA%7srI$Qu1%B0K1a7%aV(N^dJ<7Uvq;Y-F2QHiub9C0EaNF~QYJSgD^MJS@~dk;?` zOIYJVH1Q?;Cr@h;nZ=d(_{ocn% zyg8%m1y%2ye|P5XnIE1HcKzN;*Y9~JE&{=}&S>VFe3SuDa$cuRF}?0bD*u%vFK zq|R5eJ6LjXrQ~3+no%4Ri@@9#^V z@7&1wF=1S3;@u-q31-RTp?<6ls~@KVuGS6|MQR51N2q#2nHUf`Gv2II8#IqCdJN5> z2-VO?Jt7It?^i?2@)@f(MXff;`2f5wf{(tTT%#I7k5n{$NZbhczfO;yg9B|FFnstL zI4zW+Eeh}(vMJI`A_&L?tW4lOOaRHX6rM%~ zCLnL_uOZny2ZOljLmC)#NHYFY+S!HXyRa#n;7uzFB$h9E0*RHY=E@(ZoxKY+X*UzbJF@#Z8Ff8RMsYrqYxT1x`oGR3)k_I;@H|a*fMAAxQb-Fnl)vUYNoaus;*hga%J0*3W~m=-X#pMkb%Tl z0y{vPBNPH9Fhgp4SZ`LTqbx}e#5i^P&bikl(*1>D~_XP+4x!?j^f z`^-7FQ^19?bd`}>xHM$Ou{tA=%3W!!xa_M+0glP+GFg?RjJZ!Z|f6YG;811R>S1*a?);+_ZkeXtcNk0oaiHMfg0@r_#ow!vlmsO_t}L zCVRpOnTvWqRDcrd=8rOST(F1()56L5i(`;8fo%qM(M(^^QoL#@UUI%M`TC@H`ypR> z)9uInhqv#PukEyVXvE((!nU8VcqL2w^5lz?Fgcd7?FMeEAuA{ErGKmN zn}v7M@5}{SPX+g$UfnC&y*}hiZ1S3$VCd@$mp^yen_T2G6)#PAYZ?RPhy12Pw_C_k zm$z%!*L==9HhmXItLBLIUr4mIS$m)3TJXaP59@cWSu++5i{}F{ezoge$J>ruExy{r zw~yaB@3oxt>Q8Pw$cYZYcC>!X;NiI8_(<~S#+^)Bfukqc?!(1GXB7)I;oS!#3c_OS zxH4+WYuQQ^ceU7Y<(ss4CjTGa)ndn$cLd(3TBkH$nB(PDjU6L)4)RK#iC1>_-uMUi zNsDImDYm!C$awfJ3Wh|2p&t)B6>(&zg5RT9rZ@^zTwOh8Y@ZeD#{rAy1Me4=G>`&Q}>Vx}p^!#s;Z>rSJP4Yd+^2IC?U2LK|pQhLz z>5YGc7NzqU(#b8=L3)>Gt4Ksr)>Nhxb3*fgS(7EjhyeSw4;~H7!rzbVx7b3?e72PH zgL?{jK?efloQL&Q4o>;f9>35@W$+5^3y_)^B-u=PCv% z>$lEdYJ=2MW5t&o875VJw9_DkSd#grfb8%Wfe_efn>+lKEEnz0cqV5N5}q&l!VA7XX>hfV6dO^Of8XYzdRH*GSX zUt%MriW?K+sg&~FWM$8=oPb-4F4oG1UBF89J|X2Qk69;cn<-`;s-)O_HQRAT*!N+l zy9yEllR>7-NVS9&ecNvV`_xEvVdGNBRCYESm+ew~+&-f8wfPf_OH82-{&zJl=0_Tr z*mc+;wT-X+EypExzQ4`5?3C)l#>LLY1wy|Ouv_etp-xKq;29pKLn*6smy`z7pWV2X z34e|z;jX1bx zI5C&Q(%8>K`DS`}DuVdKC?+HFbi$M-k|mnU-TO8?m31#{bL#7jd$m1828ePB-&m1~ zj|N{W)E8lwQpZ9*he~CxM?My%5XRl)dbO~xtl}qb%r$KFt6$_3ko9Mh(0x-YXwkf` zHx0~Gtw>0E1$ir0g+S5%5(N;A8uV%+g2%GxCDW-{Z^p3bwH0u>y9b#_7QHHrztn5e zk7^d0$&Pa7wb_?1zj%2m?JLjTz*WPQesg8eeAa6|`_AEaJHFm=yU@!Icpo1O@MnB= zXZ_pH`pswWX({=y?-9}E`I@Fe%{MCH35jS>-%(K?K8|Ke3BJc3#QzyzRD;O>s|;fC zhCz%W5Vm0e!;?+n`WXxs?Xzft|V&Fg>}^;SD%eigGk5CB}QJg z$&7SG3@IHuY+d5CRj#`PDz9FZFn5C{i*DlzQC&a}jRUss^iu5DT2XXG^TvF_EA3F^ zN)!GNjRD(-5p5$^m~=U2IZE4bIVKyay);t$@R_8W!Nlp)#F2U0+3-kRPxaZ#{FX9t zgl_O}4gSU8t(rT#d{vM69c+BEc8Ip`p0hWZHGftHZ>}bC1QfoO>f+J}Lb$;N0d}OBSP5VfuE+D2l<)k!YI7Det5#e#D5*L2zFkLneGHa`oyWh0Ex(h+&qu-+v=3 zS9l36RPV5?$q!hs?KBI3DM~x2u3IB={KWn9*x6h1moZwA%O;F=jR~pl)@``4aKo8x=ix52};-E^0_%!vUz=p&hCJV=0VR06T0O8O-67>wdwzRapj-rMBeffv%b zVaM{%>Gc+Q1P(E~l%nZM2If3R6|e}Op$agm4%)bcm+=}(pLPOT-5~zUCj6gL{MX3) zDtUC&i1VKWK4ptIha^Lkk?^f>1JR3VMFatP{2BsOo3c2={K#t=K2w%AZ>Qh1^XBZk zm*2h&8<2ZCy^js}_Y6o;3Vo)cr89n$XrytsFL968yoas?YQ55W?Yz%g1jeddmd{fB z#>k(IKw4J8zW!c5c(@lzr_d}ms>~zBr$8z@LJp}ao*y3U)J9Db_Np~ zeF=>+-*l_bTni$?hLAJ1G>W$mM+E(f)0i9onY^D<-4u2$c)rZw zB!w=v!7yhqW7|r`wq?jY?|26WHutPr_ORxtKOXgE?0g4|-o2}qy|V4f8>!1DZZ-s} z4hE|ZuT&lO9_jP;KklnK>2sV4T28N8ME-BssO4H>>eb0BlU~>zqHD0Igz;BBK})I6 zQhKB9<|Pu|y4~p8)9XL+c(C%MukvKL94|P%du}!cw!y$=v(M7ZQoelQ#S2SYd>Iut zn!$bcS$3?O;urQ@+Y_+l`b>G(b=Ny@Zn;$xsBQC=w)^ZIep5%Vr`Olhi#wbFsQoOf zaLEKCN5QO`m8=@N#o9o@h=TvC4~g|FPWPB5?w^?t;qK(z#XYa>du89!Q@&iB@xi3s zt4X_Y3D?3?0aMni9XD`KQqJPBrS3pp-Oce^mu_DQG(eT6@XB>S@4{w^wZ*r*_^lN!V z-m@+R5bUeD_b$*XM5*eoz`LKek$a z>_Q^I!ki3|Z7fZ=c`x42x zD0N>_v)|ObhK3QYTL4IH!MbM2q>d_Ej1Q!gEHwvG95AeUt!1$ykXE)d7D%fi+fL%@ z@owE4$Y_9seqKD*yfq#Y`ec>qm@nnH*9vt7r+C}%MXTJz5PQK()n7XxPFP2{~ z4cfM>+P2)7yH)ljKILoZ@$)BwHT|nK{p+^0tIu9}cJaxj^MS%0L0j#rtriwJ zt~Ok0xHjpt*q4s^E!*CyeRu!Y_usbpAM5j;I2}AOv~ps|`+F1K^HaVP7k!US2Y1e_ z?wnz2f=#}Z!(QuQR-qEVsbnoJ`{mXbTNf|*(kd3>s6vkGeL-8rs;%Okid!eYb>^FA z?lkxg^!pz_9jqN(tsRsqba^=^P*@kV?OL_%!VP!KMsB{(vc-v?p&8}C zw1dqjvXUSf6onQOS7U%oz7=LaDMvWG#VeXoJj~of+M48cLYdK2E@-xpTsac~ z5{^*T-F|$)ai@>_Qt8!-7b>oec?F?TAS?EaGxP+-B&v6#m$jhUtWh6ro?UF#Dy1A6sl2+a*sHPcdYHa)xtbPZ){9 z8H;jW$6ALQO9+~aa%Uq%fMKjm6K(1RA(6MO>zP=sN}&Ze55dB1jbeHVi-C!e;XFx=i^NCGFReg_9 zGnS)?_-RUxP=rj9>hu6LOM@y#niN<0Reg=2vJ-LZb+(jBNyD2TaO-tkU3g0YFGQ)K z3R{%&0Hu=OWRK>6>U!*4~Z}fkO6gFT7*-WZoykb`t z;mUp87M1D>*RxehtNbd~VWl!$)_M zy;N{m-VG%%u5K=7wUy>zzisn-v!%ix$nQ<)EM_zTLuUcT@Dt}NwMFg;C#&}Xsos1d z-MP&tE0P>!o`$#3KWXGY(Ta?<#*IqPsn)K%%B8x{ELDAvwYE)4PG@m?^joBMjeDqF z7?;@XOZjxRZ?IA78(x-&gPM7h6xy5Hn&;cdI|rd`dQAGJ{7P$0?kPY> zka&RQU?q<96diY5+(G31NwtIMH|}rJszXci)wDmAJd-lAva}Lp2Xpxdo8`R7O}`?} z?H-~iM>*aul_j?d`UU9CBIH$!N51RMZ+$>oSq5mQo*D0W z6)-nYC|8mz*_Gl-?N1VQg4(ff5B5s=^(XZwW27SN-1n!*XL#D9j0Z4VA6SxmxYjam+Pn+#1bHxS>iANXY*h_B>a z>3!)c_H6>i#!fyUe&x7<_b~M@BHngv z_`W0Jd+p#EX`HaBSvQP+&6sPsV-ZZNi`TfzK%we-}T@4*zRJdq*lCM z{=bb=Qgs?V;Jn;01IwhD)lA_*C~0(Oxq#tIOG*SHE%3i^Q1ddqi*6 zvt2?rdWp>y%ox!u)W{*v>8Tg2FfltM1 z1D?{K&2Np^+Zxy``4YADp(n+@-{nd1spv5}IkK?|9PH#Mp_3sJ6N)@J9`%&`RJ5I} znLOg`SJ9~-M#uhtk0Oq-vvY$7rPCVb-v2Aja#zMfb}m3mc`zi;TU;598Nb_+`q#u- zH-YjJ_@u$pQa|)%NGCw-)f|P}$D9G5s@}mo5XVxvJMq8I3kmxF@AINDLp?9HDK<`m{DKx8-#&LgHF7bfez=RBhpq!zo5x;nfSZ672bZ$ zJlCt~(cm%4k)V@=o0V{HYMC5P-bYXY33Ds{4OgfF0FZt$e&68PoQ!K&t!s^;4RcSikHy`m(Jx?aCk z-Gz6linqx7o2YsUe@mh7!CO#Qqdi!8WTo=R?PGWP{FOYbhG0ZJgnxuLryJgOu8W#2 zJF3=B{M{q!rApJHSQI-FQhIffqEN9qT+u1)*Q$j6OCz<9AOJ<8(Or;-%Ed4AkN~NQ zI(=+}n0QoJ^53qK{11^yJkqP&Fb?C+Y3EZe;o!^E1C_{(sRekq3M<1;tng=G+i^L4 zc`o2+2s#d~I1b+O_#Dl53?Di=eZx;iOG<{4?TeyHa315RZmxj5jCVw_QM6Y-AE9`K0f4%J*}#)1y-x?z0`46$IRdW>=?H)JTOdQ29)D z{0(}FAy&tv@#l8Nv;i1(Q!WYetMp4nA$+=gl`LBipM%-cPAC8f9{aF;+IcD5!S+j& zo(VhdvT#q>oflo6%ZgF3N_*Jwb)_8^!oxaNxChTkcFE8SbUmDCHJb_Nhl)2!3zC%o zklK>h5;c>A-$P~5v1R3jhy;H|2}n?WZkK!3<(itL8%$!DP79BSeRvFVboQ{S7(@_7 z`FcfbfN2#l?LftVVhK%2vZ5&bDS1ri;B_jpL@Ft)rXk&_jJb*HQS3~kgJ~zs?buMc zaFWP+g#$_NLAXvOY?VroTW?pXe9(waPBRSv#|~y8He?m!f$24-H8EU`yVnozpjIT2Ii^OT4zsxw3HA)|OxkMIg*NVYCT zWARQrIr2m!@ay#HJ$SG^NBV&59+*%nYtxW);k0-;%rKIL5^y!olan*E?jcO7iy^E0 zN_KT^>?WB}Cl(1rVrPVqo&uPmdFe()rU5cCLE@h<7e93Fvd8Hb@~LVCYO`7yo-Ez?aJu}`v$}UtESCmbTEB2wNUqWY{xXb3D;j(^-J=#eM zCv>~FvhfQ40m+}Gt@f|2qP|IT`b!gEDF`NSSxMgFOLhd4cdR7uxLFiPuERx+$vKOs z0?AwO1he(!hXXl>z4}bL*S9l}&)(6JwOr?xPJKODU zT1BQIu{Und_sKjQI}K>;?|$ce=X>1maqi`ud(QdM-dn9dXyvr-8wa8i#dlj@YQ4Gr zRp*S8&u!x+?dvGVjDxovukYdH<~B^?n>r znO7D^O{Kx2`E<_IHq{a}mj(CDS99iLQ?1dAe1GSyt{-&$c$CZ7JGCFfr?`w#$eC~s ztYllF#sdHO+e4hOZK_$&nWh>A9F}E0)=t!v3nmJ~e9a*)?=WXNGA)fV*|X#RK^((I zty@s%+-m#lHr~pj?x9HcXsCOX@4n1g*;{S5+J5*Pe58O~C}5+EVK&)s3|LY5K4jd_ zGY15tX(fwFbuFKjJn^A_^W3W!XD<3jcv<-qPyK)ynHllF@V?Bw(kP*H7C~!T(B{KB zfn7&OU=UTEK$)Gq?xc`uUMbjoW6P6b%9x9CPjH4##B@g0x&DfPJ)-t3s6B!)eZ{`{ zreasYK3xCuq>ye4lm^ZrPwV^yI@K$r=lEU09e0xE)u{6vMCwT&q^G4HGZ1p9glB4_ z5MX*=KpJM8g5Da@mxS~s5xsjs??#@Mh^IZ|X^(i0FL;he8T0EnD8bLVxU5!=If#ye znaEK1=rGR=e{8Uz+z!rg0x>6GVWvC&;!pi1UTuRpdbN6{diHcgYhTdX1?yF8J`vIt zM0F5@JJ87M+^`zc<2O}!30mka+-oPkSKJY@l!w*jQ3keKP|7o&sCK)5Ja9guty$34 z2$@+cN4C=JAu(mx7SWf6^rfh*19jrV5a}BU^^Ks5SGc~bNOw)pXGiqqA$>XWoJL(2 zAdG6{;?>Z_t1*hY4sI$RP}C^hC;~{oR|J+QRcRqz*>v-4pT99s1+kkE&E~$W$E7bOy$Q=OdZ*3z_voUcm~m*-h`um{&}hOM+GJXYPpR zI*_xU%RPt8=RVcK7?yRE20S?=CVFTqjb0Z$nP~quUR@FsDK%M9OU>JxWgwnD)`E# zd3mJyY^eDx>KQkH{9F@-b~Tx_3*g~pn8#R zP53lDD1v#bIwS%UFvnpLoC#-0XM_34b$#PKj&J22;o+Y+B>t?h+WG&Sk;yHzx(sg& z83q2%NP#z0;6)7sXo%e65WdAp8a6#eWi*MVlb%_h2Jsfwr;Da^8Gl)e8EnPPR?Jmf zwo<(lSD&?owfTW$UhBe!P6uWV_zMG7!P9pd=c~}M31s;bUiab#R1y}iDc3AC->I(z%h_xZ0%TqKrx;4|4=ynSZ58S z2Z1@(M8Ru7MbI8G)-D)pg>9AEu`FaPTQKg97F7g?BA%lm&r!j1 zQt<46-R^0cPk$g4+1u$^HJCD*T~Bi${nooIrOfs~|64h+1eOn%?Ty$?ar$0S z)C4j1;6Y^aZ0Ndd>s@Q9qB39%z&N)H1yztMoH_X1t-5Q=Z7<6ym3F$~&sznpZsoLO z^Jnt&pM#T(MAWITf1M4>+z)-7@b}9v?cNe^ zgPnROvq9>uPm#`Ba`EM#lI-N-X7J^%Nah8aUm21y^s8(gxwr}P*GbtJ`gJ~m{-tU+ zhTf~v;|ogl64Fp!76un6Lc{5t7~&I+h7DfQ>ifA?jEyJ=jgT$DkWiS8FZU7^7j>x^ zTFl#ieVb%S?log5T0km%P)*AIRpu?6v`hZYm51@3HHBL zUM1QG8h?G5@cLQkPoc8Fmf5j53Mdc2=X&J7b8E*BgoI}1i7+XiAg?B zAWl8K>&$t}NfUG4$`o(COf^?8lH(@l z>QeyDl+V>0NHcQ{cCXUAr&9csWED$Y#qj^ZEF?EJxKV{qjbIag^|z|X17A^3J}8Dy zhMDIXT|)gef&Y3dfW~I6n1ffg#_$%k;qa`X)PIX31 zB_UHu&=zs;3%U2rw?q!~gbwuZlHO>FX*QipaUzNHS<(60Ouh#51~Uo6Nc^5RkCrt@ zqnEzUDXNrB zQLULof`i!S&-e3FhL@Ic^t>1 zo-jbHaB%F{izA1x2Jv=4EDvT44*|LyAWg@ZHP=KAc64-XA{A(Q+3vC4!33OweQ2yVv) zoRgQhHh`tP#0{aUrJAT(KYcD@C=40EF^r8<7a>{E=TW8h<|&k70Us+sAyXmX!v%gL zO11r1{CA4KRX_yJ57f*f@QonVJoK4h0e)F&+QU$9Gnz()_-#veqg4EkR0?@`65OCb zegX+a(Kul!o0KT%zSVeOFUh?3@RP?R^Z1j8?~%;i$3Oi!$^87MkMG(f6Q$jK-Qeng z^|?SK-hIJ=xB9vUxSw@hDRpgtoxZ^d=O(H^sP?hZ-q8!r;md3QZejxTJZ!(~ira;i zf^c9!-virwV8}PdUMI?BTk?b&)4&NG02P-H_oRg2tYa*M&+$P!6Pl6HzVQqFK35`2 z4@Q|*QP3aJwG6P`mj<6If#W#P53XM?kN5k=aEWoX4)ij9VXW)iYWrU;7-PXhEYz@x z4j!}}TTJfi>Kp9^ZO%uqr56==X#Lm5E)4as_;rE33s^Q6=ZK>W%=sW z=~~xqBveGICwNeff~u3nUuG??;u6e`t;3Dr6TX;?b#S1m0v~3H)_O<#0BE~@ajg4V zBaRWwnsB#=a1CtU%3^jvIKiT#?Ya*X_Knv-O@?2468ffZ<8l@!={R`_CkJuz7AE;2 zPJRT*qP+21Z~rB{84_%UsHohy7^|=nag4~jPZtJZi$76E5yjN&82lTY{3A~8;$&S1 z_Fou;KZKvCyqaZAM`&aeYfT0v}B5NG?V0uy{*bxb6e77FIbNv7QhM)!YnG9~iQpB^P&<_Z(oUP_+fCnTL8l5wF zEJBniCX;GJH&ijDOa~ydY@liRv< Date: Sun, 6 Sep 2026 05:16:59 -0400 Subject: [PATCH 18/20] Verify candidate-lock ownership and bind observations to the candidate commit The publish-time guard started `git update-ref --stdin` with start/verify/prepare and then polled for refs/heads/candidate.lock to appear. When another process already held that lock, the loop saw the other process's lock file and proceeded as if this guard owned it, while our own prepare failed unnoticed, so the replay could accept the publisher observation and record completed-offline. The guard now reads git's own transaction acknowledgements on stdout and waits for `prepare: ok`; stdout closing first (a failed verify, or a lock someone else holds) raises the existing guard error, and a five-second deadline with select kills a stuck git and raises the existing timeout error. The lock file is still checked as a regular file and the ref is still checked not to be symbolic. An experiment on a scratch bare repository confirms the behaviour this relies on: git 2.54 prints `start: ok` and `prepare: ok` for stdin transaction commands, and a second `git update-ref --stdin` against a ref another transaction has prepared does not block - it prints `start: ok`, fails with `fatal: prepare: cannot lock ref`, and closes stdout without `prepare: ok`. Offline review and publisher observations were accepted on a matching request digest and candidate tree alone. Two candidate commits can carry one tree, so an observation issued for one commit could advance a replay for another. Observations must now also carry candidate_commit_id equal to the recorded materialization's, and a missing or different value is refused with the existing mismatch error. Proof: shellcheck -x -S style scripts/test/delivery-replay.test.sh clean; bash scripts/test/delivery-replay.test.sh - 38 focused checks passed (was 36, one new case for a lock held elsewhere and one for an observation bound to another commit with this tree); bash scripts/check-rename.sh clean; git ls-files | grep -c pyc prints 0. Co-Authored-By: Claude Fable 5.1 --- README.md | 10 ++-- RESTORE.md | 8 ++- delivery/v1/replay.py | 67 +++++++++++++++++----- scripts/test/delivery-replay.test.sh | 86 ++++++++++++++++++++++++++-- work/delivery-loop-first/plan.md | 6 +- 5 files changed, 147 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index 38713ea..36c4c93 100644 --- a/README.md +++ b/README.md @@ -79,10 +79,12 @@ through the existing local Git materializer. It then checks one repo-relative candidate blob against a supplied SHA-256 and records a private, resumable state. It never executes candidate code or a user command string. -Review and publisher records are supplied offline test observations. They bind the -exact request and candidate but do not authenticate an actor or authorize a real -publication. Missing review stays waiting; a completed receipt is explicitly an -offline simulation with no authority or qualification. +Review and publisher records are supplied offline test observations. Each names the +exact request digest, candidate tree, and candidate commit, and all three must match +the recorded materialization: two candidate commits can carry one tree, so the commit +is what binds an observation to this candidate. They still do not authenticate an +actor or authorize a real publication. Missing review stays waiting; a completed +receipt is explicitly an offline simulation with no authority or qualification. ## Inactive fake adapter contract matrix diff --git a/RESTORE.md b/RESTORE.md index 294c417..70ef503 100644 --- a/RESTORE.md +++ b/RESTORE.md @@ -65,9 +65,11 @@ bash scripts/test/delivery-replay.test.sh ``` The replay is a local offline simulation. It materializes only a caller-owned -candidate, reads one fixed candidate blob, and records test observations. It does -not execute candidate code, select a profile, authenticate review, publish, merge, -deploy, or contact a provider or target. +candidate, reads one fixed candidate blob, and records test observations. Each +observation must name the request digest, candidate tree, and candidate commit of +the recorded materialization, and completion holds the candidate ref itself under a +git transaction it owns. It does not execute candidate code, select a profile, +authenticate review, publish, merge, deploy, or contact a provider or target. ### Restore the inactive default profile assembly diff --git a/delivery/v1/replay.py b/delivery/v1/replay.py index af1b10a..586c415 100755 --- a/delivery/v1/replay.py +++ b/delivery/v1/replay.py @@ -9,6 +9,7 @@ import os from pathlib import Path import re +import select import signal import shutil import stat @@ -37,6 +38,8 @@ MAX_INPUT_BYTES = 8 * 1024 * 1024 MAX_OBSERVATION_BYTES = 64 * 1024 MAX_VERIFIED_BLOB_BYTES = 1024 * 1024 +GUARD_ACKNOWLEDGEMENT_SECONDS = 5 +MAX_GUARD_LINE_BYTES = 4096 OID = re.compile(r"[0-9a-f]{40}|[0-9a-f]{64}\Z") ACTOR = re.compile(r"[a-z0-9][a-z0-9._:-]{0,127}\Z") GIT_ENVIRONMENT = { @@ -423,6 +426,33 @@ def candidate_identity(candidate_root, source_commit): "candidate_parent_commit_id": parent} +def await_guard_prepared(process): + # Ownership of the candidate ref comes from git's own transaction + # acknowledgement, never from the lock file existing: a lock another process + # holds fails our prepare, and git then closes stdout without "prepare: ok". + deadline = time.monotonic() + GUARD_ACKNOWLEDGEMENT_SECONDS + descriptor = process.stdout.fileno() + pending = b"" + while True: + remaining = deadline - time.monotonic() + if remaining <= 0: + process.kill() + raise ReplayError("candidate repository identity guard timed out") + readable, _, _ = select.select([descriptor], [], [], remaining) + if not readable: + process.kill() + raise ReplayError("candidate repository identity guard timed out") + chunk = os.read(descriptor, MAX_GUARD_LINE_BYTES) + if not chunk: + raise ReplayError("candidate repository identity guard failed") + lines = (pending + chunk).split(b"\n") + pending = lines.pop() + if len(pending) > MAX_GUARD_LINE_BYTES: + raise ReplayError("candidate repository identity guard failed") + if b"prepare: ok" in lines: + return + + @contextmanager def hold_candidate_ref(candidate_root, expected_commit): repository = Path(candidate_root).resolve() / "repository.git" @@ -430,20 +460,15 @@ def hold_candidate_ref(candidate_root, expected_commit): command = ["/usr/bin/git", f"--git-dir={repository}", "-c", "core.hooksPath=/dev/null", "update-ref", "--stdin"] process = subprocess.Popen(command, env=GIT_ENVIRONMENT, stdin=subprocess.PIPE, - stdout=subprocess.DEVNULL, stderr=subprocess.PIPE) + stdout=subprocess.PIPE, stderr=subprocess.PIPE) try: process.stdin.write( f"option no-deref\nstart\nverify refs/heads/candidate {expected_commit}\nprepare\n".encode() ) process.stdin.flush() - for _ in range(1000): - if lock_path.is_file() and not lock_path.is_symlink(): - break - if process.poll() is not None: - raise ReplayError("candidate repository identity guard failed") - time.sleep(0.001) - else: - raise ReplayError("candidate repository identity guard timed out") + await_guard_prepared(process) + if lock_path.is_symlink() or not lock_path.is_file(): + raise ReplayError("candidate repository identity guard failed") symbolic = subprocess.run( ["/usr/bin/git", f"--git-dir={repository}", "symbolic-ref", "-q", "refs/heads/candidate"], env=GIT_ENVIRONMENT, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=False @@ -465,6 +490,8 @@ def hold_candidate_ref(candidate_root, expected_commit): except subprocess.TimeoutExpired: process.terminate() process.wait(timeout=5) + if process.stdout is not None: + process.stdout.close() def reconcile_materialization(arguments, execution, input_path, identity, state_dir): @@ -528,7 +555,7 @@ def revalidate_candidate(arguments, state): state["identity"]["verifier"]["expected_sha256"]) -def observation(path, kind, identity, field): +def observation(path, kind, identity, candidate_commit_id, field): if path is None: return None source = read_bytes(path, MAX_OBSERVATION_BYTES) @@ -538,7 +565,11 @@ def observation(path, kind, identity, field): raise ReplayError("offline observation is malformed") if not isinstance(value.get("actor_id"), str) or not ACTOR.fullmatch(value["actor_id"]): raise ReplayError("offline observation actor is invalid") - if value.get("request_sha256") != identity["request_sha256"] or value.get("candidate_tree_id") != identity["candidate_tree_id"]: + # Two candidate commits can carry one tree, so the commit binds the + # observation to this exact candidate and the tree alone never does. + if value.get("request_sha256") != identity["request_sha256"] or \ + value.get("candidate_tree_id") != identity["candidate_tree_id"] or \ + value.get("candidate_commit_id") != candidate_commit_id: raise ReplayError("offline observation does not match this candidate") return {"actor_id": value["actor_id"], field: value.get(field), "sha256": source_sha} @@ -694,7 +725,10 @@ def replay_locked(arguments, state_dir): (arguments.publisher_observation, "delivery_replay_publisher_observation", "disposition", state.get("publisher")), ): if supplied is not None: - supplied_observation = observation(supplied, kind, state["identity"], field) + supplied_observation = observation( + supplied, kind, state["identity"], + state["materialization"]["candidate_commit_id"], field + ) if stop_if_interrupted(state, interrupted): return 75 if supplied_observation != recorded: @@ -743,7 +777,8 @@ def replay_locked(arguments, state_dir): revalidate_candidate(arguments, state) if stop_if_interrupted(state, interrupted): return 75 - review = observation(arguments.review_observation, "delivery_replay_review_observation", state["identity"], "verdict") + review = observation(arguments.review_observation, "delivery_replay_review_observation", + state["identity"], state["materialization"]["candidate_commit_id"], "verdict") if stop_if_interrupted(state, interrupted): return 75 if review is None: @@ -766,14 +801,16 @@ def replay_locked(arguments, state_dir): if arguments.review_observation is not None: supplied_review = observation(arguments.review_observation, "delivery_replay_review_observation", - state["identity"], "verdict") + state["identity"], + state["materialization"]["candidate_commit_id"], "verdict") if stop_if_interrupted(state, interrupted): return 75 if supplied_review != state.get("review"): raise ReplayError("supplied offline review changed after review wait") publisher = observation(arguments.publisher_observation, "delivery_replay_publisher_observation", - state["identity"], "disposition") + state["identity"], + state["materialization"]["candidate_commit_id"], "disposition") if stop_if_interrupted(state, interrupted): return 75 if publisher is None: diff --git a/scripts/test/delivery-replay.test.sh b/scripts/test/delivery-replay.test.sh index 600c064..02e6049 100755 --- a/scripts/test/delivery-replay.test.sh +++ b/scripts/test/delivery-replay.test.sh @@ -225,7 +225,7 @@ jq -e '.state.phase=="review-wait" and .authority=="none" and .offline_simulatio fail missing-review-waits request_sha=$(jq -r '.identity.request_sha256' "$tmp/changed-state/run.json") candidate_tree=$(jq -r '.identity.candidate_tree_id' "$tmp/changed-state/run.json") -candidate_commit=$(jq -r '.identity.candidate_commit_id' "$tmp/changed-state/run.json") +candidate_commit=$(jq -r '.materialization.candidate_commit_id' "$tmp/changed-state/run.json") moved_candidate=$(printf '%s\n' moved | /usr/bin/env -i HOME="$tmp/home" PATH=/usr/bin:/bin LC_ALL=C \ GIT_AUTHOR_NAME=fixture GIT_AUTHOR_EMAIL=fixture@example.invalid GIT_COMMITTER_NAME=fixture \ GIT_COMMITTER_EMAIL=fixture@example.invalid /usr/bin/git --git-dir="$tmp/changed-candidate/repository.git" \ @@ -554,8 +554,8 @@ if ! grep -Fq 'delivery replay: input is not JSON' "$tmp/invalid-journal.out" || fi pass 'invalid UTF-8 input, review, and journal records fail without a traceback' -printf '%s\n' '{"schema_version":1,"kind":"delivery_replay_review_observation","actor_id":"test.reviewer","request_sha256":"'"$request_sha"'","candidate_tree_id":"'"$candidate_tree"'","verdict":"clean"}' >"$tmp/review.json" -printf '%s\n' '{"schema_version":1,"kind":"delivery_replay_publisher_observation","actor_id":"test.publisher","request_sha256":"'"$request_sha"'","candidate_tree_id":"'"$candidate_tree"'","disposition":"offline-simulated"}' >"$tmp/publisher.json" +printf '%s\n' '{"schema_version":1,"kind":"delivery_replay_review_observation","actor_id":"test.reviewer","request_sha256":"'"$request_sha"'","candidate_tree_id":"'"$candidate_tree"'","candidate_commit_id":"'"$candidate_commit"'","verdict":"clean"}' >"$tmp/review.json" +printf '%s\n' '{"schema_version":1,"kind":"delivery_replay_publisher_observation","actor_id":"test.publisher","request_sha256":"'"$request_sha"'","candidate_tree_id":"'"$candidate_tree"'","candidate_commit_id":"'"$candidate_commit"'","disposition":"offline-simulated"}' >"$tmp/publisher.json" expect_candidate_move_rejected review-wait --review-observation "$tmp/review.json" lock_holder="$tmp/lock-holder.py" printf '%s\n' \ @@ -630,7 +630,7 @@ fi [ "$review_cancel_status" -eq 75 ] || fail review-cancel-code jq -e '(.phase=="review-wait") and (has("review")|not)' "$tmp/review-cancel-state/run.json" >/dev/null || fail review-cancel-state -printf '%s\n' '{"schema_version":1,"kind":"delivery_replay_review_observation","actor_id":123,"request_sha256":"'"$request_sha"'","candidate_tree_id":"'"$candidate_tree"'","verdict":"clean"}' >"$tmp/numeric-review.json" +printf '%s\n' '{"schema_version":1,"kind":"delivery_replay_review_observation","actor_id":123,"request_sha256":"'"$request_sha"'","candidate_tree_id":"'"$candidate_tree"'","candidate_commit_id":"'"$candidate_commit"'","verdict":"clean"}' >"$tmp/numeric-review.json" if python3 "$replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ --candidate-root "$tmp/changed-candidate" --scratch-root "$tmp/changed-scratch" --state-dir "$tmp/changed-state" \ --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" --verify-path source.txt --expected-sha256 "$expected_changed" \ @@ -747,7 +747,7 @@ jq -e '.state.phase=="completed-offline"' "$tmp/git-env.out" >/dev/null || fail [ "$(git_clean --git-dir="$tmp/changed-candidate/repository.git" rev-parse refs/heads/candidate)" = "$candidate_commit" ] || fail git-env-candidate pass 'all candidate Git operations ignore ambient repository, namespace, config, work-tree, and hook settings' -printf '%s\n' '{"schema_version":1,"kind":"delivery_replay_publisher_observation","actor_id":123,"request_sha256":"'"$request_sha"'","candidate_tree_id":"'"$candidate_tree"'","disposition":"offline-simulated"}' >"$tmp/numeric-publisher.json" +printf '%s\n' '{"schema_version":1,"kind":"delivery_replay_publisher_observation","actor_id":123,"request_sha256":"'"$request_sha"'","candidate_tree_id":"'"$candidate_tree"'","candidate_commit_id":"'"$candidate_commit"'","disposition":"offline-simulated"}' >"$tmp/numeric-publisher.json" if python3 "$replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ --candidate-root "$tmp/changed-candidate" --scratch-root "$tmp/changed-scratch" --state-dir "$tmp/changed-state" \ --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" --verify-path source.txt --expected-sha256 "$expected_changed" \ @@ -804,6 +804,62 @@ if python3 "$replay" --input "$base_input" --source-repository-id fixture.target --review-observation "$tmp/changed-review.json" --publisher-observation "$tmp/publisher.json" >"$tmp/changed-review.out" 2>&1; then fail changed-review-after-wait; fi grep -Fq 'review changed after review wait' "$tmp/changed-review.out" || fail changed-review-after-wait-error pass 'a changed supplied review cannot advance publish wait' +ref_holder="$tmp/candidate-ref-holder.py" +printf '%s\n' \ + 'import pathlib, subprocess, sys, time' \ + 'repository, commit, ready, release = sys.argv[1], sys.argv[2], pathlib.Path(sys.argv[3]), pathlib.Path(sys.argv[4])' \ + 'environment = {"PATH": "/usr/bin:/bin", "LC_ALL": "C", "GIT_CONFIG_NOSYSTEM": "1", "GIT_CONFIG_GLOBAL": "/dev/null"}' \ + 'command = ["/usr/bin/git", "--git-dir=" + repository, "update-ref", "--stdin"]' \ + 'holder = subprocess.Popen(command, env=environment, stdin=subprocess.PIPE, stdout=subprocess.PIPE)' \ + 'holder.stdin.write(("option no-deref\nstart\nverify refs/heads/candidate " + commit + "\nprepare\n").encode())' \ + 'holder.stdin.flush()' \ + 'while True:' \ + ' line = holder.stdout.readline()' \ + ' if not line: raise SystemExit("holder could not lock the candidate ref")' \ + ' if line.strip() == b"prepare: ok": break' \ + 'ready.write_text("held")' \ + 'while not release.exists(): time.sleep(0.01)' \ + 'holder.stdin.write(b"abort\n")' \ + 'holder.stdin.flush()' \ + 'holder.stdin.close()' \ + 'raise SystemExit(holder.wait())' >"$ref_holder" +mkdir -m 700 "$tmp/held-lock-state" +cp "$tmp/changed-state/materialization-input.json" "$tmp/changed-state/run.json" "$tmp/held-lock-state/" +python3 "$ref_holder" "$tmp/changed-candidate/repository.git" "$candidate_commit" \ + "$tmp/held-lock-ready" "$tmp/held-lock-release" & +ref_holder_pid=$! +held_lock_wait=0 +while [ ! -f "$tmp/held-lock-ready" ]; do + kill -0 "$ref_holder_pid" 2>/dev/null || fail held-lock-holder + held_lock_wait=$((held_lock_wait + 1)) + [ "$held_lock_wait" -le 1000 ] || fail held-lock-holder-timeout + sleep 0.01 +done +# Another process already holds the candidate ref lock, so this guard never owns +# it. The background watchdog only fires if the guard hangs instead of failing. +python3 "$replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ + --candidate-root "$tmp/changed-candidate" --scratch-root "$tmp/changed-scratch" --state-dir "$tmp/held-lock-state" \ + --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" --verify-path source.txt \ + --expected-sha256 "$expected_changed" --publisher-observation "$tmp/publisher.json" >"$tmp/held-lock.out" 2>&1 & +held_lock_pid=$! +( sleep 60; kill -9 "$held_lock_pid" 2>/dev/null ) & +held_lock_watchdog=$! +if wait "$held_lock_pid"; then fail held-lock-status; fi +kill "$held_lock_watchdog" 2>/dev/null || true +if ! grep -Fq 'delivery replay: candidate repository identity guard failed' "$tmp/held-lock.out" || + grep -Fq Traceback "$tmp/held-lock.out"; then + fail held-lock-error +fi +jq -e '.phase=="publish-wait" and (has("publisher")|not)' "$tmp/held-lock-state/run.json" >/dev/null || + fail held-lock-state +touch "$tmp/held-lock-release" +wait "$ref_holder_pid" || fail held-lock-release +python3 "$replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ + --candidate-root "$tmp/changed-candidate" --scratch-root "$tmp/changed-scratch" --state-dir "$tmp/held-lock-state" \ + --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" --verify-path source.txt \ + --expected-sha256 "$expected_changed" --publisher-observation "$tmp/publisher.json" >"$tmp/held-lock-resume.out" +jq -e '.state.phase=="completed-offline"' "$tmp/held-lock-resume.out" >/dev/null || fail held-lock-resume +pass 'a candidate ref lock held elsewhere blocks completion until it is released' python3 "$replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ --candidate-root "$tmp/changed-candidate" --scratch-root "$tmp/changed-scratch" --state-dir "$tmp/changed-state" \ --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" --verify-path source.txt --expected-sha256 "$expected_changed" \ @@ -830,7 +886,10 @@ mkdir -m 700 "$tmp/mismatch-state" "$tmp/mismatch-candidate" "$tmp/mismatch-scra python3 "$replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ --candidate-root "$tmp/mismatch-candidate" --scratch-root "$tmp/mismatch-scratch" --state-dir "$tmp/mismatch-state" \ --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" --verify-path source.txt --expected-sha256 "$expected_changed" > /dev/null -printf '%s\n' '{"schema_version":1,"kind":"delivery_replay_review_observation","actor_id":"test.reviewer","request_sha256":"'"$request_sha"'","candidate_tree_id":"'"$(printf '0%.0s' {1..40})"'","verdict":"clean"}' >"$tmp/mismatch-review.json" +mismatch_commit=$(jq -r '.materialization.candidate_commit_id' "$tmp/mismatch-state/run.json") +mismatch_tree=$(jq -r '.materialization.candidate_tree_id' "$tmp/mismatch-state/run.json") +zero_oid=$(printf '0%.0s' {1..40}) +printf '%s\n' '{"schema_version":1,"kind":"delivery_replay_review_observation","actor_id":"test.reviewer","request_sha256":"'"$request_sha"'","candidate_tree_id":"'"$zero_oid"'","candidate_commit_id":"'"$mismatch_commit"'","verdict":"clean"}' >"$tmp/mismatch-review.json" if python3 "$replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ --candidate-root "$tmp/mismatch-candidate" --scratch-root "$tmp/mismatch-scratch" --state-dir "$tmp/mismatch-state" \ --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" --verify-path source.txt --expected-sha256 "$expected_changed" \ @@ -838,6 +897,21 @@ if python3 "$replay" --input "$base_input" --source-repository-id fixture.target grep -Fq 'does not match this candidate' "$tmp/mismatch.out" || fail mismatched-review-error pass 'mismatched supplied review cannot complete the replay' +# The same tree can belong to two candidate commits, so a review naming this +# request and tree but another commit must not advance this candidate. +printf '%s\n' '{"schema_version":1,"kind":"delivery_replay_review_observation","actor_id":"test.reviewer","request_sha256":"'"$request_sha"'","candidate_tree_id":"'"$mismatch_tree"'","candidate_commit_id":"'"$zero_oid"'","verdict":"clean"}' >"$tmp/other-commit-review.json" +if python3 "$replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ + --candidate-root "$tmp/mismatch-candidate" --scratch-root "$tmp/mismatch-scratch" --state-dir "$tmp/mismatch-state" \ + --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" --verify-path source.txt --expected-sha256 "$expected_changed" \ + --review-observation "$tmp/other-commit-review.json" >"$tmp/other-commit.out" 2>&1; then fail other-commit-review; fi +if ! grep -Fq 'does not match this candidate' "$tmp/other-commit.out" || + grep -Fq Traceback "$tmp/other-commit.out"; then + fail other-commit-review-error +fi +jq -e '.phase=="review-wait" and (has("review")|not)' "$tmp/mismatch-state/run.json" >/dev/null || + fail other-commit-review-state +pass 'a review bound to another candidate commit with this tree is refused' + make_empty_input "$base_input" "$tmp/empty-final.json" source_digest=$(printf '%s\n' alpha beta | /usr/bin/shasum -a 256 | /usr/bin/awk '{print $1}') run_replay no-change "$tmp/empty-final.json" "$source_digest" >"$tmp/no-change.out" diff --git a/work/delivery-loop-first/plan.md b/work/delivery-loop-first/plan.md index 0200d0d..877e5a3 100644 --- a/work/delivery-loop-first/plan.md +++ b/work/delivery-loop-first/plan.md @@ -15,8 +15,10 @@ candidate code or a user command string. States are `materializing`, `verifying`, `review-wait`, `publish-wait`, `failed`, and `completed-offline`. Review and publisher records are supplied offline test -observations. They name the exact request and candidate plus an actor, but do not -authenticate anyone or authorize publication. Missing review remains waiting. +observations. They name the exact request digest, candidate tree, and candidate +commit plus an actor, but do not authenticate anyone or authorize publication. The +commit is required because one tree can belong to two candidate commits. Missing +review remains waiting. The slice is not profile selection, qualification, model execution, target access, deployment, release, install, merge, or production publication. It is not the From eedfb7cd662aeafc3249d8673d5c58051d7ee47a Mon Sep 17 00:00:00 2001 From: ci Date: Sun, 6 Sep 2026 08:54:08 -0400 Subject: [PATCH 19/20] Bound the source repository id before it is journaled An over-long --source-repository-id was copied into the identity and written to run.json before the materializer could reject it, and a large enough value made the journal unreadable for later replays. The id now has to satisfy the core contract's id rule before anything is written, and a saved journal with an out-of-contract id is refused. Proof: delivery-replay 39/39, shellcheck clean. Co-Authored-By: Claude Fable 5.1 --- delivery/v1/replay.py | 6 ++++++ scripts/test/delivery-replay.test.sh | 13 +++++++++++++ 2 files changed, 19 insertions(+) diff --git a/delivery/v1/replay.py b/delivery/v1/replay.py index 586c415..97a1b46 100755 --- a/delivery/v1/replay.py +++ b/delivery/v1/replay.py @@ -42,6 +42,9 @@ MAX_GUARD_LINE_BYTES = 4096 OID = re.compile(r"[0-9a-f]{40}|[0-9a-f]{64}\Z") ACTOR = re.compile(r"[a-z0-9][a-z0-9._:-]{0,127}\Z") +# The core contract's id rule; the source repository id is journaled, so it is +# bounded before anything is written. +REPOSITORY_ID = re.compile(r"[a-z0-9][a-z0-9._:-]{0,127}\Z") GIT_ENVIRONMENT = { "PATH": "/usr/bin:/bin", "LC_ALL": "C", "GIT_CONFIG_NOSYSTEM": "1", "GIT_CONFIG_GLOBAL": "/dev/null", "GIT_NO_REPLACE_OBJECTS": "1", @@ -585,6 +588,7 @@ def validate_state(state, identity): for name in ("input_sha256", "request_sha256", "driver_sha256", "materializer_sha256", "closure_helper_sha256", "jq_sha256", "run_key") ) or not isinstance(saved.get("source_repository_id"), str) or \ + not REPOSITORY_ID.fullmatch(saved["source_repository_id"]) or \ saved.get("source_hash_algorithm") not in {"sha1", "sha256"} or \ any(not isinstance(saved.get(name), str) or not OID.fullmatch(saved[name]) for name in ("source_commit_id", "source_tree_id")) or \ @@ -683,6 +687,8 @@ def replay_locked(arguments, state_dir): fcntl.flock(lock, fcntl.LOCK_EX) execution = create_execution_snapshot(repository, arguments, state_dir) sources_match = execution_sources_match(repository, arguments, execution) + if not REPOSITORY_ID.fullmatch(arguments.source_repository_id): + raise ReplayError("source repository id is invalid") input_bytes = read_bytes(arguments.input, MAX_INPUT_BYTES) input_value = parse_json(input_bytes) input_sha = digest_bytes(input_bytes) diff --git a/scripts/test/delivery-replay.test.sh b/scripts/test/delivery-replay.test.sh index 02e6049..9112ec6 100755 --- a/scripts/test/delivery-replay.test.sh +++ b/scripts/test/delivery-replay.test.sh @@ -506,6 +506,19 @@ if ! grep -Fq 'delivery replay: input is not JSON' "$tmp/huge-journal.out" || fi pass 'huge JSON integers in input, observation, and journal fail without a traceback' +long_repository_id=$(printf 'a%.0s' {1..200}) +mkdir -m 700 "$tmp/long-id-state" "$tmp/long-id-candidate" "$tmp/long-id-scratch" +if python3 "$replay" --input "$base_input" --source-repository-id "$long_repository_id" --source-git-dir "$tmp/source.git" \ + --candidate-root "$tmp/long-id-candidate" --scratch-root "$tmp/long-id-scratch" --state-dir "$tmp/long-id-state" \ + --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" --verify-path source.txt --expected-sha256 "$expected_changed" \ + >"$tmp/long-id.out" 2>&1; then fail long-repository-id; fi +if ! grep -Fq 'delivery replay: source repository id is invalid' "$tmp/long-id.out" || + grep -Fq Traceback "$tmp/long-id.out"; then + fail long-repository-id-error +fi +[ ! -e "$tmp/long-id-state/run.json" ] || fail long-repository-id-journaled +pass 'an out-of-contract source repository id is refused before anything is journaled' + printf '\377' >"$tmp/invalid-input.json" mkdir -m 700 "$tmp/invalid-input-state" "$tmp/invalid-input-candidate" "$tmp/invalid-input-scratch" if python3 "$replay" --input "$tmp/invalid-input.json" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ From c319b7a47a34c15b149fb09a1eefff6df3838d8f Mon Sep 17 00:00:00 2001 From: ci Date: Sun, 6 Sep 2026 09:09:29 -0400 Subject: [PATCH 20/20] Send fresh runs through the materializer and let the test fetch its jq A run with no journal yet reconciled against whatever candidate repository was already in the candidate root, so a pre-populated root that matched the input was adopted without the materializer's empty-root check. Reconciliation now applies only to a resumed run; a fresh run always materializes. The suite also downloads the pinned jq 1.6 itself, so it works alone on a fresh restore as the restore guide instructs. Proof: delivery-replay 40/40 (new case: a fresh run never adopts a pre-populated candidate root), shellcheck clean. Co-Authored-By: Claude Fable 5.1 --- delivery/v1/replay.py | 11 ++++++++--- scripts/test/delivery-replay.test.sh | 27 ++++++++++++++++++++++++++- 2 files changed, 34 insertions(+), 4 deletions(-) diff --git a/delivery/v1/replay.py b/delivery/v1/replay.py index 97a1b46..1c741c6 100755 --- a/delivery/v1/replay.py +++ b/delivery/v1/replay.py @@ -707,7 +707,8 @@ def replay_locked(arguments, state_dir): raise ReplayError("execution bundle does not match current dependencies") if stop_if_interrupted(state, interrupted): return 75 - if state is None: + fresh_run = state is None + if fresh_run: state = {"schema_version": 1, "kind": "delivery_replay_state", "identity": identity, "phase": "materializing", "authority": "none", "qualification": "unavailable"} atomic_bytes(input_snapshot_path, input_bytes) @@ -743,8 +744,12 @@ def replay_locked(arguments, state_dir): return 0 if state["phase"] == "materializing": try: - reconciled = reconcile_materialization(arguments, execution, input_snapshot_path, - identity, state_dir) + # Only a resumed run may adopt a candidate that is already in + # the candidate root; a fresh run always goes through the + # materializer, whose root check refuses a pre-populated root. + reconciled = None if fresh_run else reconcile_materialization( + arguments, execution, input_snapshot_path, identity, state_dir + ) state["materialization"] = reconciled or run_materializer( arguments, execution, input_snapshot_path, identity ) diff --git a/scripts/test/delivery-replay.test.sh b/scripts/test/delivery-replay.test.sh index 9112ec6..8fab3e4 100755 --- a/scripts/test/delivery-replay.test.sh +++ b/scripts/test/delivery-replay.test.sh @@ -25,7 +25,19 @@ case "$platform" in Darwin:x86_64|Darwin:arm64) asset=jq-osx-amd64; asset_sha=5c0a0a3ea600f302ee458b30317425dd9632d1ad8882259fcaf4e9b868b2b1ef ;; *) fail "unsupported host $platform" ;; esac -jq_bin="${TMPDIR:-/tmp}/ystack-portable-core-jq16/$asset" +# Run alone on a fresh restore this suite cannot rely on another suite having +# filled the shared jq 1.6 cache, so it fetches the pinned release itself. +jq_cache_dir="${TMPDIR:-/tmp}/ystack-portable-core-jq16" +/bin/mkdir -p "$jq_cache_dir" +jq_bin="$jq_cache_dir/$asset" +if [ ! -f "$jq_bin" ] || [ -L "$jq_bin" ] || [ "$(sha_file "$jq_bin")" != "$asset_sha" ]; then + download=$(/usr/bin/mktemp "$jq_cache_dir/.jq-1.6.XXXXXX") + /usr/bin/curl --proto '=https' --tlsv1.2 -fsSL \ + "https://github.com/jqlang/jq/releases/download/jq-1.6/$asset" -o "$download" + [ "$(sha_file "$download")" = "$asset_sha" ] || fail 'jq release digest' + /bin/chmod 0555 "$download" + /bin/mv "$download" "$jq_bin" +fi [ -f "$jq_bin" ] && [ ! -L "$jq_bin" ] && [ "$(sha_file "$jq_bin")" = "$asset_sha" ] || fail 'pinned jq 1.6 is required' @@ -307,6 +319,19 @@ python3 "$replay" --input "$base_input" --source-repository-id fixture.target -- jq -e '.state.phase=="review-wait"' "$tmp/reconcile-retry.out" >/dev/null || fail reconcile-retry pass 'SIGKILL after materializer output reconciles the existing candidate once' +# A fresh run (no journal yet) must not adopt a candidate already sitting in +# the candidate root, even one that matches the input: only the materializer's +# empty-root check may admit a root on a first run. +mkdir -m 700 "$tmp/fresh-reuse-state" "$tmp/fresh-reuse-scratch" +if python3 "$replay" --input "$base_input" --source-repository-id fixture.target --source-git-dir "$tmp/source.git" \ + --candidate-root "$tmp/reconcile-candidate" --scratch-root "$tmp/fresh-reuse-scratch" --state-dir "$tmp/fresh-reuse-state" \ + --closure-helper "$runtime/object-closure" --jq-bin "$jq_bin" --verify-path source.txt --expected-sha256 "$expected_changed" \ + >"$tmp/fresh-reuse.out" 2>&1; then fail fresh-reuse-accepted; fi +grep -Fq Traceback "$tmp/fresh-reuse.out" && fail fresh-reuse-traceback +jq -e '.state.phase=="failed" and .state.reason=="materialization did not complete"' "$tmp/fresh-reuse.out" >/dev/null || + fail fresh-reuse-outcome +pass 'a fresh run never adopts a pre-populated candidate root' + mkdir -m 700 "$tmp/repeated-kill-state" "$tmp/repeated-kill-candidate" "$tmp/repeated-kill-scratch" for kill_round in 1 2 3; do if python3 "$kill_wrapper" "$replay" --input "$base_input" --source-repository-id fixture.target \