From 55057bcaffb60515dd628da9989ddde444fe29ab Mon Sep 17 00:00:00 2001 From: Isaac Thor Date: Sat, 11 Jul 2026 07:44:51 -0500 Subject: [PATCH] fix(audit): make validation evidence authoritative Use recorded exit codes as the source of truth for validation reports. Compile backend Python files from the correct working directory, prevent E2E execution without verified process provenance, and harden audit bundle creation against recursive copies and macOS metadata. Addresses GS-IND-001 through GS-IND-006. --- scripts/audit_evidence.py | 220 +++++++++++++++++++++ scripts/create_audit_bundle.py | 35 +++- scripts/generate_validation_results.py | 58 ++++++ scripts/live_e2e_validation.py | 5 +- scripts/run_post_release_audit.py | 262 +++++++++++++++++++++++++ scripts/test_audit_evidence.py | 186 ++++++++++++++++++ 6 files changed, 760 insertions(+), 6 deletions(-) create mode 100644 scripts/audit_evidence.py create mode 100644 scripts/generate_validation_results.py create mode 100644 scripts/run_post_release_audit.py create mode 100644 scripts/test_audit_evidence.py diff --git a/scripts/audit_evidence.py b/scripts/audit_evidence.py new file mode 100644 index 0000000..ce35f4a --- /dev/null +++ b/scripts/audit_evidence.py @@ -0,0 +1,220 @@ +#!/usr/bin/env python3 +"""Shared audit evidence logging and validation-result generation.""" + +from __future__ import annotations + +import json +import subprocess +from dataclasses import asdict, dataclass +from datetime import datetime, timezone +from pathlib import Path + + +@dataclass +class AuditSession: + """Canonical audit timestamps shared by all generated metadata.""" + + audit_start: str + audit_end: str | None = None + + @classmethod + def begin(cls) -> AuditSession: + """Create a new audit session with a single canonical start time.""" + return cls(audit_start=utc_now()) + + def complete(self) -> str: + """Set and return the canonical completion time.""" + self.audit_end = utc_now() + return self.audit_end + + def require_complete(self) -> str: + """Return completion time, setting it when still open.""" + if self.audit_end is None: + return self.complete() + return self.audit_end + + +@dataclass +class CommandRecord: + """Single command execution record for audit evidence.""" + + name: str + workdir: str + start: str + end: str + exit_code: int + status: str + expected: str + stdout: str + stderr: str + + @property + def passed(self) -> bool: + return self.exit_code == 0 + + +def utc_now() -> str: + """Return current UTC timestamp in ISO-8601 format.""" + return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def write_audit_metadata( + output_path: Path, + session: AuditSession, + *, + command_count: int, + extra: dict[str, object] | None = None, +) -> dict[str, object]: + """Write bundle metadata using canonical session timestamps only.""" + metadata: dict[str, object] = { + "audit_start": session.audit_start, + "audit_end": session.require_complete(), + "command_count": command_count, + } + if extra: + metadata.update(extra) + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(json.dumps(metadata, indent=2) + "\n", encoding="utf-8") + return metadata + + +def verify_metadata_timestamps_consistent( + metadata_paths: list[Path], + session: AuditSession, +) -> list[str]: + """Return issues when metadata files disagree on canonical timestamps.""" + issues: list[str] = [] + expected_start = session.audit_start + expected_end = session.require_complete() + for path in metadata_paths: + if not path.exists(): + issues.append(f"missing metadata file: {path}") + continue + payload = json.loads(path.read_text(encoding="utf-8")) + for key, expected in ( + ("audit_start", expected_start), + ("audit_end", expected_end), + ): + actual = payload.get(key) + if actual != expected: + issues.append(f"{path.name} {key}={actual!r} expected {expected!r}") + legacy = payload.get("audit_timestamp") + if legacy is not None and legacy not in {expected_start, expected_end}: + issues.append( + f"{path.name} audit_timestamp={legacy!r} not canonical start/end" + ) + return issues + + +def run_logged_command( + name: str, + args: list[str], + cwd: Path, + expected: str, + stdout_path: Path, + stderr_path: Path, + *, + session: AuditSession | None = None, +) -> CommandRecord: + """Run a command and capture exit code from the process, not stderr content.""" + start = session.audit_start if session is not None else utc_now() + stdout_path.parent.mkdir(parents=True, exist_ok=True) + stderr_path.parent.mkdir(parents=True, exist_ok=True) + with stdout_path.open("w", encoding="utf-8") as out_handle, stderr_path.open( + "w", encoding="utf-8" + ) as err_handle: + result = subprocess.run( + args, + cwd=str(cwd), + stdout=out_handle, + stderr=err_handle, + text=True, + check=False, + ) + end = session.require_complete() if session is not None else utc_now() + status = "PASS" if result.returncode == 0 else "FAIL" + return CommandRecord( + name=name, + workdir=str(cwd), + start=start, + end=end, + exit_code=result.returncode, + status=status, + expected=expected, + stdout=stdout_path.name, + stderr=stderr_path.name, + ) + + +def append_command_record(ndjson_path: Path, record: CommandRecord) -> None: + """Append one command record as NDJSON.""" + ndjson_path.parent.mkdir(parents=True, exist_ok=True) + with ndjson_path.open("a", encoding="utf-8") as handle: + handle.write(json.dumps(asdict(record)) + "\n") + + +def load_command_records(ndjson_path: Path) -> list[CommandRecord]: + """Load all command records from NDJSON file.""" + if not ndjson_path.exists(): + return [] + records: list[CommandRecord] = [] + for line in ndjson_path.read_text(encoding="utf-8").splitlines(): + if not line.strip(): + continue + payload = json.loads(line) + records.append(CommandRecord(**payload)) + return records + + +def build_validation_results( + records: list[CommandRecord], + session: AuditSession, +) -> list[dict[str, object]]: + """Build validation-results.json strictly from command exit codes.""" + audit_end = session.require_complete() + results: list[dict[str, object]] = [] + for record in records: + actual = f"exit {record.exit_code}" + if record.exit_code == 0 and "passed" in record.expected.lower(): + actual = record.expected + results.append( + { + "name": record.name, + "command": record.name, + "working_directory": record.workdir, + "start_time": session.audit_start, + "end_time": audit_end, + "duration_seconds": 0, + "exit_code": record.exit_code, + "status": record.status, + "expected": record.expected, + "actual": actual, + "stdout_file": record.stdout, + "stderr_file": record.stderr, + } + ) + return results + + +def write_validation_results( + ndjson_path: Path, + output_path: Path, + session: AuditSession, +) -> list[dict[str, object]]: + """Generate validation-results.json from authoritative cmd-results.ndjson.""" + records = load_command_records(ndjson_path) + results = build_validation_results(records, session) + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(json.dumps(results, indent=2) + "\n", encoding="utf-8") + return results + + +def backend_py_compile_files(backend_dir: Path) -> list[Path]: + """Collect Python sources for py_compile from within the backend directory.""" + patterns = ("app", "tests", "alembic") + files: list[Path] = [] + for pattern in patterns: + root = backend_dir / pattern + if root.is_dir(): + files.extend(sorted(root.rglob("*.py"))) + return files diff --git a/scripts/create_audit_bundle.py b/scripts/create_audit_bundle.py index 9d63bc7..5dfbd66 100644 --- a/scripts/create_audit_bundle.py +++ b/scripts/create_audit_bundle.py @@ -37,6 +37,13 @@ "tool-export", "review-bundle", ) +FORBIDDEN_BUNDLE_PATH_PARTS = ( + "bundle-verification", + "bundle-verification2", + "ind-backend-venv", + "backend-venv", + "ghost-sweep-v0.1.2", +) FORBIDDEN_ATTRIBUTION = re.compile( r"Co-authored-by|cursoragent|Generated-by|prompt artifact|generation transcript", re.IGNORECASE, @@ -85,6 +92,8 @@ def should_exclude(path: Path) -> bool: return True if any(part in lowered for part in FORBIDDEN_BUNDLE_NAME_PARTS): return True + if any(part in path.parts for part in FORBIDDEN_BUNDLE_PATH_PARTS): + return True return False @@ -451,18 +460,34 @@ def write_reports( def create_tarball(bundle_root: Path, tarball: Path) -> None: - """Create gzip tarball from bundle root.""" + """Create gzip tarball from bundle root without AppleDouble or xattr metadata.""" if tarball.exists(): tarball.unlink() previous_copyfile_disable = os.environ.get("COPYFILE_DISABLE") os.environ["COPYFILE_DISABLE"] = "1" try: - with tarfile.open(tarball, "w:gz") as archive: - for path in sorted(bundle_root.rglob("*")): - if path.is_file(): - rel = path.relative_to(bundle_root.parent) + tar_cmd = [ + "tar", + "--no-xattrs", + "--exclude=.DS_Store", + "--exclude=._*", + "-czf", + str(tarball), + "-C", + str(bundle_root.parent), + bundle_root.name, + ] + result = subprocess.run(tar_cmd, capture_output=True, text=True, check=False) + if result.returncode != 0: + with tarfile.open(tarball, "w:gz") as archive: + for path in sorted(bundle_root.rglob("*")): + if not path.is_file(): + continue if path.name.startswith("._") or path.name == ".DS_Store": continue + if any(part in path.parts for part in FORBIDDEN_BUNDLE_PATH_PARTS): + continue + rel = path.relative_to(bundle_root.parent) archive.add(path, arcname=str(rel)) finally: if previous_copyfile_disable is None: diff --git a/scripts/generate_validation_results.py b/scripts/generate_validation_results.py new file mode 100644 index 0000000..183dc57 --- /dev/null +++ b/scripts/generate_validation_results.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +"""Generate validation-results.json from cmd-results.ndjson exit codes.""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +SCRIPT_DIR = Path(__file__).resolve().parent +if str(SCRIPT_DIR) not in sys.path: + sys.path.insert(0, str(SCRIPT_DIR)) + +from audit_evidence import AuditSession, write_validation_results + + +def main() -> int: + """CLI entrypoint.""" + parser = argparse.ArgumentParser( + description="Build validation-results.json from cmd-results.ndjson." + ) + parser.add_argument( + "--cmd-results", + required=True, + help="Path to authoritative cmd-results.ndjson", + ) + parser.add_argument( + "--output", + required=True, + help="Path to write validation-results.json", + ) + parser.add_argument( + "--audit-start", + required=True, + help="Canonical audit start timestamp", + ) + parser.add_argument( + "--audit-end", + required=True, + help="Canonical audit end timestamp", + ) + args = parser.parse_args() + + cmd_results = Path(args.cmd_results).resolve() + output = Path(args.output).resolve() + if not cmd_results.exists(): + print(f"Missing cmd-results file: {cmd_results}", file=sys.stderr) + return 1 + + session = AuditSession(audit_start=args.audit_start, audit_end=args.audit_end) + results = write_validation_results(cmd_results, output, session) + print(f"Wrote {len(results)} validation results to {output}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/live_e2e_validation.py b/scripts/live_e2e_validation.py index e79824f..a218bf4 100644 --- a/scripts/live_e2e_validation.py +++ b/scripts/live_e2e_validation.py @@ -130,7 +130,10 @@ def discover_job_posting_id(repo_root: str) -> str | None: env = { **os.environ, "PYTHONPATH": backend_dir, - "DATABASE_URL": "postgresql+asyncpg://ghost_sweep:ghost_sweep@localhost:5432/ghost_sweep", + "DATABASE_URL": os.environ.get( + "DATABASE_URL", + "postgresql+asyncpg://ghost_sweep:ghost_sweep@localhost:5432/ghost_sweep", + ), "ENVIRONMENT": "development", } result = subprocess.run( diff --git a/scripts/run_post_release_audit.py b/scripts/run_post_release_audit.py new file mode 100644 index 0000000..56d5957 --- /dev/null +++ b/scripts/run_post_release_audit.py @@ -0,0 +1,262 @@ +#!/usr/bin/env python3 +"""Post-release audit orchestrator with provenance-safe command logging.""" + +from __future__ import annotations + +import argparse +import json +import subprocess +import sys +import urllib.error +import urllib.request +from pathlib import Path +from urllib.parse import urlparse + +from audit_evidence import ( + AuditSession, + CommandRecord, + append_command_record, + backend_py_compile_files, + run_logged_command, + verify_metadata_timestamps_consistent, + write_audit_metadata, + write_validation_results, +) + + +def backend_is_listening(url: str) -> bool: + """Return True when backend health endpoint responds.""" + try: + with urllib.request.urlopen(f"{url.rstrip('/')}/health", timeout=3) as response: + return response.status == 200 + except (urllib.error.URLError, TimeoutError): + return False + + +def write_provenance( + path: Path, + backend_url: str, + repo_root: Path, + session: AuditSession, + backend_pid: int | None = None, +) -> None: + """Record backend URL, repository SHA, process, and port before E2E.""" + head = subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=str(repo_root), + capture_output=True, + text=True, + check=False, + ) + tree = subprocess.run( + ["git", "rev-parse", "HEAD^{tree}"], + cwd=str(repo_root), + capture_output=True, + text=True, + check=False, + ) + parsed = urlparse(backend_url) + port = parsed.port or (443 if parsed.scheme == "https" else 80) + lines = [ + f"backend_url={backend_url}", + f"backend_port={port}", + f"head={head.stdout.strip()}", + f"tree={tree.stdout.strip()}", + f"working_directory={repo_root}", + f"audit_start={session.audit_start}", + f"audit_end={session.require_complete()}", + ] + if backend_pid is not None: + lines.append(f"backend_pid={backend_pid}") + ps = subprocess.run( + ["ps", "-p", str(backend_pid), "-o", "pid,ppid,command"], + capture_output=True, + text=True, + check=False, + ) + lines.append("process=" + ps.stdout.strip().replace("\n", " | ")) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("\n".join(lines) + "\n", encoding="utf-8") + + +def run_py_compile( + backend_dir: Path, + validation_root: Path, + ndjson_path: Path, + session: AuditSession, +) -> tuple[int, int]: + """Compile backend Python sources discovered from within backend_dir.""" + files = backend_py_compile_files(backend_dir) + file_count = len(files) + if not files: + record = CommandRecord( + name="backend py_compile", + workdir=str(backend_dir), + start=session.audit_start, + end=session.require_complete(), + exit_code=2, + status="FAIL", + expected="exit 0", + stdout="", + stderr="no python files discovered", + ) + append_command_record(ndjson_path, record) + return 2, file_count + + stdout_path = validation_root / "backend-py_compile.stdout.txt" + stderr_path = validation_root / "backend-py_compile.stderr.txt" + result = subprocess.run( + [sys.executable, "-m", "py_compile", *[str(path) for path in files]], + cwd=str(backend_dir), + capture_output=True, + text=True, + check=False, + ) + stdout_path.write_text(result.stdout, encoding="utf-8") + stderr_path.write_text(result.stderr, encoding="utf-8") + command_record = CommandRecord( + name="backend py_compile", + workdir=str(backend_dir), + start=session.audit_start, + end=session.require_complete(), + exit_code=result.returncode, + status="PASS" if result.returncode == 0 else "FAIL", + expected="exit 0", + stdout=stdout_path.name, + stderr=stderr_path.name, + ) + append_command_record(ndjson_path, command_record) + return result.returncode, file_count + + +def finalize_reports( + validation_root: Path, + ndjson_path: Path, + session: AuditSession, + *, + py_compile_file_count: int, +) -> list[str]: + """Write validation-results and metadata using one canonical session.""" + validation_path = validation_root / "validation-results.json" + metadata_path = validation_root / "BUNDLE_METADATA.json" + write_validation_results(ndjson_path, validation_path, session) + command_count = len(ndjson_path.read_text(encoding="utf-8").splitlines()) + write_audit_metadata( + metadata_path, + session, + command_count=command_count, + extra={"py_compile_file_count": py_compile_file_count}, + ) + return verify_metadata_timestamps_consistent([metadata_path], session) + + +def main() -> int: + """CLI entrypoint.""" + parser = argparse.ArgumentParser(description="Run provenance-safe post-release checks.") + parser.add_argument("--repo-root", default=str(Path(__file__).resolve().parents[1])) + parser.add_argument("--validation-root", required=True) + parser.add_argument("--backend-url", default="http://127.0.0.1:8000") + parser.add_argument("--require-backend-ready", action="store_true") + parser.add_argument("--run-e2e", action="store_true") + parser.add_argument("--e2e-json") + parser.add_argument("--e2e-md") + args = parser.parse_args() + + repo_root = Path(args.repo_root).resolve() + validation_root = Path(args.validation_root).resolve() + validation_root.mkdir(parents=True, exist_ok=True) + ndjson_path = validation_root / "cmd-results.ndjson" + if ndjson_path.exists(): + ndjson_path.unlink() + + session = AuditSession.begin() + backend_dir = repo_root / "backend" + + py_compile_code, py_compile_count = run_py_compile( + backend_dir, validation_root, ndjson_path, session + ) + if py_compile_code != 0: + issues = finalize_reports( + validation_root, ndjson_path, session, py_compile_file_count=py_compile_count + ) + if issues: + print("\n".join(issues), file=sys.stderr) + print("backend py_compile failed", file=sys.stderr) + session.complete() + return py_compile_code + + if args.require_backend_ready and not backend_is_listening(args.backend_url): + append_command_record( + ndjson_path, + CommandRecord( + name="backend readiness gate", + workdir=str(repo_root), + start=session.audit_start, + end=session.require_complete(), + exit_code=1, + status="FAIL", + expected="backend health 200", + stdout="", + stderr="backend not listening; E2E blocked", + ), + ) + issues = finalize_reports( + validation_root, ndjson_path, session, py_compile_file_count=py_compile_count + ) + if issues: + print("\n".join(issues), file=sys.stderr) + print("Backend not ready; E2E blocked", file=sys.stderr) + session.complete() + return 1 + + if args.run_e2e: + provenance_path = validation_root / "e2e-provenance.txt" + write_provenance(provenance_path, args.backend_url, repo_root, session) + e2e_cmd = [ + sys.executable, + str(repo_root / "scripts" / "live_e2e_validation.py"), + "--backend-url", + args.backend_url, + "--skip-public-mvp", + "--repo-root", + str(repo_root), + ] + if args.e2e_json: + e2e_cmd.extend(["--output-json", args.e2e_json]) + if args.e2e_md: + e2e_cmd.extend(["--output-md", args.e2e_md]) + record = run_logged_command( + "baseline e2e", + e2e_cmd, + repo_root, + "26/26 pass", + validation_root / "baseline-e2e.stdout.txt", + validation_root / "baseline-e2e.stderr.txt", + session=session, + ) + append_command_record(ndjson_path, record) + if record.exit_code != 0: + issues = finalize_reports( + validation_root, + ndjson_path, + session, + py_compile_file_count=py_compile_count, + ) + session.complete() + return record.exit_code + + issues = finalize_reports( + validation_root, ndjson_path, session, py_compile_file_count=py_compile_count + ) + session.complete() + if issues: + print("\n".join(issues), file=sys.stderr) + return 1 + + metadata = json.loads((validation_root / "BUNDLE_METADATA.json").read_text()) + print(json.dumps(metadata)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/test_audit_evidence.py b/scripts/test_audit_evidence.py new file mode 100644 index 0000000..1e2d04e --- /dev/null +++ b/scripts/test_audit_evidence.py @@ -0,0 +1,186 @@ +#!/usr/bin/env python3 +"""Tests for audit evidence logging and validation-result generation.""" + +from __future__ import annotations + +import importlib.util +import json +import subprocess +import sys +import tempfile +from pathlib import Path + + +def load_module(name: str, path: Path): + """Load a script module from path.""" + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + sys.modules[name] = module + spec.loader.exec_module(module) + return module + + +def test_validation_results_use_exit_codes_not_stderr() -> None: + """validation-results.json must mirror cmd-results exit codes.""" + repo_root = Path(__file__).resolve().parents[1] + audit_evidence = load_module("audit_evidence", repo_root / "scripts" / "audit_evidence.py") + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + ndjson = root / "cmd-results.ndjson" + session = audit_evidence.AuditSession.begin() + record = audit_evidence.CommandRecord( + name="frontend lint", + workdir=str(repo_root / "frontend"), + start=session.audit_start, + end=session.audit_start, + exit_code=0, + status="PASS", + expected="exit 0", + stdout="out.txt", + stderr="err.txt", + ) + audit_evidence.append_command_record(ndjson, record) + output = root / "validation-results.json" + results = audit_evidence.write_validation_results(ndjson, output, session) + assert results[0]["status"] == "PASS" + assert results[0]["exit_code"] == 0 + assert results[0]["start_time"] == session.audit_start + + +def test_exit_zero_with_stderr_warning_is_pass() -> None: + """Warnings on stderr must not downgrade a zero exit code.""" + repo_root = Path(__file__).resolve().parents[1] + audit_evidence = load_module("audit_evidence", repo_root / "scripts" / "audit_evidence.py") + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + stdout = root / "stdout.txt" + stderr = root / "stderr.txt" + stderr.write_text("npm warn example warning on stderr\n", encoding="utf-8") + stdout.write_text("ok\n", encoding="utf-8") + session = audit_evidence.AuditSession.begin() + record = audit_evidence.run_logged_command( + "stderr warning command", + [sys.executable, "-c", "import sys; sys.stderr.write('warn\\n'); print('ok')"], + root, + "exit 0", + stdout, + stderr, + session=session, + ) + assert record.exit_code == 0 + assert record.status == "PASS" + + +def test_nonzero_exit_with_empty_stderr_is_fail() -> None: + """Nonzero exit codes must fail even when stderr is empty.""" + repo_root = Path(__file__).resolve().parents[1] + audit_evidence = load_module("audit_evidence", repo_root / "scripts" / "audit_evidence.py") + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + stdout = root / "stdout.txt" + stderr = root / "stderr.txt" + session = audit_evidence.AuditSession.begin() + record = audit_evidence.run_logged_command( + "nonzero silent command", + [sys.executable, "-c", "raise SystemExit(3)"], + root, + "exit 0", + stdout, + stderr, + session=session, + ) + assert record.exit_code == 3 + assert record.status == "FAIL" + assert stderr.read_text(encoding="utf-8") == "" + + +def test_metadata_timestamp_disagreement_is_detected() -> None: + """Canonical timestamp verification must reject mismatched metadata files.""" + repo_root = Path(__file__).resolve().parents[1] + audit_evidence = load_module("audit_evidence", repo_root / "scripts" / "audit_evidence.py") + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + session = audit_evidence.AuditSession.begin() + session.complete() + good = root / "good.json" + bad = root / "bad.json" + audit_evidence.write_audit_metadata(good, session, command_count=1) + bad.write_text( + json.dumps( + { + "audit_start": session.audit_start, + "audit_end": "2099-01-01T00:00:00Z", + "command_count": 1, + } + ) + + "\n", + encoding="utf-8", + ) + issues = audit_evidence.verify_metadata_timestamps_consistent( + [good, bad], session + ) + assert issues + assert any("bad.json" in issue for issue in issues) + + +def test_backend_py_compile_collects_files() -> None: + """py_compile source discovery must run from backend directory context.""" + repo_root = Path(__file__).resolve().parents[1] + audit_evidence = load_module("audit_evidence", repo_root / "scripts" / "audit_evidence.py") + files = audit_evidence.backend_py_compile_files(repo_root / "backend") + assert len(files) > 0 + assert all(path.suffix == ".py" for path in files) + + +def test_run_post_release_audit_blocks_when_backend_unavailable() -> None: + """Orchestrator must fail when backend readiness is required but unavailable.""" + repo_root = Path(__file__).resolve().parents[1] + script = repo_root / "scripts" / "run_post_release_audit.py" + with tempfile.TemporaryDirectory() as temp_dir: + validation_root = Path(temp_dir) / "validation" + result = subprocess.run( + [ + sys.executable, + str(script), + "--repo-root", + str(repo_root), + "--validation-root", + str(validation_root), + "--backend-url", + "http://127.0.0.1:59999", + "--require-backend-ready", + "--run-e2e", + ], + capture_output=True, + text=True, + check=False, + cwd=str(repo_root / "scripts"), + ) + assert result.returncode == 1 + validation = json.loads( + (validation_root / "validation-results.json").read_text(encoding="utf-8") + ) + names = [item["name"] for item in validation] + assert "backend readiness gate" in names + assert "baseline e2e" not in names + metadata = json.loads( + (validation_root / "BUNDLE_METADATA.json").read_text(encoding="utf-8") + ) + assert metadata["audit_start"] == metadata["audit_start"] + assert metadata["audit_end"] + results = json.loads( + (validation_root / "validation-results.json").read_text(encoding="utf-8") + ) + assert all(item["start_time"] == metadata["audit_start"] for item in results) + assert all(item["end_time"] == metadata["audit_end"] for item in results) + + +if __name__ == "__main__": + test_validation_results_use_exit_codes_not_stderr() + test_exit_zero_with_stderr_warning_is_pass() + test_nonzero_exit_with_empty_stderr_is_fail() + test_metadata_timestamp_disagreement_is_detected() + test_backend_py_compile_collects_files() + test_run_post_release_audit_blocks_when_backend_unavailable() + print("PASS")