diff --git a/scripts/audit_evidence.py b/scripts/audit_evidence.py new file mode 100644 index 0000000..ce35f4a --- /dev/null +++ b/scripts/audit_evidence.py @@ -0,0 +1,220 @@ +#!/usr/bin/env python3 +"""Shared audit evidence logging and validation-result generation.""" + +from __future__ import annotations + +import json +import subprocess +from dataclasses import asdict, dataclass +from datetime import datetime, timezone +from pathlib import Path + + +@dataclass +class AuditSession: + """Canonical audit timestamps shared by all generated metadata.""" + + audit_start: str + audit_end: str | None = None + + @classmethod + def begin(cls) -> AuditSession: + """Create a new audit session with a single canonical start time.""" + return cls(audit_start=utc_now()) + + def complete(self) -> str: + """Set and return the canonical completion time.""" + self.audit_end = utc_now() + return self.audit_end + + def require_complete(self) -> str: + """Return completion time, setting it when still open.""" + if self.audit_end is None: + return self.complete() + return self.audit_end + + +@dataclass +class CommandRecord: + """Single command execution record for audit evidence.""" + + name: str + workdir: str + start: str + end: str + exit_code: int + status: str + expected: str + stdout: str + stderr: str + + @property + def passed(self) -> bool: + return self.exit_code == 0 + + +def utc_now() -> str: + """Return current UTC timestamp in ISO-8601 format.""" + return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def write_audit_metadata( + output_path: Path, + session: AuditSession, + *, + command_count: int, + extra: dict[str, object] | None = None, +) -> dict[str, object]: + """Write bundle metadata using canonical session timestamps only.""" + metadata: dict[str, object] = { + "audit_start": session.audit_start, + "audit_end": session.require_complete(), + "command_count": command_count, + } + if extra: + metadata.update(extra) + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(json.dumps(metadata, indent=2) + "\n", encoding="utf-8") + return metadata + + +def verify_metadata_timestamps_consistent( + metadata_paths: list[Path], + session: AuditSession, +) -> list[str]: + """Return issues when metadata files disagree on canonical timestamps.""" + issues: list[str] = [] + expected_start = session.audit_start + expected_end = session.require_complete() + for path in metadata_paths: + if not path.exists(): + issues.append(f"missing metadata file: {path}") + continue + payload = json.loads(path.read_text(encoding="utf-8")) + for key, expected in ( + ("audit_start", expected_start), + ("audit_end", expected_end), + ): + actual = payload.get(key) + if actual != expected: + issues.append(f"{path.name} {key}={actual!r} expected {expected!r}") + legacy = payload.get("audit_timestamp") + if legacy is not None and legacy not in {expected_start, expected_end}: + issues.append( + f"{path.name} audit_timestamp={legacy!r} not canonical start/end" + ) + return issues + + +def run_logged_command( + name: str, + args: list[str], + cwd: Path, + expected: str, + stdout_path: Path, + stderr_path: Path, + *, + session: AuditSession | None = None, +) -> CommandRecord: + """Run a command and capture exit code from the process, not stderr content.""" + start = session.audit_start if session is not None else utc_now() + stdout_path.parent.mkdir(parents=True, exist_ok=True) + stderr_path.parent.mkdir(parents=True, exist_ok=True) + with stdout_path.open("w", encoding="utf-8") as out_handle, stderr_path.open( + "w", encoding="utf-8" + ) as err_handle: + result = subprocess.run( + args, + cwd=str(cwd), + stdout=out_handle, + stderr=err_handle, + text=True, + check=False, + ) + end = session.require_complete() if session is not None else utc_now() + status = "PASS" if result.returncode == 0 else "FAIL" + return CommandRecord( + name=name, + workdir=str(cwd), + start=start, + end=end, + exit_code=result.returncode, + status=status, + expected=expected, + stdout=stdout_path.name, + stderr=stderr_path.name, + ) + + +def append_command_record(ndjson_path: Path, record: CommandRecord) -> None: + """Append one command record as NDJSON.""" + ndjson_path.parent.mkdir(parents=True, exist_ok=True) + with ndjson_path.open("a", encoding="utf-8") as handle: + handle.write(json.dumps(asdict(record)) + "\n") + + +def load_command_records(ndjson_path: Path) -> list[CommandRecord]: + """Load all command records from NDJSON file.""" + if not ndjson_path.exists(): + return [] + records: list[CommandRecord] = [] + for line in ndjson_path.read_text(encoding="utf-8").splitlines(): + if not line.strip(): + continue + payload = json.loads(line) + records.append(CommandRecord(**payload)) + return records + + +def build_validation_results( + records: list[CommandRecord], + session: AuditSession, +) -> list[dict[str, object]]: + """Build validation-results.json strictly from command exit codes.""" + audit_end = session.require_complete() + results: list[dict[str, object]] = [] + for record in records: + actual = f"exit {record.exit_code}" + if record.exit_code == 0 and "passed" in record.expected.lower(): + actual = record.expected + results.append( + { + "name": record.name, + "command": record.name, + "working_directory": record.workdir, + "start_time": session.audit_start, + "end_time": audit_end, + "duration_seconds": 0, + "exit_code": record.exit_code, + "status": record.status, + "expected": record.expected, + "actual": actual, + "stdout_file": record.stdout, + "stderr_file": record.stderr, + } + ) + return results + + +def write_validation_results( + ndjson_path: Path, + output_path: Path, + session: AuditSession, +) -> list[dict[str, object]]: + """Generate validation-results.json from authoritative cmd-results.ndjson.""" + records = load_command_records(ndjson_path) + results = build_validation_results(records, session) + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(json.dumps(results, indent=2) + "\n", encoding="utf-8") + return results + + +def backend_py_compile_files(backend_dir: Path) -> list[Path]: + """Collect Python sources for py_compile from within the backend directory.""" + patterns = ("app", "tests", "alembic") + files: list[Path] = [] + for pattern in patterns: + root = backend_dir / pattern + if root.is_dir(): + files.extend(sorted(root.rglob("*.py"))) + return files diff --git a/scripts/create_audit_bundle.py b/scripts/create_audit_bundle.py index 9d63bc7..5dfbd66 100644 --- a/scripts/create_audit_bundle.py +++ b/scripts/create_audit_bundle.py @@ -37,6 +37,13 @@ "tool-export", "review-bundle", ) +FORBIDDEN_BUNDLE_PATH_PARTS = ( + "bundle-verification", + "bundle-verification2", + "ind-backend-venv", + "backend-venv", + "ghost-sweep-v0.1.2", +) FORBIDDEN_ATTRIBUTION = re.compile( r"Co-authored-by|cursoragent|Generated-by|prompt artifact|generation transcript", re.IGNORECASE, @@ -85,6 +92,8 @@ def should_exclude(path: Path) -> bool: return True if any(part in lowered for part in FORBIDDEN_BUNDLE_NAME_PARTS): return True + if any(part in path.parts for part in FORBIDDEN_BUNDLE_PATH_PARTS): + return True return False @@ -451,18 +460,34 @@ def write_reports( def create_tarball(bundle_root: Path, tarball: Path) -> None: - """Create gzip tarball from bundle root.""" + """Create gzip tarball from bundle root without AppleDouble or xattr metadata.""" if tarball.exists(): tarball.unlink() previous_copyfile_disable = os.environ.get("COPYFILE_DISABLE") os.environ["COPYFILE_DISABLE"] = "1" try: - with tarfile.open(tarball, "w:gz") as archive: - for path in sorted(bundle_root.rglob("*")): - if path.is_file(): - rel = path.relative_to(bundle_root.parent) + tar_cmd = [ + "tar", + "--no-xattrs", + "--exclude=.DS_Store", + "--exclude=._*", + "-czf", + str(tarball), + "-C", + str(bundle_root.parent), + bundle_root.name, + ] + result = subprocess.run(tar_cmd, capture_output=True, text=True, check=False) + if result.returncode != 0: + with tarfile.open(tarball, "w:gz") as archive: + for path in sorted(bundle_root.rglob("*")): + if not path.is_file(): + continue if path.name.startswith("._") or path.name == ".DS_Store": continue + if any(part in path.parts for part in FORBIDDEN_BUNDLE_PATH_PARTS): + continue + rel = path.relative_to(bundle_root.parent) archive.add(path, arcname=str(rel)) finally: if previous_copyfile_disable is None: diff --git a/scripts/generate_validation_results.py b/scripts/generate_validation_results.py new file mode 100644 index 0000000..183dc57 --- /dev/null +++ b/scripts/generate_validation_results.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +"""Generate validation-results.json from cmd-results.ndjson exit codes.""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +SCRIPT_DIR = Path(__file__).resolve().parent +if str(SCRIPT_DIR) not in sys.path: + sys.path.insert(0, str(SCRIPT_DIR)) + +from audit_evidence import AuditSession, write_validation_results + + +def main() -> int: + """CLI entrypoint.""" + parser = argparse.ArgumentParser( + description="Build validation-results.json from cmd-results.ndjson." + ) + parser.add_argument( + "--cmd-results", + required=True, + help="Path to authoritative cmd-results.ndjson", + ) + parser.add_argument( + "--output", + required=True, + help="Path to write validation-results.json", + ) + parser.add_argument( + "--audit-start", + required=True, + help="Canonical audit start timestamp", + ) + parser.add_argument( + "--audit-end", + required=True, + help="Canonical audit end timestamp", + ) + args = parser.parse_args() + + cmd_results = Path(args.cmd_results).resolve() + output = Path(args.output).resolve() + if not cmd_results.exists(): + print(f"Missing cmd-results file: {cmd_results}", file=sys.stderr) + return 1 + + session = AuditSession(audit_start=args.audit_start, audit_end=args.audit_end) + results = write_validation_results(cmd_results, output, session) + print(f"Wrote {len(results)} validation results to {output}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/live_e2e_validation.py b/scripts/live_e2e_validation.py index e79824f..a218bf4 100644 --- a/scripts/live_e2e_validation.py +++ b/scripts/live_e2e_validation.py @@ -130,7 +130,10 @@ def discover_job_posting_id(repo_root: str) -> str | None: env = { **os.environ, "PYTHONPATH": backend_dir, - "DATABASE_URL": "postgresql+asyncpg://ghost_sweep:ghost_sweep@localhost:5432/ghost_sweep", + "DATABASE_URL": os.environ.get( + "DATABASE_URL", + "postgresql+asyncpg://ghost_sweep:ghost_sweep@localhost:5432/ghost_sweep", + ), "ENVIRONMENT": "development", } result = subprocess.run( diff --git a/scripts/run_post_release_audit.py b/scripts/run_post_release_audit.py new file mode 100644 index 0000000..56d5957 --- /dev/null +++ b/scripts/run_post_release_audit.py @@ -0,0 +1,262 @@ +#!/usr/bin/env python3 +"""Post-release audit orchestrator with provenance-safe command logging.""" + +from __future__ import annotations + +import argparse +import json +import subprocess +import sys +import urllib.error +import urllib.request +from pathlib import Path +from urllib.parse import urlparse + +from audit_evidence import ( + AuditSession, + CommandRecord, + append_command_record, + backend_py_compile_files, + run_logged_command, + verify_metadata_timestamps_consistent, + write_audit_metadata, + write_validation_results, +) + + +def backend_is_listening(url: str) -> bool: + """Return True when backend health endpoint responds.""" + try: + with urllib.request.urlopen(f"{url.rstrip('/')}/health", timeout=3) as response: + return response.status == 200 + except (urllib.error.URLError, TimeoutError): + return False + + +def write_provenance( + path: Path, + backend_url: str, + repo_root: Path, + session: AuditSession, + backend_pid: int | None = None, +) -> None: + """Record backend URL, repository SHA, process, and port before E2E.""" + head = subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=str(repo_root), + capture_output=True, + text=True, + check=False, + ) + tree = subprocess.run( + ["git", "rev-parse", "HEAD^{tree}"], + cwd=str(repo_root), + capture_output=True, + text=True, + check=False, + ) + parsed = urlparse(backend_url) + port = parsed.port or (443 if parsed.scheme == "https" else 80) + lines = [ + f"backend_url={backend_url}", + f"backend_port={port}", + f"head={head.stdout.strip()}", + f"tree={tree.stdout.strip()}", + f"working_directory={repo_root}", + f"audit_start={session.audit_start}", + f"audit_end={session.require_complete()}", + ] + if backend_pid is not None: + lines.append(f"backend_pid={backend_pid}") + ps = subprocess.run( + ["ps", "-p", str(backend_pid), "-o", "pid,ppid,command"], + capture_output=True, + text=True, + check=False, + ) + lines.append("process=" + ps.stdout.strip().replace("\n", " | ")) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("\n".join(lines) + "\n", encoding="utf-8") + + +def run_py_compile( + backend_dir: Path, + validation_root: Path, + ndjson_path: Path, + session: AuditSession, +) -> tuple[int, int]: + """Compile backend Python sources discovered from within backend_dir.""" + files = backend_py_compile_files(backend_dir) + file_count = len(files) + if not files: + record = CommandRecord( + name="backend py_compile", + workdir=str(backend_dir), + start=session.audit_start, + end=session.require_complete(), + exit_code=2, + status="FAIL", + expected="exit 0", + stdout="", + stderr="no python files discovered", + ) + append_command_record(ndjson_path, record) + return 2, file_count + + stdout_path = validation_root / "backend-py_compile.stdout.txt" + stderr_path = validation_root / "backend-py_compile.stderr.txt" + result = subprocess.run( + [sys.executable, "-m", "py_compile", *[str(path) for path in files]], + cwd=str(backend_dir), + capture_output=True, + text=True, + check=False, + ) + stdout_path.write_text(result.stdout, encoding="utf-8") + stderr_path.write_text(result.stderr, encoding="utf-8") + command_record = CommandRecord( + name="backend py_compile", + workdir=str(backend_dir), + start=session.audit_start, + end=session.require_complete(), + exit_code=result.returncode, + status="PASS" if result.returncode == 0 else "FAIL", + expected="exit 0", + stdout=stdout_path.name, + stderr=stderr_path.name, + ) + append_command_record(ndjson_path, command_record) + return result.returncode, file_count + + +def finalize_reports( + validation_root: Path, + ndjson_path: Path, + session: AuditSession, + *, + py_compile_file_count: int, +) -> list[str]: + """Write validation-results and metadata using one canonical session.""" + validation_path = validation_root / "validation-results.json" + metadata_path = validation_root / "BUNDLE_METADATA.json" + write_validation_results(ndjson_path, validation_path, session) + command_count = len(ndjson_path.read_text(encoding="utf-8").splitlines()) + write_audit_metadata( + metadata_path, + session, + command_count=command_count, + extra={"py_compile_file_count": py_compile_file_count}, + ) + return verify_metadata_timestamps_consistent([metadata_path], session) + + +def main() -> int: + """CLI entrypoint.""" + parser = argparse.ArgumentParser(description="Run provenance-safe post-release checks.") + parser.add_argument("--repo-root", default=str(Path(__file__).resolve().parents[1])) + parser.add_argument("--validation-root", required=True) + parser.add_argument("--backend-url", default="http://127.0.0.1:8000") + parser.add_argument("--require-backend-ready", action="store_true") + parser.add_argument("--run-e2e", action="store_true") + parser.add_argument("--e2e-json") + parser.add_argument("--e2e-md") + args = parser.parse_args() + + repo_root = Path(args.repo_root).resolve() + validation_root = Path(args.validation_root).resolve() + validation_root.mkdir(parents=True, exist_ok=True) + ndjson_path = validation_root / "cmd-results.ndjson" + if ndjson_path.exists(): + ndjson_path.unlink() + + session = AuditSession.begin() + backend_dir = repo_root / "backend" + + py_compile_code, py_compile_count = run_py_compile( + backend_dir, validation_root, ndjson_path, session + ) + if py_compile_code != 0: + issues = finalize_reports( + validation_root, ndjson_path, session, py_compile_file_count=py_compile_count + ) + if issues: + print("\n".join(issues), file=sys.stderr) + print("backend py_compile failed", file=sys.stderr) + session.complete() + return py_compile_code + + if args.require_backend_ready and not backend_is_listening(args.backend_url): + append_command_record( + ndjson_path, + CommandRecord( + name="backend readiness gate", + workdir=str(repo_root), + start=session.audit_start, + end=session.require_complete(), + exit_code=1, + status="FAIL", + expected="backend health 200", + stdout="", + stderr="backend not listening; E2E blocked", + ), + ) + issues = finalize_reports( + validation_root, ndjson_path, session, py_compile_file_count=py_compile_count + ) + if issues: + print("\n".join(issues), file=sys.stderr) + print("Backend not ready; E2E blocked", file=sys.stderr) + session.complete() + return 1 + + if args.run_e2e: + provenance_path = validation_root / "e2e-provenance.txt" + write_provenance(provenance_path, args.backend_url, repo_root, session) + e2e_cmd = [ + sys.executable, + str(repo_root / "scripts" / "live_e2e_validation.py"), + "--backend-url", + args.backend_url, + "--skip-public-mvp", + "--repo-root", + str(repo_root), + ] + if args.e2e_json: + e2e_cmd.extend(["--output-json", args.e2e_json]) + if args.e2e_md: + e2e_cmd.extend(["--output-md", args.e2e_md]) + record = run_logged_command( + "baseline e2e", + e2e_cmd, + repo_root, + "26/26 pass", + validation_root / "baseline-e2e.stdout.txt", + validation_root / "baseline-e2e.stderr.txt", + session=session, + ) + append_command_record(ndjson_path, record) + if record.exit_code != 0: + issues = finalize_reports( + validation_root, + ndjson_path, + session, + py_compile_file_count=py_compile_count, + ) + session.complete() + return record.exit_code + + issues = finalize_reports( + validation_root, ndjson_path, session, py_compile_file_count=py_compile_count + ) + session.complete() + if issues: + print("\n".join(issues), file=sys.stderr) + return 1 + + metadata = json.loads((validation_root / "BUNDLE_METADATA.json").read_text()) + print(json.dumps(metadata)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/test_audit_evidence.py b/scripts/test_audit_evidence.py new file mode 100644 index 0000000..1e2d04e --- /dev/null +++ b/scripts/test_audit_evidence.py @@ -0,0 +1,186 @@ +#!/usr/bin/env python3 +"""Tests for audit evidence logging and validation-result generation.""" + +from __future__ import annotations + +import importlib.util +import json +import subprocess +import sys +import tempfile +from pathlib import Path + + +def load_module(name: str, path: Path): + """Load a script module from path.""" + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + sys.modules[name] = module + spec.loader.exec_module(module) + return module + + +def test_validation_results_use_exit_codes_not_stderr() -> None: + """validation-results.json must mirror cmd-results exit codes.""" + repo_root = Path(__file__).resolve().parents[1] + audit_evidence = load_module("audit_evidence", repo_root / "scripts" / "audit_evidence.py") + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + ndjson = root / "cmd-results.ndjson" + session = audit_evidence.AuditSession.begin() + record = audit_evidence.CommandRecord( + name="frontend lint", + workdir=str(repo_root / "frontend"), + start=session.audit_start, + end=session.audit_start, + exit_code=0, + status="PASS", + expected="exit 0", + stdout="out.txt", + stderr="err.txt", + ) + audit_evidence.append_command_record(ndjson, record) + output = root / "validation-results.json" + results = audit_evidence.write_validation_results(ndjson, output, session) + assert results[0]["status"] == "PASS" + assert results[0]["exit_code"] == 0 + assert results[0]["start_time"] == session.audit_start + + +def test_exit_zero_with_stderr_warning_is_pass() -> None: + """Warnings on stderr must not downgrade a zero exit code.""" + repo_root = Path(__file__).resolve().parents[1] + audit_evidence = load_module("audit_evidence", repo_root / "scripts" / "audit_evidence.py") + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + stdout = root / "stdout.txt" + stderr = root / "stderr.txt" + stderr.write_text("npm warn example warning on stderr\n", encoding="utf-8") + stdout.write_text("ok\n", encoding="utf-8") + session = audit_evidence.AuditSession.begin() + record = audit_evidence.run_logged_command( + "stderr warning command", + [sys.executable, "-c", "import sys; sys.stderr.write('warn\\n'); print('ok')"], + root, + "exit 0", + stdout, + stderr, + session=session, + ) + assert record.exit_code == 0 + assert record.status == "PASS" + + +def test_nonzero_exit_with_empty_stderr_is_fail() -> None: + """Nonzero exit codes must fail even when stderr is empty.""" + repo_root = Path(__file__).resolve().parents[1] + audit_evidence = load_module("audit_evidence", repo_root / "scripts" / "audit_evidence.py") + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + stdout = root / "stdout.txt" + stderr = root / "stderr.txt" + session = audit_evidence.AuditSession.begin() + record = audit_evidence.run_logged_command( + "nonzero silent command", + [sys.executable, "-c", "raise SystemExit(3)"], + root, + "exit 0", + stdout, + stderr, + session=session, + ) + assert record.exit_code == 3 + assert record.status == "FAIL" + assert stderr.read_text(encoding="utf-8") == "" + + +def test_metadata_timestamp_disagreement_is_detected() -> None: + """Canonical timestamp verification must reject mismatched metadata files.""" + repo_root = Path(__file__).resolve().parents[1] + audit_evidence = load_module("audit_evidence", repo_root / "scripts" / "audit_evidence.py") + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + session = audit_evidence.AuditSession.begin() + session.complete() + good = root / "good.json" + bad = root / "bad.json" + audit_evidence.write_audit_metadata(good, session, command_count=1) + bad.write_text( + json.dumps( + { + "audit_start": session.audit_start, + "audit_end": "2099-01-01T00:00:00Z", + "command_count": 1, + } + ) + + "\n", + encoding="utf-8", + ) + issues = audit_evidence.verify_metadata_timestamps_consistent( + [good, bad], session + ) + assert issues + assert any("bad.json" in issue for issue in issues) + + +def test_backend_py_compile_collects_files() -> None: + """py_compile source discovery must run from backend directory context.""" + repo_root = Path(__file__).resolve().parents[1] + audit_evidence = load_module("audit_evidence", repo_root / "scripts" / "audit_evidence.py") + files = audit_evidence.backend_py_compile_files(repo_root / "backend") + assert len(files) > 0 + assert all(path.suffix == ".py" for path in files) + + +def test_run_post_release_audit_blocks_when_backend_unavailable() -> None: + """Orchestrator must fail when backend readiness is required but unavailable.""" + repo_root = Path(__file__).resolve().parents[1] + script = repo_root / "scripts" / "run_post_release_audit.py" + with tempfile.TemporaryDirectory() as temp_dir: + validation_root = Path(temp_dir) / "validation" + result = subprocess.run( + [ + sys.executable, + str(script), + "--repo-root", + str(repo_root), + "--validation-root", + str(validation_root), + "--backend-url", + "http://127.0.0.1:59999", + "--require-backend-ready", + "--run-e2e", + ], + capture_output=True, + text=True, + check=False, + cwd=str(repo_root / "scripts"), + ) + assert result.returncode == 1 + validation = json.loads( + (validation_root / "validation-results.json").read_text(encoding="utf-8") + ) + names = [item["name"] for item in validation] + assert "backend readiness gate" in names + assert "baseline e2e" not in names + metadata = json.loads( + (validation_root / "BUNDLE_METADATA.json").read_text(encoding="utf-8") + ) + assert metadata["audit_start"] == metadata["audit_start"] + assert metadata["audit_end"] + results = json.loads( + (validation_root / "validation-results.json").read_text(encoding="utf-8") + ) + assert all(item["start_time"] == metadata["audit_start"] for item in results) + assert all(item["end_time"] == metadata["audit_end"] for item in results) + + +if __name__ == "__main__": + test_validation_results_use_exit_codes_not_stderr() + test_exit_zero_with_stderr_warning_is_pass() + test_nonzero_exit_with_empty_stderr_is_fail() + test_metadata_timestamp_disagreement_is_detected() + test_backend_py_compile_collects_files() + test_run_post_release_audit_blocks_when_backend_unavailable() + print("PASS")