diff --git a/.gitignore b/.gitignore index e64aae5..553e46c 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ runs/ system/ .worktrees/ +experiments/ *.local __pycache__/ .pytest_cache/ diff --git a/AGENTS.md b/AGENTS.md index a5d3feb..a87d45e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -12,6 +12,8 @@ Build a repeatable, read-only telemetry and benchmark harness for the Surface Bo - Suspend tests must be operator-gated until explicitly approved. - Do not collect secrets or unrelated personal data. - Preserve raw artifacts; reports must be reproducible from them. +- Kernel workflow tools must not install packages, sign UKIs, edit Limine, set a boot default, or boot a candidate without explicit human approval. +- Preserve the known-good `linux-surface` package and boot entry while testing separately named `linux-eiger` candidates. ## Workflow diff --git a/README.md b/README.md index 1c0250d..a29a18e 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,10 @@ Project bootstrap. The implementation plan is in `docs/plan/eiger-telemetry.md`. Idle, browse, video, burst, and suspend/S0ix. AutoFDO/perf profiling is deliberately deferred until workload definitions are stable. +## Kernel development workflow + +A safe Arch/linux-surface → `linux-eiger` build, Secure-Boot/Limine boot, benchmark, review, and rollback workflow is in [`docs/kernel-development-workflow.md`](docs/kernel-development-workflow.md). Its read-only build-provenance preflight tool is documented in [`kernel/README.md`](kernel/README.md). + ## Delivery workflow Work is managed on the Hermes Kanban board and implemented in isolated worktrees. Changes flow through PM specification, implementation, independent QA/review, and a human merge gate. See `AGENTS.md` and `CONTRIBUTING.md`. diff --git a/docs/kernel-development-workflow.md b/docs/kernel-development-workflow.md new file mode 100644 index 0000000..b5ccbcc --- /dev/null +++ b/docs/kernel-development-workflow.md @@ -0,0 +1,255 @@ +# Eiger Kernel Development and Measurement Workflow + +This runbook defines the safe path from a known-good `linux-surface` baseline to a custom Eiger kernel, then to evidence-based promotion. It is tailored to this host: + +- Arch Linux / Omarchy +- `linux-surface` 6.19.8.arch1-3 baseline +- Limine boot loader +- Secure Boot enabled +- measured UKI boot path +- Surface Book 2 with two independently reported batteries + +The workflow is deliberately split into **build**, **boot**, and **measurement** gates. A package that builds is not automatically bootable. A kernel that boots is not automatically comparable. A benchmark that completes is not automatically evidence of an improvement. + +## Non-negotiable rules + +1. Keep the vendor `linux-surface` package and its known-good UKI/boot entry intact until an Eiger candidate has passed the complete acceptance sequence. +2. Build as an unprivileged user. Do not use `sudo make install` and do not replace the active kernel in place. +3. Package custom work under a distinct package base/name such as `linux-eiger` and `linux-eiger-headers`; use a distinct kernel local version such as `-eiger.r1`. +4. Boot selection, package installation, Limine/UKI updates, Secure-Boot signing, kernel removal, and release/publishing are human gates. +5. Kernel comparison changes one intended independent variable at a time. Every candidate has a pinned source revision, patch manifest, configuration hash, package hash, and boot identity. +6. Benchmark collection is read-only. Experimental controls are recorded, never silently tuned. + +## Roles and durable artifacts + +| Stage | Owner | Required durable artifact | +| --- | --- | --- | +| Hypothesis | PM + owner | issue/card, expected metric and risk | +| Baseline | QA | accepted raw run directories + report | +| Source/config | engineer | source revision, patch series, config hash/diff | +| Build | engineer | preflight JSON, build log, package SHA-256, package list | +| Install/boot | owner | package transaction log, signing evidence, boot entry, rollback entry | +| Smoke test | QA | boot fingerprint, hardware/journal checks | +| Experiment | QA | paired benchmark artifacts and exclusion ledger | +| Review/promotion | reviewer + owner | comparison report, decision record, human merge/release approval | + +## 0. One-time preparation + +### Source policy + +Base Eiger packaging on the official `linux-surface` Arch packaging tree, which tracks the Arch Linux kernel packaging plus Surface patches. Pin the exact Git commit used for each experiment; do not build from an unnamed moving checkout. + +Keep local sources outside the telemetry repository, for example: + +```text +~/src/linux-surface-packaging/ # pinned upstream packaging checkout +~/src/linux-eiger-packaging/ # Eiger packaging fork/worktree +~/build/eiger// # disposable build output/logs +~/eiger-telemetry/ # protocol, collector, reports, immutable runs +``` + +The `eiger-telemetry` repository must store manifests, configuration fragments, patch references, checksums, and reports—not multi-gigabyte kernel sources, object files, packages, or raw personal telemetry. + +### Build dependencies + +The current host has `make`, `gcc`, `bison`, and `pahole`, but strict preflight currently reports missing `bc` and `flex`. Install only the reviewed build dependencies before attempting a package build. On Arch, the typical baseline is `base-devel bc flex` plus any package-specific dependencies declared by the pinned PKGBUILD. + +Do not install dependencies during a formal benchmark matrix. Dependency changes belong to a recorded preparation phase followed by a new baseline fingerprint. + +### Secure Boot and boot safety + +This host has Secure Boot enabled and boots a measured UKI through Limine. Before an Eiger candidate is installed, document: + +- how the existing `linux-surface` package creates/signs its UKI; +- the custom Eiger package's UKI path and signing step; +- the exact Limine entry title for both candidate and known-good baseline; +- one-shot boot selection/recovery procedure; +- how to recover from a failed candidate boot without removing `linux-surface`. + +Do not assume `sbctl` automatically signs a custom package. Verify the generated artifact and signing status after installation. This is a human-gated operation. + +## 1. Create a hypothesis card + +Every kernel change starts with a single falsifiable statement, for example: + +> On the Surface Book 2, patch set X reduces idle wakeups and raises package C10 residency without regressing burst energy or resume reliability. + +The card records: + +- intended source/config/patch change; +- expected metric direction and scenario; +- primary metric and guardrail metrics; +- kernel A (baseline) and B (candidate) labels; +- known risks and the rollback entry; +- benchmark protocol revision and required number of paired runs. + +Do not bundle unrelated scheduler, driver, compiler, config, or firmware changes into one candidate. + +## 2. Capture the baseline before changing source + +1. Boot the baseline `linux-surface` entry. +2. Run `eiger-kernel-preflight` against the pinned baseline source/config and save its JSON alongside the experiment record. +3. Capture a full system fingerprint. +4. Collect the protocol-defined baseline matrix for the relevant power profile and observed charge state. +5. Preserve raw runs, hash manifests, summaries, excluded runs, and environment controls. +6. Require at least three accepted A/B-comparable repetitions before making a performance claim. Five is the target. + +The first provisional capture is a smoke artifact only. It must not be used as a baseline claim because it lacks controlled conditions and battery energy resolution. + +## 3. Create a reproducible candidate + +For each candidate, create an experiment directory such as: + +```text +experiments/2026-08-31-idle-wakeup-r1/ +├── experiment.json +├── baseline-preflight.json +├── candidate-preflight.json +├── source.lock # upstream URL + exact revision +├── patches/series # references/checksums, not copied upstream trees +├── config/ +│ ├── baseline.config.sha256 +│ ├── candidate.config +│ └── config.diff +├── build/ +│ ├── command.txt +│ ├── environment.allowlist.json +│ ├── build.log +│ ├── packages.sha256 +│ └── package-file-list.txt +└── results/ # links/IDs to immutable telemetry runs +``` + +Required candidate identity: + +- pinned linux-surface packaging revision and upstream kernel revision; +- patch series names and SHA-256 values; +- candidate config SHA-256 and audited config diff; +- compiler and linker versions; build command; source-tree cleanliness; +- package filename/version, package SHA-256, and module `pkgbase` value; +- expected `uname -r` / local version. + +The preflight command is intentionally read-only: + +```sh +kernel/bin/eiger-kernel-preflight \ + --source ~/src/linux-eiger-packaging/kernel/src/linux-* \ + --config ~/src/linux-eiger-packaging/kernel/src/linux-*/.config \ + --output experiments//candidate-preflight.json \ + --baseline-label surface-6.19.8 \ + --candidate-label eiger-6.19.8-r1 \ + --strict +``` + +It creates a provenance artifact, checks a source `Makefile` and exact configuration hash, records source Git identity/dirty state, checks build-tool availability, and warns about insufficient output-filesystem space. It does not compile, install, invoke sudo, or modify the source tree. + +## 4. Build packages, never install the raw kernel + +From a clean, pinned Arch package worktree: + +1. Change `pkgbase` and package names to `linux-eiger`; do not conflict with `linux-surface`. +2. Apply exactly the experiment's named patch/config changes. +3. Run strict preflight and preserve the JSON result. +4. Build as the normal user with the pinned package recipe, recording the exact command and complete output log. +5. Generate SHA-256 checksums and package file lists. +6. Verify the package includes a distinct kernel/module path and, if relevant, matching headers. +7. Do not run `pacman -U`, modify `/boot`, or boot the candidate until a reviewer has inspected the build artifacts and the owner approves the install gate. + +A build failure is data: retain the log and preflight context. Do not repair a source/config issue by making undocumented changes. + +## 5. Human-gated installation and one-shot boot + +After approval: + +1. Confirm backup and known-good `linux-surface` package/boot entry remain available. +2. Install the `linux-eiger` packages without removing `linux-surface`. +3. Verify generated UKI/initramfs location, signing status, and Limine entry using the actual system hooks. +4. Record package transaction output and artifact hashes. +5. Set a one-shot candidate boot, never change the permanent default first. +6. Boot candidate B and verify `uname -r`, boot ID, cmdline, module `pkgbase`, and UKI/entry identity. +7. If it fails to boot or breaks essential hardware, select the known-good baseline entry and record the failure. Do not continue performance testing. + +## 6. Candidate smoke gate + +Before performance collection, QA verifies a minimal non-performance acceptance set: + +- boot and filesystem mount success; +- display, keyboard/trackpad, Wi-Fi, audio, camera/tablet or other relevant Surface hardware; +- battery/AC telemetry still visible for both batteries; +- no unexpected kernel errors/warnings during settling; +- fingerprint captured with candidate identity; +- privileged `turbostat` path available through the reviewed narrow wrapper, if package-power metrics are required; +- suspend remains separately gated and is not started automatically. + +The smoke gate catches regressions. It does not establish an efficiency improvement. + +## 7. Measure A/B performance and power + +Use `docs/benchmark-protocol.md` and its power/charge-state extension. The comparability tuple is: + +```text +kernel × scenario × power_profile × observed_charge_state × repetition +``` + +For each pair: + +1. Precompute and record a balanced A/B ordering seed. +2. Boot the named kernel condition, then settle. +3. Confirm observed power profile and natural charge state. Skip—not substitute—when they do not match. +4. Run explicit workload/phase markers and read-only capture. +5. Verify hashes, duration, completeness, accepted/excluded status, and all control values. +6. Reboot before switching kernel condition. +7. Retain outliers and excluded runs with reasons. + +Required metrics depend on the hypothesis but generally include: + +- battery energy/power where reliable, per battery and combined; +- package/core/GFX/RAM watts from privileged turbostat, never mislabelled as laptop watts; +- C-state/package-state/GPU RC6 residency; +- wakeup, interrupt, softirq, and PCI runtime-PM deltas; +- thermal maximum and throttling evidence; +- completion time and energy-per-task for burst workloads; +- suspend elapsed time, S0ix counters, and battery loss only in the separately approved suspend flow. + +## 8. Analysis and decision + +The report must: + +- show all raw run IDs, accepted pairs, skipped/excluded pairs, and reasons; +- reject comparisons with changed protocol, collector, source/config identity, controls, kernel boot identity, profile, or charge state; +- show median, mean, dispersion, sample count, and individual pair deltas; +- distinguish a correlation from a causal claim; +- state metric availability limitations and observer effects; +- report regressions in guardrails even when the primary metric improves. + +A candidate is **not promoted** if it has fewer than three accepted pairs, lacks the primary measurement, regresses reliability, or has unexplained control/identity differences. + +## 9. Promotion and rollback + +Human review chooses one of: + +- **reject:** retain evidence; remove only the experimental package after confirming the known-good kernel boots; +- **iterate:** create a new candidate card with one documented delta; +- **provisional keep:** retain both boot entries and continue long-term historian observation; +- **promote:** set the Eiger entry as default only after a documented human approval and rollback test. + +Never delete the baseline kernel or its signed boot path as part of an experiment. Keep the last known-good package and package checksum available for rollback. + +## Mandatory human gates + +| Gate | Why it is human-only | +| --- | --- | +| Installing local kernel packages | Changes bootable system state | +| Secure-Boot signing/enrollment | Changes security/boot trust state | +| Editing Limine entries/defaults | Can make the host unbootable | +| First boot of a candidate | Requires recovery judgement | +| Suspend experiment | May strand the device or conceal a resume failure | +| Merging/publishing a kernel package or release | External/release commitment | + +## Workflow status on this host + +- Baseline kernel and boot path: identified. +- Preflight tooling: implemented and tested on the workflow branch. +- Telemetry collector: integrated in a separate uncommitted worktree but QA has open defects; not ready for formal comparison. +- Power/charge preflight, narrow turbostat wrapper, overnight scheduling, historian, markers: separate Kanban work remains queued/in progress. +- First custom compile: intentionally blocked until package-source/config selection, missing build dependencies, and the Secure-Boot/UKI installation plan are reviewed. diff --git a/kernel/README.md b/kernel/README.md new file mode 100644 index 0000000..e8ffe5a --- /dev/null +++ b/kernel/README.md @@ -0,0 +1,25 @@ +# Kernel workflow tooling + +This directory contains **read-only preparation tools** for Eiger kernel experiments. It does not build, install, sign, boot, or remove a kernel. + +## `eiger-kernel-preflight` + +Generate a machine-readable provenance artifact before a build: + +```sh +kernel/bin/eiger-kernel-preflight \ + --source ~/src/linux-eiger-packaging/kernel/src/linux-* \ + --config ~/src/linux-eiger-packaging/kernel/src/linux-*/.config \ + --output experiments//candidate-preflight.json \ + --baseline-label surface-6.19.8 \ + --candidate-label eiger-6.19.8-r1 \ + --strict +``` + +It validates safe experiment labels, source-tree structure, the supplied kernel configuration hash, source Git identity/dirty state, required build-tool availability, and output-filesystem free space. It writes an atomic JSON artifact. + +`--strict` exits nonzero if any required build tool is unavailable. The tool never invokes a build command, `sudo`, package manager, bootloader, signer, or telemetry collector. + +Required tools are `make`, `gcc`, `bc`, `bison`, `flex`, and `pahole`. `clang` and `ccache` are recorded as optional. The exact package recipe may require additional dependencies; treat its pinned PKGBUILD as authoritative. + +See [`../docs/kernel-development-workflow.md`](../docs/kernel-development-workflow.md) for the complete build, boot, measurement, review, and rollback process. diff --git a/kernel/bin/eiger-kernel-preflight b/kernel/bin/eiger-kernel-preflight new file mode 100755 index 0000000..a827046 --- /dev/null +++ b/kernel/bin/eiger-kernel-preflight @@ -0,0 +1,161 @@ +#!/usr/bin/env python3 +"""Read-only preflight for a reproducible Eiger kernel experiment.""" +from __future__ import annotations + +import argparse +import datetime as dt +import hashlib +import json +import os +import pathlib +import platform +import re +import shutil +import subprocess +import sys +import tempfile +from typing import Any + +SCHEMA_VERSION = "kernel-preflight-v1" +LABEL_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$") +REQUIRED_BUILD_TOOLS = ("make", "gcc", "bc", "bison", "flex", "pahole") +OPTIONAL_BUILD_TOOLS = ("clang", "ccache") + + +def sha256_file(path: pathlib.Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def command_output(args: list[str], cwd: pathlib.Path) -> str | None: + try: + result = subprocess.run(args, cwd=cwd, text=True, capture_output=True, check=False) + except OSError: + return None + return result.stdout.strip() if result.returncode == 0 else None + + +def git_identity(source: pathlib.Path) -> dict[str, Any]: + revision = command_output(["git", "rev-parse", "HEAD"], source) + if revision is None: + return {"repository": False, "revision": "NA", "dirty": "NA"} + dirty = command_output(["git", "status", "--porcelain"], source) + return {"repository": True, "revision": revision, "dirty": bool(dirty)} + + +def collect_tools(search_path: str) -> dict[str, dict[str, Any]]: + original_path = os.environ.get("PATH") + os.environ["PATH"] = search_path + try: + entries: dict[str, dict[str, Any]] = {} + for name in REQUIRED_BUILD_TOOLS + OPTIONAL_BUILD_TOOLS: + found = shutil.which(name) + entries[name] = { + "required": name in REQUIRED_BUILD_TOOLS, + "available": found is not None, + "path": found or "NA", + } + return entries + finally: + if original_path is None: + os.environ.pop("PATH", None) + else: + os.environ["PATH"] = original_path + + +def atomic_json_write(path: pathlib.Path, document: dict[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + fd, temp_name = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent) + temp_path = pathlib.Path(temp_name) + try: + with os.fdopen(fd, "w", encoding="utf-8", newline="\n") as stream: + json.dump(document, stream, indent=2, sort_keys=True) + stream.write("\n") + temp_path.replace(path) + except BaseException: + temp_path.unlink(missing_ok=True) + raise + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--source", required=True, type=pathlib.Path, help="kernel source directory") + parser.add_argument("--config", required=True, type=pathlib.Path, help="exact kernel .config to build") + parser.add_argument("--output", required=True, type=pathlib.Path, help="preflight JSON output path") + parser.add_argument("--baseline-label", required=True) + parser.add_argument("--candidate-label", required=True) + parser.add_argument("--strict", action="store_true", help="fail when a required build tool is missing") + parser.add_argument("--tool-path", default=os.environ.get("PATH", ""), help="PATH to use for tool discovery") + return parser.parse_args() + + +def validate(args: argparse.Namespace) -> list[str]: + errors: list[str] = [] + for name, value in (("baseline label", args.baseline_label), ("candidate label", args.candidate_label)): + if not LABEL_RE.fullmatch(value): + errors.append(f"invalid label for {name}: {value!r}") + if not args.source.is_dir(): + errors.append(f"source directory does not exist: {args.source}") + elif not (args.source / "Makefile").is_file(): + errors.append(f"kernel source Makefile is missing: {args.source / 'Makefile'}") + if not args.config.is_file(): + errors.append(f"kernel config does not exist: {args.config}") + return errors + + +def main() -> int: + args = parse_args() + errors = validate(args) + if errors: + print("\n".join(errors), file=sys.stderr) + return 2 + + tools = collect_tools(args.tool_path) + missing_required = sorted(name for name, detail in tools.items() if detail["required"] and not detail["available"]) + stat = os.statvfs(args.output.parent if args.output.parent.exists() else pathlib.Path.cwd()) + free_bytes = stat.f_bavail * stat.f_frsize + warnings: list[str] = [] + failures: list[str] = [] + if missing_required: + if args.strict: + failures.append("missing_required_build_tools") + else: + warnings.append("missing_required_build_tools") + if free_bytes < 25 * 1024**3: + warnings.append("less_than_25_gib_free_at_output_filesystem") + + document: dict[str, Any] = { + "schema_version": SCHEMA_VERSION, + "status": "failed" if failures else "passed", + "failures": failures, + "warnings": warnings, + "created_utc": dt.datetime.now(dt.timezone.utc).isoformat().replace("+00:00", "Z"), + "experiment": { + "baseline_label": args.baseline_label, + "candidate_label": args.candidate_label, + "strict": args.strict, + }, + "source": { + "path": str(args.source.resolve()), + "makefile_present": True, + "config_path": str(args.config.resolve()), + "config_sha256": sha256_file(args.config), + "git": git_identity(args.source), + }, + "build_tools": tools, + "storage": {"output_path": str(args.output.resolve()), "free_bytes": free_bytes}, + "host": {"hostname": platform.node(), "kernel": platform.release(), "machine": platform.machine()}, + } + atomic_json_write(args.output, document) + if failures: + print(f"preflight failed; artifact written to {args.output}", file=sys.stderr) + return 1 + print(args.output) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/kernel/templates/experiment.env.example b/kernel/templates/experiment.env.example new file mode 100644 index 0000000..a9cdd14 --- /dev/null +++ b/kernel/templates/experiment.env.example @@ -0,0 +1,23 @@ +# Copy into an untracked experiment directory, fill in, and preserve with artifacts. +# Do not store credentials, secrets, or a sudo password here. + +EXPERIMENT_ID=2026-08-31-idle-wakeup-r1 +BASELINE_LABEL=surface-6.19.8 +CANDIDATE_LABEL=eiger-6.19.8-r1 + +# Pin the source checkout and capture its revision separately in source.lock. +PACKAGING_SOURCE=$HOME/src/linux-eiger-packaging +KERNEL_CONFIG=$HOME/src/linux-eiger-packaging/kernel/src/linux-*/.config +EXPERIMENT_ROOT=$HOME/build/eiger/$EXPERIMENT_ID +PREFLIGHT_OUTPUT=$EXPERIMENT_ROOT/candidate-preflight.json + +# Measurement identity: must match both members of an A/B pair. +PROTOCOL_VERSION=benchmark-v1 +SCENARIO=idle +POWER_PROFILE=balanced +OBSERVED_CHARGE_STATE=battery-disconnected +REPETITIONS=5 +ORDER_SEED=20260831 + +# Build values are evidence, not controls. Record actual commands/logs separately. +MAKEFLAGS=-j8 diff --git a/tests/test_kernel_preflight.py b/tests/test_kernel_preflight.py new file mode 100644 index 0000000..8d31459 --- /dev/null +++ b/tests/test_kernel_preflight.py @@ -0,0 +1,116 @@ +from __future__ import annotations + +import json +import pathlib +import subprocess +import tempfile +import unittest + + +REPO = pathlib.Path(__file__).resolve().parents[1] +PREFLIGHT = REPO / "kernel" / "bin" / "eiger-kernel-preflight" + + +class KernelPreflightTests(unittest.TestCase): + def make_source_tree(self, root: pathlib.Path) -> tuple[pathlib.Path, pathlib.Path]: + source = root / "linux" + source.mkdir() + (source / "Makefile").write_text("VERSION = 6\n") + config = source / ".config" + config.write_text("CONFIG_LOCALVERSION=\"-eiger-test\"\n") + return source, config + + def test_preflight_writes_reproducible_provenance_artifact(self) -> None: + with tempfile.TemporaryDirectory() as td: + temp = pathlib.Path(td) + source, config = self.make_source_tree(temp) + output = temp / "preflight.json" + completed = subprocess.run( + [ + str(PREFLIGHT), + "--source", + str(source), + "--config", + str(config), + "--output", + str(output), + "--baseline-label", + "surface-6.19.8", + "--candidate-label", + "eiger-6.19.8-r1", + ], + text=True, + capture_output=True, + check=False, + ) + self.assertEqual(completed.returncode, 0, completed.stderr) + artifact = json.loads(output.read_text()) + self.assertEqual(artifact["schema_version"], "kernel-preflight-v1") + self.assertEqual(artifact["experiment"]["baseline_label"], "surface-6.19.8") + self.assertEqual(artifact["experiment"]["candidate_label"], "eiger-6.19.8-r1") + self.assertEqual(artifact["source"]["config_sha256"], artifact["source"]["config_sha256"]) + self.assertTrue(artifact["source"]["makefile_present"]) + self.assertIn("make", artifact["build_tools"]) + self.assertIn("passed", artifact["status"]) + + def test_preflight_rejects_unsafe_experiment_labels_without_writing_output(self) -> None: + with tempfile.TemporaryDirectory() as td: + temp = pathlib.Path(td) + source, config = self.make_source_tree(temp) + output = temp / "preflight.json" + completed = subprocess.run( + [ + str(PREFLIGHT), + "--source", + str(source), + "--config", + str(config), + "--output", + str(output), + "--baseline-label", + "surface baseline", + "--candidate-label", + "eiger-6.19.8-r1", + ], + text=True, + capture_output=True, + check=False, + ) + self.assertEqual(completed.returncode, 2) + self.assertFalse(output.exists()) + self.assertIn("invalid label", completed.stderr) + + def test_strict_mode_fails_when_required_build_tool_is_missing(self) -> None: + with tempfile.TemporaryDirectory() as td: + temp = pathlib.Path(td) + source, config = self.make_source_tree(temp) + output = temp / "preflight.json" + completed = subprocess.run( + [ + str(PREFLIGHT), + "--source", + str(source), + "--config", + str(config), + "--output", + str(output), + "--baseline-label", + "surface-6.19.8", + "--candidate-label", + "eiger-6.19.8-r1", + "--strict", + "--tool-path", + "/definitely-empty-path", + ], + text=True, + capture_output=True, + check=False, + ) + self.assertEqual(completed.returncode, 1) + artifact = json.loads(output.read_text()) + self.assertEqual(artifact["status"], "failed") + self.assertIn("missing_required_build_tools", artifact["failures"]) + + +if __name__ == "__main__": + unittest.main()