From 5082f3f3ad3f47428d16d8d7e5bc6dea7264526e Mon Sep 17 00:00:00 2001 From: Balogun Feranmi Date: Thu, 17 Sep 2026 09:58:15 +0100 Subject: [PATCH] test(e2e): cover GrubSetKargs with story 138 (#233) Story 138 drives a one-step high-risk GrubSetKargs plan (append quiet, delete splash), closing the last Ubuntu-only action gap. The suite is now 138 stories (54 atomic + 84 Ubuntu) with 0 Ubuntu-only uncovered actions; committed live-VM evidence stays at 79/79 per LTS, with the five additional stories (134-138) not yet in a recorded run. The public-claims contract test is adapted to the derived-zero gap: stale nonzero gap prose is rejected against derived zero, and the universal claim is rejected against a fixture gap reopened by neutralising the story quote. --- CONTRIBUTING.md | 4 +- docs/introduction.md | 6 +- tests/e2e/stories/story-138.sh | 21 ++ tests/release/public-claims.test.sh | 287 ++++++++++++++++------------ 4 files changed, 188 insertions(+), 130 deletions(-) create mode 100755 tests/e2e/stories/story-138.sh diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8f27e2a7..8395ce23 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -44,10 +44,10 @@ table below. | Area | Why it matters | Difficulty | |---|---|---| -| **Ubuntu LTS support** | The current suite is 83 Ubuntu stories. Committed live-VM evidence covers 79 of those Ubuntu stories on each LTS release, with a committed replay twin that reproduces each run: 22.04, 24.04 and 26.04 all at 79/79. The four additional stories are not yet included in a committed live-VM run. `ubuntu-vm.sh` accepts `UBUNTU_RELEASE=jammy\|noble\|resolute`. Remaining: story coverage for the cross-family actions, and one Debian-only action still has no story: `GrubSetKargs`. | medium | +| **Ubuntu LTS support** | The current suite is 84 Ubuntu stories. Committed live-VM evidence covers 79 of those Ubuntu stories on each LTS release, with a committed replay twin that reproduces each run: 22.04, 24.04 and 26.04 all at 79/79. The five additional stories are not yet included in a committed live-VM run. `ubuntu-vm.sh` accepts `UBUNTU_RELEASE=jammy\|noble\|resolute`. Remaining: story coverage for the cross-family actions, and every Debian-only action has a story. | medium | | **Distro detection coverage** | Robust `/etc/os-release` parsing for every release we claim to support. Pure-function tests against real fixture files, no integration mocks. The existing fixtures at the bottom of `crates/sysknife-core/src/distro.rs` show the shape. | easy | | **Action catalogue gaps** | Add a typed action (for example `EnableFirewallZone`). Small and isolated, and every PR carries the policy entry, the risk level and the tests. | easy | -| **E2E story coverage** | Real prompts, real LLM, real daemon. The suite is 137 stories: 54 atomic + 83 Ubuntu. What is left is the cross-family middle: of the action names available on both families, 61 are still untouched by any story, plus 10 Fedora-only and 1 Ubuntu-only ones. See #233 for the clustered map. | medium | +| **E2E story coverage** | Real prompts, real LLM, real daemon. The suite is 138 stories: 54 atomic + 84 Ubuntu. What is left is the cross-family middle: of the action names available on both families, 61 are still untouched by any story, plus 10 Fedora-only and 0 Ubuntu-only ones. See #233 for the clustered map. | medium | | **Fedora Atomic validation** | The action families exist and `DistroId::is_supported()` returns true for Atomic 41 and up. Nobody has run `tests/e2e/atomic-vm.sh` against a current release. Needs Fedora Atomic hardware or a VM host. | tedious | | **Demo recording on real hardware** | Replace the bundled demo GIF with a 30-second recording on real Ubuntu 26.04 with rollback visible. | easy | diff --git a/docs/introduction.md b/docs/introduction.md index 647b5fa1..84276b5b 100644 --- a/docs/introduction.md +++ b/docs/introduction.md @@ -104,13 +104,13 @@ SysKnife, not an afterthought. > **ℹ️ Distro support** > -> The current suite is 83 Ubuntu stories. All three Ubuntu LTS releases have a +> The current suite is 84 Ubuntu stories. All three Ubuntu LTS releases have a > committed live-VM run covering 79 of those Ubuntu stories, in > `tests/evidence/story-runs/`: -> 22.04, 24.04 and 26.04 all at 79/79. The four additional stories are not yet +> 22.04, 24.04 and 26.04 all at 79/79. The five additional stories are not yet > included in a committed live-VM run. Each run has a replay twin that > reproduces it, serving every call with zero misses. -> One Debian-only action still has no story. +> Every Debian-only action has a story. > Fedora Atomic is supported by the rpm-ostree action family, but a current > Silverblue 44 VM run is a release gate. Plain Fedora remains experimental > until the `dnf` action family ships. diff --git a/tests/e2e/stories/story-138.sh b/tests/e2e/stories/story-138.sh new file mode 100755 index 00000000..b69dd9d0 --- /dev/null +++ b/tests/e2e/stories/story-138.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +# Story 138 (ubuntu, high-risk): Append quiet and remove splash in GRUB +# Intent: "in GRUB, append quiet to the kernel command line and delete splash" +# Distro: ubuntu +set -euo pipefail +INTENT="in GRUB, append quiet to the kernel command line and delete splash" +echo "=== Story 138 (ubuntu): GrubSetKargs ===" +PLAN=$(sysknife --dry-run --json "$INTENT" 2>/tmp/sysknife-story-138-stderr.log) +echo "$PLAN" | jq . + +STEP_COUNT=$(echo "$PLAN" | jq '.plan.steps | length') +if [[ "$STEP_COUNT" != "1" ]]; then echo "FAIL: expected 1 step, got $STEP_COUNT"; exit 1; fi +STEP=$(echo "$PLAN" | jq '.plan.steps[0] | select(.action == "GrubSetKargs")') +if [[ -z "$STEP" || "$STEP" == "null" ]]; then echo "FAIL: expected GrubSetKargs"; exit 1; fi +RISK=$(echo "$STEP" | jq -r '.risk') +if [[ "$RISK" != "high" ]]; then echo "FAIL: expected risk high, got $RISK"; exit 1; fi +APPEND=$(echo "$STEP" | jq -c '.params.append') +if [[ "$APPEND" != '["quiet"]' ]]; then echo "FAIL: expected append=[\"quiet\"], got $APPEND"; exit 1; fi +DELETE=$(echo "$STEP" | jq -c '.params.delete') +if [[ "$DELETE" != '["splash"]' ]]; then echo "FAIL: expected delete=[\"splash\"], got $DELETE"; exit 1; fi +echo "PASS: Story 138" diff --git a/tests/release/public-claims.test.sh b/tests/release/public-claims.test.sh index f9eb5359..75b875a3 100755 --- a/tests/release/public-claims.test.sh +++ b/tests/release/public-claims.test.sh @@ -259,57 +259,41 @@ assert_rejected_with_diagnostic \ "$wrong_atomic" "atomic" "derived $derived_atomic" cp "$repo_root/docs/introduction.md" "$fixture/docs/introduction.md" -# The zero-uncovered direction: close the whole Debian-only gap in the story -# evidence (quoting every derived-uncovered Ubuntu action) while the prose -# still states a nonzero gap. The stale gap sentence must be rejected against -# derived zero. Nothing here names today's actions or count; all of it is -# derived from the fixture. +# The stale-gap direction with the gap closed: the tree leaves zero Ubuntu-only +# actions uncovered, so a nonzero gap sentence written into the prose must be +# rejected against derived zero. Nothing here retypes today's counts; the +# premise (derived zero) and the published figure are both read out. read -r gap_published gap_derived < <( - python3 - "$fixture/docs/action-reference.md" "$fixture/tests/e2e/stories" \ - "$fixture/CONTRIBUTING.md" "$repo_root/scripts/check_evidence_claims.py" <<'PY' + python3 - "$fixture" "$repo_root/scripts/check_evidence_claims.py" <<'PY' import importlib.util import re import sys from pathlib import Path -catalogue = Path(sys.argv[1]).read_text(encoding="utf-8") -story_dir = Path(sys.argv[2]) -rows = re.findall( - r"^\| `([A-Za-z0-9_]+)` \|.*?\| (All|Ubuntu|Fedora) \|", - catalogue, - re.MULTILINE, -) -if not rows: - raise SystemExit("could not derive catalogue rows for the gap mutation") -story_files = sorted(story_dir.glob("story-*.sh")) -if not story_files: - raise SystemExit("could not derive story files for the gap mutation") -named = set() -for story in story_files: - named.update( - re.findall( - r'"([A-Za-z0-9_]+)"', - story.read_text(encoding="utf-8", errors="replace"), - ) - ) -gap = sorted(name for name, family in rows if family == "Ubuntu" and name not in named) -if not gap: - raise SystemExit("fixture has no Debian-only gap to close") -target = story_files[0] -target.write_text( - target.read_text(encoding="utf-8") - + "".join(f'\n# coverage fixture: "{action}"\n' for action in gap), - encoding="utf-8", -) -spec = importlib.util.spec_from_file_location("checker", sys.argv[4]) +spec = importlib.util.spec_from_file_location("checker", sys.argv[2]) mod = importlib.util.module_from_spec(spec) spec.loader.exec_module(mod) -derived = mod.uncovered_action_counts(story_dir.parents[2])["Ubuntu"] -text = Path(sys.argv[3]).read_text(encoding="utf-8") -match = re.search(mod.DEBIAN_GAP_PROSE, re.sub(r"\s+", " ", text), re.IGNORECASE) -if not match: - raise SystemExit("could not find the Debian-only gap prose in the fixture") -print(mod._claim_count(match.group("count")), derived) +fixture = Path(sys.argv[1]) +derived = mod.uncovered_action_counts(fixture)["Ubuntu"] +if derived != 0: + raise SystemExit( + f"fixture gap is not zero before the stale-prose mutation: {derived}" + ) +path = fixture / "CONTRIBUTING.md" +text = path.read_text(encoding="utf-8") +old = "and every Debian-only action has a story." +new = "and one Debian-only action still has no story: `GrubSetKargs`." +if text.count(old) != 1: + raise SystemExit("universal Debian-only sentence was not unique in the fixture") +path.write_text(text.replace(old, new), encoding="utf-8") +found = re.search( + mod.DEBIAN_GAP_PROSE, + re.sub(r"\s+", " ", path.read_text(encoding="utf-8")), + re.IGNORECASE, +) +if not found: + raise SystemExit("stale-prose mutation did not apply") +print(mod._claim_count(found.group("count")), derived) PY ) if [[ -z "${gap_published:-}" || -z "${gap_derived:-}" ]]; then @@ -321,161 +305,214 @@ if [[ "$gap_derived" != "0" ]]; then "$gap_derived" >&2 exit 1 fi +if ! grep -Fq 'one Debian-only action still has no story' \ + "$fixture/CONTRIBUTING.md"; then + printf 'FAIL: stale-prose mutation did not apply\n' >&2 + exit 1 +fi assert_rejected_with_diagnostic \ 'stale nonzero gap prose against a derived zero gap' \ "published $gap_published" "derived $gap_derived" -cp "$repo_root"/tests/e2e/stories/story-*.sh "$fixture/tests/e2e/stories/" +cp "$repo_root/CONTRIBUTING.md" "$fixture/CONTRIBUTING.md" if ! "$checker" "$fixture" >/dev/null 2>&1; then printf 'FAIL: restored gap fixture rejected — mutation result is meaningless\n' >&2 exit 1 fi # The universal "every Debian-only action" claim is screened in every claim -# file, not just CONTRIBUTING. Reintroduce it in docs/introduction.md — where -# it actually stood until this change — and prove the global rule rejects it -# against the derived count, which is read out of the fixture, not retyped. -read -r debian_uncovered < <( - python3 - "$repo_root/scripts/check_evidence_claims.py" "$fixture" <<'PY' +# file, not just CONTRIBUTING. The tree now covers the whole gap, so reopen +# one in the fixture evidence — by neutralising the quoted action name in the +# fixture story that carries the coverage — and prove the global rule rejects +# the pristine universal prose against the derived count. The premise (derived +# zero before, derived one after) is read out of the fixture, not retyped, and +# the diagnostic must name introduction.md: the CONTRIBUTING mismatch the same +# evidence also produces is not sufficient proof for this rule. +read -r debian_before debian_uncovered < <( + python3 - "$fixture" \ + "$repo_root/scripts/check_evidence_claims.py" <<'PY' import importlib.util import sys from pathlib import Path -spec = importlib.util.spec_from_file_location("checker", sys.argv[1]) +spec = importlib.util.spec_from_file_location("checker", sys.argv[2]) mod = importlib.util.module_from_spec(spec) spec.loader.exec_module(mod) -print(mod.uncovered_action_counts(Path(sys.argv[2]))["Ubuntu"]) +fixture = Path(sys.argv[1]) +before = mod.uncovered_action_counts(fixture)["Ubuntu"] +if before != 0: + raise SystemExit( + f"fixture gap is not zero before the uncover mutation: {before}" + ) +story_dir = fixture / "tests/e2e/stories" +holders = sorted( + path for path in story_dir.glob("story-*.sh") + if '"GrubSetKargs"' in path.read_text(encoding="utf-8", errors="replace") +) +if len(holders) != 1: + raise SystemExit( + "expected exactly one fixture story to quote GrubSetKargs, " + f"found {len(holders)}" + ) +target = holders[0] +text = target.read_text(encoding="utf-8") +if text.count('"GrubSetKargs"') != 1: + raise SystemExit("GrubSetKargs quote was not unique in the fixture story") +target.write_text( + text.replace('"GrubSetKargs"', '"GrubSetKargsFixture"'), encoding="utf-8" +) +after = mod.uncovered_action_counts(fixture)["Ubuntu"] +if after != 1: + raise SystemExit( + f"uncover mutation did not reopen a gap of one (derived {after})" + ) +print(before, after) PY ) -if [[ -z "${debian_uncovered:-}" ]]; then +if [[ -z "${debian_before:-}" || -z "${debian_uncovered:-}" ]]; then printf 'FAIL: universal-claim mutation produced no derived count\n' >&2 exit 1 fi -printf '\nEvery Debian-only action has a story.\n' >> "$fixture/docs/introduction.md" -if ! grep -Fq 'Every Debian-only action has a story' "$fixture/docs/introduction.md"; then - printf 'FAIL: universal-claim mutation did not apply\n' >&2 +if ! grep -R -Fq '"GrubSetKargsFixture"' \ + "$fixture/tests/e2e/stories"; then + printf 'FAIL: uncover mutation did not apply\n' >&2 exit 1 fi assert_rejected_with_diagnostic \ - 'universal Debian-only claim in another screened file' \ + 'universal Debian-only claim against a reopened gap' \ 'introduction.md' "leaves $debian_uncovered Ubuntu-only" -cp "$repo_root/docs/introduction.md" "$fixture/docs/introduction.md" +cp "$repo_root"/tests/e2e/stories/story-*.sh "$fixture/tests/e2e/stories/" +if ! "$checker" "$fixture" >/dev/null 2>&1; then + printf 'FAIL: restored uncover fixture rejected — mutation result is meaningless\n' >&2 + exit 1 +fi -# The introduction's gap count is derived, not trusted: bump it with the tree -# untouched and prove rejection names the file and both figures. The premise -# itself is verified first — the published count must equal the derived one -# before the bump, or the assertion below would blame the checker for a -# fixture that was already stale. -read -r intro_published intro_new < <( - python3 - "$fixture/docs/introduction.md" "$fixture" \ +# The introduction's Debian-only prose is derived, not trusted: with the tree +# untouched (derived gap zero), restate its universal sentence as a stale +# nonzero gap and prove rejection names the file and both figures. +read -r intro_published intro_derived < <( + python3 - "$fixture" \ "$repo_root/scripts/check_evidence_claims.py" <<'PY' import importlib.util import re import sys from pathlib import Path -spec = importlib.util.spec_from_file_location("checker", sys.argv[3]) +spec = importlib.util.spec_from_file_location("checker", sys.argv[2]) mod = importlib.util.module_from_spec(spec) spec.loader.exec_module(mod) -path = Path(sys.argv[1]) +fixture = Path(sys.argv[1]) +derived = mod.uncovered_action_counts(fixture)["Ubuntu"] +if derived != 0: + raise SystemExit( + f"fixture gap is not zero before the introduction mutation: {derived}" + ) +path = fixture / "docs/introduction.md" text = path.read_text(encoding="utf-8") -match = re.search(mod.DEBIAN_GAP_PROSE, re.sub(r"\s+", " ", text), re.IGNORECASE) -if not match: - raise SystemExit("could not find the Debian-only gap prose in the fixture") -old = mod._claim_count(match.group("count")) -derived = mod.uncovered_action_counts(Path(sys.argv[2]))["Ubuntu"] -if old != derived: - raise SystemExit(f"fixture gap prose already stale: published {old}, derived {derived}") -new = old + 1 -raw = re.search(mod.DEBIAN_GAP_PROSE, text, re.IGNORECASE) -path.write_text( - text[: raw.start("count")] + str(new) + text[raw.end("count") :], - encoding="utf-8", +old = "Every Debian-only action has a story." +new = "One Debian-only action still has no story." +if text.count(old) != 1: + raise SystemExit("universal Debian-only sentence was not unique in the fixture") +path.write_text(text.replace(old, new), encoding="utf-8") +found = re.search( + mod.DEBIAN_GAP_PROSE, + re.sub(r"\s+", " ", path.read_text(encoding="utf-8")), + re.IGNORECASE, ) -print(old, new) +if not found: + raise SystemExit("introduction-count mutation did not apply") +print(mod._claim_count(found.group("count")), derived) PY ) -if [[ -z "${intro_published:-}" || -z "${intro_new:-}" ]]; then +if [[ -z "${intro_published:-}" || -z "${intro_derived:-}" ]]; then printf 'FAIL: introduction-count mutation produced no values\n' >&2 exit 1 fi -if ! grep -Eq "$intro_new Debian-only (actions still have|action still has) no story" \ +if ! grep -Fq 'One Debian-only action still has no story' \ "$fixture/docs/introduction.md"; then printf 'FAIL: introduction-count mutation did not apply\n' >&2 exit 1 fi assert_rejected_with_diagnostic \ 'introduction gap count that disagrees with the tree' \ - 'introduction.md' "published $intro_new" "derived $intro_published" + 'introduction.md' "published $intro_published" "derived $intro_derived" cp "$repo_root/docs/introduction.md" "$fixture/docs/introduction.md" -# ...and the other direction for the same sentence: cover one -# derived-uncovered Ubuntu action while the introduction prose stays stale. The -# prose premise is re-verified, the candidate is derived from the fixture, and +# ...and the other direction for the same sentence: reopen a gap of one in the +# fixture evidence while the introduction prose states a different nonzero +# count. The candidate gap and both figures are derived from the fixture, and # the diagnostic must name this file — the CONTRIBUTING mismatch the same # evidence also produces is not sufficient proof for this rule. -read -r intro_before intro_after intro_action < <( - python3 - "$fixture/docs/action-reference.md" "$fixture/tests/e2e/stories" \ - "$fixture/docs/introduction.md" "$fixture" \ +read -r intro_before intro_after < <( + python3 - "$fixture" \ "$repo_root/scripts/check_evidence_claims.py" <<'PY' import importlib.util import re import sys from pathlib import Path -catalogue = Path(sys.argv[1]).read_text(encoding="utf-8") -story_dir = Path(sys.argv[2]) -rows = re.findall( - r"^\| `([A-Za-z0-9_]+)` \|.*?\| (All|Ubuntu|Fedora) \|", - catalogue, - re.MULTILINE, -) -if not rows: - raise SystemExit("could not derive catalogue rows for the intro mutation") -story_files = sorted(story_dir.glob("story-*.sh")) -if not story_files: - raise SystemExit("could not derive story files for the intro mutation") -named = set() -for story in story_files: - named.update( - re.findall( - r'"([A-Za-z0-9_]+)"', - story.read_text(encoding="utf-8", errors="replace"), - ) - ) -gap = sorted(name for name, family in rows if family == "Ubuntu" and name not in named) -if not gap: - raise SystemExit("fixture has no Debian-only gap to narrow") -spec = importlib.util.spec_from_file_location("checker", sys.argv[5]) +spec = importlib.util.spec_from_file_location("checker", sys.argv[2]) mod = importlib.util.module_from_spec(spec) spec.loader.exec_module(mod) -before = mod.uncovered_action_counts(Path(sys.argv[4]))["Ubuntu"] -intro_text = Path(sys.argv[3]).read_text(encoding="utf-8") -intro_match = re.search( - mod.DEBIAN_GAP_PROSE, re.sub(r"\s+", " ", intro_text), re.IGNORECASE +fixture = Path(sys.argv[1]) +before = mod.uncovered_action_counts(fixture)["Ubuntu"] +if before != 0: + raise SystemExit( + f"fixture gap is not zero before the intro-evidence mutation: {before}" + ) +story_dir = fixture / "tests/e2e/stories" +holders = sorted( + path for path in story_dir.glob("story-*.sh") + if '"GrubSetKargs"' in path.read_text(encoding="utf-8", errors="replace") ) -if not intro_match or mod._claim_count(intro_match.group("count")) != before: - raise SystemExit("introduction prose does not state the pre-mutation gap") -action = gap[0] -target = story_files[0] +if len(holders) != 1: + raise SystemExit( + "expected exactly one fixture story to quote GrubSetKargs, " + f"found {len(holders)}" + ) +target = holders[0] +story_text = target.read_text(encoding="utf-8") +if story_text.count('"GrubSetKargs"') != 1: + raise SystemExit("GrubSetKargs quote was not unique in the fixture story") target.write_text( - target.read_text(encoding="utf-8") + f'\n# coverage fixture: "{action}"\n', + story_text.replace('"GrubSetKargs"', '"GrubSetKargsFixture"'), encoding="utf-8", ) -print(before, before - 1, action) +after = mod.uncovered_action_counts(fixture)["Ubuntu"] +if after != 1: + raise SystemExit( + f"intro-evidence mutation did not reopen a gap of one (derived {after})" + ) +intro_path = fixture / "docs/introduction.md" +intro_text = intro_path.read_text(encoding="utf-8") +old = "Every Debian-only action has a story." +new = "Two Debian-only actions still have no story." +if intro_text.count(old) != 1: + raise SystemExit("universal Debian-only sentence was not unique in the fixture") +intro_path.write_text(intro_text.replace(old, new), encoding="utf-8") +found = re.search( + mod.DEBIAN_GAP_PROSE, + re.sub(r"\s+", " ", intro_path.read_text(encoding="utf-8")), + re.IGNORECASE, +) +if not found or mod._claim_count(found.group("count")) != 2: + raise SystemExit("introduction-evidence mutation did not apply") +print(mod._claim_count(found.group("count")), after) PY ) -if [[ -z "${intro_before:-}" || -z "${intro_after:-}" || -z "${intro_action:-}" ]]; then +if [[ -z "${intro_before:-}" || -z "${intro_after:-}" ]]; then printf 'FAIL: introduction-evidence mutation produced no values\n' >&2 exit 1 fi -if ! grep -R -Fq "\"$intro_action\"" "$fixture/tests/e2e/stories"; then - printf 'FAIL: introduction-evidence mutation did not apply for %s\n' "$intro_action" >&2 +if ! grep -Fq 'Two Debian-only actions still have no story' \ + "$fixture/docs/introduction.md"; then + printf 'FAIL: introduction-evidence mutation did not apply\n' >&2 exit 1 fi assert_rejected_with_diagnostic \ 'story evidence that makes the introduction gap stale' \ 'introduction.md' "published $intro_before" "derived $intro_after" cp "$repo_root"/tests/e2e/stories/story-*.sh "$fixture/tests/e2e/stories/" +cp "$repo_root/docs/introduction.md" "$fixture/docs/introduction.md" if ! "$checker" "$fixture" >/dev/null 2>&1; then printf 'FAIL: restored intro-evidence fixture rejected — mutation result is meaningless\n' >&2 exit 1