From 2d5b9e7f3dd4b0483511dc8123711464f9cc2132 Mon Sep 17 00:00:00 2001 From: matthias schedel Date: Mon, 3 Aug 2026 11:11:09 +0200 Subject: [PATCH 1/2] fix: add acceptance verifier registry --- .github/acceptance-verifiers.json | 9 ++++++ justfile | 4 +-- scripts/test_acceptance_verifier_registry.py | 31 ++++++++++++++++++++ 3 files changed, 42 insertions(+), 2 deletions(-) create mode 100644 .github/acceptance-verifiers.json create mode 100644 scripts/test_acceptance_verifier_registry.py diff --git a/.github/acceptance-verifiers.json b/.github/acceptance-verifiers.json new file mode 100644 index 0000000000..b837d9e031 --- /dev/null +++ b/.github/acceptance-verifiers.json @@ -0,0 +1,9 @@ +{ + "schema": "acceptance-verifiers/v1", + "verifiers": [ + { + "id": "herdr-check-ubuntu", + "check_name": "check (ubuntu-latest)" + } + ] +} diff --git a/justfile b/justfile index cceeb24681..f7f9e9017b 100644 --- a/justfile +++ b/justfile @@ -3,7 +3,7 @@ # Run tests test: cargo nextest run --locked --status-level fail --final-status-level fail --failure-output final --success-output never - python3 -m unittest scripts.test_agent_detection_manifest_check scripts.test_changelog scripts.test_config_reference_check scripts.test_docs_translation_parity scripts.test_hermes_integration_asset scripts.test_package_windows_conpty scripts.test_pr_gate_workflow scripts.test_preview scripts.test_vendor_libghostty_vt scripts.test_vendor_portable_pty scripts.test_watch_pr_checks + python3 -m unittest scripts.test_acceptance_verifier_registry scripts.test_agent_detection_manifest_check scripts.test_changelog scripts.test_config_reference_check scripts.test_docs_translation_parity scripts.test_hermes_integration_asset scripts.test_package_windows_conpty scripts.test_pr_gate_workflow scripts.test_preview scripts.test_vendor_libghostty_vt scripts.test_vendor_portable_pty scripts.test_watch_pr_checks just integration-assets-test just plugin-marketplace-test @@ -38,7 +38,7 @@ windows-lint: # Check formatting + run unit tests + Windows target lint + maintenance script tests [unix] check: ci windows-lint - python3 -m unittest scripts.test_agent_detection_manifest_check scripts.test_changelog scripts.test_config_reference_check scripts.test_docs_translation_parity scripts.test_hermes_integration_asset scripts.test_package_windows_conpty scripts.test_pr_gate_workflow scripts.test_preview scripts.test_vendor_libghostty_vt scripts.test_vendor_portable_pty scripts.test_watch_pr_checks + python3 -m unittest scripts.test_acceptance_verifier_registry scripts.test_agent_detection_manifest_check scripts.test_changelog scripts.test_config_reference_check scripts.test_docs_translation_parity scripts.test_hermes_integration_asset scripts.test_package_windows_conpty scripts.test_pr_gate_workflow scripts.test_preview scripts.test_vendor_libghostty_vt scripts.test_vendor_portable_pty scripts.test_watch_pr_checks @echo "docs reminder: if this changes user-facing behavior, make sure the relevant release docs are updated or called out before release." # Run independent final verification concurrently and print one summary. diff --git a/scripts/test_acceptance_verifier_registry.py b/scripts/test_acceptance_verifier_registry.py new file mode 100644 index 0000000000..173ccaf23a --- /dev/null +++ b/scripts/test_acceptance_verifier_registry.py @@ -0,0 +1,31 @@ +import json +import unittest +from pathlib import Path + + +REPO_ROOT = Path(__file__).resolve().parent.parent +REGISTRY_PATH = REPO_ROOT / ".github" / "acceptance-verifiers.json" +CI_WORKFLOW_PATH = REPO_ROOT / ".github" / "workflows" / "ci.yml" + + +class AcceptanceVerifierRegistryTests(unittest.TestCase): + def test_registry_allows_the_trusted_ubuntu_herdr_check(self) -> None: + registry = json.loads(REGISTRY_PATH.read_text(encoding="utf-8")) + + self.assertEqual(registry["schema"], "acceptance-verifiers/v1") + self.assertEqual( + registry["verifiers"], + [{ + "id": "herdr-check-ubuntu", + "check_name": "check (ubuntu-latest)", + }], + ) + + workflow = CI_WORKFLOW_PATH.read_text(encoding="utf-8") + self.assertIn("name: check (${{ matrix.os }})", workflow) + self.assertIn("- os: ubuntu-latest", workflow) + self.assertIn("run: just ci '${{ matrix.nextest_filter }}'", workflow) + + +if __name__ == "__main__": + unittest.main() From ec2a0c255f28e5f231a83e3e90d54c33ab739f86 Mon Sep 17 00:00:00 2001 From: matthias schedel Date: Mon, 3 Aug 2026 11:26:18 +0200 Subject: [PATCH 2/2] test: add terminal QA adapter --- .qa/preview-adapter.json | 5 + docs/qa/user-journeys.md | 62 +++++++++++ justfile | 4 +- scripts/qa-preview-adapter.sh | 122 +++++++++++++++++++++ scripts/qa-sidebar-lifecycle-journey.sh | 101 ++++++++++++++++++ scripts/test_qa_preview_adapter.py | 136 ++++++++++++++++++++++++ 6 files changed, 428 insertions(+), 2 deletions(-) create mode 100644 .qa/preview-adapter.json create mode 100644 docs/qa/user-journeys.md create mode 100755 scripts/qa-preview-adapter.sh create mode 100755 scripts/qa-sidebar-lifecycle-journey.sh create mode 100644 scripts/test_qa_preview_adapter.py diff --git a/.qa/preview-adapter.json b/.qa/preview-adapter.json new file mode 100644 index 0000000000..af09172b1b --- /dev/null +++ b/.qa/preview-adapter.json @@ -0,0 +1,5 @@ +{ + "schema": "preview-adapter/v1", + "command": ["./scripts/qa-preview-adapter.sh"], + "journeys_document": "docs/qa/user-journeys.md" +} diff --git a/docs/qa/user-journeys.md b/docs/qa/user-journeys.md new file mode 100644 index 0000000000..740086667c --- /dev/null +++ b/docs/qa/user-journeys.md @@ -0,0 +1,62 @@ +# Sidebar lifecycle QA journeys + +Herdr is a terminal application. These journeys provide executable behavioral +evidence, not a browser preview or visual screenshot claim. Every command must +run against the full Git `HEAD` it names and writes a durable, head-bound local +assertion artifact under `QA_ARTIFACT_DIR` (or an explicit `--artifact-dir`). + +Use the declaration at `.qa/preview-adapter.json` to create a terminal contract +card. Its `preview_url` is intentionally empty: Herdr has no web preview. + +## sidebar-all-tab-retention + +The sidebar projection retains every tab in expanded and compact presentation: +inactive, completed, singleton, multi-pane, and agentless tabs remain present. +Compact mode changes presentation only; it does not choose only the latest +active tab. + +Evidence: `initial_sidebar_projection_keeps_every_workspace_tab_in_expanded_and_collapsed_views`. + +## same-session-title-replacement + +A later `UserPromptSubmit` from the same agent session selects its latest +meaningful final paragraph for one fresh persisted title. A terse/meta +continuation retains that session's initial briefing subject instead of +manufacturing a generic label. It does not append a subtitle or duplicate +title. + +Evidence: `terse_later_user_prompt_submit_retains_the_session_initial_title` +and `work_title_initial_briefings_are_session_scoped_and_later_objectives_replace_them`. + +## reopen-clears-done-without-reorder + +Opening a completed pane clears its done presentation while the canonical +workspace/tab order remains stable. + +Evidence: `api_pane_focus_marks_already_focused_done_pane_seen` and +`initial_sidebar_projection_keeps_every_workspace_tab_in_expanded_and_collapsed_views`. + +## working-latches-until-genuine-completion + +Authoritative lifecycle working state remains latched when fallback idle evidence +appears. A genuine completion transition is the only event that changes it to +done. + +Evidence: `fallback_idle_does_not_override_full_lifecycle_hook_working` and +`full_internal_event_queue_eventually_applies_working_to_idle_transition`. + +## Running one journey + +```sh +scripts/qa-sidebar-lifecycle-journey.sh \ + --repo-dir "$PWD" \ + --head "$(git rev-parse HEAD)" \ + --flow sidebar-all-tab-retention \ + --artifact-dir .local/qa/sidebar-all-tab-retention +``` + +The result is a `qa-journey-result/v1` JSON document on stdout. Exit `0` means +the named assertion passed, `64` is invalid input, `66` is an exact-head binding +failure, and any other nonzero code is an assertion failure. The runner never +starts a Herdr session, changes a live configuration or hook, or emits prompt +content. diff --git a/justfile b/justfile index f7f9e9017b..bd97c2f3ec 100644 --- a/justfile +++ b/justfile @@ -3,7 +3,7 @@ # Run tests test: cargo nextest run --locked --status-level fail --final-status-level fail --failure-output final --success-output never - python3 -m unittest scripts.test_acceptance_verifier_registry scripts.test_agent_detection_manifest_check scripts.test_changelog scripts.test_config_reference_check scripts.test_docs_translation_parity scripts.test_hermes_integration_asset scripts.test_package_windows_conpty scripts.test_pr_gate_workflow scripts.test_preview scripts.test_vendor_libghostty_vt scripts.test_vendor_portable_pty scripts.test_watch_pr_checks + python3 -m unittest scripts.test_acceptance_verifier_registry scripts.test_agent_detection_manifest_check scripts.test_changelog scripts.test_config_reference_check scripts.test_docs_translation_parity scripts.test_hermes_integration_asset scripts.test_package_windows_conpty scripts.test_pr_gate_workflow scripts.test_preview scripts.test_qa_preview_adapter scripts.test_vendor_libghostty_vt scripts.test_vendor_portable_pty scripts.test_watch_pr_checks just integration-assets-test just plugin-marketplace-test @@ -38,7 +38,7 @@ windows-lint: # Check formatting + run unit tests + Windows target lint + maintenance script tests [unix] check: ci windows-lint - python3 -m unittest scripts.test_acceptance_verifier_registry scripts.test_agent_detection_manifest_check scripts.test_changelog scripts.test_config_reference_check scripts.test_docs_translation_parity scripts.test_hermes_integration_asset scripts.test_package_windows_conpty scripts.test_pr_gate_workflow scripts.test_preview scripts.test_vendor_libghostty_vt scripts.test_vendor_portable_pty scripts.test_watch_pr_checks + python3 -m unittest scripts.test_acceptance_verifier_registry scripts.test_agent_detection_manifest_check scripts.test_changelog scripts.test_config_reference_check scripts.test_docs_translation_parity scripts.test_hermes_integration_asset scripts.test_package_windows_conpty scripts.test_pr_gate_workflow scripts.test_preview scripts.test_qa_preview_adapter scripts.test_vendor_libghostty_vt scripts.test_vendor_portable_pty scripts.test_watch_pr_checks @echo "docs reminder: if this changes user-facing behavior, make sure the relevant release docs are updated or called out before release." # Run independent final verification concurrently and print one summary. diff --git a/scripts/qa-preview-adapter.sh b/scripts/qa-preview-adapter.sh new file mode 100755 index 0000000000..10633b77fd --- /dev/null +++ b/scripts/qa-preview-adapter.sh @@ -0,0 +1,122 @@ +#!/usr/bin/env bash +# Emit a terminal-only QA card for the checked-out Herdr revision. +set -euo pipefail + +repo_dir="${QA_REPO_DIR:-$PWD}" +pr="" +head="" +mode="" +format="" + +usage() { + echo "usage: qa-preview-adapter.sh [--repo-dir ] --pr --head --mode --format json" >&2 +} + +require_value() { + [[ $# -ge 2 && -n "$2" ]] || { usage; exit 64; } +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --repo-dir) require_value "$@"; repo_dir="$2"; shift 2 ;; + --pr) require_value "$@"; pr="$2"; shift 2 ;; + --head) require_value "$@"; head="$2"; shift 2 ;; + --mode) require_value "$@"; mode="$2"; shift 2 ;; + --format) require_value "$@"; format="$2"; shift 2 ;; + *) usage; exit 64 ;; + esac +done + +[[ -d "$repo_dir" ]] || { echo "qa-preview-adapter: --repo-dir must name a directory" >&2; exit 64; } +[[ "$pr" =~ ^[1-9][0-9]*$ ]] || { echo "qa-preview-adapter: --pr must be a positive number" >&2; exit 64; } +[[ "$head" =~ ^[0-9a-fA-F]{40}$ ]] || { echo "qa-preview-adapter: full --head required" >&2; exit 64; } +[[ "$mode" == "qa" || "$mode" == "card" ]] || { echo "qa-preview-adapter: --mode must be qa or card" >&2; exit 64; } +[[ "$format" == "json" ]] || { echo "qa-preview-adapter: --format must be json" >&2; exit 64; } + +head="$(printf '%s' "$head" | tr '[:upper:]' '[:lower:]')" +repo_root="$(git -C "$repo_dir" rev-parse --show-toplevel 2>/dev/null)" \ + || { echo "qa-preview-adapter: --repo-dir is not a Git checkout" >&2; exit 66; } +repo_root="$(cd "$repo_root" && pwd -P)" +actual_head="$(git -C "$repo_root" rev-parse HEAD)" +[[ "$actual_head" == "$head" ]] \ + || { echo "qa-preview-adapter: checkout is not the requested exact head" >&2; exit 66; } + +declaration="$repo_root/.qa/preview-adapter.json" +journeys="$repo_root/docs/qa/user-journeys.md" +journey_runner="$repo_root/scripts/qa-sidebar-lifecycle-journey.sh" +[[ -f "$declaration" && -f "$journeys" && -x "$journey_runner" ]] \ + || { echo "qa-preview-adapter: declaration, journeys, or flow runner is unavailable" >&2; exit 66; } + +python3 - "$declaration" "$head" <<'PY' +import json +import sys + +declaration = json.load(open(sys.argv[1], encoding="utf-8")) +if declaration != { + "schema": "preview-adapter/v1", + "command": ["./scripts/qa-preview-adapter.sh"], + "journeys_document": "docs/qa/user-journeys.md", +}: + raise SystemExit("qa-preview-adapter: invalid repository declaration") + +head = sys.argv[2] +flows = [ + ( + "sidebar-all-tab-retention", + "Sidebar retains every tab in expanded and compact presentation", + "Exercise the sidebar projection with inactive, completed, singleton, multi-pane, and agentless tabs.", + "Both presentations retain the same canonical tab identities; compact mode changes presentation only.", + ["src/ui/sidebar.rs", "src/app/state.rs"], + ), + ( + "same-session-title-replacement", + "Later prompts replace the one title for the same session", + "Submit a later UserPromptSubmit for an existing agent session.", + "The persisted tab title is replaced once, with no subtitle or duplicate title.", + ["src/app/api/panes.rs", "src/app/state.rs"], + ), + ( + "reopen-clears-done-without-reorder", + "Opening completed work clears done without moving its tab", + "Focus a completed pane while preserving canonical workspace and tab identity order.", + "Done clears on focus; its row identity and position remain unchanged.", + ["src/app/api/panes.rs", "src/ui/sidebar.rs"], + ), + ( + "working-latches-until-genuine-completion", + "Working remains latched until a genuine completion transition", + "Apply fallback idle evidence after an authoritative lifecycle working event, then complete the turn.", + "Fallback idle does not flicker working; genuine completion changes working to done.", + ["src/terminal/state.rs", "src/app/mod.rs"], + ), +] +card = { + "schema": "preview-card/v1", + "head_sha": head, + # Herdr is a terminal application. An empty URL is deliberate: no web preview exists. + "preview_url": "", + "card_markdown": ( + "## Terminal contract ready\n\n" + "This change has no web preview. Run the named repository-owned " + "nonvisual journeys; each emits a head-bound local assertion artifact." + ), + "required_flows": [ + { + "id": flow_id, + "title": title, + "actor": "Herdr operator", + "action": action, + "expected": expected, + "dependency_surfaces": surfaces, + "visual_required": False, + "automation": { + "schema": "qa-automation/v1", + "command": ["./scripts/qa-sidebar-lifecycle-journey.sh", "--flow", flow_id], + }, + } + for flow_id, title, action, expected, surfaces in flows + ], + "artifacts": [], +} +print(json.dumps(card, sort_keys=True, separators=(",", ":"))) +PY diff --git a/scripts/qa-sidebar-lifecycle-journey.sh b/scripts/qa-sidebar-lifecycle-journey.sh new file mode 100755 index 0000000000..85af1437ea --- /dev/null +++ b/scripts/qa-sidebar-lifecycle-journey.sh @@ -0,0 +1,101 @@ +#!/usr/bin/env bash +# Run a single head-bound, nonvisual sidebar/lifecycle contract journey. +set -euo pipefail + +repo_dir="${QA_REPO_DIR:-$PWD}" +head="${QA_EXACT_HEAD:-}" +flow="${QA_FLOW_ID:-}" +artifact_dir="${QA_ARTIFACT_DIR:-}" + +usage() { + echo "usage: qa-sidebar-lifecycle-journey.sh --repo-dir --head --flow --artifact-dir " >&2 +} + +require_value() { + [[ $# -ge 2 && -n "$2" ]] || { usage; exit 64; } +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --repo-dir) require_value "$@"; repo_dir="$2"; shift 2 ;; + --head) require_value "$@"; head="$2"; shift 2 ;; + --flow) require_value "$@"; flow="$2"; shift 2 ;; + --artifact-dir) require_value "$@"; artifact_dir="$2"; shift 2 ;; + *) usage; exit 64 ;; + esac +done + +[[ "$head" =~ ^[0-9a-fA-F]{40}$ ]] || { echo "qa-sidebar-lifecycle-journey: full --head required" >&2; exit 64; } +[[ -n "$artifact_dir" ]] || { echo "qa-sidebar-lifecycle-journey: --artifact-dir required" >&2; exit 64; } +head="$(printf '%s' "$head" | tr '[:upper:]' '[:lower:]')" +repo_root="$(git -C "$repo_dir" rev-parse --show-toplevel 2>/dev/null)" \ + || { echo "qa-sidebar-lifecycle-journey: --repo-dir is not a Git checkout" >&2; exit 66; } +repo_root="$(cd "$repo_root" && pwd -P)" +actual_head="$(git -C "$repo_root" rev-parse HEAD)" +[[ "$actual_head" == "$head" ]] \ + || { echo "qa-sidebar-lifecycle-journey: checkout is not the requested exact head" >&2; exit 66; } + +case "$flow" in + sidebar-all-tab-retention) + filters=("initial_sidebar_projection_keeps_every_workspace_tab_in_expanded_and_collapsed_views") + ;; + same-session-title-replacement) + filters=( + "terse_later_user_prompt_submit_retains_the_session_initial_title" + "work_title_initial_briefings_are_session_scoped_and_later_objectives_replace_them" + ) + ;; + reopen-clears-done-without-reorder) + filters=( + "api_pane_focus_marks_already_focused_done_pane_seen" + "initial_sidebar_projection_keeps_every_workspace_tab_in_expanded_and_collapsed_views" + ) + ;; + working-latches-until-genuine-completion) + filters=( + "fallback_idle_does_not_override_full_lifecycle_hook_working" + "full_internal_event_queue_eventually_applies_working_to_idle_transition" + ) + ;; + *) echo "qa-sidebar-lifecycle-journey: unknown flow $flow" >&2; exit 64 ;; +esac + +artifact_dir="$(mkdir -p "$artifact_dir" && cd "$artifact_dir" && pwd -P)" +log_path="$artifact_dir/${flow}.log" +for filter in "${filters[@]}"; do + if ! ( + cd "$repo_root" + CARGO_TARGET_DIR="$artifact_dir/cargo-target" just test-one "$filter" + ) >>"$log_path" 2>&1; then + echo "qa-sidebar-lifecycle-journey: assertion failed for $flow" >&2 + exit 1 + fi +done + +artifact_path="$artifact_dir/${flow}.json" +python3 - "$artifact_path" "$head" "$flow" "${filters[@]}" <<'PY' +import json +import sys + +path, head, flow, *filters = sys.argv[1:] +with open(path, "w", encoding="utf-8") as output: + json.dump( + { + "schema": "herdr-terminal-contract/v1", + "head_sha": head, + "flow_id": flow, + "assertions": filters, + "visual_evidence": "not-required", + }, + output, + sort_keys=True, + separators=(",", ":"), + ) + output.write("\n") +print(json.dumps({ + "schema": "qa-journey-result/v1", + "status": "PASS", + "head_sha": head, + "artifacts": [{"id": f"{flow}-assertions", "type": "assertion", "path": path}], +}, sort_keys=True, separators=(",", ":"))) +PY diff --git a/scripts/test_qa_preview_adapter.py b/scripts/test_qa_preview_adapter.py new file mode 100644 index 0000000000..2a1ce265af --- /dev/null +++ b/scripts/test_qa_preview_adapter.py @@ -0,0 +1,136 @@ +import json +import os +import subprocess +import unittest +from unittest import mock +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +ADAPTER = ROOT / "scripts" / "qa-preview-adapter.sh" +JOURNEY = ROOT / "scripts" / "qa-sidebar-lifecycle-journey.sh" +DECLARATION = ROOT / ".qa" / "preview-adapter.json" +JOURNEYS = ROOT / "docs" / "qa" / "user-journeys.md" + + +class QaPreviewAdapterTests(unittest.TestCase): + def setUp(self) -> None: + self.head = subprocess.check_output( + ["git", "rev-parse", "HEAD"], cwd=ROOT, text=True + ).strip() + + def adapter(self, *args: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [str(ADAPTER), *args], + cwd=ROOT, + text=True, + capture_output=True, + check=False, + ) + + def test_declaration_points_to_executable_terminal_adapter_and_journeys(self) -> None: + declaration = json.loads(DECLARATION.read_text(encoding="utf-8")) + self.assertEqual(declaration["schema"], "preview-adapter/v1") + self.assertEqual(declaration["command"], ["./scripts/qa-preview-adapter.sh"]) + self.assertEqual(declaration["journeys_document"], "docs/qa/user-journeys.md") + self.assertTrue(ADAPTER.is_file()) + self.assertTrue(os.access(ADAPTER, os.X_OK)) + self.assertTrue(JOURNEY.is_file()) + self.assertTrue(os.access(JOURNEY, os.X_OK)) + self.assertTrue(JOURNEYS.is_file()) + + def test_card_is_exact_head_terminal_contract_with_named_nonvisual_flows(self) -> None: + with mock.patch.dict(os.environ, {"QA_SECRET_TOKEN": "never-print"}): + result = self.adapter( + "--repo-dir", str(ROOT), "--pr", "13", "--head", self.head, + "--mode", "qa", "--format", "json", + ) + self.assertEqual(result.returncode, 0, result.stderr) + card = json.loads(result.stdout) + self.assertEqual(card["schema"], "preview-card/v1") + self.assertEqual(card["head_sha"], self.head) + self.assertEqual(card["preview_url"], "") + self.assertNotIn("http", card["card_markdown"].lower()) + self.assertNotIn("never-print", result.stdout) + self.assertEqual(card["artifacts"], []) + self.assertEqual( + [flow["id"] for flow in card["required_flows"]], + [ + "sidebar-all-tab-retention", + "same-session-title-replacement", + "reopen-clears-done-without-reorder", + "working-latches-until-genuine-completion", + ], + ) + for flow in card["required_flows"]: + self.assertFalse(flow["visual_required"]) + self.assertEqual(flow["automation"]["schema"], "qa-automation/v1") + self.assertEqual( + flow["automation"]["command"], + ["./scripts/qa-sidebar-lifecycle-journey.sh", "--flow", flow["id"]], + ) + + def test_declared_command_uses_its_process_working_directory(self) -> None: + result = subprocess.run( + [ + str(ADAPTER), "--pr", "13", "--head", self.head, + "--mode", "qa", "--format", "json", + ], + cwd=ROOT, + text=True, + capture_output=True, + check=False, + ) + self.assertEqual(result.returncode, 0, result.stderr) + card = json.loads(result.stdout) + self.assertEqual(card["head_sha"], self.head) + + def test_adapter_rejects_invalid_arguments_and_checkout_binding(self) -> None: + short_head = self.adapter( + "--repo-dir", str(ROOT), "--pr", "13", "--head", self.head[:12], + "--mode", "qa", "--format", "json", + ) + self.assertEqual(short_head.returncode, 64) + self.assertIn("full --head", short_head.stderr) + + wrong_head = self.adapter( + "--repo-dir", str(ROOT), "--pr", "13", "--head", "0" * 40, + "--mode", "qa", "--format", "json", + ) + self.assertEqual(wrong_head.returncode, 66) + self.assertIn("not the requested exact head", wrong_head.stderr) + + invalid_pr = self.adapter( + "--repo-dir", str(ROOT), "--pr", "0", "--head", self.head, + "--mode", "qa", "--format", "json", + ) + self.assertEqual(invalid_pr.returncode, 64) + + invalid_mode = self.adapter( + "--repo-dir", str(ROOT), "--pr", "13", "--head", self.head, + "--mode", "preview", "--format", "json", + ) + self.assertEqual(invalid_mode.returncode, 64) + + invalid_format = self.adapter( + "--repo-dir", str(ROOT), "--pr", "13", "--head", self.head, + "--mode", "qa", "--format", "text", + ) + self.assertEqual(invalid_format.returncode, 64) + + def test_journey_rejects_a_wrong_head_before_running_product_assertions(self) -> None: + artifact_root = ROOT / ".local" / "qa-preview-adapter-test" + failed = subprocess.run( + [str(JOURNEY), "--repo-dir", str(ROOT), "--head", "0" * 40, + "--flow", "sidebar-all-tab-retention", "--artifact-dir", str(artifact_root)], + cwd=ROOT, + text=True, + capture_output=True, + check=False, + ) + self.assertEqual(failed.returncode, 66) + self.assertIn("not the requested exact head", failed.stderr) + + +if __name__ == "__main__": + unittest.main()