diff --git a/.ci/test-impact/impact_map.json b/.ci/test-impact/impact_map.json new file mode 100644 index 000000000..ac07bc329 --- /dev/null +++ b/.ci/test-impact/impact_map.json @@ -0,0 +1,21 @@ +{ + "schema_version": 1, + "source_paths": { + "backend/app/core/s3_validation.py": { + "test_modules": [ + "tests/test_config.py", + "tests/test_artifact_store_conformance.py", + "tests/test_s3_artifact_store.py" + ], + "reason": "S3 configuration validation is exercised in the shared-foundation partition." + } + }, + "path_prefixes": { + ".commitrail/": { + "test_modules": [ + "tests/projects/review_policy/test_semantics.py" + ], + "reason": "Commitrail policy semantics are tested in the shared-foundation partition." + } + } +} diff --git a/.commitrail/INDEX.md b/.commitrail/INDEX.md index 610b1e226..0f9f4590f 100644 --- a/.commitrail/INDEX.md +++ b/.commitrail/INDEX.md @@ -17,6 +17,7 @@ for current product capability. | [WS-REV-001](initiatives/WS-REV-001/OVERVIEW.md) | Planned | Shared acceptance/source and existing fence foundations; human hidden review work remains independently dependency-gated | | [WS-QUAL-002](initiatives/WS-QUAL-002/OVERVIEW.md) | Planned | Populate subsystem ownership before changed-line mutation work | | [WS-QUAL-003](initiatives/WS-QUAL-003/OVERVIEW.md) | Planned | Audit and prune test proof, add missing safety cases, decompose oversized test modules | +| [WS-CI-006](initiatives/WS-CI-006/OVERVIEW.md) | Planned | Shadow-mode semantic-lane impact report beside the unchanged full required suite; gate changes require later evidence and review | | [WS-XINT-002](initiatives/WS-XINT-002/OVERVIEW.md) | Planned | Remaining ART/AUTH activation edges only | | [WS-XINT-003](initiatives/WS-XINT-003/OVERVIEW.md) | Planned | Resume activation only against exact merged REV behavior | | WS-POL-002 | Superseded | Future guide inference belongs to WS-POL-003; reframe remaining executor work against current specifications | diff --git a/.commitrail/initiatives/WS-CI-006/OVERVIEW.md b/.commitrail/initiatives/WS-CI-006/OVERVIEW.md new file mode 100644 index 000000000..6c0dfc531 --- /dev/null +++ b/.commitrail/initiatives/WS-CI-006/OVERVIEW.md @@ -0,0 +1,51 @@ +# WS-CI-006 — Change-impact backend verification + +- Disposition: Planned +- Next usable boundary: [WS-CI-006-01](WS-CI-006-01.md) + +## Intent + +Reduce routine backend PR feedback by selecting tests according to demonstrated +change impact, without making incomplete test evidence authoritative. Today +the full Backend suite remains required. The first usable step is a shadow +report whose recommendation is compared with the complete suite on the same +exact PR target. + +## Current evidence and limits + +- `backend/scripts/test_lane_catalogue.py` assigns every discovered test module + to the complete backend run. Shared, project and task node collections are + partitioned across two, three and three jobs respectively; a module in a + partitioned group requires every shard in its group. +- `.github/workflows/backend.yml` continues to run all nine lanes, authorization + preflight, MinIO, real API proof, and evidence aggregation on every PR. The + added `impact-report` job is informational and cannot control lane execution. +- Run [36724982896](https://github.com/Flow-Research/workstream/actions/runs/36724982896) + completed 7,918 tests with zero skips/deselections in about 44 minutes. This + is one observed run, not a universal baseline. +- Initial explicit mappings are intentionally narrow: S3 validation maps to + three reviewed test modules, and Commitrail-only changes map to the policy + semantics module. Their lane owners are derived from the canonical catalogue; + changed test modules also select every lane partition that owns them. Any + unmapped path recommends all nine lanes. +- The report is generated by the PR candidate and is not independent policy + evidence. PRs that change the selector, map, catalogue or workflow cannot + validate their own changes; no lane is omitted from actual CI. The report job + is not a dependency of the required aggregate, lane fan-in or API end-to-end + proof, so its failure cannot suppress those checks. Branch protection remains + the authority over which standalone job statuses are merge requirements. + +## Direction after shadow evidence + +1. Observe recommendations beside complete test results on the same exact + target across representative PRs. +2. Expand mappings only after tracing production owners, downstream consumers, + shared fixtures, integration services and the lane catalogue. Unknown impact + remains full-suite. +3. Evaluate whether the accumulated evidence supports a separate bounded change + to required test execution. That decision must preserve the full suite on + `main` and broad/cross-cutting changes; this initiative does not pre-approve + a CI gate reduction or promise every PR completes in five minutes. + +No external selector service, historical mutable test signal, test deletion, +coverage quota, arbitrary sharding, or product behavior change is in scope. diff --git a/.commitrail/initiatives/WS-CI-006/WS-CI-006-01.md b/.commitrail/initiatives/WS-CI-006/WS-CI-006-01.md new file mode 100644 index 000000000..e45758b51 --- /dev/null +++ b/.commitrail/initiatives/WS-CI-006/WS-CI-006-01.md @@ -0,0 +1,164 @@ +# WS-CI-006-01 — Shadow-mode backend test-impact report + +- Initiative: `WS-CI-006` +- Durable disposition: `Planned` +- Intended merge outcome: Backend CI reports a deterministic, exact-PR proposed semantic-lane selection and rationale while the existing complete Backend suite remains unchanged and blocking on every PR and `main` push. This change does not enable selective gating. + +## Intent + +Backend PRs currently pay for the complete suite even when a change has a +bounded impact. Before changing that gate, collect exact-head evidence about +which semantic lanes a conservative change-impact map would select. Use the +existing lane catalogue as the first coarse unit; later work may select within +a lane only after the shadow reports establish a sound owner-to-test closure. + +The observed baseline is run +[36724982896](https://github.com/Flow-Research/workstream/actions/runs/36724982896): +7,918 tests, zero skipped/deselected, about 44 minutes wall time. This is one +run, not a universal timing estimate. The target is under five minutes for +small, well-understood changes after a later separately reviewed gating phase; +this shadow change makes no runtime reduction or timing claim. + +## Current behavior and lane semantics + +- `backend/scripts/test_lane_catalogue.py` assigns every discovered test module + to the complete run. Shared, project, and task groups are partitioned across + multiple jobs. A module appearing in a partitioned group requires every + shard in that group to cover all of its node IDs. +- `.github/workflows/backend.yml` runs all nine lanes, the authorization + preflight, MinIO-backed tests, real API proof and aggregate evidence on every + PR and `main` push. The required `test` result and full-suite policy remain + unchanged in this PR. +- A shadow report may recommend lanes but cannot control `if` conditions, + services, test commands, required jobs, fan-in, or merge status. Unknown, + broad, malformed, or unclassified input reports `all lanes`. + +## Bounded change + +### Allowed files + +- `.github/workflows/backend.yml` for a PR-only classifier/report job, and + `.github/workflows/agent-gates.yml` to run repository-level selector tests; + the existing backend full-suite graph and commands stay blocking. +- `.ci/test-impact/impact_map.json` and + `scripts/backend_test_impact.py` for deterministic shadow classification and + exact-target report generation. +- `scripts/test_backend_test_impact.py` for selector behavior regressions in + the repository-level lightweight gate suite. +- `backend/scripts/test_lane_catalogue.py` only as the authoritative inventory + read by the selector; `scripts/git_delta.py` and + `scripts/test_git_delta.py` only for shared byte-safe Git delta primitives. +- `scripts/test_lightweight_agent_gates.py` for backend workflow-shape and + fan-in regressions. +- `docs/operations_backend_testing.md`, the WS-CI-006 initiative overview, + this record, and the Commitrail index for the shadow-only operating contract. +- `docs/roadmap_status.md` only if its current CI capability statement needs + correction to describe this intended merged state. + +### Prohibited changes + +- No changes to Backend test bodies, assertions, collection, + skip/deselect behavior, coverage policy, current lane partitioning, services, + test commands, required status checks, branch protection, or merge rules. +- The selector regression suite may be added to the existing repository-level + Agent Gates test command; this does not change Backend test collection. +- No selector-driven workflow conditions, lane omissions, workflow-level path + filters, test execution service, third-party impact product, or mutable + historical selection authority. +- No test deletion or claim that shadow output reduces CI time. + +## Shadow selection contract + +- Resolve changed paths from the exact PR base/head and merge base; bind a + successfully resolved report to the base SHA, head SHA, execution SHA/tree, + changed-path digest, selector/map version and digests, selected semantic + lanes and rationale. If target identity or changed paths cannot be resolved, + mark unavailable facts as unavailable and recommend all lanes. +- Use the current semantic lane catalogue to map test modules to every lane + shard that owns their nodes. A changed test module is included in the proposed + impact closure. Shared fixtures, schema/migrations, dependencies, workflow, + lane catalogue, map/selector changes, unknown source paths, or unavailable + Git evidence conservatively recommend all nine lanes. +- Initial source mapping is deliberately limited to the reviewed S3 validation + owner `backend/app/core/s3_validation.py`, mapped to + `tests/test_config.py`, `tests/test_artifact_store_conformance.py`, and + `tests/test_s3_artifact_store.py`. Resolve their owning lanes from the + canonical catalogue; do not copy shard membership into the impact map. + Changes to other application source recommend all lanes until additional + consumer closures are demonstrated and explicitly mapped. +- `.commitrail/**` remains in the changed-path report and maps to + `tests/projects/review_policy/test_semantics.py`; derive its owner lanes from + the canonical catalogue. Mixed changes union this with source selection. If + any other path is unclassified, recommend all lanes. Documentation, skills, + and agent-policy paths are not implicitly exempted. +- The report explains every selected lane and every omitted lane. Omission is + allowed in the *recommendation only* when the exact mapping explains why; + missing evidence yields all lanes. The classifier is observational: CI still + executes all nine lanes and all existing integration/preflight jobs. +- The selector and map are part of the PR candidate in this shadow phase. Their + report is candidate-produced diagnostic evidence, not a trusted test policy + or independent audit receipt. A PR that changes the selector, map, catalogue, + or workflow cannot validate those changed inputs; any later gating change + must establish trusted selection policy separately. +- The report is uploaded and linked from the PR workflow summary. It is + descriptive evidence for comparing the proposed lane set with the complete + run from the same PR head; it never attests to another commit. + +## Acceptance criteria + +- [ ] The current full Backend workflow still runs unchanged on every PR and + `main` push, including all nine lanes, preflight, API/integration proof, and + the required `test` aggregate. +- [ ] A successfully resolved target report includes exact base/head/execution + tree, changed paths, selector/map identity, selected lanes, omitted lanes and + per-lane reasons. If target facts cannot be established, it clearly marks + them unavailable and recommends all lanes. The artifact and summary identify + the supplied PR head without asserting unverified facts. +- [ ] The initial S3 source change recommends both shared-foundation shards; + each mapped test module resolves to every partition owning its nodes. +- [ ] Commitrail-only changes recommend the shared-foundation shards containing + the policy-semantics tests; mixed known changes union their closures. +- [ ] Unknown paths, shared test support, migrations/schema, dependency and CI + machinery changes, malformed input, stale or missing Git objects, and + selector errors recommend all nine lanes rather than a partial set. +- [ ] Adversarial tests cover malformed/empty path lists, duplicate paths, + unknown files, renames from unknown source paths, changed tests, shared + fixtures, each protected map/selector input, stale or mismatched PR targets, + and missing lane ownership. +- [ ] Workflow regression tests prove the classifier output cannot condition, + skip, replace, or weaken any full-suite job or required check; in particular, + the report job is not a dependency of the required aggregate, lane fan-in or + API end-to-end step. +- [ ] A hosted PR run shows the shadow report beside complete passing test + evidence for the same head. Subsequent naturally occurring PRs provide the + representative comparison set; do not infer safety from synthetic paths or + a single S3-only example. +- [ ] Full-suite completeness, real integration checks, authorization, + concurrency, migration and rollback proof remain blocking. Coverage remains + diagnostic only. + +## Risk and review routing + +- Risk class: `L1` CI/workflow integrity. +- Required tracks: `ci_integrity`, `qa`, `test_delta`, `security`, + `documentation`, and `reuse_dedup`, selected through the reviewer matrix. +- Human review focus: proof the report is observational only, partition-aware + lane selection, exact-head binding, and unchanged full-suite enforcement. + +## Evidence + +Local focused verification uses the selector behavior tests, the lightweight +workflow-shape and fan-in tests, Ruff on changed Python files, Commitrail +validation, Markdown-link validation and the stale-wording scan. The PR must +also complete the existing hosted Backend workflow on the exact candidate; its +nine lanes, API drill, infrastructure and evidence checks remain unchanged. + +## Reconciliation + +- Current source: nine complete semantic lanes and their integration/fan-in + remain required on PRs and `main`. +- Next boundary: collect shadow classifications against full runs across + representative changes, then review the map and timing evidence before + planning any separate selective-gating change. +- No contribution instructions or product capability claims are relaxed by + this change. No main-push check is removed. diff --git a/.github/workflows/agent-gates.yml b/.github/workflows/agent-gates.yml index 66237449b..d209b3180 100644 --- a/.github/workflows/agent-gates.yml +++ b/.github/workflows/agent-gates.yml @@ -73,4 +73,5 @@ jobs: scripts.test_commitrail_contribution_paths scripts.test_commitrail_archive_batch scripts.test_commitrail_markdown_structure + scripts.test_backend_test_impact scripts.test_lightweight_agent_gates diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml index 9cd448bd3..8a436e7c7 100644 --- a/.github/workflows/backend.yml +++ b/.github/workflows/backend.yml @@ -139,6 +139,48 @@ jobs: --ledger ../.ci/auth-boundaries/TEST_STRUCTURE_DEBT.json python -m scripts.behavior_ownership validate + impact-report: + if: ${{ github.event_name == 'pull_request' }} + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 + with: + persist-credentials: false + fetch-depth: 0 + + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 + with: + python-version: "3.12" + + - name: Bind and classify the exact pull request target + env: + PR_BASE_SHA: ${{ github.event.pull_request.base.sha }} + PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: >- + python scripts/backend_test_impact.py + --json .ci/test-impact/report.json + --markdown .ci/test-impact/report.md + + - name: Upload exact-target impact report + id: report + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: backend-test-impact-${{ github.sha }}-${{ github.run_attempt }} + path: | + .ci/test-impact/report.json + .ci/test-impact/report.md + if-no-files-found: error + retention-days: 7 + + - name: Link report artifact in run summary + env: + REPORT_URL: ${{ steps.report.outputs.artifact-url }} + run: | + printf '\n[Download exact-head test-impact report](%s)\n' "${REPORT_URL}" >> "${GITHUB_STEP_SUMMARY}" + lanes: needs: minio-image runs-on: ubuntu-latest diff --git a/docs/operations_backend_testing.md b/docs/operations_backend_testing.md index ab85af6f4..5c7e6c9a3 100644 --- a/docs/operations_backend_testing.md +++ b/docs/operations_backend_testing.md @@ -110,6 +110,28 @@ If provisioning fails, confirm the local PostgreSQL provisioning credential can ## Hosted semantic-lane full-suite proof +For pull requests, the workflow also publishes a test-impact shadow report in +the Backend run summary and as a seven-day artifact. When exact Git target and +changed-path evidence resolves, it records the PR base/head, merge base, +checked-out execution SHA/tree, changed paths, selector/map/catalogue digests, +and lane-level selection reasons. If that evidence cannot be established, it +marks unavailable fields and recommends all lanes rather than presenting an +unverified exact-target classification. The initial +map is deliberately narrow; changed test modules use the existing lane +catalogue, while shared fixtures, migrations/schema, dependencies, workflow or +catalogue changes and any unmapped path recommend all nine lanes. A +classification error records an all-lanes fallback. The report is observational: +all nine matrix lanes, authorization preflight, API/integration proof and +evidence fan-in remain required and run independently of its recommendation. +The report job is not a dependency of the required aggregate, lane fan-in or API +end-to-end proof; report failure cannot suppress those checks. Branch protection +remains the authority over which standalone job statuses are merge requirements. +Do not use a shadow report as evidence that omitted lanes passed. A later +change to CI selection policy requires representative same-head comparisons, +trusted selection policy, and its own review. Because the report is generated +from the PR candidate, a PR changing its selector, map, catalogue, or workflow +does not validate that changed input. + The required GitHub check remains `Backend / test`. Nine matrix jobs each own a digest-pinned PostgreSQL service container, a pinned-source MinIO image, and exactly one dependency lane. A step-level curl health loop admits MinIO diff --git a/docs/roadmap_status.md b/docs/roadmap_status.md index bf078863a..26c6c631f 100644 --- a/docs/roadmap_status.md +++ b/docs/roadmap_status.md @@ -189,8 +189,10 @@ cannot be reused as post-submission review-gate evidence. See the - Cross-module behavior is moving through explicit public ports under the modular-monolith boundary. New private edges are prohibited and touched debt is reduced incrementally. -- GitHub CI distributes the backend suite across semantic lanes, rejects - skipped/deselected tests and requires behavior, boundary and real API proof. +- GitHub CI distributes the backend suite across semantic lanes and reports a + PR-only shadow impact recommendation; all nine full-suite lanes remain + required. It rejects skipped/deselected tests and requires behavior, boundary + and real API proof. Coverage is diagnostic only, with no percentage gate or test-count target. Redundant coverage-only reruns are removed; their tests remain in full-suite lanes. Its nine-lane allocation uses three project lanes, three task lanes, two diff --git a/scripts/backend_test_impact.py b/scripts/backend_test_impact.py new file mode 100644 index 000000000..5134f0863 --- /dev/null +++ b/scripts/backend_test_impact.py @@ -0,0 +1,295 @@ +"""Produce a conservative, observational backend test-impact recommendation.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +from pathlib import Path, PurePosixPath +import sys +from typing import Any + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) +sys.path.insert(1, str(ROOT / "backend")) + +from scripts.git_delta import resolve_commit, resolve_merge_base, run_checked, run_checked_bytes # noqa: E402 +from scripts.test_lane_catalogue import LANES # noqa: E402 + +MAP_PATH = ROOT / ".ci/test-impact/impact_map.json" +CATALOGUE_PATH = ROOT / "backend/scripts/test_lane_catalogue.py" +SCRIPT_PATH = Path(__file__).resolve() +ALL_LANES = tuple(lane.name for lane in LANES) +SELECTOR_VERSION = 1 +SHA_LENGTH = 40 + + +class SelectionError(RuntimeError): + """Raised when exact-target evidence cannot be established.""" + + +def _sha256(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def _changed_paths(base: str, head: str, *, repository_root: Path = ROOT) -> tuple[str, list[str]]: + if len(base) != SHA_LENGTH or len(head) != SHA_LENGTH: + raise SelectionError("PR base and head must be full commit SHAs") + merge_base = resolve_merge_base(base, head, repository_root=repository_root) + raw = run_checked_bytes( + ["git", "diff", "--no-renames", "--name-only", "-z", f"{merge_base}...{head}"], + repository_root=repository_root, + ) + paths = [item.decode("utf-8", errors="strict") for item in raw.split(b"\0") if item] + if not paths: + raise SelectionError("target diff contains no changed paths") + if len(paths) != len(set(paths)): + raise SelectionError("target diff contains duplicate paths") + for path in paths: + parsed = PurePosixPath(path) + if parsed.is_absolute() or ".." in parsed.parts or "\\" in path: + raise SelectionError(f"unsafe changed path: {path!r}") + return merge_base, sorted(paths) + + +def _test_path_owner(path: str) -> tuple[str, ...] | None: + if not path.startswith("backend/tests/") or not path.endswith(".py"): + return None + module_path = path.removeprefix("backend/") + owners = tuple(lane.name for lane in LANES if module_path in lane.modules) + return owners or None + + +def classify(paths: list[str], impact_map: dict[str, Any]) -> tuple[list[str], dict[str, list[str]]]: + """Return lane recommendations and auditable per-lane causes.""" + source_paths = impact_map.get("source_paths") + prefixes = impact_map.get("path_prefixes") + if not all(isinstance(item, dict) for item in (source_paths, prefixes)): + raise SelectionError("impact map has an invalid structure") + + selected: set[str] = set() + reasons: dict[str, list[str]] = {lane: [] for lane in ALL_LANES} + full_run_reasons: list[str] = [] + + for path in paths: + mapping: dict[str, Any] | None = None + if path in source_paths: + mapping = source_paths[path] + else: + matching_prefixes = [prefix for prefix in prefixes if path.startswith(prefix)] + if matching_prefixes: + mapping = prefixes[max(matching_prefixes, key=len)] + + if mapping is not None: + modules = mapping.get("test_modules") + if not isinstance(modules, list) or not modules: + raise SelectionError(f"invalid test-module mapping for {path}") + owners: set[str] = set() + for module in modules: + if not isinstance(module, str): + raise SelectionError(f"invalid mapped test module for {path}") + module_owners = {lane.name for lane in LANES if module in lane.modules} + if not module_owners: + raise SelectionError(f"mapped test module has no lane owner: {module}") + owners.update(module_owners) + lanes = [lane for lane in ALL_LANES if lane in owners] + reason = f"{path}: {mapping.get('reason', 'explicit impact mapping')}" + selected.update(owners) + for lane in lanes: + reasons[lane].append(reason) + continue + + test_owners = _test_path_owner(path) + if test_owners is not None: + selected.update(test_owners) + for lane in test_owners: + reasons[lane].append(f"{path}: changed test module belongs to this semantic lane") + continue + + full_run_reasons.append(f"{path}: no reviewed impact mapping; recommend all lanes") + + if full_run_reasons: + selected = set(ALL_LANES) + for lane in ALL_LANES: + reasons[lane].extend(full_run_reasons) + elif not selected: + selected = set(ALL_LANES) + for lane in ALL_LANES: + reasons[lane].append("no safely classifiable changed path; recommend all lanes") + + return [lane for lane in ALL_LANES if lane in selected], reasons + + +def _inline_code(value: object) -> str: + """Render untrusted values as one-line code without Markdown delimiters.""" + visible: list[str] = [] + for character in str(value): + codepoint = ord(character) + if character == "`": + visible.append(r"\x60") + elif character == "\n": + visible.append(r"\n") + elif character == "\r": + visible.append(r"\r") + elif codepoint < 32 or codepoint == 127: + visible.append(rf"\x{codepoint:02x}") + else: + visible.append(character) + return f"`{''.join(visible)}`" + + +def _markdown(report: dict[str, Any]) -> str: + lines = [ + "## Backend test-impact shadow report", + "", + "This is an advisory recommendation only. The complete required backend suite still runs.", + "", + f"- PR base: {_inline_code(report['base_sha'])}", + f"- PR head: {_inline_code(report['head_sha'])}", + "- Workflow execution SHA/tree: " + f"{_inline_code(report['execution_sha'])} / {_inline_code(report['execution_tree_sha'])}", + f"- Merge base: {_inline_code(report['merge_base'])}", + f"- Selector version: {_inline_code(report['selector_version'])}", + f"- Changed-path SHA-256: {_inline_code(report['changed_paths_sha256'])}", + f"- Lane catalogue SHA-256: {_inline_code(report['lane_catalogue_sha256'])}", + f"- Impact map SHA-256: {_inline_code(report['impact_map_sha256'])}", + f"- Classification status: {_inline_code(report['classification_status'])}", + "", + "### Recommended lanes", + "", + ] + for lane in report["lanes"]: + if lane["selected"]: + reason_text = "; ".join(_inline_code(reason) for reason in lane["reasons"]) + lines.append(f"- {_inline_code(lane['name'])} — {reason_text}") + else: + lines.append( + f"- {_inline_code(lane['name'])} — omitted: " + f"{_inline_code(lane['omission_reason'])}" + ) + lines.extend(["", "### Changed paths", ""]) + lines.extend(f"- {_inline_code(path)}" for path in report["changed_paths"]) + if report.get("classification_error"): + lines.extend(["", f"Fallback detail: {_inline_code(report['classification_error'])}"]) + return "\n".join(lines) + "\n" + + +def build_report(base: str, head: str, execution_sha: str) -> dict[str, Any]: + """Bind the recommendation to exact Git targets and selector inputs.""" + if resolve_commit("HEAD", repository_root=ROOT) != execution_sha: + raise SelectionError("checked-out execution SHA does not match the workflow target") + tree_sha = run_checked( + ["git", "rev-parse", f"{execution_sha}^{{tree}}"], repository_root=ROOT + ).strip() + execution_parents = run_checked( + ["git", "show", "-s", "--format=%P", execution_sha], repository_root=ROOT + ).split() + if execution_parents != [base, head]: + raise SelectionError("workflow execution commit is not the exact PR base/head merge") + merge_base, paths = _changed_paths(base, head) + if ( + resolve_commit(base, repository_root=ROOT) != base + or resolve_commit(head, repository_root=ROOT) != head + ): + raise SelectionError("base or head does not resolve to the requested commit") + map_digest: str | None = None + classification_error: str | None = None + try: + map_raw = MAP_PATH.read_bytes() + map_digest = _sha256(map_raw) + impact_map = json.loads(map_raw) + if not isinstance(impact_map, dict) or impact_map.get("schema_version") != 1: + raise SelectionError("impact map has an unsupported schema") + selected, reasons = classify(paths, impact_map) + classification_status = "classified" + except Exception as exc: # noqa: BLE001 - any classifier fault recommends all lanes. + classification_error = f"{type(exc).__name__}: {exc}" + selected = list(ALL_LANES) + reasons = { + lane: ["classification failed after exact target resolution; recommend all lanes"] + for lane in ALL_LANES + } + classification_status = "fallback_all_lanes" + lanes = [] + for name in ALL_LANES: + is_selected = name in selected + lanes.append( + { + "name": name, + "selected": is_selected, + "reasons": reasons[name] if is_selected else [], + "omission_reason": "all changed paths have reviewed owners outside this lane" if not is_selected else None, + } + ) + return { + "schema_version": 1, + "selector_version": SELECTOR_VERSION, + "classification_status": classification_status, + "base_sha": base, + "head_sha": head, + "execution_sha": execution_sha, + "execution_tree_sha": tree_sha, + "merge_base": merge_base, + "changed_paths": paths, + "changed_paths_sha256": _sha256("\0".join(paths).encode()), + "selector_sha256": _sha256(SCRIPT_PATH.read_bytes()), + "impact_map_sha256": map_digest, + "lane_catalogue_sha256": _sha256(CATALOGUE_PATH.read_bytes()), + "selected_lanes": selected, + "lanes": lanes, + "classification_error": classification_error, + } + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--base", default=os.environ.get("PR_BASE_SHA", "")) + parser.add_argument("--head", default=os.environ.get("PR_HEAD_SHA", "")) + parser.add_argument("--execution-sha", default=os.environ.get("GITHUB_SHA", "")) + parser.add_argument("--json", type=Path, required=True) + parser.add_argument("--markdown", type=Path, required=True) + args = parser.parse_args() + + error: str | None = None + try: + report = build_report(args.base, args.head, args.execution_sha) + except Exception as exc: # noqa: BLE001 - any classifier fault falls back to all lanes. + error = f"{type(exc).__name__}: {exc}" + report = { + "schema_version": 1, + "selector_version": SELECTOR_VERSION, + "classification_status": "fallback_all_lanes", + "base_sha": args.base, + "head_sha": args.head, + "execution_sha": args.execution_sha, + "execution_tree_sha": None, + "merge_base": None, + "changed_paths": [], + "changed_paths_sha256": None, + "selector_sha256": _sha256(SCRIPT_PATH.read_bytes()), + "impact_map_sha256": _sha256(MAP_PATH.read_bytes()) if MAP_PATH.is_file() else None, + "lane_catalogue_sha256": _sha256(CATALOGUE_PATH.read_bytes()), + "selected_lanes": list(ALL_LANES), + "lanes": [ + {"name": lane, "selected": True, "reasons": ["selection evidence unavailable; fail safe to all lanes"], "omission_reason": None} + for lane in ALL_LANES + ], + "classification_error": error, + } + args.json.parent.mkdir(parents=True, exist_ok=True) + args.markdown.parent.mkdir(parents=True, exist_ok=True) + args.json.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8") + args.markdown.write_text(_markdown(report), encoding="utf-8") + summary_path = os.environ.get("GITHUB_STEP_SUMMARY") + if summary_path: + with Path(summary_path).open("a", encoding="utf-8") as summary: + summary.write(args.markdown.read_text(encoding="utf-8")) + print(f"Impact report: {args.json}") + if error: + print(f"Impact classification fell back safely to all lanes: {error}", file=sys.stderr) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/git_delta.py b/scripts/git_delta.py index 2045fa221..449a79091 100644 --- a/scripts/git_delta.py +++ b/scripts/git_delta.py @@ -41,6 +41,31 @@ def run_checked( return result.stdout +def run_checked_bytes( + command: list[str], + *, + repository_root: Path | None = None, + timeout_seconds: float = 10, +) -> bytes: + """Return byte-exact stdout for Git, preserving NUL-delimited path data.""" + try: + result = subprocess.run( + command, + cwd=repository_root, + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + timeout=timeout_seconds, + ) + except (OSError, subprocess.TimeoutExpired) as exc: + code = "GIT_TIMEOUT" if isinstance(exc, subprocess.TimeoutExpired) else "GIT_EXEC_ERROR" + raise GitCommandError(code, command, str(exc)) from exc + if result.returncode != 0: + detail = result.stderr.decode("utf-8", errors="replace").strip() + raise GitCommandError("GIT_COMMAND_FAILED", command, detail) + return result.stdout + + def resolve_commit(ref: str, *, repository_root: Path | None = None) -> str: """Resolve a ref to one full commit SHA or fail closed.""" command = ["git", "rev-parse", "--verify", f"{ref}^{{commit}}"] diff --git a/scripts/test_backend_test_impact.py b/scripts/test_backend_test_impact.py new file mode 100644 index 000000000..14a918cc6 --- /dev/null +++ b/scripts/test_backend_test_impact.py @@ -0,0 +1,429 @@ +"""Regressions for the observational backend impact recommendation.""" + +from __future__ import annotations + +import json +from pathlib import Path +import re +import subprocess +import sys +import tempfile +import unittest +from unittest.mock import patch + +import scripts.backend_test_impact as selector +from scripts.backend_test_impact import ALL_LANES, classify +from scripts.test_lane_catalogue import LANES, PARTITIONED_SHARED_LANES + +ROOT = Path(__file__).resolve().parents[1] +IMPACT_MAP = json.loads((ROOT / ".ci/test-impact/impact_map.json").read_text()) + + +def test_exact_s3_owner_recommends_both_shared_partitions() -> None: + selected, _ = classify(["backend/app/core/s3_validation.py"], IMPACT_MAP) + + assert selected == list(PARTITIONED_SHARED_LANES) + + +def test_committrail_only_recommends_shared_semantics_partitions() -> None: + selected, _ = classify([".commitrail/changes/ci-example.md"], IMPACT_MAP) + + assert selected == list(PARTITIONED_SHARED_LANES) + + +def test_mapped_source_and_test_changes_union_their_lane_closures() -> None: + selected, _ = classify( + [ + "backend/app/core/s3_validation.py", + "backend/tests/test_projects.py", + ], + IMPACT_MAP, + ) + + assert set(PARTITIONED_SHARED_LANES) <= set(selected) + assert {"project_lifecycle_a", "project_lifecycle_b", "project_lifecycle_c"} <= set(selected) + + +def test_changed_test_module_selects_every_partition_that_owns_it() -> None: + owners = tuple( + lane.name for lane in LANES if "tests/test_s3_artifact_store.py" in lane.modules + ) + + selected, _ = classify(["backend/tests/test_s3_artifact_store.py"], IMPACT_MAP) + + assert owners == PARTITIONED_SHARED_LANES + assert selected == list(owners) + + +def test_unknown_source_fixture_and_unmapped_test_fail_safe_to_all_lanes() -> None: + for path in ( + "backend/app/modules/tasks/service.py", + "backend/tests/conftest.py", + "backend/tests/test_not_in_catalogue.py", + "docs/operations_backend_testing.md", + ".ci/test-impact/impact_map.json", + "scripts/backend_test_impact.py", + "backend/scripts/test_lane_catalogue.py", + ".github/workflows/backend.yml", + ): + selected, _ = classify([path], IMPACT_MAP) + assert selected == list(ALL_LANES), path + + +def test_rename_from_unmapped_source_keeps_deleted_path_and_fails_safe(tmp_path: Path) -> None: + _git(tmp_path, "init") + _git(tmp_path, "config", "user.email", "test@example.com") + _git(tmp_path, "config", "user.name", "Test") + old_path = tmp_path / "backend/app/unknown.py" + old_path.parent.mkdir(parents=True) + old_path.write_text("same content\n", encoding="utf-8") + _git(tmp_path, "add", "backend/app/unknown.py") + _git(tmp_path, "commit", "-m", "add unmapped source") + base = _git(tmp_path, "rev-parse", "HEAD") + + mapped_path = tmp_path / "backend/app/core/s3_validation.py" + mapped_path.parent.mkdir(parents=True) + old_path.rename(mapped_path) + _git(tmp_path, "add", "--all") + _git(tmp_path, "commit", "-m", "rename into mapped source") + head = _git(tmp_path, "rev-parse", "HEAD") + + _, paths = selector._changed_paths(base, head, repository_root=tmp_path) + selected, _ = classify(paths, IMPACT_MAP) + + assert paths == [ + "backend/app/core/s3_validation.py", + "backend/app/unknown.py", + ] + assert selected == list(ALL_LANES) + + +def test_mixed_known_and_unknown_paths_fail_safe_to_all_lanes() -> None: + selected, reasons = classify( + ["backend/app/core/s3_validation.py", "backend/requirements.lock"], IMPACT_MAP + ) + + assert selected == list(ALL_LANES) + assert all("no reviewed impact mapping" in ";".join(reasons[lane]) for lane in ALL_LANES) + + +def test_empty_path_list_recommends_every_lane() -> None: + selected, _ = classify([], IMPACT_MAP) + + assert selected == list(ALL_LANES) + + +def test_report_binds_execution_tree_and_exact_pr_merge_parents() -> None: + base = "a" * 40 + head = "b" * 40 + execution = "c" * 40 + tree = "d" * 40 + + def resolve_commit(ref: str, *, repository_root: Path) -> str: + assert repository_root == ROOT + if ref == "HEAD": + return execution + if ref == base: + return base + if ref == head: + return head + raise AssertionError(ref) + + def run_checked(command: list[str], *, repository_root: Path) -> str: + assert repository_root == ROOT + if command == ["git", "rev-parse", f"{execution}^{{tree}}"]: + return tree + if command == ["git", "show", "-s", "--format=%P", execution]: + return f"{base} {head}" + raise AssertionError(command) + + with ( + patch.object(selector, "resolve_commit", side_effect=resolve_commit), + patch.object(selector, "run_checked", side_effect=run_checked), + patch.object(selector, "resolve_merge_base", return_value=base), + patch.object(selector, "run_checked_bytes", return_value=b".commitrail/change.md\0"), + ): + report = selector.build_report(base, head, execution) + + assert report["base_sha"] == base + assert report["head_sha"] == head + assert report["execution_sha"] == execution + assert report["execution_tree_sha"] == tree + assert report["merge_base"] == base + assert report["changed_paths"] == [".commitrail/change.md"] + assert report["changed_paths_sha256"] == selector._sha256(b".commitrail/change.md") + assert report["selector_sha256"] == selector._sha256( + selector.SCRIPT_PATH.read_bytes() + ) + assert report["impact_map_sha256"] == selector._sha256( + selector.MAP_PATH.read_bytes() + ) + assert report["lane_catalogue_sha256"] == selector._sha256( + selector.CATALOGUE_PATH.read_bytes() + ) + assert report["selected_lanes"] == list(PARTITIONED_SHARED_LANES) + assert report["lanes"][0]["reasons"] == [ + ".commitrail/change.md: Commitrail policy semantics are tested in the shared-foundation partition." + ] + assert report["lanes"][1]["reasons"] == report["lanes"][0]["reasons"] + assert all( + lane["omission_reason"] + for lane in report["lanes"] + if not lane["selected"] + ) + + +def test_report_rejects_execution_commit_with_stale_pr_parents() -> None: + base = "a" * 40 + head = "b" * 40 + execution = "c" * 40 + + def resolve_commit(ref: str, *, repository_root: Path) -> str: + if ref == "HEAD": + return execution + raise AssertionError(ref) + + def run_checked(command: list[str], *, repository_root: Path) -> str: + if command == ["git", "rev-parse", f"{execution}^{{tree}}"]: + return "d" * 40 + if command == ["git", "show", "-s", "--format=%P", execution]: + return f"{base} {'e' * 40}" + raise AssertionError(command) + + with ( + patch.object(selector, "resolve_commit", side_effect=resolve_commit), + patch.object(selector, "run_checked", side_effect=run_checked), + ): + try: + selector.build_report(base, head, execution) + except selector.SelectionError as exc: + assert "not the exact PR base/head merge" in str(exc) + else: + raise AssertionError("stale target accepted") + + +def test_classifier_failure_preserves_exact_target_and_changed_path_evidence() -> None: + base = "a" * 40 + head = "b" * 40 + execution = "c" * 40 + tree = "d" * 40 + path = "backend/app/core/s3_validation.py" + + def resolve_commit(ref: str, *, repository_root: Path) -> str: + del repository_root + return {"HEAD": execution, base: base, head: head}[ref] + + def run_checked(command: list[str], *, repository_root: Path) -> str: + del repository_root + if command[1] == "rev-parse": + return tree + if command[1] == "show": + return f"{base} {head}" + raise AssertionError(command) + + with ( + patch.object(selector, "resolve_commit", side_effect=resolve_commit), + patch.object(selector, "run_checked", side_effect=run_checked), + patch.object(selector, "resolve_merge_base", return_value=base), + patch.object(selector, "run_checked_bytes", return_value=f"{path}\0".encode()), + patch.object(selector, "classify", side_effect=selector.SelectionError("bad map")), + ): + report = selector.build_report(base, head, execution) + + assert report["classification_status"] == "fallback_all_lanes" + assert report["base_sha"] == base + assert report["head_sha"] == head + assert report["execution_sha"] == execution + assert report["execution_tree_sha"] == tree + assert report["merge_base"] == base + assert report["changed_paths"] == [path] + assert report["changed_paths_sha256"] == selector._sha256(path.encode()) + assert report["selector_sha256"] + assert report["impact_map_sha256"] + assert report["lane_catalogue_sha256"] + assert report["selected_lanes"] == list(ALL_LANES) + assert report["classification_error"] == "SelectionError: bad map" + + +def test_markdown_report_escapes_untrusted_paths_and_reasons() -> None: + injected = "evil`\n\n## Forged status" + rendered = selector._markdown( + { + "base_sha": "base", + "head_sha": "head", + "execution_sha": "execution", + "execution_tree_sha": "tree", + "merge_base": "merge", + "selector_version": 1, + "changed_paths_sha256": "digest", + "lane_catalogue_sha256": "catalogue", + "impact_map_sha256": "map", + "classification_status": "classified", + "lanes": [ + {"name": "lane", "selected": True, "reasons": [injected]}, + ], + "changed_paths": [injected], + } + ) + + assert "\n## Forged status" not in rendered + assert r"\x60\n\n## Forged status" in rendered + + +def test_duplicate_or_unsafe_git_paths_fail_classification() -> None: + with ( + patch.object(selector, "resolve_merge_base", return_value="a" * 40), + patch.object(selector, "run_checked_bytes", return_value=b"backend/app.py\0backend/app.py\0"), + ): + try: + selector._changed_paths("a" * 40, "b" * 40) + except selector.SelectionError as exc: + assert "duplicate paths" in str(exc) + else: + raise AssertionError("duplicate Git paths accepted") + + with ( + patch.object(selector, "resolve_merge_base", return_value="a" * 40), + patch.object(selector, "run_checked_bytes", return_value=b"../outside\0"), + ): + try: + selector._changed_paths("a" * 40, "b" * 40) + except selector.SelectionError as exc: + assert "unsafe changed path" in str(exc) + else: + raise AssertionError("unsafe Git path accepted") + + +def test_cli_reports_all_lane_fallback_when_target_evidence_is_unavailable(tmp_path: Path) -> None: + report_path = tmp_path / "report.json" + markdown_path = tmp_path / "report.md" + result = subprocess.run( + [ + sys.executable, + str(ROOT / "scripts/backend_test_impact.py"), + "--base", + "a" * 40, + "--head", + "b" * 40, + "--execution-sha", + "c" * 40, + "--json", + str(report_path), + "--markdown", + str(markdown_path), + ], + cwd=ROOT, + capture_output=True, + check=False, + text=True, + ) + + assert result.returncode == 0 + report = json.loads(report_path.read_text(encoding="utf-8")) + assert report["classification_status"] == "fallback_all_lanes" + assert report["base_sha"] == "a" * 40 + assert report["head_sha"] == "b" * 40 + assert report["execution_sha"] == "c" * 40 + assert report["selected_lanes"] == list(ALL_LANES) + assert report["execution_tree_sha"] is None + assert report["merge_base"] is None + assert report["changed_paths"] == [] + assert report["changed_paths_sha256"] is None + assert report["selector_sha256"] == selector._sha256(selector.SCRIPT_PATH.read_bytes()) + assert report["impact_map_sha256"] == selector._sha256(selector.MAP_PATH.read_bytes()) + assert report["lane_catalogue_sha256"] == selector._sha256( + selector.CATALOGUE_PATH.read_bytes() + ) + markdown = markdown_path.read_text(encoding="utf-8") + assert f"`{'a' * 40}`" in markdown + assert f"`{'b' * 40}`" in markdown + assert "selection evidence unavailable" in markdown_path.read_text(encoding="utf-8") + + +def test_backend_workflow_keeps_report_out_of_lane_execution_control() -> None: + workflow = (ROOT / ".github/workflows/backend.yml").read_text(encoding="utf-8") + impact_job = workflow.split("\n impact-report:\n", 1)[1].split("\n lanes:\n", 1)[0] + lane_job = workflow.split("\n lanes:\n", 1)[1].split("\n test:\n", 1)[0] + aggregate_job = workflow.split("\n test:\n", 1)[1] + + lane_header = lane_job.split(" services:", 1)[0] + assert not re.search(r"(?m)^\s*if\s*:", lane_header) + assert "impact-report" not in lane_job + assert "selected_lanes" not in lane_job + assert "if: ${{ github.event_name == 'pull_request' }}" in impact_job + assert "python scripts/backend_test_impact.py" in impact_job + impact_step = impact_job.split( + " - name: Bind and classify the exact pull request target\n", 1 + )[1] + assert "run: >-\n python scripts/backend_test_impact.py" in impact_step + assert "backend-test-impact-${{ github.sha }}-${{ github.run_attempt }}" in impact_job + assert "needs: [auth-boundary-preflight, lanes, minio-image]" in aggregate_job + assert "impact-report" not in aggregate_job + assert "Require preflight and every semantic lane" in aggregate_job + assert "API contract real API e2e" in aggregate_job + api_step = aggregate_job.split(" - name: API contract real API e2e\n", 1)[1].split( + "\n - name:", 1 + )[0] + assert not re.search(r"(?m)^\s*if\s*:", api_step) + assert "scripts/run_isolated_tests.py" in api_step + + +def _git(repository: Path, *arguments: str) -> str: + return subprocess.check_output( + ["git", *arguments], cwd=repository, text=True, stderr=subprocess.STDOUT + ).strip() + + +class BackendTestImpactTests(unittest.TestCase): + """Run repository CI selector checks in the lightweight standard suite.""" + + def test_exact_s3_owner(self) -> None: + test_exact_s3_owner_recommends_both_shared_partitions() + + def test_committrail_only(self) -> None: + test_committrail_only_recommends_shared_semantics_partitions() + + def test_mapped_source_and_test_union(self) -> None: + test_mapped_source_and_test_changes_union_their_lane_closures() + + def test_changed_test_partition_owners(self) -> None: + test_changed_test_module_selects_every_partition_that_owns_it() + + def test_unknown_source_and_test_fallback(self) -> None: + test_unknown_source_fixture_and_unmapped_test_fail_safe_to_all_lanes() + + def test_rename_fallback(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + test_rename_from_unmapped_source_keeps_deleted_path_and_fails_safe( + Path(temporary) + ) + + def test_mixed_unknown_fallback(self) -> None: + test_mixed_known_and_unknown_paths_fail_safe_to_all_lanes() + + def test_empty_path_fallback(self) -> None: + test_empty_path_list_recommends_every_lane() + + def test_exact_report_binding(self) -> None: + test_report_binds_execution_tree_and_exact_pr_merge_parents() + + def test_stale_execution_parent_rejected(self) -> None: + test_report_rejects_execution_commit_with_stale_pr_parents() + + def test_classification_failure_keeps_target_evidence(self) -> None: + test_classifier_failure_preserves_exact_target_and_changed_path_evidence() + + def test_markdown_values_are_escaped(self) -> None: + test_markdown_report_escapes_untrusted_paths_and_reasons() + + def test_duplicate_and_unsafe_paths_rejected(self) -> None: + test_duplicate_or_unsafe_git_paths_fail_classification() + + def test_unavailable_target_report(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + test_cli_reports_all_lane_fallback_when_target_evidence_is_unavailable( + Path(temporary) + ) + + def test_workflow_keeps_report_advisory(self) -> None: + test_backend_workflow_keeps_report_out_of_lane_execution_control() diff --git a/scripts/test_git_delta.py b/scripts/test_git_delta.py index c0453f699..d05a632ae 100644 --- a/scripts/test_git_delta.py +++ b/scripts/test_git_delta.py @@ -11,6 +11,7 @@ from scripts.git_delta import committed_changed_files from scripts.git_delta import diff_text from scripts.git_delta import numstat +from scripts.git_delta import run_checked_bytes class GitDeltaTests(unittest.TestCase): @@ -48,6 +49,27 @@ def test_committed_delta_is_sorted_and_local_changes_are_optional(self) -> None: self.assertEqual(numstat(base, head, repository_root=root, include_local=False)[:2], (2, 0)) self.assertIn("+++ b/a.txt", diff_text(base, head, repository_root=root, include_local=False)) + def test_checked_byte_output_preserves_nul_delimited_paths(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + self._git(root, "init") + self._git(root, "config", "user.email", "test@example.com") + self._git(root, "config", "user.name", "Test") + (root / "base.txt").write_text("base\n", encoding="utf-8") + self._git(root, "add", "base.txt") + self._git(root, "commit", "-m", "base") + unusual_path = "line\nbreak.txt" + (root / unusual_path).write_text("content\n", encoding="utf-8") + self._git(root, "add", unusual_path) + self._git(root, "commit", "-m", "add unusual path") + + result = run_checked_bytes( + ["git", "diff", "--name-only", "-z", "HEAD^", "HEAD"], + repository_root=root, + ) + + self.assertEqual(result, b"line\nbreak.txt\0") + @staticmethod def _git(root: Path, *arguments: str) -> str: return subprocess.check_output( diff --git a/scripts/test_lightweight_agent_gates.py b/scripts/test_lightweight_agent_gates.py index 2a3d88d08..b13e41f41 100644 --- a/scripts/test_lightweight_agent_gates.py +++ b/scripts/test_lightweight_agent_gates.py @@ -4,6 +4,7 @@ import os import re +import shlex import subprocess import tempfile import textwrap @@ -19,6 +20,20 @@ from scripts.check_stale_workstream_wording import forbidden_path_failures +def _run_command_tokens(step: str) -> list[str]: + """Read active argv from one GitHub Actions folded run block.""" + run_block = re.search(r"(?ms)^ run: >-\n((?: {10,}[^\n]*\n)+)", step) + if run_block is None: + return [] + folded_command = " ".join( + line[10:].strip() for line in run_block[1].splitlines() if line.strip() + ) + shell = shlex.shlex(folded_command, posix=True) + shell.whitespace_split = True + shell.commenters = "#" + return list(shell) + + class LightweightAgentGateTests(unittest.TestCase): """Keep the retained checks executable and cover their core parsing rules.""" @@ -115,6 +130,9 @@ def test_stale_artifact_rejects_unknown_phase(self) -> None: def test_backend_uses_distributed_semantic_lanes_and_stable_fan_in(self) -> None: workflow = Path(".github/workflows/backend.yml").read_text(encoding="utf-8") agent_gates = Path(".github/workflows/agent-gates.yml").read_text(encoding="utf-8") + lightweight_gate_step = agent_gates.split( + " - name: Lightweight gate regression tests\n", 1 + )[1].split("\n - name:", 1)[0] gate_requirements = Path(".github/requirements/agent-gates.txt").read_text( encoding="utf-8" ) @@ -142,6 +160,22 @@ def test_backend_uses_distributed_semantic_lanes_and_stable_fan_in(self) -> None " needs: [auth-boundary-preflight, lanes, minio-image]", workflow ) self.assertIn("Require preflight and every semantic lane", workflow) + lanes = workflow.split("\n lanes:\n", 1)[1].split("\n test:\n", 1)[0] + aggregate = workflow.split("\n test:\n", 1)[1] + self.assertNotIn("impact-report", lanes) + self.assertNotIn("impact-report", aggregate) + self.assertIn("API contract real API e2e", aggregate) + api_e2e = aggregate.split(" - name: API contract real API e2e\n", 1)[1].split( + "\n - name:", 1 + )[0] + lane_header = lanes.split(" services:", 1)[0] + self.assertNotRegex(lane_header, r"(?m)^\s*if\s*:") + self.assertNotRegex(api_e2e, r"(?m)^\s*if\s*:") + immediate_lane_guard = " lanes:\n if: ${{ false }}\n runs-on: ubuntu-latest\n" + immediate_api_guard = " - name: API contract real API e2e\n if: ${{ false }}\n" + self.assertRegex(immediate_lane_guard, r"(?m)^\s*if\s*:") + self.assertRegex(immediate_api_guard, r"(?m)^\s*if\s*:") + self.assertIn("scripts/run_isolated_tests.py", api_e2e) self.assertIn("python -m scripts.merge_test_lane_evidence", workflow) self.assertIn("scripts/validate_test_lane_evidence.py", workflow) self.assertIn( @@ -166,6 +200,20 @@ def test_backend_uses_distributed_semantic_lanes_and_stable_fan_in(self) -> None self.assertIn("scripts.test_commitrail_contribution_paths", agent_gates) self.assertIn('WORKSTREAM_BASE_SHA: ${{ github.event.pull_request.base.sha }}', agent_gates) self.assertNotIn("scripts.test_chunk_state_sync", agent_gates) + command = _run_command_tokens(lightweight_gate_step) + self.assertEqual(command[:4], ["python3", "-m", "unittest", "-v"]) + modules = command[4:] + self.assertTrue(modules) + self.assertTrue(all(re.fullmatch(r"scripts\.[a-z][a-z0-9_]*", item) for item in modules)) + self.assertIn("scripts.test_backend_test_impact", modules) + commented_selector = lightweight_gate_step.replace( + " scripts.test_backend_test_impact\n", + " # remaining text is intentionally shell-commented\n" + " scripts.test_backend_test_impact\n", + ) + self.assertNotIn( + "scripts.test_backend_test_impact", _run_command_tokens(commented_selector) + ) self.assertIn("--require-hashes", agent_gates) self.assertIn("-r .github/requirements/agent-gates.txt", agent_gates) for package in (