From a0bb58816de0d7793ca07ed9ae137ee38c511c64 Mon Sep 17 00:00:00 2001 From: huangruiteng Date: Fri, 11 Sep 2026 18:44:04 +0800 Subject: [PATCH 1/3] ci: qualify exact-change test selection in shadow mode Signed-off-by: huangruiteng --- .github/workflows/python-tests.yml | 114 ++++++++++++++- docs/development/ci-impact-selection.md | 131 +++++++++++++++++ docs/development/testing-and-quality.md | 8 + scripts/ci/impact_plan.py | 153 +++++++++++++++++++ scripts/ci/impact_shadow.py | 130 +++++++++++++++++ scripts/ci/review_gate.py | 52 +++---- scripts/ci/test_impact_plan.py | 186 ++++++++++++++++++++++++ scripts/ci/test_impact_shadow.py | 110 ++++++++++++++ 8 files changed, 854 insertions(+), 30 deletions(-) create mode 100644 docs/development/ci-impact-selection.md create mode 100644 scripts/ci/impact_plan.py create mode 100644 scripts/ci/impact_shadow.py create mode 100644 scripts/ci/test_impact_plan.py create mode 100644 scripts/ci/test_impact_shadow.py diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index cb3a7e1247..b886273e4d 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -32,6 +32,8 @@ jobs: timeout-minutes: 3 outputs: core_tests: ${{ steps.classify.outputs.core_tests }} + impact_profile: ${{ steps.classify.outputs.impact_profile }} + shadow_profile: ${{ steps.classify.outputs.shadow_profile }} steps: - uses: actions/checkout@v7 with: @@ -40,21 +42,38 @@ jobs: with: python-version: "3.11" - name: Validate merge gate semantics - run: python -m unittest discover -s scripts/ci -p 'test_review_gate.py' + run: python -m unittest discover -s scripts/ci -p 'test_*.py' - name: Classify the exact pull-request change id: classify env: EVENT_NAME: ${{ github.event_name }} BASE_SHA: ${{ github.event.pull_request.base.sha }} HEAD_SHA: ${{ github.event.pull_request.head.sha }} + CHECKOUT_SHA: ${{ github.sha }} shell: bash run: | set -euo pipefail if [[ "$EVENT_NAME" == pull_request ]]; then - python scripts/ci/review_gate.py classify --base "$BASE_SHA" --head "$HEAD_SHA" >> "$GITHUB_OUTPUT" + python scripts/ci/review_gate.py classify --base "$BASE_SHA" --head "$HEAD_SHA" --plan impact-plan.json >> "$GITHUB_OUTPUT" else - echo 'core_tests=true' >> "$GITHUB_OUTPUT" + python scripts/ci/review_gate.py classify --base "$CHECKOUT_SHA" --head "$CHECKOUT_SHA" --non-pr --plan impact-plan.json >> "$GITHUB_OUTPUT" fi + python - <<'PY' + import json, os + from pathlib import Path + plan = json.loads(Path("impact-plan.json").read_text()) + with open(os.environ["GITHUB_STEP_SUMMARY"], "a") as summary: + summary.write("## CI impact plan (shadow rollout)\n\n") + summary.write(f"Candidate: **{plan['candidate_profile']}**; execution: **{plan['execution_mode']}**.\n\n") + summary.write(plan['reason'] + ".\n\n") + summary.write("Full-suite checks and coverage remain authoritative. See the plan artifact for the exact revisions and selected files.\n") + PY + - uses: actions/upload-artifact@v7 + with: + name: ci-impact-plan + path: impact-plan.json + if-no-files-found: error + retention-days: 7 checks: needs: changes @@ -226,8 +245,17 @@ jobs: --splits 2 --group ${{ matrix.shard }} --splitting-algorithm least_duration --durations=25 --durations-min=1 + --junitxml=junit.xml --cov=loopx --cov-report=term + - name: Upload full-shard outcomes for selection audit + if: always() + uses: actions/upload-artifact@v7 + with: + name: python-junit-${{ matrix.shard }} + path: junit.xml + if-no-files-found: error + retention-days: 7 - name: Upload shard coverage uses: actions/upload-artifact@v7 with: @@ -237,6 +265,82 @@ jobs: if-no-files-found: error retention-days: 3 + impact-tests: + needs: changes + if: needs.changes.outputs.shadow_profile == 'vision' + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + - uses: actions/setup-python@v6 + with: + python-version: "3.11" + cache: pip + - uses: actions/setup-node@v6 + with: + node-version: "24" + cache: npm + cache-dependency-path: package-lock.json + - run: | + python -m pip install --disable-pip-version-check -e ".[test]" + npm ci --ignore-scripts + - uses: actions/download-artifact@v7 + with: + name: ci-impact-plan + - name: Execute the complete candidate profile without full-suite coverage claims + run: python scripts/ci/impact_shadow.py run --plan impact-plan.json + - uses: actions/upload-artifact@v7 + if: always() + with: + name: ci-impact-selected + path: impact-results/ + if-no-files-found: error + retention-days: 7 + + impact-shadow: + needs: [changes, impact-tests, test-shard] + if: always() && needs.changes.outputs.shadow_profile == 'vision' + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Require candidate execution + env: + SELECTED_RESULT: ${{ needs.impact-tests.result }} + run: | + test "$SELECTED_RESULT" = success + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + - uses: actions/setup-python@v6 + with: + python-version: "3.11" + - uses: actions/download-artifact@v7 + with: + name: ci-impact-plan + - uses: actions/download-artifact@v7 + with: + name: ci-impact-selected + path: impact-results + - uses: actions/download-artifact@v7 + with: + pattern: python-junit-* + path: full-reports + - name: Compare exact test identities and outcomes + run: python scripts/ci/impact_shadow.py audit --plan impact-plan.json + - uses: actions/upload-artifact@v7 + if: always() + with: + name: ci-impact-comparison + path: impact-comparison.json + if-no-files-found: error + retention-days: 7 + - name: Full execution failures cannot qualify the shadow + env: + FULL_RESULT: ${{ needs.test-shard.result }} + run: test "$FULL_RESULT" = success + pytest: # Keep the required check name; a skipped/failed shard must not turn it green. if: always() && needs.changes.outputs.core_tests == 'true' @@ -429,7 +533,7 @@ jobs: merge-gate: # Always publish one stable outcome, including documentation-only PRs. if: always() - needs: [changes, pytest, node-minimum-compatibility, stage2c-correctness-e2e, windows-powershell] + needs: [changes, pytest, node-minimum-compatibility, stage2c-correctness-e2e, windows-powershell, impact-shadow] runs-on: ubuntu-latest timeout-minutes: 3 steps: @@ -440,4 +544,4 @@ jobs: - name: Reject incomplete or unsuccessful qualification env: NEEDS_JSON: ${{ toJSON(needs) }} - run: python scripts/ci/review_gate.py verify + run: python scripts/ci/review_gate.py verify --shadow diff --git a/docs/development/ci-impact-selection.md b/docs/development/ci-impact-selection.md new file mode 100644 index 0000000000..eb3db3cc62 --- /dev/null +++ b/docs/development/ci-impact-selection.md @@ -0,0 +1,131 @@ +# CI Impact Selection / CI 影响范围选择 + +## Current rollout: shadow, not selective merge authority + +The PR workflow retains its full core qualification for every executable, +policy, runtime-prompt or unknown change. The existing documentation-only +exemption is unchanged. `scripts/ci/impact_plan.py` now also proposes a bounded +candidate test profile and explains why it was selected. A candidate never +authorizes skipping the full suite in this rollout. + +当前先交付 **shadow 对照阶段**：不是用一次绿灯就宣布可以安全跳过全量。 +原有文档豁免不变，代码和未知改动仍跑全量；额外实际运行候选集合、比较测试身份 +和结果、记录耗时。此阶段增加少量并行工作，尚不承诺降低 PR 的总体耗时。 + +```text +merge-base … exact PR head + → NUL-delimited Git changes (renames appear as deletion + addition) + → candidate + reason + immutable revisions + ├── existing full qualification + full coverage + └── selected Python tests + real CLI smokes (shadow only) + ↓ + compare exact collected test identities and outcomes + ↓ + stable merge-gate: full success AND required shadow success +``` + +## First reviewed boundary: vision checkpoint + +The initial `vision` profile recognizes exact existing source/test/smoke paths, +not filenames containing “vision”. Its Python inventory spans checkpoint +authoring, readback, refresh recovery/isolation, public safety, replan admission, +terminal succession, quota and settlement. Real CLI smokes exercise bounded +write/read behavior, closed-vision successor routing and status/quota latency. +Full TS tests/typechecking and CLI output-budget qualification remain common +checks; they are not duplicated into another test framework. + +This is the cross-domain behavior demonstrated by the vision authoring-budget +change, not a claim that every goal-domain edit can use the same slice. A new +profile needs its own reviewed owning boundary and caller inventory. + +首批只选择边界明确的 vision checkpoint。它的消费者横跨 refresh、quota、终态与 +结算，所以不能只跑同目录测试。测试目标复用仓库已有测试和 smoke；canary 的风险 +目录可辅助维护清单，但自由文本关键词匹配、`max_checks` 截断不能作为 CI 免责依据。 + +| Change | Candidate / execution | +| --- | --- | +| Allowlisted Markdown only | `docs`; existing explicit core skips | +| Only recognized vision boundary paths | `vision`; full suite plus shadow | +| Known new test within that complete inventory | `vision`; that test is included | +| Runtime additions/deletions, renames, type changes | `full` | +| Shared decoder/dispatcher/schema, dependencies, fixtures outside the inventory | `full` | +| CI policy/workflow changes | `full` plus candidate rehearsal | +| Empty diff, non-PR execution or any unmapped path | `full` | +| Missing Git base or malformed input | Classification fails; no successful exemption | + +Both sides of renames count. A mixed PR takes the conservative union: one +unmapped code path makes the whole candidate full. Noncanonical paths and +unrecognized Git statuses cannot become documentation exemptions. GitHub +outputs contain only closed profile names and booleans, never changed filenames +or shell commands supplied by a PR. + +## Evidence and gate semantics + +Each workflow run publishes: + +- `ci-impact-plan`: base/head/merge-base/tested-checkout revisions, changed + paths, complete selected inventory, reason and actual execution mode. +- `ci-impact-selected` when applicable: JUnit outcomes and a bounded execution + receipt with command exit codes, elapsed seconds and a digest of the plan. +- `python-junit-1` and `python-junit-2`: full-shard outcomes. +- `ci-impact-comparison` when applicable: selected/full counts, missing cases, + outcome differences and failures outside the selected set. It does not copy + failure text, stdout or private runtime evidence into its summary. + +The runner rejects stale checkout identities, changed inventories, missing +checks and attempts to reinterpret shadow as selective authority. The audit +requires both full reports, unique test identities, nonempty collection from +every selected file, successful selected tests (a skip is not a pass), matching +full outcomes, successful CLI smokes and no failure outside the selected set. +Any failed/cancelled/missing required job keeps the stable `merge-gate` red. + +Coverage remains unambiguous: only the two **full** Python shards feed the +existing coverage floor and Sonar report. Shadow runs neither upload partial +coverage under full-suite artifact names nor borrow coverage from another SHA. +Plan/rehearsal artifacts are diagnostics, not a replacement for full coverage. + +覆盖率仍由同一版本的两个全量分片产生；精简集合的“绿”不能冒充完整覆盖率， +也不能用旧版本 coverage 补齐。shadow 通过只证明本次实际执行与对照成立，不证明 +未来永远不会漏测。缺失、跳过、取消、结果不一致都必须明确失败，不能转成免责。 + +## Qualify locally + +```bash +python -m unittest discover -s scripts/ci -p 'test_*.py' +python scripts/ci/review_gate.py classify --base origin/main --head HEAD --plan impact-plan.json +python scripts/ci/impact_shadow.py run --plan impact-plan.json +python scripts/ci/impact_shadow.py audit --plan impact-plan.json --full-dir full-reports +``` + +Run the selected commands only for plans with `shadow_profile=vision`. Install +the repository test dependencies and supported Node runtime first. The audit +expects `full-reports/python-junit-{1,2}/junit.xml` downloaded from the same +workflow run; missing reports are not a local pass. Keep generated plans, +JUnit files and receipts outside tracked source files. + +## Activation and expansion criteria + +Before a follow-up enables selective-only PR execution for a profile: + +1. Review successful shadow evidence across representative changes to its write + rule, read projection and cross-domain semantics, not just a constant edit. +2. Prove sensitivity with deliberately omitted checks, changed outcomes and + real-entrypoint semantic regressions. Investigate failures outside the + candidate instead of mechanically accepting the observed selection. +3. Run both old/full and proposed selective workflow paths, including docs, + mixed changes, missing reports, renamed/deleted tests and merge aggregation. +4. Bind execution to the exact plan/checkout; evaluate exemption rules from a + trusted base policy. A PR that changes the selector or its test inventory + must qualify fully and cannot approve its own narrower exemption. +5. Preserve full qualification on main and existing full-public nightly/release + sweeps. Keep a force-full escape hatch; paid model behavior tests remain + explicitly activated release/manual work, not ordinary PR discovery. + +Then expand one proven domain at a time. UI, installer, provider and scheduler +changes still require full qualification here. In particular, provider changes +retain their real-backend requirements; this planner never waives PostgreSQL +qualification or other authority-boundary evidence. + +下一阶段先依据证据启用一个范围，再扩展到 UI、安装器等领域；不预先添加尚未验证 +的免责。未知仍全量、selector 自身变更仍全量、主干与低频全量兜底保留。付费模型 +测试的触发频率不变，本功能也不修改 LoopX Goal、Todo、runtime 或 automation。 diff --git a/docs/development/testing-and-quality.md b/docs/development/testing-and-quality.md index 5dababc6c3..a16816b105 100644 --- a/docs/development/testing-and-quality.md +++ b/docs/development/testing-and-quality.md @@ -110,6 +110,14 @@ Changes to the classifier or workflow need both code-path and documentation-only qualification. Keep required check names stable and never require a workflow-level path-filtered check that cannot report on every PR. +The [CI impact-selection shadow rollout](ci-impact-selection.md) additionally +proposes a reviewed vision-checkpoint slice and compares its actual test +identities/results with the full shards. It does not yet skip core work or +replace full-suite coverage. Missing or failed required shadow evidence also +fails `merge-gate`; policy changes rehearse the candidate while retaining full +qualification. CI 影响范围选择目前处于 shadow 阶段：真实运行、对照并积累证据， +不把候选清单当成跳过全量的授权。 + PRs opened before activation may need a branch update to produce the new required check; an old green suite alone does not supply a missing aggregate. diff --git a/scripts/ci/impact_plan.py b/scripts/ci/impact_plan.py new file mode 100644 index 0000000000..b611f26089 --- /dev/null +++ b/scripts/ci/impact_plan.py @@ -0,0 +1,153 @@ +"""Exact-path, fail-closed CI impact planning; candidate plans are shadow-only.""" + +from __future__ import annotations + +from dataclasses import dataclass +import json +import os +from pathlib import Path, PurePosixPath +import subprocess + + +SCHEMA = "loopx_ci_impact_plan_v1" +ROOT_DOCS = {"README.md", "README.zh-CN.md", "CHANGELOG.md", "CONTRIBUTING.md"} + +# This is a reviewed behavioral boundary, not a filename/keyword search. The +# checkpoint has callers across refresh, quota, settlement and terminal routing. +VISION_SOURCES = ( + "loopx/control_plane/goals/vision_checkpoint.ts", + "loopx/control_plane/goals/goal_vision.py", +) +VISION_TESTS = ( + "tests/control_plane/test_vision_checkpoint_runtime.py", + "tests/control_plane/test_vision_budget_cli.py", + "tests/control_plane/test_goal_vision_succession.py", + "tests/control_plane/test_goal_vision_blocked_successor.py", + "tests/control_plane/test_vision_wait_coverage.py", + "tests/control_plane/test_refresh_checkpoint_recovery.py", + "tests/control_plane/test_refresh_checkpoint_isolation.py", + "tests/control_plane/test_refresh_state_replan_gate.py", + "tests/control_plane/test_goal_frontier_replan_rules.py", + "tests/control_plane/test_goal_terminal_no_followup.py", + "tests/control_plane/test_autonomous_replan_ack.py", + "tests/control_plane/test_monitor_replan_agent_scope.py", + "tests/control_plane/test_monitor_followthrough_contract.py", + "tests/control_plane/test_quota_settlement_cli.py", + "tests/control_plane/test_quota_cli_projection.py", + "tests/control_plane/test_public_safe_text_owner_parity.py", + "tests/control_plane/test_run_context_retention.py", + "tests/cli_commands/test_quota_turn_envelope_validation_failure.py", +) +VISION_SMOKES = ( + "examples/project/goal-vision-refresh-state-budget-smoke.py", + "examples/project/goal-vision-path-delta-smoke.py", + "examples/project/goal-vision-replan-contract-smoke.py", + "examples/project/goal-vision-closed-successor-smoke.py", + "examples/control_plane/status-quota-perf-budget-smoke.py", +) +VISION_TS_TESTS = ( + "tests/control_plane_ts/vision_checkpoint.test.ts", + "tests/control_plane_ts/vision_wait_coverage.test.ts", + "tests/control_plane_ts/refresh_recovery.test.ts", + "tests/control_plane_ts/replan_settlement.test.ts", + "tests/control_plane_ts/turn_settlement.test.ts", +) +VISION_PATHS = frozenset((*VISION_SOURCES, *VISION_TESTS, *VISION_SMOKES, *VISION_TS_TESTS)) + + +@dataclass(frozen=True) +class Change: + status: str + path: str + + +def is_document(path: str) -> bool: + return path in ROOT_DOCS or (path.startswith("docs/") and PurePosixPath(path).suffix == ".md") + + +def candidate(changes: list[Change], *, pull_request: bool = True) -> tuple[str, str]: + if not pull_request: + return "full", "non-PR events always qualify the full suite" + if not changes: + return "full", "empty or unavailable impact is not an exemption" + for change in changes: + path = PurePosixPath(change.path) + if path.is_absolute() or ".." in path.parts or str(path) != change.path: + return "full", "non-canonical path requires full qualification" + if change.status not in {"A", "M", "D"}: + return "full", "type changes or unrecognized Git statuses require full qualification" + code = [change for change in changes if not is_document(change.path)] + if not code: + return "docs", "only existing documentation exemptions changed" + if any(change.path not in VISION_PATHS for change in code): + return "full", "unmapped or shared-boundary change requires full qualification" + if any(change.status == "D" or (change.status == "A" and change.path in VISION_SOURCES) for change in code): + return "full", "runtime additions, deletions and renames require full qualification" + return "vision", "checkpoint authoring and its read/refresh/replan/settlement consumers" + + +def git(*args: str) -> bytes: + return subprocess.check_output(["git", *args]) + + +def revision(ref: str) -> str: + return git("rev-parse", "--verify", "--end-of-options", f"{ref}^{{commit}}").decode().strip() + + +def diff_changes(base: str, head: str) -> list[Change]: + raw = git("diff", "--name-status", "--no-renames", "-z", base, head, "--").split(b"\0") + if raw[-1] != b"" or (len(raw) - 1) % 2: + raise ValueError("malformed NUL-delimited Git change list") + return [Change(os.fsdecode(raw[i]), os.fsdecode(raw[i + 1])) for i in range(0, len(raw) - 1, 2)] + + +def plan(base: str, head: str, *, pull_request: bool = True) -> dict: + base_sha, head_sha = revision(base), revision(head) + merge_base = git("merge-base", base_sha, head_sha).decode().strip() + changes = diff_changes(merge_base, head_sha) + profile, reason = candidate(changes, pull_request=pull_request) + # CI-policy edits rehearse the candidate too, but can never select less than + # full qualification. This also exercises the new job before rollout. + policy_changed = any(item.path.startswith("scripts/ci/") or item.path == ".github/workflows/python-tests.yml" for item in changes) + shadow = "vision" if pull_request and (profile == "vision" or policy_changed) else "none" + return { + "schema_version": SCHEMA, + "base_sha": base_sha, + "head_sha": head_sha, + "merge_base_sha": merge_base, + "checkout_sha": revision("HEAD"), + "candidate_profile": profile, + "shadow_profile": shadow, + "execution_mode": "docs" if profile == "docs" else "full_with_shadow" if shadow == "vision" else "full", + "reason": reason, + "changes": [{"status": item.status, "path": item.path} for item in changes], + "pytest_files": list(VISION_TESTS) if shadow == "vision" else [], + "smoke_files": list(VISION_SMOKES) if shadow == "vision" else [], + "shared_checks": [] if profile == "docs" else ["full TS tests and typecheck", "lint", "CLI output budgets"], + "coverage_scope": "full suite remains authoritative; shadow is selected-only", + "selection_is_merge_authority": False, + } + + +def write_plan(packet: dict, output: str) -> None: + Path(output).write_text(json.dumps(packet, ensure_ascii=True, indent=2) + "\n", encoding="utf-8") + + +def validate_shadow_plan(packet: dict) -> None: + if packet.get("schema_version") != SCHEMA or packet.get("candidate_profile") not in {"vision", "full"} or packet.get("shadow_profile") != "vision": + raise ValueError("not a supported vision shadow plan") + if packet.get("execution_mode") != "full_with_shadow" or packet.get("selection_is_merge_authority") is not False: + raise ValueError("this rollout cannot authorize selective-only qualification") + if packet.get("pytest_files") != list(VISION_TESTS) or packet.get("smoke_files") != list(VISION_SMOKES): + raise ValueError("plan does not contain the complete reviewed test inventory") + if packet.get("checkout_sha") != revision("HEAD"): + raise ValueError("plan belongs to a different tested checkout") + for ref in ("base_sha", "head_sha", "merge_base_sha"): + sha = packet.get(ref, "") + if not isinstance(sha, str) or len(sha) != 40 or any(char not in "0123456789abcdef" for char in sha): + raise ValueError("plan contains an invalid immutable revision") + if plan(packet["base_sha"], packet["head_sha"]) != packet: + raise ValueError("plan cannot be reproduced from its exact Git revisions") + for path in (*VISION_TESTS, *VISION_SMOKES): + if not Path(path).is_file() or Path(path).is_symlink(): + raise ValueError(f"selected check is missing or not a regular file: {path}") diff --git a/scripts/ci/impact_shadow.py b/scripts/ci/impact_shadow.py new file mode 100644 index 0000000000..e6bcc00ded --- /dev/null +++ b/scripts/ci/impact_shadow.py @@ -0,0 +1,130 @@ +"""Execute and audit selected CI tests without weakening full-suite gates.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path +import subprocess +import sys +import time +import xml.etree.ElementTree as ET + +from impact_plan import VISION_SMOKES, VISION_TESTS, validate_shadow_plan + + +def digest(packet: dict) -> str: + return hashlib.sha256(json.dumps(packet, sort_keys=True).encode()).hexdigest() + + +def execute(packet: dict, output: Path) -> int: + validate_shadow_plan(packet) + output.mkdir(parents=True, exist_ok=True) + commands = [ + ("pytest", [sys.executable, "-m", "pytest", "-q", "-n", "2", "--dist", "loadfile", + *VISION_TESTS, f"--junitxml={output / 'selected.xml'}"], 900), + *((path, [sys.executable, path], 240) for path in VISION_SMOKES), + ] + results = [] + for name, argv, timeout in commands: + started = time.monotonic() + try: + code = subprocess.run(argv, timeout=timeout, check=False).returncode + except subprocess.TimeoutExpired: + code = 124 + results.append({"check": name, "exit_code": code, "seconds": round(time.monotonic() - started, 3)}) + receipt = { + "schema_version": "loopx_ci_shadow_execution_v1", + "plan_sha256": digest(packet), + "checkout_sha": packet["checkout_sha"], + "coverage_scope": "selected_only", + "checks": results, + } + (output / "execution.json").write_text(json.dumps(receipt, indent=2) + "\n", encoding="utf-8") + return int(any(item["exit_code"] for item in results)) + + +def cases(path: Path) -> dict[tuple[str, str], str]: + """JUnit outcomes, never failure text, stdout or machine-local paths.""" + result = {} + root = ET.parse(path).getroot() + if root.tag not in {"testsuite", "testsuites"}: + raise ValueError("not a JUnit report") + for item in root.iter("testcase"): + key = (item.get("classname", ""), item.get("name", "")) + if not all(key) or key in result: + raise ValueError("missing or duplicate JUnit test identity") + state = "failed" if item.find("failure") is not None or item.find("error") is not None else ( + "skipped" if item.find("skipped") is not None else "passed") + result[key] = state + if not result: + raise ValueError("empty JUnit report is not qualification") + return result + + +def compare(selected: dict, full_reports: list[dict], files: tuple[str, ...]) -> dict: + full = {} + for report in full_reports: + if set(full) & set(report): + raise ValueError("full shards contain duplicate test identities") + full.update(report) + prefixes = tuple(path.removesuffix(".py").replace("/", ".") for path in files) + for path, prefix in zip(files, prefixes): + if not any(key[0] == prefix or key[0].startswith(prefix + ".") for key in selected): + raise ValueError(f"selected file collected no tests: {path}") + missing = set(selected) - set(full) + omitted = {key for key in full if any(key[0] == prefix or key[0].startswith(prefix + ".") for prefix in prefixes)} - set(selected) + differences = {key for key in selected.keys() & full.keys() if selected[key] != full[key]} + selected_not_passed = {key for key, state in selected.items() if state != "passed"} + unselected_failures = {key for key, state in full.items() if state == "failed" and key not in selected} + report = { + "selected_test_count": len(selected), + "full_test_count": len(full), + "missing_from_full_count": len(missing), + "missing_from_selected_count": len(omitted), + "outcome_difference_count": len(differences), + "selected_not_passed_count": len(selected_not_passed), + "unselected_failure_count": len(unselected_failures), + "ok": not (missing or omitted or differences or selected_not_passed or unselected_failures), + "limitation": "Agreement on this revision does not prove the absence of future selection gaps.", + } + return report + + +def audit(packet: dict, selected_dir: Path, full_dir: Path, output: Path) -> int: + validate_shadow_plan(packet) + receipt = json.loads((selected_dir / "execution.json").read_text(encoding="utf-8")) + if receipt.get("schema_version") != "loopx_ci_shadow_execution_v1" or receipt.get("plan_sha256") != digest(packet): + raise ValueError("execution receipt belongs to a different plan") + if receipt.get("checkout_sha") != packet["checkout_sha"] or receipt.get("coverage_scope") != "selected_only": + raise ValueError("execution receipt has invalid revision or coverage provenance") + checks = receipt.get("checks", []) + if [item.get("check") for item in checks] != ["pytest", *VISION_SMOKES] or any(item.get("exit_code") != 0 for item in checks): + raise ValueError("selected tests or required real-entrypoint smokes did not succeed") + result = compare(cases(selected_dir / "selected.xml"), [ + cases(full_dir / f"python-junit-{shard}" / "junit.xml") for shard in (1, 2) + ], VISION_TESTS) + result.update({"schema_version": "loopx_ci_shadow_comparison_v1", "plan_sha256": digest(packet), + "checkout_sha": packet["checkout_sha"], "coverage_scope": "selected_only"}) + output.write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8") + print(json.dumps(result, sort_keys=True)) + return int(not result["ok"]) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("operation", choices=("run", "audit")) + parser.add_argument("--plan", required=True) + parser.add_argument("--selected-dir", default="impact-results") + parser.add_argument("--full-dir", default="full-reports") + parser.add_argument("--output", default="impact-comparison.json") + args = parser.parse_args() + packet = json.loads(Path(args.plan).read_text(encoding="utf-8")) + if args.operation == "run": + return execute(packet, Path(args.selected_dir)) + return audit(packet, Path(args.selected_dir), Path(args.full_dir), Path(args.output)) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/ci/review_gate.py b/scripts/ci/review_gate.py index 38c36ef8e8..39cbae1fed 100644 --- a/scripts/ci/review_gate.py +++ b/scripts/ci/review_gate.py @@ -5,9 +5,7 @@ import argparse import json import os -from pathlib import PurePosixPath -import subprocess - +from impact_plan import Change, candidate, plan, write_plan CORE_JOBS = ( "pytest", @@ -15,7 +13,6 @@ "stage2c-correctness-e2e", "windows-powershell", ) -ROOT_DOCS = {"README.md", "README.zh-CN.md", "CHANGELOG.md", "CONTRIBUTING.md"} def requires_core_tests(paths: list[str]) -> bool: @@ -23,17 +20,12 @@ def requires_core_tests(paths: list[str]) -> bool: # Disable rename detection at the caller so both old and new paths count. if not paths: return True - return any( - not ( - path in ROOT_DOCS - or (path.startswith("docs/") and PurePosixPath(path).suffix == ".md") - ) - for path in paths - ) + return candidate([Change("M", path) for path in paths])[0] != "docs" -def verify(needs: object) -> None: - if not isinstance(needs, dict) or set(needs) != {"changes", *CORE_JOBS}: +def verify(needs: object, *, shadow: bool = False) -> None: + expected_jobs = {"changes", *CORE_JOBS, *(["impact-shadow"] if shadow else [])} + if not isinstance(needs, dict) or set(needs) != expected_jobs: raise ValueError("missing or unexpected merge-gate dependencies") changes = needs["changes"] if not isinstance(changes, dict) or changes.get("result") != "success": @@ -47,6 +39,17 @@ def verify(needs: object) -> None: job = needs[name] if not isinstance(job, dict) or job.get("result") != expected: raise ValueError(f"{name} must be {expected}") + if shadow: + profile = outputs.get("impact_profile") + if profile not in {"docs", "full", "vision"} or (profile == "docs") != (classification == "false"): + raise ValueError("missing or contradictory impact profile") + shadow_profile = outputs.get("shadow_profile") + if shadow_profile not in {"vision", "none"} or (profile == "docs" and shadow_profile != "none") or (profile == "vision" and shadow_profile != "vision"): + raise ValueError("missing or contradictory shadow profile") + expected_shadow = "success" if shadow_profile == "vision" else "skipped" + job = needs["impact-shadow"] + if not isinstance(job, dict) or job.get("result") != expected_shadow: + raise ValueError(f"impact-shadow must be {expected_shadow}") def main() -> None: @@ -55,21 +58,20 @@ def main() -> None: classify = sub.add_parser("classify") classify.add_argument("--base", required=True) classify.add_argument("--head", required=True) - sub.add_parser("verify") + classify.add_argument("--plan", help="write the non-authoritative impact plan") + classify.add_argument("--non-pr", action="store_true") + sub.add_parser("verify").add_argument("--shadow", action="store_true") args = parser.parse_args() if args.command == "classify": - # SHAs come through environment variables, never interpolated shell code. - # Git failure propagates; it must never become a documentation-only pass. - base = subprocess.check_output( - ["git", "merge-base", args.base, args.head], text=True - ).strip() - raw = subprocess.check_output( - ["git", "diff", "--name-only", "--no-renames", "-z", base, args.head, "--"] - ) - paths = [os.fsdecode(path) for path in raw.split(b"\0") if path] - print(f"core_tests={str(requires_core_tests(paths)).lower()}") + packet = plan(args.base, args.head, pull_request=not args.non_pr) + if args.plan: + write_plan(packet, args.plan) + print(f"core_tests={str(packet['candidate_profile'] != 'docs').lower()}") + if args.plan: + print(f"impact_profile={packet['candidate_profile']}") + print(f"shadow_profile={packet['shadow_profile']}") else: - verify(json.loads(os.environ["NEEDS_JSON"])) + verify(json.loads(os.environ["NEEDS_JSON"]), shadow=args.shadow) print("merge-gate: qualification complete") diff --git a/scripts/ci/test_impact_plan.py b/scripts/ci/test_impact_plan.py new file mode 100644 index 0000000000..c7104d9f46 --- /dev/null +++ b/scripts/ci/test_impact_plan.py @@ -0,0 +1,186 @@ +"""Selection rules are conservative; exact Git changes are the input oracle.""" + +from __future__ import annotations + +import copy +import json +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest +from unittest.mock import patch + +from impact_plan import Change, VISION_SOURCES, VISION_TESTS, candidate, validate_shadow_plan +from review_gate import CORE_JOBS, verify + + +def gate(profile="vision", shadow="vision"): + core = profile != "docs" + return { + "changes": {"result": "success", "outputs": {"core_tests": str(core).lower(), + "impact_profile": profile, "shadow_profile": shadow}}, + **{name: {"result": "success" if core else "skipped"} for name in CORE_JOBS}, + "impact-shadow": {"result": "success" if shadow == "vision" else "skipped"}, + } + + +class ImpactRuleTests(unittest.TestCase): + def test_sources_select_cross_domain_consumers(self): + for source in VISION_SOURCES: + self.assertEqual(candidate([Change("M", source)])[0], "vision") + for required in ( + "tests/control_plane/test_refresh_state_replan_gate.py", + "tests/control_plane/test_goal_terminal_no_followup.py", + "tests/control_plane/test_quota_settlement_cli.py", + "tests/control_plane/test_vision_budget_cli.py", + ): + self.assertIn(required, VISION_TESTS) + self.assertEqual(len(VISION_TESTS), len(set(VISION_TESTS))) + + def test_additional_test_is_selected_but_new_runtime_is_full(self): + self.assertEqual(candidate([Change("A", VISION_TESTS[0])])[0], "vision") + self.assertEqual(candidate([Change("A", VISION_SOURCES[0])])[0], "full") + + def test_unknown_shared_and_ci_paths_cannot_hide_in_a_known_change(self): + for path in ( + "loopx/control_plane/runtime_decode.ts", "loopx/control_plane/effect_runtime_handlers.ts", + "loopx/control_plane/coordination/todo_update.ts", "loopx/cli.py", "tests/conftest.py", + "scripts/ci/impact_plan.py", ".github/workflows/python-tests.yml", "pyproject.toml", + "package-lock.json", "new-vision-checkpoint.ts", "docs/executable.py", "loopx/prompt.md", + ): + with self.subTest(path=path): + self.assertEqual(candidate([Change("M", VISION_SOURCES[0]), Change("M", path)])[0], "full") + + def test_deletion_type_change_rename_and_bad_status_are_full(self): + for status in ("D", "T", "R100", "U", ""): + self.assertEqual(candidate([Change(status, VISION_SOURCES[0])])[0], "full") + self.assertEqual(candidate([Change("D", VISION_SOURCES[0]), Change("A", "docs/copied.md")])[0], "full") + + def test_docs_maintain_the_existing_exemption_not_a_keyword_heuristic(self): + self.assertEqual(candidate([Change("M", "docs/vision.md"), Change("D", "README.md")])[0], "docs") + for path in ("docs/not-a-doc.ts", "docs/../loopx/code.md", "/docs/readme.md", "docs//readme.md"): + self.assertEqual(candidate([Change("M", path)])[0], "full") + self.assertEqual(candidate([])[0], "full") + + def test_main_and_scheduled_contexts_never_select_less(self): + for changes in ([], [Change("M", "docs/a.md")], [Change("M", VISION_SOURCES[0])]): + self.assertEqual(candidate(changes, pull_request=False)[0], "full") + + def test_full_gate_is_retained_for_every_candidate(self): + self.assertEqual(set(CORE_JOBS), {"pytest", "node-minimum-compatibility", "stage2c-correctness-e2e", "windows-powershell"}) + for profile, shadow in (("vision", "vision"), ("full", "vision"), ("full", "none"), ("docs", "none")): + verify(gate(profile, shadow), shadow=True) + for name in (*CORE_JOBS, "impact-shadow"): + for state in ("failure", "cancelled", "skipped", "neutral", None): + value = gate() + value[name]["result"] = state + with self.subTest(name=name, state=state), self.assertRaises(ValueError): + verify(value, shadow=True) + + def test_missing_and_contradictory_profiles_fail_closed(self): + for value in (gate("docs", "vision"), gate("vision", "none"), gate("invented", "none")): + with self.assertRaises(ValueError): + verify(value, shadow=True) + value = gate("docs", "none") + value["impact-shadow"]["result"] = "success" + with self.assertRaises(ValueError): + verify(value, shadow=True) + for field in ("core_tests", "impact_profile", "shadow_profile"): + value = gate() + del value["changes"]["outputs"][field] + with self.assertRaises(ValueError): + verify(value, shadow=True) + for name in ("impact-shadow", "changes", *CORE_JOBS): + value = gate() + del value[name] + with self.assertRaises(ValueError): + verify(value, shadow=True) + + +class RealGitImpactTests(unittest.TestCase): + def test_real_diff_and_cli_bind_revisions_and_preserve_all_paths(self): + script = str(Path(__file__).with_name("review_gate.py")) + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + env = {key: value for key, value in os.environ.items() + if not key.startswith(("PYTEST", "COVERAGE", "COV_CORE"))} + + def git(*args): + return subprocess.check_output(["git", *args], cwd=root, env=env, text=True).strip() + + def write(path, content): + target = root / path + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(content) + + def classify(base, head="HEAD", *extra): + result = subprocess.run([sys.executable, script, "classify", "--base", base, "--head", head, + "--plan", str(root / "result.json"), *extra], cwd=root, env=env, capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stderr) + return result.stdout, json.loads((root / "result.json").read_text()) + + git("init", "-q") + git("config", "user.name", "CI Fixture") + git("config", "user.email", "ci@example.invalid") + git("config", "core.hooksPath", str(root / "no-hooks")) + write(VISION_SOURCES[0], "// baseline\n") + git("add", VISION_SOURCES[0]) + git("commit", "-qm", "baseline") + base = git("rev-parse", "HEAD") + write(VISION_SOURCES[0], "// changed\n") + git("commit", "-qam", "vision change") + output, packet = classify(base) + self.assertIn("core_tests=true", output) + self.assertIn("impact_profile=vision", output) + self.assertEqual(packet["head_sha"], git("rev-parse", "HEAD")) + self.assertEqual(packet["merge_base_sha"], base) + self.assertEqual(packet["execution_mode"], "full_with_shadow") + self.assertFalse(packet["selection_is_merge_authority"]) + self.assertEqual(classify(base, "HEAD", "--non-pr")[1]["shadow_profile"], "none") + # A newline-bearing filename is one path, never a GITHUB_OUTPUT line. + strange = "unknown\ncore_tests=false" + write(strange, "fixture\n") + git("add", strange) + git("commit", "-qm", "unknown path") + output, packet = classify(base) + self.assertNotIn("core_tests=false", output) + self.assertIn(strange, [item["path"] for item in packet["changes"]]) + self.assertEqual(packet["candidate_profile"], "full") + write("scripts/ci/impact_plan.py", "# policy changed\n") + git("add", "scripts/ci/impact_plan.py") + git("commit", "-qm", "policy qualification") + packet = classify(base)[1] + self.assertEqual((packet["candidate_profile"], packet["shadow_profile"]), ("full", "vision")) + before_rename = git("rev-parse", "HEAD") + (root / "docs").mkdir() + git("mv", VISION_SOURCES[0], "docs/moved.md") + git("commit", "-qm", "rename") + packet = classify(before_rename)[1] + self.assertEqual(packet["candidate_profile"], "full") + self.assertEqual({item["status"] for item in packet["changes"]}, {"A", "D"}) + failed = subprocess.run([sys.executable, script, "classify", "--base", "missing-base", "--head", "HEAD", + "--plan", str(root / "invalid.json")], cwd=root, env=env, capture_output=True, text=True) + self.assertNotEqual(failed.returncode, 0) + self.assertFalse((root / "invalid.json").exists()) + + def test_plan_validation_rejects_wrong_revision_inventory_and_authority(self): + from impact_plan import SCHEMA, VISION_SMOKES + packet = {"schema_version": SCHEMA, "candidate_profile": "vision", "shadow_profile": "vision", + "execution_mode": "full_with_shadow", "selection_is_merge_authority": False, + "checkout_sha": "a" * 40, "base_sha": "b" * 40, "head_sha": "a" * 40, + "merge_base_sha": "b" * 40, "pytest_files": list(VISION_TESTS), "smoke_files": list(VISION_SMOKES)} + with patch("impact_plan.revision", return_value="a" * 40), patch("impact_plan.plan", return_value=packet), patch("impact_plan.Path.is_file", return_value=True), patch("impact_plan.Path.is_symlink", return_value=False): + validate_shadow_plan(packet) + for field, value in (("checkout_sha", "c" * 40), ("pytest_files", []), ("smoke_files", []), + ("selection_is_merge_authority", True), ("execution_mode", "targeted"), + ("head_sha", "HEAD"), ("base_sha", "c" * 40)): + changed = copy.deepcopy(packet) + changed[field] = value + with self.subTest(field=field), self.assertRaises(ValueError): + validate_shadow_plan(changed) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/ci/test_impact_shadow.py b/scripts/ci/test_impact_shadow.py new file mode 100644 index 0000000000..2f8f7bf155 --- /dev/null +++ b/scripts/ci/test_impact_shadow.py @@ -0,0 +1,110 @@ +"""Negative or absent evidence cannot qualify a candidate test selection.""" + +from __future__ import annotations + +import json +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch + +from impact_plan import VISION_SMOKES, VISION_TESTS +from impact_shadow import audit, cases, compare, digest, execute + + +class ShadowComparisonTests(unittest.TestCase): + def test_same_cases_and_outcomes_match_without_claiming_full_coverage(self): + chosen = {("tests.test_a", "test_ok[x]"): "passed"} + report = compare(chosen, [chosen, {("tests.test_b", "test_other"): "passed"}], ("tests/test_a.py",)) + self.assertTrue(report["ok"]) + self.assertEqual(report["selected_test_count"], 1) + self.assertEqual(report["full_test_count"], 2) + self.assertIn("does not prove", report["limitation"]) + + def test_omissions_different_outcomes_and_unselected_failures_are_not_passes(self): + key, other = ("tests.test_a", "test_a"), ("tests.test_b", "test_b") + for selected, full, metric in ( + ({key: "passed"}, {other: "passed"}, "missing_from_full_count"), + ({key: "passed"}, {key: "failed"}, "outcome_difference_count"), + ({key: "skipped"}, {key: "skipped"}, "selected_not_passed_count"), + ({key: "failed"}, {key: "failed"}, "selected_not_passed_count"), + ({key: "passed"}, {key: "passed", other: "failed"}, "unselected_failure_count"), + ({key: "passed"}, {key: "passed", ("tests.test_a", "test_omitted"): "passed"}, "missing_from_selected_count"), + ): + with self.subTest(metric=metric): + result = compare(selected, [full], ("tests/test_a.py",)) + self.assertFalse(result["ok"]) + self.assertEqual(result[metric], 1) + + def test_empty_collection_and_duplicate_full_shards_fail(self): + with self.assertRaises(ValueError): + compare({}, [{}], ("tests/test_a.py",)) + chosen = {("tests.test_a", "test_a"): "passed"} + with self.assertRaises(ValueError): + compare(chosen, [chosen, chosen], ("tests/test_a.py",)) + with self.assertRaises(ValueError): + compare(chosen, [chosen], ("tests/test_a.py", "tests/test_b.py")) + + def test_junit_uses_identity_and_state_not_failure_text(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "junit.xml" + path.write_text('' + 'not exported' + '') + self.assertEqual(list(cases(path).values()), ["passed", "failed", "skipped"]) + for text in ('', '', '', + ''): + path.write_text(text) + with self.assertRaises(ValueError): + cases(path) + + def test_failed_command_keeps_negative_receipt_and_does_not_invent_success(self): + with tempfile.TemporaryDirectory() as directory, patch("impact_shadow.validate_shadow_plan"), patch("impact_shadow.subprocess.run") as run: + run.return_value.returncode = 1 + packet = {"checkout_sha": "a" * 40} + self.assertEqual(execute(packet, Path(directory)), 1) + receipt = json.loads((Path(directory) / "execution.json").read_text()) + self.assertEqual(receipt["coverage_scope"], "selected_only") + self.assertEqual([item["check"] for item in receipt["checks"]], ["pytest", *VISION_SMOKES]) + self.assertTrue(all(item["exit_code"] == 1 for item in receipt["checks"])) + + def test_wrong_plan_and_missing_smokes_cannot_use_a_green_junit(self): + with tempfile.TemporaryDirectory() as directory, patch("impact_shadow.validate_shadow_plan"): + root = Path(directory) + packet = {"checkout_sha": "a" * 40} + receipt = {"schema_version": "loopx_ci_shadow_execution_v1", "checkout_sha": "a" * 40, + "plan_sha256": digest(packet), "coverage_scope": "selected_only", "checks": []} + for change in ({}, {"plan_sha256": "wrong"}, {"coverage_scope": "full"}, {"checkout_sha": "b" * 40}): + (root / "execution.json").write_text(json.dumps({**receipt, **change})) + with self.assertRaises(ValueError): + audit(packet, root, root, root / "comparison.json") + self.assertFalse((root / "comparison.json").exists()) + + def test_real_report_audit_requires_both_shards_and_complete_inventory(self): + with tempfile.TemporaryDirectory() as directory, patch("impact_shadow.validate_shadow_plan"): + root = Path(directory) + packet = {"checkout_sha": "a" * 40} + receipt = {"schema_version": "loopx_ci_shadow_execution_v1", "checkout_sha": "a" * 40, + "plan_sha256": digest(packet), "coverage_scope": "selected_only", + "checks": [{"check": name, "exit_code": 0} for name in ("pytest", *VISION_SMOKES)]} + (root / "execution.json").write_text(json.dumps(receipt)) + rows = [f'' for path in VISION_TESTS] + + def report(path, cases): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("" + "".join(cases) + "") + + report(root / "selected.xml", rows) + report(root / "python-junit-1/junit.xml", rows[:1]) + with self.assertRaises(FileNotFoundError): + audit(packet, root, root, root / "result.json") + report(root / "python-junit-2/junit.xml", rows[1:]) + self.assertEqual(audit(packet, root, root, root / "result.json"), 0) + result = json.loads((root / "result.json").read_text()) + self.assertTrue(result["ok"]) + self.assertEqual(result["selected_test_count"], len(VISION_TESTS)) + self.assertEqual(result["coverage_scope"], "selected_only") + + +if __name__ == "__main__": + unittest.main() From 89303c617b1050a3094d7d5ec75ba52b914f78a9 Mon Sep 17 00:00:00 2001 From: huangruiteng Date: Fri, 11 Sep 2026 18:58:25 +0800 Subject: [PATCH 2/3] test(ci): cover shadow dependencies in workflow regression Signed-off-by: huangruiteng --- tests/test_python_ci_workflow.py | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/tests/test_python_ci_workflow.py b/tests/test_python_ci_workflow.py index bb5b41450a..6648a50ea3 100644 --- a/tests/test_python_ci_workflow.py +++ b/tests/test_python_ci_workflow.py @@ -111,10 +111,10 @@ def test_merge_gate_runs_on_all_prs_and_checks_every_core_aggregate() -> None: assert "if: always()" in gate assert ( "needs: [changes, pytest, node-minimum-compatibility, " - "stage2c-correctness-e2e, windows-powershell]" + "stage2c-correctness-e2e, windows-powershell, impact-shadow]" ) in gate assert "NEEDS_JSON: ${{ toJSON(needs) }}" in gate - assert "run: python scripts/ci/review_gate.py verify" in gate + assert "run: python scripts/ci/review_gate.py verify --shadow" in gate assert "continue-on-error" not in gate for name in ("checks", "test-shard", "stage2c-suite", "windows-powershell"): job = WORKFLOW.split(f" {name}:\n", 1)[1].split(" steps:", 1)[0] @@ -122,6 +122,24 @@ def test_merge_gate_runs_on_all_prs_and_checks_every_core_aggregate() -> None: assert "if: needs.changes.outputs.core_tests == 'true'" in job +def test_shadow_workflow_keeps_full_coverage_and_requires_exact_artifacts() -> None: + selected = WORKFLOW.split(" impact-tests:\n", 1)[1].split(" impact-shadow:\n", 1)[0] + audit = WORKFLOW.split(" impact-shadow:\n", 1)[1].split(" pytest:\n", 1)[0] + assert "if: needs.changes.outputs.shadow_profile == 'vision'" in selected + assert "impact_shadow.py run --plan impact-plan.json" in selected + assert "--cov" not in selected and "coverage-xml" not in selected + assert "needs: [changes, impact-tests, test-shard]" in audit + assert "if: always() && needs.changes.outputs.shadow_profile == 'vision'" in audit + for artifact in ("ci-impact-plan", "ci-impact-selected", "python-junit-*"): + assert artifact in audit + assert "impact_shadow.py audit --plan impact-plan.json" in audit + assert 'test "$SELECTED_RESULT" = success' in audit + assert 'test "$FULL_RESULT" = success' in audit + assert "continue-on-error" not in selected + audit + assert "--junitxml=junit.xml" in WORKFLOW + assert "name: python-junit-${{ matrix.shard }}" in WORKFLOW + + def test_two_shards_execute_each_test_once_and_merge_portable_coverage( tmp_path: Path, ) -> None: From be0fe25754a41144084b923d223963406aa124cc Mon Sep 17 00:00:00 2001 From: huangruiteng Date: Fri, 11 Sep 2026 21:56:58 +0800 Subject: [PATCH 3/3] ci: exempt client-only changes and distribute heavy suites Signed-off-by: huangruiteng --- .github/workflows/python-tests.yml | 165 ++++++++---------- docs/development/ci-impact-selection.md | 207 +++++++++-------------- docs/development/testing-and-quality.md | 34 ++-- scripts/ci/impact_plan.py | 159 +++++++----------- scripts/ci/impact_shadow.py | 130 -------------- scripts/ci/module_shard.py | 40 +++++ scripts/ci/review_gate.py | 74 ++++---- scripts/ci/test_impact_plan.py | 215 +++++++----------------- scripts/ci/test_impact_shadow.py | 110 ------------ scripts/ci/test_review_gate.py | 156 +++++------------ tests/test_python_ci_workflow.py | 155 +++++++++-------- 11 files changed, 489 insertions(+), 956 deletions(-) delete mode 100644 scripts/ci/impact_shadow.py create mode 100644 scripts/ci/module_shard.py delete mode 100644 scripts/ci/test_impact_shadow.py diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index b886273e4d..4189a98b22 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -2,6 +2,8 @@ name: Python Tests on: pull_request: + types: [opened, synchronize, reopened, ready_for_review, labeled, unlabeled] + workflow_dispatch: push: branches: - main @@ -32,8 +34,10 @@ jobs: timeout-minutes: 3 outputs: core_tests: ${{ steps.classify.outputs.core_tests }} - impact_profile: ${{ steps.classify.outputs.impact_profile }} - shadow_profile: ${{ steps.classify.outputs.shadow_profile }} + change_kind: ${{ steps.classify.outputs.change_kind }} + python_tests: ${{ steps.classify.outputs.python_tests }} + stage2c_tests: ${{ steps.classify.outputs.stage2c_tests }} + presentation_tests: ${{ steps.classify.outputs.presentation_tests }} steps: - uses: actions/checkout@v7 with: @@ -50,11 +54,14 @@ jobs: BASE_SHA: ${{ github.event.pull_request.base.sha }} HEAD_SHA: ${{ github.event.pull_request.head.sha }} CHECKOUT_SHA: ${{ github.sha }} + FORCE_FULL: ${{ contains(github.event.pull_request.labels.*.name, 'ci:full') }} shell: bash run: | set -euo pipefail + extra=() + if [[ "$FORCE_FULL" == true ]]; then extra+=(--force-full); fi if [[ "$EVENT_NAME" == pull_request ]]; then - python scripts/ci/review_gate.py classify --base "$BASE_SHA" --head "$HEAD_SHA" --plan impact-plan.json >> "$GITHUB_OUTPUT" + python scripts/ci/review_gate.py classify --base "$BASE_SHA" --head "$HEAD_SHA" --plan impact-plan.json "${extra[@]}" >> "$GITHUB_OUTPUT" else python scripts/ci/review_gate.py classify --base "$CHECKOUT_SHA" --head "$CHECKOUT_SHA" --non-pr --plan impact-plan.json >> "$GITHUB_OUTPUT" fi @@ -63,10 +70,11 @@ jobs: from pathlib import Path plan = json.loads(Path("impact-plan.json").read_text()) with open(os.environ["GITHUB_STEP_SUMMARY"], "a") as summary: - summary.write("## CI impact plan (shadow rollout)\n\n") - summary.write(f"Candidate: **{plan['candidate_profile']}**; execution: **{plan['execution_mode']}**.\n\n") + summary.write("## CI job exemptions\n\n") + summary.write(f"Change kind: **{plan['change_kind']}**; Python shards: **{plan['python_shards']}**.\n\n") summary.write(plan['reason'] + ".\n\n") - summary.write("Full-suite checks and coverage remain authoritative. See the plan artifact for the exact revisions and selected files.\n") + summary.write(f"Python: {plan['python_tests']}; Stage2c: {plan['stage2c_tests']}; frontend: {plan['presentation_tests']}.\n\n") + summary.write(plan["coverage_scope"] + ". Unknown/mixed changes and main run full. Label ci:full forces full qualification.\n") PY - uses: actions/upload-artifact@v7 with: @@ -212,13 +220,13 @@ jobs: test-shard: needs: changes - if: needs.changes.outputs.core_tests == 'true' + if: needs.changes.outputs.python_tests == 'true' runs-on: ubuntu-latest timeout-minutes: 30 strategy: fail-fast: false matrix: - shard: [1, 2] + shard: [1, 2, 3, 4] steps: - uses: actions/checkout@v7 with: @@ -242,7 +250,7 @@ jobs: # Each runner retains the measured two-worker pool. run: >- python -m pytest -q -n 2 -m "not stage2c_e2e" - --splits 2 --group ${{ matrix.shard }} + --splits 4 --group ${{ matrix.shard }} --splitting-algorithm least_duration --durations=25 --durations-min=1 --junitxml=junit.xml @@ -265,85 +273,9 @@ jobs: if-no-files-found: error retention-days: 3 - impact-tests: - needs: changes - if: needs.changes.outputs.shadow_profile == 'vision' - runs-on: ubuntu-latest - timeout-minutes: 20 - steps: - - uses: actions/checkout@v7 - with: - fetch-depth: 0 - - uses: actions/setup-python@v6 - with: - python-version: "3.11" - cache: pip - - uses: actions/setup-node@v6 - with: - node-version: "24" - cache: npm - cache-dependency-path: package-lock.json - - run: | - python -m pip install --disable-pip-version-check -e ".[test]" - npm ci --ignore-scripts - - uses: actions/download-artifact@v7 - with: - name: ci-impact-plan - - name: Execute the complete candidate profile without full-suite coverage claims - run: python scripts/ci/impact_shadow.py run --plan impact-plan.json - - uses: actions/upload-artifact@v7 - if: always() - with: - name: ci-impact-selected - path: impact-results/ - if-no-files-found: error - retention-days: 7 - - impact-shadow: - needs: [changes, impact-tests, test-shard] - if: always() && needs.changes.outputs.shadow_profile == 'vision' - runs-on: ubuntu-latest - timeout-minutes: 5 - steps: - - name: Require candidate execution - env: - SELECTED_RESULT: ${{ needs.impact-tests.result }} - run: | - test "$SELECTED_RESULT" = success - - uses: actions/checkout@v7 - with: - fetch-depth: 0 - - uses: actions/setup-python@v6 - with: - python-version: "3.11" - - uses: actions/download-artifact@v7 - with: - name: ci-impact-plan - - uses: actions/download-artifact@v7 - with: - name: ci-impact-selected - path: impact-results - - uses: actions/download-artifact@v7 - with: - pattern: python-junit-* - path: full-reports - - name: Compare exact test identities and outcomes - run: python scripts/ci/impact_shadow.py audit --plan impact-plan.json - - uses: actions/upload-artifact@v7 - if: always() - with: - name: ci-impact-comparison - path: impact-comparison.json - if-no-files-found: error - retention-days: 7 - - name: Full execution failures cannot qualify the shadow - env: - FULL_RESULT: ${{ needs.test-shard.result }} - run: test "$FULL_RESULT" = success - pytest: # Keep the required check name; a skipped/failed shard must not turn it green. - if: always() && needs.changes.outputs.core_tests == 'true' + if: always() && needs.changes.outputs.python_tests == 'true' needs: [changes, checks, test-shard] runs-on: ubuntu-latest timeout-minutes: 10 @@ -367,9 +299,13 @@ jobs: path: coverage-shards - name: Combine complete coverage and enforce the existing floor run: | - test -s coverage-shards/python-coverage-1/.coverage - test -s coverage-shards/python-coverage-2/.coverage - python -m coverage combine coverage-shards/python-coverage-1/.coverage coverage-shards/python-coverage-2/.coverage + shards=() + for shard in 1 2 3 4; do + path="coverage-shards/python-coverage-${shard}/.coverage" + test -s "$path" + shards+=("$path") + done + python -m coverage combine "${shards[@]}" python -m coverage report --fail-under=19.6 python -m coverage xml -o coverage.xml - uses: actions/upload-artifact@v7 @@ -387,14 +323,18 @@ jobs: stage2c-suite: needs: changes - if: needs.changes.outputs.core_tests == 'true' - name: stage2c (${{ matrix.suite }}) + if: needs.changes.outputs.stage2c_tests == 'true' + name: stage2c (${{ matrix.suite }} ${{ matrix.shard }}) runs-on: ubuntu-latest timeout-minutes: 30 strategy: fail-fast: false matrix: - suite: [e2e, mutants, installed] + include: + - {suite: e2e, shard: 1} + - {suite: e2e, shard: 2} + - {suite: mutants, shard: 0} + - {suite: installed, shard: 0} steps: - uses: actions/checkout@v7 - uses: actions/setup-python@v6 @@ -436,7 +376,7 @@ jobs: env: LOOPX_SHADOW_COMPARISON_OUTPUT: .local/stage2c-observables # Keep each module's shared workspace and ordered parity rows on one worker. - run: python -m pytest -q -n 4 --dist loadfile -m stage2c_e2e --durations=20 --junitxml=stage2c-e2e.xml + run: python -m pytest -q -n 2 --dist loadfile -m stage2c_e2e -p scripts.ci.module_shard --ci-module-shards 2 --ci-module-shard ${{ matrix.shard }} --durations=20 --junitxml=stage2c-e2e.xml - name: Reject deliberate correctness regressions if: matrix.suite == 'mutants' run: python examples/shared-goal-authority-e2e/mutants.py --output .local/stage2c-mutants @@ -453,7 +393,7 @@ jobs: if: always() uses: actions/upload-artifact@v7 with: - name: stage2c-correctness-evidence-${{ matrix.suite }} + name: stage2c-correctness-evidence-${{ matrix.suite }}-${{ matrix.shard }} include-hidden-files: true if-no-files-found: error path: | @@ -465,7 +405,7 @@ jobs: stage2c-correctness-e2e: # Preserve the public check name and reject failed, cancelled or skipped lanes. - if: always() && needs.changes.outputs.core_tests == 'true' + if: always() && needs.changes.outputs.stage2c_tests == 'true' needs: [changes, stage2c-suite] runs-on: ubuntu-latest timeout-minutes: 2 @@ -477,7 +417,7 @@ jobs: windows-powershell: needs: changes - if: needs.changes.outputs.core_tests == 'true' + if: needs.changes.outputs.python_tests == 'true' runs-on: windows-latest timeout-minutes: 20 steps: @@ -530,10 +470,39 @@ jobs: tests/control_plane_ts/local_authority_provider.test.ts tests/control_plane_ts/sqlite_runtime_admission.test.ts + presentation: + needs: changes + if: needs.changes.outputs.presentation_tests == 'true' + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-python@v6 + with: + python-version: "3.11" + cache: pip + - uses: actions/setup-node@v6 + with: + node-version: "24" + cache: npm + cache-dependency-path: apps/presentation/dashboard/package-lock.json + - run: python -m pip install --disable-pip-version-check -e ".[test]" + - name: Build and test the actual packaged Dashboard + working-directory: apps/presentation/dashboard + run: | + npm ci --ignore-scripts + npm run build:chat + if [[ -n "$(git -C ../../.. status --short --untracked-files=all -- loopx/web/chat)" ]]; then + echo "Packaged Dashboard differs from its source build." >&2 + exit 1 + fi + ./node_modules/.bin/playwright install --with-deps chromium + npm run smoke:personal-workspace-packaged + merge-gate: # Always publish one stable outcome, including documentation-only PRs. if: always() - needs: [changes, pytest, node-minimum-compatibility, stage2c-correctness-e2e, windows-powershell, impact-shadow] + needs: [changes, checks, pytest, node-minimum-compatibility, stage2c-correctness-e2e, windows-powershell, presentation] runs-on: ubuntu-latest timeout-minutes: 3 steps: @@ -544,4 +513,4 @@ jobs: - name: Reject incomplete or unsuccessful qualification env: NEEDS_JSON: ${{ toJSON(needs) }} - run: python scripts/ci/review_gate.py verify --shadow + run: python scripts/ci/review_gate.py verify diff --git a/docs/development/ci-impact-selection.md b/docs/development/ci-impact-selection.md index eb3db3cc62..61a57d3fe0 100644 --- a/docs/development/ci-impact-selection.md +++ b/docs/development/ci-impact-selection.md @@ -1,131 +1,88 @@ -# CI Impact Selection / CI 影响范围选择 - -## Current rollout: shadow, not selective merge authority - -The PR workflow retains its full core qualification for every executable, -policy, runtime-prompt or unknown change. The existing documentation-only -exemption is unchanged. `scripts/ci/impact_plan.py` now also proposes a bounded -candidate test profile and explains why it was selected. A candidate never -authorizes skipping the full suite in this rollout. - -当前先交付 **shadow 对照阶段**：不是用一次绿灯就宣布可以安全跳过全量。 -原有文档豁免不变，代码和未知改动仍跑全量；额外实际运行候选集合、比较测试身份 -和结果、记录耗时。此阶段增加少量并行工作，尚不承诺降低 PR 的总体耗时。 - -```text -merge-base … exact PR head - → NUL-delimited Git changes (renames appear as deletion + addition) - → candidate + reason + immutable revisions - ├── existing full qualification + full coverage - └── selected Python tests + real CLI smokes (shadow only) - ↓ - compare exact collected test identities and outcomes - ↓ - stable merge-gate: full success AND required shadow success -``` - -## First reviewed boundary: vision checkpoint - -The initial `vision` profile recognizes exact existing source/test/smoke paths, -not filenames containing “vision”. Its Python inventory spans checkpoint -authoring, readback, refresh recovery/isolation, public safety, replan admission, -terminal succession, quota and settlement. Real CLI smokes exercise bounded -write/read behavior, closed-vision successor routing and status/quota latency. -Full TS tests/typechecking and CLI output-budget qualification remain common -checks; they are not duplicated into another test framework. - -This is the cross-domain behavior demonstrated by the vision authoring-budget -change, not a claim that every goal-domain edit can use the same slice. A new -profile needs its own reviewed owning boundary and caller inventory. - -首批只选择边界明确的 vision checkpoint。它的消费者横跨 refresh、quota、终态与 -结算，所以不能只跑同目录测试。测试目标复用仓库已有测试和 smoke；canary 的风险 -目录可辅助维护清单，但自由文本关键词匹配、`max_checks` 截断不能作为 CI 免责依据。 - -| Change | Candidate / execution | -| --- | --- | -| Allowlisted Markdown only | `docs`; existing explicit core skips | -| Only recognized vision boundary paths | `vision`; full suite plus shadow | -| Known new test within that complete inventory | `vision`; that test is included | -| Runtime additions/deletions, renames, type changes | `full` | -| Shared decoder/dispatcher/schema, dependencies, fixtures outside the inventory | `full` | -| CI policy/workflow changes | `full` plus candidate rehearsal | -| Empty diff, non-PR execution or any unmapped path | `full` | -| Missing Git base or malformed input | Classification fails; no successful exemption | - -Both sides of renames count. A mixed PR takes the conservative union: one -unmapped code path makes the whole candidate full. Noncanonical paths and -unrecognized Git statuses cannot become documentation exemptions. GitHub -outputs contain only closed profile names and booleans, never changed filenames -or shell commands supplied by a PR. - -## Evidence and gate semantics - -Each workflow run publishes: - -- `ci-impact-plan`: base/head/merge-base/tested-checkout revisions, changed - paths, complete selected inventory, reason and actual execution mode. -- `ci-impact-selected` when applicable: JUnit outcomes and a bounded execution - receipt with command exit codes, elapsed seconds and a digest of the plan. -- `python-junit-1` and `python-junit-2`: full-shard outcomes. -- `ci-impact-comparison` when applicable: selected/full counts, missing cases, - outcome differences and failures outside the selected set. It does not copy - failure text, stdout or private runtime evidence into its summary. - -The runner rejects stale checkout identities, changed inventories, missing -checks and attempts to reinterpret shadow as selective authority. The audit -requires both full reports, unique test identities, nonempty collection from -every selected file, successful selected tests (a skip is not a pass), matching -full outcomes, successful CLI smokes and no failure outside the selected set. -Any failed/cancelled/missing required job keeps the stable `merge-gate` red. - -Coverage remains unambiguous: only the two **full** Python shards feed the -existing coverage floor and Sonar report. Shadow runs neither upload partial -coverage under full-suite artifact names nor borrow coverage from another SHA. -Plan/rehearsal artifacts are diagnostics, not a replacement for full coverage. - -覆盖率仍由同一版本的两个全量分片产生；精简集合的“绿”不能冒充完整覆盖率， -也不能用旧版本 coverage 补齐。shadow 通过只证明本次实际执行与对照成立，不证明 -未来永远不会漏测。缺失、跳过、取消、结果不一致都必须明确失败，不能转成免责。 - -## Qualify locally +# CI job exemptions / 按职责免跑重型 CI + +## Policy, not a hand-maintained test selection + +Every PR receives the same stable merge gate. The classifier reads the complete +NUL-delimited Git diff at immutable base/head revisions; a PR title, label or +author description cannot claim that runtime changes are “just UI”. + +| Whole PR | Common TS/lint/contracts | Full Python / Windows | Stage2c | Packaged Dashboard | +| --- | --- | --- | --- | --- | +| Existing Markdown-only documentation exemption | Skip | Skip | Skip | Existing Frontstage workflow | +| Client Dashboard source/assets only, optionally with docs | Run | Skip | Skip | Required build, freshness and browser smoke | +| Backend, prompt, tests, dependency, build, CI policy, mixed or unknown | Run | Run | Run | Also run for CI-policy rehearsal or forced-full UI | +| main push / manual run | Run | Run | Run | Existing surface workflows; CI-policy rehearsal when applicable | + +The presentation boundary is deliberately small: Dashboard `src/`, `public/` +and packaged `loopx/web/chat/`, with explicit client-code/image/font extensions. +Package manifests, Vite/build configuration, native desktop code, backend Python +and arbitrary JSON are not exempt. Both sides of renames are classified; moving +runtime code into a UI directory stays full. Symlinks/type changes cannot qualify. + +The new exemption is available only when the selector, gate and workflow blobs +match the already-reviewed target branch. Changing CI policy cannot exempt its +own PR. Missing policy or uncertain ownership runs full; missing Git revisions +fail classification. The pre-existing documentation exemption remains supported. + +本方案不是为每类 PR 维护一套测试清单，而是明确重型 job 的职责。纯前端变化不需要 +重跑后端持久化与崩溃恢复矩阵，但前端自己的实际构建与浏览器验收成为必需项。 +预算、静态宿主 prompt 和 Python/TS 逻辑暂不享受免跑；它们仍可能改变核心行为。 +新增一种豁免只需审阅其业务边界和保留的验收，不要求列举全部替代测试文件。 + +## Four complete Python shards + +Full Python qualification uses four runners with two xdist workers each: +`--splits 4 --group N --splitting-algorithm least_duration`. It still partitions +the whole collection, excluding only the separately executed Stage2c marker. +No tests are removed. Without timing history the splitter uses equal weights; +four-way parallelism is not a claim of perfect duration balancing. + +The aggregate requires every shard to succeed and all four coverage files to +exist before combining them. The existing full-suite coverage floor remains. +No Python coverage artifact or Sonar run is manufactured when Python is exempt. + +全量 Python 从 2 个分片扩大到 4 个，每片仍为 2 个 worker，不提高单机进程争抢。 +真实 pytest-split/xdist/coverage 回归覆盖分片集合互斥、并集完整、四份报告合并以及 +缺失任意报告时拒绝通过。分片增加会增加安装开销与同时占用的 runner；应看实际 +critical path 和 runner-minutes，而不是宣称“4 片必然快一倍”。 + +## Override and evidence + +Add the **`ci:full`** PR label to force full qualification. Label addition/removal +reruns the workflow. Manually dispatching Python Tests also runs full. The label +can only add checks, never waive them. Main retains full qualification. + +The `ci-impact-plan` artifact and job summary report exact revisions, change kind, +per-job execution flags, reason and coverage scope. The merge gate requires +success for required jobs and an explicit skip for exempt ones; failure, +cancellation, missing outputs, contradictory flags or unexpected skips fail. + +Stage2c retains all correctness cases: its E2E lane uses two runners with two +workers each, while mutants and installed-package lanes remain separate. The +small pytest plugin assigns whole modules using deterministic largest-first +test-count balancing and retains collection order within each module. It does +not split a stateful module across machines or workers. This is not timing-based +optimal scheduling: one very large module can still dominate a shard. + +Stage2c E2E 从单 runner 的 4 个 worker 改为两个 runner 各 2 个 worker；总 worker +数不增加，但不再挤在同一台机器。按完整模块分片，并保留 loadfile 与模块内顺序。 +真实回归以共享状态、顺序敏感的模块验证两路并集完整且互斥，避免盲目按单测试分片。 + +The first implementation retains minimum-Node checks and removes the earlier vision selected-test runner, +selected/full comparison machinery and its extra shadow workload. The prior +shadow results remain historical evidence, not a permanent extra CI obligation. + +## Qualification ```bash python -m unittest discover -s scripts/ci -p 'test_*.py' +python -m pytest tests/test_python_ci_workflow.py tests/test_sonarcloud_workflow.py -q python scripts/ci/review_gate.py classify --base origin/main --head HEAD --plan impact-plan.json -python scripts/ci/impact_shadow.py run --plan impact-plan.json -python scripts/ci/impact_shadow.py audit --plan impact-plan.json --full-dir full-reports +python scripts/ci/review_gate.py classify --base origin/main --head HEAD --force-full --plan impact-plan.json ``` -Run the selected commands only for plans with `shadow_profile=vision`. Install -the repository test dependencies and supported Node runtime first. The audit -expects `full-reports/python-junit-{1,2}/junit.xml` downloaded from the same -workflow run; missing reports are not a local pass. Keep generated plans, -JUnit files and receipts outside tracked source files. - -## Activation and expansion criteria - -Before a follow-up enables selective-only PR execution for a profile: - -1. Review successful shadow evidence across representative changes to its write - rule, read projection and cross-domain semantics, not just a constant edit. -2. Prove sensitivity with deliberately omitted checks, changed outcomes and - real-entrypoint semantic regressions. Investigate failures outside the - candidate instead of mechanically accepting the observed selection. -3. Run both old/full and proposed selective workflow paths, including docs, - mixed changes, missing reports, renamed/deleted tests and merge aggregation. -4. Bind execution to the exact plan/checkout; evaluate exemption rules from a - trusted base policy. A PR that changes the selector or its test inventory - must qualify fully and cannot approve its own narrower exemption. -5. Preserve full qualification on main and existing full-public nightly/release - sweeps. Keep a force-full escape hatch; paid model behavior tests remain - explicitly activated release/manual work, not ordinary PR discovery. - -Then expand one proven domain at a time. UI, installer, provider and scheduler -changes still require full qualification here. In particular, provider changes -retain their real-backend requirements; this planner never waives PostgreSQL -qualification or other authority-boundary evidence. - -下一阶段先依据证据启用一个范围，再扩展到 UI、安装器等领域；不预先添加尚未验证 -的免责。未知仍全量、selector 自身变更仍全量、主干与低频全量兜底保留。付费模型 -测试的触发频率不变，本功能也不修改 LoopX Goal、Todo、runtime 或 automation。 +Use repository-supported Python/test dependencies. Keep generated plans and +JUnit/coverage artifacts outside tracked source. Hosted CI must qualify the real +workflow after policy changes; unit checks do not prove runner scheduling or +latency. Paid model tests remain release/manual only. No Goal, automation, +authority provider, runtime permission or live state is changed by this policy. diff --git a/docs/development/testing-and-quality.md b/docs/development/testing-and-quality.md index a16816b105..cd7adfc50c 100644 --- a/docs/development/testing-and-quality.md +++ b/docs/development/testing-and-quality.md @@ -77,9 +77,11 @@ golden 来让测试通过。 `python-tests.yml` publishes `merge-gate` for every pull request. Code, workflow, policy and unknown paths require the existing `pytest` aggregate -(including TypeScript checks and both Python shards), Stage 2C correctness +(including TypeScript checks and all four Python shards), Stage 2C correctness aggregate, and Windows tests to succeed. Failed, cancelled, missing or -unexpectedly skipped results cannot pass the gate. +unexpectedly skipped results cannot pass the gate. The client-only exception +below retains common checks and substitutes packaged Dashboard qualification +for unrelated backend jobs. For a change limited to allowlisted root Markdown or `docs/**/*.md`, the classifier explicitly skips the expensive core jobs and the aggregate checks @@ -110,13 +112,13 @@ Changes to the classifier or workflow need both code-path and documentation-only qualification. Keep required check names stable and never require a workflow-level path-filtered check that cannot report on every PR. -The [CI impact-selection shadow rollout](ci-impact-selection.md) additionally -proposes a reviewed vision-checkpoint slice and compares its actual test -identities/results with the full shards. It does not yet skip core work or -replace full-suite coverage. Missing or failed required shadow evidence also -fails `merge-gate`; policy changes rehearse the candidate while retaining full -qualification. CI 影响范围选择目前处于 shadow 阶段：真实运行、对照并积累证据， -不把候选清单当成跳过全量的授权。 +The [job exemption policy](ci-impact-selection.md) additionally permits pure +Dashboard-client changes to skip backend Python/Windows and Stage2c, while +requiring common checks and the real packaged Dashboard build/browser smoke. +Mixed, prompt, dependency and unknown changes stay full. Full Python runs four +complete shards and combines all four coverage files. The `ci:full` label forces +full qualification; main stays full. No selected-only report impersonates full +coverage. 新的前端豁免由目标分支已审阅的策略控制；CI 自身变更仍全量验证。 PRs opened before activation may need a branch update to produce the new required check; an old green suite alone does not supply a missing aggregate. @@ -343,7 +345,7 @@ network latency, provider availability, or a two-hour matrix. 它刻意不包含真实模型调用和 full smoke catalog，因此普通迭代不依赖凭证、网络 时延、模型服务可用性或两小时级测试矩阵。 -The Linux suite uses two hosted runners with two xdist workers each. +The Linux suite uses four hosted runners with two xdist workers each. `pytest-split` partitions the complete collection using `least_duration`; without a timing file, tests have equal weight and alternate between shards. Lint, type checks, and the CLI budget run separately. The required `pytest` @@ -353,18 +355,18 @@ individual shards. Relative coverage paths make reports portable across runners. The reusable Sonar workflow consumes that same run's XML and never reruns pytest or reads cross-run artifacts. Missing Sonar tokens still skip analysis successfully; test jobs receive no Sonar secret. The trigger is the union of -the former Python and Sonar paths, so app-only and Sonar-configuration changes -also run this lane, including on forks without a token. +the former Python and Sonar paths. Client-only PRs use the exemption above; +Sonar-configuration changes remain full, including on forks without a token. -Linux 全套测试分到两台 hosted runner，每台保留两个 xdist worker。`pytest-split` +Linux 全套测试分到四台 hosted runner，每台保留两个 xdist worker。`pytest-split` 按完整 collection 分片；没有历史耗时时，等权测试交替分配。lint、类型检查和 CLI 预算独立执行。必需的 `pytest` 汇总检查会拒绝失败／跳过的分片和缺失的 coverage， 合并后再执行原有 19.6% 门槛；不要求单个分片达到全套覆盖率。coverage 使用相对路径， Sonar 只复用同一次 run 的 XML，不重复测试、不跨 run 取产物。缺少 token 仍成功跳过 -Sonar，测试 job 不接收 Sonar secret。触发范围取原有两套 workflow 的并集，因此仅改 -前端或 Sonar 配置也走此通道，包括没有 token 的 fork。 +Sonar，测试 job 不接收 Sonar secret。触发范围取原有两套 workflow 的并集；纯前端 +PR 使用前述豁免，Sonar 配置变更仍全量运行，包括没有 token 的 fork。 -Reproduce one shard locally with `python -m pytest -q -n 2 --splits 2 --group 1 +Reproduce one shard locally with `python -m pytest -q -n 2 --splits 4 --group 1 --splitting-algorithm least_duration --cov=loopx`. Omit the split arguments to run the complete suite locally. 全量本地测试仍省略分片参数即可。 diff --git a/scripts/ci/impact_plan.py b/scripts/ci/impact_plan.py index b611f26089..7ea54b91d6 100644 --- a/scripts/ci/impact_plan.py +++ b/scripts/ci/impact_plan.py @@ -1,4 +1,4 @@ -"""Exact-path, fail-closed CI impact planning; candidate plans are shadow-only.""" +"""Classify whole-PR ownership; exempt unrelated heavy jobs, never select tests.""" from __future__ import annotations @@ -8,51 +8,12 @@ from pathlib import Path, PurePosixPath import subprocess - -SCHEMA = "loopx_ci_impact_plan_v1" +SCHEMA = "loopx_ci_job_plan_v1" ROOT_DOCS = {"README.md", "README.zh-CN.md", "CHANGELOG.md", "CONTRIBUTING.md"} - -# This is a reviewed behavioral boundary, not a filename/keyword search. The -# checkpoint has callers across refresh, quota, settlement and terminal routing. -VISION_SOURCES = ( - "loopx/control_plane/goals/vision_checkpoint.ts", - "loopx/control_plane/goals/goal_vision.py", -) -VISION_TESTS = ( - "tests/control_plane/test_vision_checkpoint_runtime.py", - "tests/control_plane/test_vision_budget_cli.py", - "tests/control_plane/test_goal_vision_succession.py", - "tests/control_plane/test_goal_vision_blocked_successor.py", - "tests/control_plane/test_vision_wait_coverage.py", - "tests/control_plane/test_refresh_checkpoint_recovery.py", - "tests/control_plane/test_refresh_checkpoint_isolation.py", - "tests/control_plane/test_refresh_state_replan_gate.py", - "tests/control_plane/test_goal_frontier_replan_rules.py", - "tests/control_plane/test_goal_terminal_no_followup.py", - "tests/control_plane/test_autonomous_replan_ack.py", - "tests/control_plane/test_monitor_replan_agent_scope.py", - "tests/control_plane/test_monitor_followthrough_contract.py", - "tests/control_plane/test_quota_settlement_cli.py", - "tests/control_plane/test_quota_cli_projection.py", - "tests/control_plane/test_public_safe_text_owner_parity.py", - "tests/control_plane/test_run_context_retention.py", - "tests/cli_commands/test_quota_turn_envelope_validation_failure.py", -) -VISION_SMOKES = ( - "examples/project/goal-vision-refresh-state-budget-smoke.py", - "examples/project/goal-vision-path-delta-smoke.py", - "examples/project/goal-vision-replan-contract-smoke.py", - "examples/project/goal-vision-closed-successor-smoke.py", - "examples/control_plane/status-quota-perf-budget-smoke.py", -) -VISION_TS_TESTS = ( - "tests/control_plane_ts/vision_checkpoint.test.ts", - "tests/control_plane_ts/vision_wait_coverage.test.ts", - "tests/control_plane_ts/refresh_recovery.test.ts", - "tests/control_plane_ts/replan_settlement.test.ts", - "tests/control_plane_ts/turn_settlement.test.ts", -) -VISION_PATHS = frozenset((*VISION_SOURCES, *VISION_TESTS, *VISION_SMOKES, *VISION_TS_TESTS)) +POLICY_PATHS = ("scripts/ci/impact_plan.py", "scripts/ci/review_gate.py", ".github/workflows/python-tests.yml") +OUTPUTS = ("core_tests", "python_tests", "stage2c_tests", "presentation_tests") +PRESENTATION_ROOTS = ("apps/presentation/dashboard/src/", "apps/presentation/dashboard/public/", "loopx/web/chat/") +PRESENTATION_SUFFIXES = {".ts", ".tsx", ".js", ".mjs", ".css", ".html", ".svg", ".png", ".jpg", ".jpeg", ".webp", ".ico", ".woff", ".woff2"} @dataclass(frozen=True) @@ -67,23 +28,26 @@ def is_document(path: str) -> bool: def candidate(changes: list[Change], *, pull_request: bool = True) -> tuple[str, str]: if not pull_request: - return "full", "non-PR events always qualify the full suite" + return "full", "main and manual runs retain full qualification" if not changes: - return "full", "empty or unavailable impact is not an exemption" + return "full", "empty or unavailable diff is not an exemption" for change in changes: path = PurePosixPath(change.path) - if path.is_absolute() or ".." in path.parts or str(path) != change.path: - return "full", "non-canonical path requires full qualification" - if change.status not in {"A", "M", "D"}: - return "full", "type changes or unrecognized Git statuses require full qualification" - code = [change for change in changes if not is_document(change.path)] + if path.is_absolute() or ".." in path.parts or str(path) != change.path or change.status not in {"A", "M", "D"}: + return "full", "noncanonical paths or type changes require full qualification" + code = [item for item in changes if not is_document(item.path)] if not code: - return "docs", "only existing documentation exemptions changed" - if any(change.path not in VISION_PATHS for change in code): - return "full", "unmapped or shared-boundary change requires full qualification" - if any(change.status == "D" or (change.status == "A" and change.path in VISION_SOURCES) for change in code): - return "full", "runtime additions, deletions and renames require full qualification" - return "vision", "checkpoint authoring and its read/refresh/replan/settlement consumers" + return "docs", "documentation only; existing runtime exemption" + if all(item.path.startswith(PRESENTATION_ROOTS) and PurePosixPath(item.path).suffix in PRESENTATION_SUFFIXES for item in code): + return "presentation", "client-only source/assets; retain frontend build/browser and common checks" + return "full", "runtime, prompts, tests, dependencies, build policy or unknown paths may affect backend behavior" + + +def job_flags(kind: str, *, presentation: bool = False) -> dict[str, bool]: + if kind not in {"docs", "presentation", "full"}: + raise ValueError("unknown CI change kind") + return {"core_tests": kind != "docs", "python_tests": kind == "full", + "stage2c_tests": kind == "full", "presentation_tests": kind == "presentation" or presentation} def git(*args: str) -> bytes: @@ -97,57 +61,48 @@ def revision(ref: str) -> str: def diff_changes(base: str, head: str) -> list[Change]: raw = git("diff", "--name-status", "--no-renames", "-z", base, head, "--").split(b"\0") if raw[-1] != b"" or (len(raw) - 1) % 2: - raise ValueError("malformed NUL-delimited Git change list") + raise ValueError("malformed NUL-delimited Git changes") return [Change(os.fsdecode(raw[i]), os.fsdecode(raw[i + 1])) for i in range(0, len(raw) - 1, 2)] -def plan(base: str, head: str, *, pull_request: bool = True) -> dict: +def trusted_policy(base: str, head: str) -> bool: + # A PR cannot introduce its own exemption. No manifest or per-test inventory. + for path in POLICY_PATHS: + try: + old = subprocess.check_output(["git", "rev-parse", f"{base}:{path}"], stderr=subprocess.DEVNULL) + new = subprocess.check_output(["git", "rev-parse", f"{head}:{path}"], stderr=subprocess.DEVNULL) + except subprocess.CalledProcessError: + return False + if old != new: + return False + return True + + +def plan(base: str, head: str, *, pull_request: bool = True, force_full: bool = False) -> dict: base_sha, head_sha = revision(base), revision(head) merge_base = git("merge-base", base_sha, head_sha).decode().strip() changes = diff_changes(merge_base, head_sha) - profile, reason = candidate(changes, pull_request=pull_request) - # CI-policy edits rehearse the candidate too, but can never select less than - # full qualification. This also exercises the new job before rollout. - policy_changed = any(item.path.startswith("scripts/ci/") or item.path == ".github/workflows/python-tests.yml" for item in changes) - shadow = "vision" if pull_request and (profile == "vision" or policy_changed) else "none" - return { - "schema_version": SCHEMA, - "base_sha": base_sha, - "head_sha": head_sha, - "merge_base_sha": merge_base, - "checkout_sha": revision("HEAD"), - "candidate_profile": profile, - "shadow_profile": shadow, - "execution_mode": "docs" if profile == "docs" else "full_with_shadow" if shadow == "vision" else "full", - "reason": reason, - "changes": [{"status": item.status, "path": item.path} for item in changes], - "pytest_files": list(VISION_TESTS) if shadow == "vision" else [], - "smoke_files": list(VISION_SMOKES) if shadow == "vision" else [], - "shared_checks": [] if profile == "docs" else ["full TS tests and typecheck", "lint", "CLI output budgets"], - "coverage_scope": "full suite remains authoritative; shadow is selected-only", - "selection_is_merge_authority": False, - } + kind, reason = candidate(changes, pull_request=pull_request) + presentation = kind == "presentation" + policy_change = any(item.path.startswith("scripts/ci/") or item.path == POLICY_PATHS[-1] for item in changes) + if kind != "full": + for item in changes: + for sha in (merge_base, head_sha): + mode = git("ls-tree", "--format=%(objectmode)", sha, "--", item.path).decode().strip() + if mode and mode not in {"100644", "100755"}: + kind, reason = "full", "symlinks and nonregular Git objects cannot grant an exemption" + if force_full: + kind, reason = "full", "explicit force-full override" + elif kind == "presentation" and not trusted_policy(base_sha, head_sha): + kind, reason = "full", "presentation exemption policy is not already trusted on the target branch" + flags = job_flags(kind, presentation=presentation or policy_change) + return {"schema_version": SCHEMA, "base_sha": base_sha, "head_sha": head_sha, + "merge_base_sha": merge_base, "checkout_sha": revision("HEAD"), "change_kind": kind, + "force_full": force_full, "reason": reason, **flags, + "changes": [{"status": item.status, "path": item.path} for item in changes], + "coverage_scope": "full Python suite" if flags["python_tests"] else "no Python coverage claim", + "python_shards": 4 if flags["python_tests"] else 0} def write_plan(packet: dict, output: str) -> None: Path(output).write_text(json.dumps(packet, ensure_ascii=True, indent=2) + "\n", encoding="utf-8") - - -def validate_shadow_plan(packet: dict) -> None: - if packet.get("schema_version") != SCHEMA or packet.get("candidate_profile") not in {"vision", "full"} or packet.get("shadow_profile") != "vision": - raise ValueError("not a supported vision shadow plan") - if packet.get("execution_mode") != "full_with_shadow" or packet.get("selection_is_merge_authority") is not False: - raise ValueError("this rollout cannot authorize selective-only qualification") - if packet.get("pytest_files") != list(VISION_TESTS) or packet.get("smoke_files") != list(VISION_SMOKES): - raise ValueError("plan does not contain the complete reviewed test inventory") - if packet.get("checkout_sha") != revision("HEAD"): - raise ValueError("plan belongs to a different tested checkout") - for ref in ("base_sha", "head_sha", "merge_base_sha"): - sha = packet.get(ref, "") - if not isinstance(sha, str) or len(sha) != 40 or any(char not in "0123456789abcdef" for char in sha): - raise ValueError("plan contains an invalid immutable revision") - if plan(packet["base_sha"], packet["head_sha"]) != packet: - raise ValueError("plan cannot be reproduced from its exact Git revisions") - for path in (*VISION_TESTS, *VISION_SMOKES): - if not Path(path).is_file() or Path(path).is_symlink(): - raise ValueError(f"selected check is missing or not a regular file: {path}") diff --git a/scripts/ci/impact_shadow.py b/scripts/ci/impact_shadow.py deleted file mode 100644 index e6bcc00ded..0000000000 --- a/scripts/ci/impact_shadow.py +++ /dev/null @@ -1,130 +0,0 @@ -"""Execute and audit selected CI tests without weakening full-suite gates.""" - -from __future__ import annotations - -import argparse -import hashlib -import json -from pathlib import Path -import subprocess -import sys -import time -import xml.etree.ElementTree as ET - -from impact_plan import VISION_SMOKES, VISION_TESTS, validate_shadow_plan - - -def digest(packet: dict) -> str: - return hashlib.sha256(json.dumps(packet, sort_keys=True).encode()).hexdigest() - - -def execute(packet: dict, output: Path) -> int: - validate_shadow_plan(packet) - output.mkdir(parents=True, exist_ok=True) - commands = [ - ("pytest", [sys.executable, "-m", "pytest", "-q", "-n", "2", "--dist", "loadfile", - *VISION_TESTS, f"--junitxml={output / 'selected.xml'}"], 900), - *((path, [sys.executable, path], 240) for path in VISION_SMOKES), - ] - results = [] - for name, argv, timeout in commands: - started = time.monotonic() - try: - code = subprocess.run(argv, timeout=timeout, check=False).returncode - except subprocess.TimeoutExpired: - code = 124 - results.append({"check": name, "exit_code": code, "seconds": round(time.monotonic() - started, 3)}) - receipt = { - "schema_version": "loopx_ci_shadow_execution_v1", - "plan_sha256": digest(packet), - "checkout_sha": packet["checkout_sha"], - "coverage_scope": "selected_only", - "checks": results, - } - (output / "execution.json").write_text(json.dumps(receipt, indent=2) + "\n", encoding="utf-8") - return int(any(item["exit_code"] for item in results)) - - -def cases(path: Path) -> dict[tuple[str, str], str]: - """JUnit outcomes, never failure text, stdout or machine-local paths.""" - result = {} - root = ET.parse(path).getroot() - if root.tag not in {"testsuite", "testsuites"}: - raise ValueError("not a JUnit report") - for item in root.iter("testcase"): - key = (item.get("classname", ""), item.get("name", "")) - if not all(key) or key in result: - raise ValueError("missing or duplicate JUnit test identity") - state = "failed" if item.find("failure") is not None or item.find("error") is not None else ( - "skipped" if item.find("skipped") is not None else "passed") - result[key] = state - if not result: - raise ValueError("empty JUnit report is not qualification") - return result - - -def compare(selected: dict, full_reports: list[dict], files: tuple[str, ...]) -> dict: - full = {} - for report in full_reports: - if set(full) & set(report): - raise ValueError("full shards contain duplicate test identities") - full.update(report) - prefixes = tuple(path.removesuffix(".py").replace("/", ".") for path in files) - for path, prefix in zip(files, prefixes): - if not any(key[0] == prefix or key[0].startswith(prefix + ".") for key in selected): - raise ValueError(f"selected file collected no tests: {path}") - missing = set(selected) - set(full) - omitted = {key for key in full if any(key[0] == prefix or key[0].startswith(prefix + ".") for prefix in prefixes)} - set(selected) - differences = {key for key in selected.keys() & full.keys() if selected[key] != full[key]} - selected_not_passed = {key for key, state in selected.items() if state != "passed"} - unselected_failures = {key for key, state in full.items() if state == "failed" and key not in selected} - report = { - "selected_test_count": len(selected), - "full_test_count": len(full), - "missing_from_full_count": len(missing), - "missing_from_selected_count": len(omitted), - "outcome_difference_count": len(differences), - "selected_not_passed_count": len(selected_not_passed), - "unselected_failure_count": len(unselected_failures), - "ok": not (missing or omitted or differences or selected_not_passed or unselected_failures), - "limitation": "Agreement on this revision does not prove the absence of future selection gaps.", - } - return report - - -def audit(packet: dict, selected_dir: Path, full_dir: Path, output: Path) -> int: - validate_shadow_plan(packet) - receipt = json.loads((selected_dir / "execution.json").read_text(encoding="utf-8")) - if receipt.get("schema_version") != "loopx_ci_shadow_execution_v1" or receipt.get("plan_sha256") != digest(packet): - raise ValueError("execution receipt belongs to a different plan") - if receipt.get("checkout_sha") != packet["checkout_sha"] or receipt.get("coverage_scope") != "selected_only": - raise ValueError("execution receipt has invalid revision or coverage provenance") - checks = receipt.get("checks", []) - if [item.get("check") for item in checks] != ["pytest", *VISION_SMOKES] or any(item.get("exit_code") != 0 for item in checks): - raise ValueError("selected tests or required real-entrypoint smokes did not succeed") - result = compare(cases(selected_dir / "selected.xml"), [ - cases(full_dir / f"python-junit-{shard}" / "junit.xml") for shard in (1, 2) - ], VISION_TESTS) - result.update({"schema_version": "loopx_ci_shadow_comparison_v1", "plan_sha256": digest(packet), - "checkout_sha": packet["checkout_sha"], "coverage_scope": "selected_only"}) - output.write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8") - print(json.dumps(result, sort_keys=True)) - return int(not result["ok"]) - - -def main() -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("operation", choices=("run", "audit")) - parser.add_argument("--plan", required=True) - parser.add_argument("--selected-dir", default="impact-results") - parser.add_argument("--full-dir", default="full-reports") - parser.add_argument("--output", default="impact-comparison.json") - args = parser.parse_args() - packet = json.loads(Path(args.plan).read_text(encoding="utf-8")) - if args.operation == "run": - return execute(packet, Path(args.selected_dir)) - return audit(packet, Path(args.selected_dir), Path(args.full_dir), Path(args.output)) - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/ci/module_shard.py b/scripts/ci/module_shard.py new file mode 100644 index 0000000000..92adb06167 --- /dev/null +++ b/scripts/ci/module_shard.py @@ -0,0 +1,40 @@ +"""Shard complete pytest modules; preserve ordered shared-fixture tests.""" +from collections import defaultdict + +import pytest + + +def assign_modules(counts: dict[str, int], shards: int) -> dict[str, int]: + """Deterministic largest-module-first balancing, without timing-cache authority.""" + if shards < 1: + raise ValueError("shards must be positive") + loads = [0] * shards + result = {} + for name, count in sorted(counts.items(), key=lambda item: (-item[1], item[0])): + slot = min(range(shards), key=lambda index: (loads[index], index)) + result[name] = slot + 1 + loads[slot] += count + return result + + +def pytest_addoption(parser): + parser.addoption("--ci-module-shards", type=int, default=1) + parser.addoption("--ci-module-shard", type=int, default=1) + + +@pytest.hookimpl(trylast=True) +def pytest_collection_modifyitems(config, items): + shards = config.getoption("--ci-module-shards") + shard = config.getoption("--ci-module-shard") + if shards < 1 or not 1 <= shard <= shards: + raise pytest.UsageError("invalid module shard coordinates") + counts = defaultdict(int) + for item in items: + counts[item.nodeid.split("::", 1)[0]] += 1 + assigned = assign_modules(counts, shards) + selected, deselected = [], [] + for item in items: + target = selected if assigned[item.nodeid.split("::", 1)[0]] == shard else deselected + target.append(item) + items[:] = selected + config.hook.pytest_deselected(items=deselected) diff --git a/scripts/ci/review_gate.py b/scripts/ci/review_gate.py index 39cbae1fed..9ca7e5ffbd 100644 --- a/scripts/ci/review_gate.py +++ b/scripts/ci/review_gate.py @@ -1,55 +1,45 @@ -"""Conservative PR classification and fail-closed core CI aggregation.""" +"""Per-job qualification: an intentional exemption is not an accidental skip.""" from __future__ import annotations import argparse import json import os -from impact_plan import Change, candidate, plan, write_plan +from impact_plan import Change, OUTPUTS, candidate, job_flags, plan, write_plan -CORE_JOBS = ( - "pytest", - "node-minimum-compatibility", - "stage2c-correctness-e2e", - "windows-powershell", -) +JOB_OUTPUT = { + "checks": "core_tests", + "pytest": "python_tests", + "node-minimum-compatibility": "core_tests", + "stage2c-correctness-e2e": "stage2c_tests", + "windows-powershell": "python_tests", + "presentation": "presentation_tests", +} +CORE_JOBS = tuple(JOB_OUTPUT) def requires_core_tests(paths: list[str]) -> bool: - # Unknown paths, executable documentation, policy and runtime prompts run CI. - # Disable rename detection at the caller so both old and new paths count. - if not paths: - return True return candidate([Change("M", path) for path in paths])[0] != "docs" -def verify(needs: object, *, shadow: bool = False) -> None: - expected_jobs = {"changes", *CORE_JOBS, *(["impact-shadow"] if shadow else [])} - if not isinstance(needs, dict) or set(needs) != expected_jobs: +def verify(needs: object) -> None: + if not isinstance(needs, dict) or set(needs) != {"changes", *CORE_JOBS}: raise ValueError("missing or unexpected merge-gate dependencies") changes = needs["changes"] if not isinstance(changes, dict) or changes.get("result") != "success": raise ValueError("change classification did not succeed") outputs = changes.get("outputs") - classification = outputs.get("core_tests") if isinstance(outputs, dict) else None - if classification not in ("true", "false"): - raise ValueError("missing or invalid core-test classification") - expected = "success" if classification == "true" else "skipped" - for name in CORE_JOBS: + if not isinstance(outputs, dict) or any(outputs.get(key) not in {"true", "false"} for key in OUTPUTS): + raise ValueError("missing or invalid job classification") + kind = outputs.get("change_kind") + expected = job_flags(kind, presentation=kind == "full" and outputs["presentation_tests"] == "true") + if any(outputs[key] != str(value).lower() for key, value in expected.items()): + raise ValueError("contradictory job exemptions") + for name, output in JOB_OUTPUT.items(): + required = "success" if outputs[output] == "true" else "skipped" job = needs[name] - if not isinstance(job, dict) or job.get("result") != expected: - raise ValueError(f"{name} must be {expected}") - if shadow: - profile = outputs.get("impact_profile") - if profile not in {"docs", "full", "vision"} or (profile == "docs") != (classification == "false"): - raise ValueError("missing or contradictory impact profile") - shadow_profile = outputs.get("shadow_profile") - if shadow_profile not in {"vision", "none"} or (profile == "docs" and shadow_profile != "none") or (profile == "vision" and shadow_profile != "vision"): - raise ValueError("missing or contradictory shadow profile") - expected_shadow = "success" if shadow_profile == "vision" else "skipped" - job = needs["impact-shadow"] - if not isinstance(job, dict) or job.get("result") != expected_shadow: - raise ValueError(f"impact-shadow must be {expected_shadow}") + if not isinstance(job, dict) or job.get("result") != required: + raise ValueError(f"{name} must be {required}") def main() -> None: @@ -58,20 +48,22 @@ def main() -> None: classify = sub.add_parser("classify") classify.add_argument("--base", required=True) classify.add_argument("--head", required=True) - classify.add_argument("--plan", help="write the non-authoritative impact plan") + classify.add_argument("--plan") classify.add_argument("--non-pr", action="store_true") - sub.add_parser("verify").add_argument("--shadow", action="store_true") + classify.add_argument("--force-full", action="store_true") + sub.add_parser("verify") args = parser.parse_args() if args.command == "classify": - packet = plan(args.base, args.head, pull_request=not args.non_pr) + packet = plan(args.base, args.head, pull_request=not args.non_pr, force_full=args.force_full) if args.plan: write_plan(packet, args.plan) - print(f"core_tests={str(packet['candidate_profile'] != 'docs').lower()}") - if args.plan: - print(f"impact_profile={packet['candidate_profile']}") - print(f"shadow_profile={packet['shadow_profile']}") + print(f"change_kind={packet['change_kind']}") + for key in OUTPUTS: + print(f"{key}={str(packet[key]).lower()}") + else: + print(f"core_tests={str(packet['core_tests']).lower()}") else: - verify(json.loads(os.environ["NEEDS_JSON"]), shadow=args.shadow) + verify(json.loads(os.environ["NEEDS_JSON"])) print("merge-gate: qualification complete") diff --git a/scripts/ci/test_impact_plan.py b/scripts/ci/test_impact_plan.py index c7104d9f46..80c71fc63a 100644 --- a/scripts/ci/test_impact_plan.py +++ b/scripts/ci/test_impact_plan.py @@ -1,8 +1,5 @@ -"""Selection rules are conservative; exact Git changes are the input oracle.""" - +"""Whole-PR boundaries, trusted policy and real Git negative controls.""" from __future__ import annotations - -import copy import json import os from pathlib import Path @@ -10,176 +7,92 @@ import sys import tempfile import unittest -from unittest.mock import patch - -from impact_plan import Change, VISION_SOURCES, VISION_TESTS, candidate, validate_shadow_plan -from review_gate import CORE_JOBS, verify - - -def gate(profile="vision", shadow="vision"): - core = profile != "docs" - return { - "changes": {"result": "success", "outputs": {"core_tests": str(core).lower(), - "impact_profile": profile, "shadow_profile": shadow}}, - **{name: {"result": "success" if core else "skipped"} for name in CORE_JOBS}, - "impact-shadow": {"result": "success" if shadow == "vision" else "skipped"}, - } - - -class ImpactRuleTests(unittest.TestCase): - def test_sources_select_cross_domain_consumers(self): - for source in VISION_SOURCES: - self.assertEqual(candidate([Change("M", source)])[0], "vision") - for required in ( - "tests/control_plane/test_refresh_state_replan_gate.py", - "tests/control_plane/test_goal_terminal_no_followup.py", - "tests/control_plane/test_quota_settlement_cli.py", - "tests/control_plane/test_vision_budget_cli.py", - ): - self.assertIn(required, VISION_TESTS) - self.assertEqual(len(VISION_TESTS), len(set(VISION_TESTS))) - - def test_additional_test_is_selected_but_new_runtime_is_full(self): - self.assertEqual(candidate([Change("A", VISION_TESTS[0])])[0], "vision") - self.assertEqual(candidate([Change("A", VISION_SOURCES[0])])[0], "full") - - def test_unknown_shared_and_ci_paths_cannot_hide_in_a_known_change(self): - for path in ( - "loopx/control_plane/runtime_decode.ts", "loopx/control_plane/effect_runtime_handlers.ts", - "loopx/control_plane/coordination/todo_update.ts", "loopx/cli.py", "tests/conftest.py", - "scripts/ci/impact_plan.py", ".github/workflows/python-tests.yml", "pyproject.toml", - "package-lock.json", "new-vision-checkpoint.ts", "docs/executable.py", "loopx/prompt.md", - ): +from impact_plan import Change, POLICY_PATHS, candidate + + +class ImpactTests(unittest.TestCase): + def test_presentation_and_docs_union(self): + for status in ("A", "M", "D"): + for path in ("apps/presentation/dashboard/src/App.tsx", "loopx/web/chat/assets/chat.js", + "apps/presentation/dashboard/public/icon.svg"): + self.assertEqual(candidate([Change(status, path), Change("M", "docs/design.md")])[0], "presentation") + + def test_one_shared_unknown_prompt_test_or_dependency_forces_full(self): + for path in ("loopx/cli.py", "loopx/control_plane/goals/vision_checkpoint.ts", + "loopx/claude_goal_mode/commands/loopx.md", "tests/test_ui.py", + "apps/presentation/dashboard/package-lock.json", "apps/presentation/dashboard/vite.config.ts", + "apps/desktop/src/main.rs", "scripts/ci/impact_plan.py", ".github/workflows/python-tests.yml", + "loopx/web/chat/backend.py", "loopx/web/chat/config.json", "unknown", "docs/build.py"): with self.subTest(path=path): - self.assertEqual(candidate([Change("M", VISION_SOURCES[0]), Change("M", path)])[0], "full") - - def test_deletion_type_change_rename_and_bad_status_are_full(self): - for status in ("D", "T", "R100", "U", ""): - self.assertEqual(candidate([Change(status, VISION_SOURCES[0])])[0], "full") - self.assertEqual(candidate([Change("D", VISION_SOURCES[0]), Change("A", "docs/copied.md")])[0], "full") + self.assertEqual(candidate([Change("M", "loopx/web/chat/index.html"), Change("M", path)])[0], "full") - def test_docs_maintain_the_existing_exemption_not_a_keyword_heuristic(self): - self.assertEqual(candidate([Change("M", "docs/vision.md"), Change("D", "README.md")])[0], "docs") - for path in ("docs/not-a-doc.ts", "docs/../loopx/code.md", "/docs/readme.md", "docs//readme.md"): - self.assertEqual(candidate([Change("M", path)])[0], "full") - self.assertEqual(candidate([])[0], "full") + def test_invalid_empty_non_pr_and_move_out_of_runtime_are_not_exempt(self): + for changes in ([], [Change("T", "docs/a.md")], [Change("M", "docs/../code.md")], + [Change("M", "/docs/a.md")], [Change("M", "docs//a.md")], + [Change("D", "loopx/code.py"), Change("A", "docs/code.md")]): + self.assertEqual(candidate(changes)[0], "full") + self.assertEqual(candidate([Change("M", "docs/a.md")], pull_request=False)[0], "full") - def test_main_and_scheduled_contexts_never_select_less(self): - for changes in ([], [Change("M", "docs/a.md")], [Change("M", VISION_SOURCES[0])]): - self.assertEqual(candidate(changes, pull_request=False)[0], "full") - - def test_full_gate_is_retained_for_every_candidate(self): - self.assertEqual(set(CORE_JOBS), {"pytest", "node-minimum-compatibility", "stage2c-correctness-e2e", "windows-powershell"}) - for profile, shadow in (("vision", "vision"), ("full", "vision"), ("full", "none"), ("docs", "none")): - verify(gate(profile, shadow), shadow=True) - for name in (*CORE_JOBS, "impact-shadow"): - for state in ("failure", "cancelled", "skipped", "neutral", None): - value = gate() - value[name]["result"] = state - with self.subTest(name=name, state=state), self.assertRaises(ValueError): - verify(value, shadow=True) - - def test_missing_and_contradictory_profiles_fail_closed(self): - for value in (gate("docs", "vision"), gate("vision", "none"), gate("invented", "none")): - with self.assertRaises(ValueError): - verify(value, shadow=True) - value = gate("docs", "none") - value["impact-shadow"]["result"] = "success" - with self.assertRaises(ValueError): - verify(value, shadow=True) - for field in ("core_tests", "impact_profile", "shadow_profile"): - value = gate() - del value["changes"]["outputs"][field] - with self.assertRaises(ValueError): - verify(value, shadow=True) - for name in ("impact-shadow", "changes", *CORE_JOBS): - value = gate() - del value[name] - with self.assertRaises(ValueError): - verify(value, shadow=True) - - -class RealGitImpactTests(unittest.TestCase): - def test_real_diff_and_cli_bind_revisions_and_preserve_all_paths(self): + def test_real_git_policy_force_full_and_type_boundaries(self): script = str(Path(__file__).with_name("review_gate.py")) with tempfile.TemporaryDirectory() as directory: root = Path(directory) - env = {key: value for key, value in os.environ.items() - if not key.startswith(("PYTEST", "COVERAGE", "COV_CORE"))} - + env = {k: v for k, v in os.environ.items() if not k.startswith(("PYTEST", "COVERAGE", "COV_CORE"))} def git(*args): return subprocess.check_output(["git", *args], cwd=root, env=env, text=True).strip() - def write(path, content): target = root / path target.parent.mkdir(parents=True, exist_ok=True) target.write_text(content) - - def classify(base, head="HEAD", *extra): - result = subprocess.run([sys.executable, script, "classify", "--base", base, "--head", head, - "--plan", str(root / "result.json"), *extra], cwd=root, env=env, capture_output=True, text=True) + def classify(base, *extra): + result = subprocess.run([sys.executable, script, "classify", "--base", base, "--head", "HEAD", + "--plan", str(root / "plan.json"), *extra], cwd=root, env=env, capture_output=True, text=True) self.assertEqual(result.returncode, 0, result.stderr) - return result.stdout, json.loads((root / "result.json").read_text()) - + return json.loads((root / "plan.json").read_text()), result.stdout git("init", "-q") git("config", "user.name", "CI Fixture") git("config", "user.email", "ci@example.invalid") git("config", "core.hooksPath", str(root / "no-hooks")) - write(VISION_SOURCES[0], "// baseline\n") - git("add", VISION_SOURCES[0]) - git("commit", "-qm", "baseline") + path = "loopx/web/chat/index.html" + write(path, "old") + for policy in POLICY_PATHS: + write(policy, "reviewed policy") + git("add", ".") + git("commit", "-qm", "base") base = git("rev-parse", "HEAD") - write(VISION_SOURCES[0], "// changed\n") - git("commit", "-qam", "vision change") - output, packet = classify(base) - self.assertIn("core_tests=true", output) - self.assertIn("impact_profile=vision", output) + write(path, "new") + git("commit", "-qam", "presentation") + packet, _ = classify(base) + self.assertEqual(packet["change_kind"], "presentation") + self.assertFalse(packet["python_tests"]) + self.assertFalse(packet["stage2c_tests"]) + self.assertTrue(packet["presentation_tests"]) self.assertEqual(packet["head_sha"], git("rev-parse", "HEAD")) - self.assertEqual(packet["merge_base_sha"], base) - self.assertEqual(packet["execution_mode"], "full_with_shadow") - self.assertFalse(packet["selection_is_merge_authority"]) - self.assertEqual(classify(base, "HEAD", "--non-pr")[1]["shadow_profile"], "none") - # A newline-bearing filename is one path, never a GITHUB_OUTPUT line. - strange = "unknown\ncore_tests=false" - write(strange, "fixture\n") + for flag in ("--force-full", "--non-pr"): + forced, _ = classify(base, flag) + self.assertTrue(forced["python_tests"]) + self.assertTrue(forced["stage2c_tests"]) + self.assertEqual(forced["python_shards"], 4) + # A policy change cannot authorize its own exemption. + write(POLICY_PATHS[0], "changed policy") + git("commit", "-qam", "policy") + self.assertEqual(classify(base)[0]["change_kind"], "full") + strange = "unknown\npython_tests=false" + write(strange, "fixture") git("add", strange) - git("commit", "-qm", "unknown path") - output, packet = classify(base) - self.assertNotIn("core_tests=false", output) + git("commit", "-qm", "unusual path") + packet, output = classify(base) + self.assertNotIn("python_tests=false", output) self.assertIn(strange, [item["path"] for item in packet["changes"]]) - self.assertEqual(packet["candidate_profile"], "full") - write("scripts/ci/impact_plan.py", "# policy changed\n") - git("add", "scripts/ci/impact_plan.py") - git("commit", "-qm", "policy qualification") - packet = classify(base)[1] - self.assertEqual((packet["candidate_profile"], packet["shadow_profile"]), ("full", "vision")) - before_rename = git("rev-parse", "HEAD") + before_link = git("rev-parse", "HEAD") (root / "docs").mkdir() - git("mv", VISION_SOURCES[0], "docs/moved.md") - git("commit", "-qm", "rename") - packet = classify(before_rename)[1] - self.assertEqual(packet["candidate_profile"], "full") - self.assertEqual({item["status"] for item in packet["changes"]}, {"A", "D"}) - failed = subprocess.run([sys.executable, script, "classify", "--base", "missing-base", "--head", "HEAD", - "--plan", str(root / "invalid.json")], cwd=root, env=env, capture_output=True, text=True) + (root / "docs/link.md").symlink_to("../loopx/web/chat/index.html") + git("add", "docs/link.md") + git("commit", "-qm", "symlink") + self.assertEqual(classify(before_link)[0]["change_kind"], "full") + failed = subprocess.run([sys.executable, script, "classify", "--base", "missing", "--head", "HEAD"], + cwd=root, env=env, capture_output=True) self.assertNotEqual(failed.returncode, 0) - self.assertFalse((root / "invalid.json").exists()) - - def test_plan_validation_rejects_wrong_revision_inventory_and_authority(self): - from impact_plan import SCHEMA, VISION_SMOKES - packet = {"schema_version": SCHEMA, "candidate_profile": "vision", "shadow_profile": "vision", - "execution_mode": "full_with_shadow", "selection_is_merge_authority": False, - "checkout_sha": "a" * 40, "base_sha": "b" * 40, "head_sha": "a" * 40, - "merge_base_sha": "b" * 40, "pytest_files": list(VISION_TESTS), "smoke_files": list(VISION_SMOKES)} - with patch("impact_plan.revision", return_value="a" * 40), patch("impact_plan.plan", return_value=packet), patch("impact_plan.Path.is_file", return_value=True), patch("impact_plan.Path.is_symlink", return_value=False): - validate_shadow_plan(packet) - for field, value in (("checkout_sha", "c" * 40), ("pytest_files", []), ("smoke_files", []), - ("selection_is_merge_authority", True), ("execution_mode", "targeted"), - ("head_sha", "HEAD"), ("base_sha", "c" * 40)): - changed = copy.deepcopy(packet) - changed[field] = value - with self.subTest(field=field), self.assertRaises(ValueError): - validate_shadow_plan(changed) if __name__ == "__main__": diff --git a/scripts/ci/test_impact_shadow.py b/scripts/ci/test_impact_shadow.py deleted file mode 100644 index 2f8f7bf155..0000000000 --- a/scripts/ci/test_impact_shadow.py +++ /dev/null @@ -1,110 +0,0 @@ -"""Negative or absent evidence cannot qualify a candidate test selection.""" - -from __future__ import annotations - -import json -from pathlib import Path -import tempfile -import unittest -from unittest.mock import patch - -from impact_plan import VISION_SMOKES, VISION_TESTS -from impact_shadow import audit, cases, compare, digest, execute - - -class ShadowComparisonTests(unittest.TestCase): - def test_same_cases_and_outcomes_match_without_claiming_full_coverage(self): - chosen = {("tests.test_a", "test_ok[x]"): "passed"} - report = compare(chosen, [chosen, {("tests.test_b", "test_other"): "passed"}], ("tests/test_a.py",)) - self.assertTrue(report["ok"]) - self.assertEqual(report["selected_test_count"], 1) - self.assertEqual(report["full_test_count"], 2) - self.assertIn("does not prove", report["limitation"]) - - def test_omissions_different_outcomes_and_unselected_failures_are_not_passes(self): - key, other = ("tests.test_a", "test_a"), ("tests.test_b", "test_b") - for selected, full, metric in ( - ({key: "passed"}, {other: "passed"}, "missing_from_full_count"), - ({key: "passed"}, {key: "failed"}, "outcome_difference_count"), - ({key: "skipped"}, {key: "skipped"}, "selected_not_passed_count"), - ({key: "failed"}, {key: "failed"}, "selected_not_passed_count"), - ({key: "passed"}, {key: "passed", other: "failed"}, "unselected_failure_count"), - ({key: "passed"}, {key: "passed", ("tests.test_a", "test_omitted"): "passed"}, "missing_from_selected_count"), - ): - with self.subTest(metric=metric): - result = compare(selected, [full], ("tests/test_a.py",)) - self.assertFalse(result["ok"]) - self.assertEqual(result[metric], 1) - - def test_empty_collection_and_duplicate_full_shards_fail(self): - with self.assertRaises(ValueError): - compare({}, [{}], ("tests/test_a.py",)) - chosen = {("tests.test_a", "test_a"): "passed"} - with self.assertRaises(ValueError): - compare(chosen, [chosen, chosen], ("tests/test_a.py",)) - with self.assertRaises(ValueError): - compare(chosen, [chosen], ("tests/test_a.py", "tests/test_b.py")) - - def test_junit_uses_identity_and_state_not_failure_text(self): - with tempfile.TemporaryDirectory() as directory: - path = Path(directory) / "junit.xml" - path.write_text('' - 'not exported' - '') - self.assertEqual(list(cases(path).values()), ["passed", "failed", "skipped"]) - for text in ('', '', '', - ''): - path.write_text(text) - with self.assertRaises(ValueError): - cases(path) - - def test_failed_command_keeps_negative_receipt_and_does_not_invent_success(self): - with tempfile.TemporaryDirectory() as directory, patch("impact_shadow.validate_shadow_plan"), patch("impact_shadow.subprocess.run") as run: - run.return_value.returncode = 1 - packet = {"checkout_sha": "a" * 40} - self.assertEqual(execute(packet, Path(directory)), 1) - receipt = json.loads((Path(directory) / "execution.json").read_text()) - self.assertEqual(receipt["coverage_scope"], "selected_only") - self.assertEqual([item["check"] for item in receipt["checks"]], ["pytest", *VISION_SMOKES]) - self.assertTrue(all(item["exit_code"] == 1 for item in receipt["checks"])) - - def test_wrong_plan_and_missing_smokes_cannot_use_a_green_junit(self): - with tempfile.TemporaryDirectory() as directory, patch("impact_shadow.validate_shadow_plan"): - root = Path(directory) - packet = {"checkout_sha": "a" * 40} - receipt = {"schema_version": "loopx_ci_shadow_execution_v1", "checkout_sha": "a" * 40, - "plan_sha256": digest(packet), "coverage_scope": "selected_only", "checks": []} - for change in ({}, {"plan_sha256": "wrong"}, {"coverage_scope": "full"}, {"checkout_sha": "b" * 40}): - (root / "execution.json").write_text(json.dumps({**receipt, **change})) - with self.assertRaises(ValueError): - audit(packet, root, root, root / "comparison.json") - self.assertFalse((root / "comparison.json").exists()) - - def test_real_report_audit_requires_both_shards_and_complete_inventory(self): - with tempfile.TemporaryDirectory() as directory, patch("impact_shadow.validate_shadow_plan"): - root = Path(directory) - packet = {"checkout_sha": "a" * 40} - receipt = {"schema_version": "loopx_ci_shadow_execution_v1", "checkout_sha": "a" * 40, - "plan_sha256": digest(packet), "coverage_scope": "selected_only", - "checks": [{"check": name, "exit_code": 0} for name in ("pytest", *VISION_SMOKES)]} - (root / "execution.json").write_text(json.dumps(receipt)) - rows = [f'' for path in VISION_TESTS] - - def report(path, cases): - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text("" + "".join(cases) + "") - - report(root / "selected.xml", rows) - report(root / "python-junit-1/junit.xml", rows[:1]) - with self.assertRaises(FileNotFoundError): - audit(packet, root, root, root / "result.json") - report(root / "python-junit-2/junit.xml", rows[1:]) - self.assertEqual(audit(packet, root, root, root / "result.json"), 0) - result = json.loads((root / "result.json").read_text()) - self.assertTrue(result["ok"]) - self.assertEqual(result["selected_test_count"], len(VISION_TESTS)) - self.assertEqual(result["coverage_scope"], "selected_only") - - -if __name__ == "__main__": - unittest.main() diff --git a/scripts/ci/test_review_gate.py b/scripts/ci/test_review_gate.py index 7b7e9c7d82..0c71c9cbea 100644 --- a/scripts/ci/test_review_gate.py +++ b/scripts/ci/test_review_gate.py @@ -1,131 +1,63 @@ -"""The required gate must not report success for absent or skipped core work.""" - +"""An intentionally exempt job must be skipped; all required jobs must pass.""" from __future__ import annotations - import copy -import os -from pathlib import Path -import subprocess -import sys -import tempfile import unittest - -from review_gate import CORE_JOBS, requires_core_tests, verify +from impact_plan import job_flags +from review_gate import CORE_JOBS, JOB_OUTPUT, requires_core_tests, verify -def needs(core: bool = True) -> dict: - return { - "changes": {"result": "success", "outputs": {"core_tests": str(core).lower()}}, - **{job: {"result": "success" if core else "skipped"} for job in CORE_JOBS}, - } +def needs(kind="full", *, presentation=False): + flags = job_flags(kind, presentation=presentation) + return {"changes": {"result": "success", "outputs": {"change_kind": kind, + **{key: str(value).lower() for key, value in flags.items()}}}, + **{job: {"result": "success" if flags[output] else "skipped"} for job, output in JOB_OUTPUT.items()}} -class ReviewGateTests(unittest.TestCase): - def test_documentation_only(self): +class GateTests(unittest.TestCase): + def test_legal_profiles_and_policy_rehearsal(self): + for kind in ("docs", "presentation", "full"): + verify(needs(kind)) + verify(needs("full", presentation=True)) self.assertFalse(requires_core_tests(["README.md", "docs/guide.md"])) - - def test_unknown_and_executable_paths_require_tests(self): - for path in ( - "loopx/prompt.md", "docs/build.py", "AGENTS.md", ".github/CODEOWNERS", - ".github/workflows/python-tests.yml", "scripts/ci/review_gate.py", - "packages/new-package/index.ts", "new-config.json", "docs/demo.js", - ): - with self.subTest(path=path): - self.assertTrue(requires_core_tests(["docs/guide.md", path])) - - def test_empty_diff_is_not_an_exemption(self): + self.assertTrue(requires_core_tests(["loopx/prompt.md"])) self.assertTrue(requires_core_tests([])) - def test_rename_out_of_code_is_not_an_exemption(self): - self.assertTrue(requires_core_tests(["loopx/code.py", "docs/code.md"])) - - def test_core_success(self): - verify(needs()) - - def test_explicit_documentation_skip(self): - verify(needs(False)) - - def test_every_unsuccessful_core_result_is_rejected(self): - for job in CORE_JOBS: - for result in ("skipped", "failure", "cancelled", "neutral", "", None): - value = needs() - value[job]["result"] = result - with self.subTest(job=job, result=result), self.assertRaises(ValueError): + def test_every_missing_failure_cancel_or_unexpected_skip_is_rejected(self): + for kind in ("docs", "presentation", "full"): + good = needs(kind) + for job in CORE_JOBS: + for state in ("success", "skipped", "failure", "cancelled", "neutral", None): + if state == good[job]["result"]: + continue + value = copy.deepcopy(good) + value[job]["result"] = state + with self.subTest(kind=kind, job=job, state=state), self.assertRaises(ValueError): + verify(value) + for job in ("changes", *CORE_JOBS): + value = copy.deepcopy(good) + del value[job] + with self.assertRaises(ValueError): verify(value) - def test_missing_or_extra_dependencies_are_rejected(self): - for job in ("changes", *CORE_JOBS): - value = needs() - del value[job] - with self.subTest(job=job), self.assertRaises(ValueError): - verify(value) + def test_bad_and_contradictory_classification_cannot_skip_work(self): + for kind in ("docs", "presentation", "full"): + for field in ("change_kind", "core_tests", "python_tests", "stage2c_tests", "presentation_tests"): + for replacement in (None, "", True, "unknown"): + value = needs(kind) + value["changes"]["outputs"][field] = replacement + with self.assertRaises((ValueError, TypeError)): + verify(value) + value = needs("full") + value["changes"]["outputs"]["stage2c_tests"] = "false" with self.assertRaises(ValueError): - verify({**needs(), "extra": {"result": "success"}}) - - def test_invalid_classification_is_rejected(self): - for result in ("failure", "cancelled", "skipped"): + verify(value) + for state in ("failure", "skipped", "cancelled"): value = needs() - value["changes"]["result"] = result + value["changes"]["result"] = state with self.assertRaises(ValueError): verify(value) - for output in ({}, {"core_tests": ""}, {"core_tests": True}, None): - value = needs() - value["changes"]["outputs"] = output - with self.assertRaises(ValueError): - verify(value) - - def test_docs_cannot_hide_a_failed_job(self): - for job in CORE_JOBS: - value = copy.deepcopy(needs(False)) - value[job]["result"] = "failure" - with self.assertRaises(ValueError): - verify(value) - - def test_real_git_diff_handles_code_deletion_and_missing_base(self): - script = str(Path(__file__).with_name("review_gate.py")) - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - # pytest-cov injects subprocess startup variables. The synthetic - # checkout must not contribute temporary source paths to real CI. - env = {key: value for key, value in os.environ.items() - if not key.startswith(("PYTEST", "COVERAGE", "COV_CORE"))} - - def git(*args): - return subprocess.check_output( - ["git", *args], cwd=root, env=env, text=True - ).strip() - - git("init", "-q") - git("config", "user.name", "CI Fixture") - git("config", "user.email", "ci@example.invalid") - git("config", "core.hooksPath", str(root / "no-hooks")) - (root / "docs").mkdir() - (root / "loopx").mkdir() - (root / "docs/readme.md").write_text("baseline\n") - (root / "loopx/code.py").write_text("pass\n") - git("add", "docs/readme.md", "loopx/code.py") - git("commit", "-qm", "baseline") - base = git("rev-parse", "HEAD") - (root / "docs/readme.md").write_text("documentation only\n") - git("commit", "-qam", "docs") - result = subprocess.check_output( - [sys.executable, script, "classify", "--base", base, "--head", "HEAD"], - cwd=root, env=env, text=True, - ) - self.assertEqual(result.strip(), "core_tests=false") - git("mv", "loopx/code.py", "docs/code.md") - git("commit", "-qm", "move code into docs") - result = subprocess.check_output( - [sys.executable, script, "classify", "--base", base, "--head", "HEAD"], - cwd=root, env=env, text=True, - ) - self.assertEqual(result.strip(), "core_tests=true") - failed = subprocess.run( - [sys.executable, script, "classify", "--base", "missing-ref", "--head", "HEAD"], - cwd=root, env=env, capture_output=True, text=True, - ) - self.assertNotEqual(failed.returncode, 0) - self.assertNotIn("core_tests=false", failed.stdout) + with self.assertRaises(ValueError): + verify({**needs(), "extra": {"result": "success"}}) if __name__ == "__main__": diff --git a/tests/test_python_ci_workflow.py b/tests/test_python_ci_workflow.py index 6648a50ea3..8b8fea7f9f 100644 --- a/tests/test_python_ci_workflow.py +++ b/tests/test_python_ci_workflow.py @@ -11,9 +11,8 @@ import pytest -WORKFLOW = ( - Path(__file__).resolve().parents[1] / ".github" / "workflows" / "python-tests.yml" -).read_text(encoding="utf-8") +WORKFLOW_ROOT = Path(__file__).resolve().parents[1] +WORKFLOW = (WORKFLOW_ROOT / ".github/workflows/python-tests.yml").read_text(encoding="utf-8") @pytest.mark.parametrize("result", ["success", "failure", "cancelled", "skipped", ""]) @@ -21,7 +20,7 @@ def test_stage2c_gate_requires_all_lanes(result: str) -> None: gate = WORKFLOW.split(" stage2c-correctness-e2e:", 1)[1].split(" windows-powershell:", 1)[0] assert "if: always()" in gate assert "needs: [changes, stage2c-suite]" in gate - assert "if: always() && needs.changes.outputs.core_tests == 'true'" in gate + assert "if: always() && needs.changes.outputs.stage2c_tests == 'true'" in gate assert "STAGE2C_RESULT: ${{ needs.stage2c-suite.result }}" in gate script = gate.split("run: ", 1)[1].strip() actual = subprocess.run( @@ -31,8 +30,9 @@ def test_stage2c_gate_requires_all_lanes(result: str) -> None: assert (actual.returncode == 0) == (result == "success") suite = WORKFLOW.split(" stage2c-suite:", 1)[1].split(" stage2c-correctness-e2e:", 1)[0] assert "fail-fast: false" in suite - assert "suite: [e2e, mutants, installed]" in suite - assert "if: needs.changes.outputs.core_tests == 'true'" in suite.split(" steps:", 1)[0] + for entry in ("{suite: e2e, shard: 1}", "{suite: e2e, shard: 2}", "{suite: mutants, shard: 0}", "{suite: installed, shard: 0}"): + assert entry in suite + assert "if: needs.changes.outputs.stage2c_tests == 'true'" in suite.split(" steps:", 1)[0] steps = {step.splitlines()[0]: step for step in suite.split(" - name: ")[1:]} for name, lane in [ ("Qualify real CLI, mixed writers, process death, and recovery", "e2e"), @@ -45,44 +45,58 @@ def test_stage2c_gate_requires_all_lanes(result: str) -> None: assert "--case" not in steps["Reject deliberate correctness regressions"] artifact = steps["Retain bounded acceptance evidence"] assert "if: always()" in artifact - assert "name: stage2c-correctness-evidence-${{ matrix.suite }}" in artifact + assert "name: stage2c-correctness-evidence-${{ matrix.suite }}-${{ matrix.shard }}" in artifact assert "if-no-files-found: error" in artifact def test_stage2c_workers_preserve_module_state_and_execute_every_row(tmp_path: Path) -> None: step = WORKFLOW.split("name: Qualify real CLI, mixed writers, process death, and recovery", 1)[1] command = step.split("run: ", 1)[1].splitlines()[0] - args = shlex.split(command) - args[0] = sys.executable - # Actual workflow command against two modules with order-sensitive shared state. - # Per-test distribution would break the module fixture's accumulated state. - for name in ("first", "second"): - (tmp_path / f"test_{name}.py").write_text( - "import os\nfrom pathlib import Path\nimport pytest\n" - "pytestmark = pytest.mark.stage2c_e2e\n" - "@pytest.fixture(scope='module')\ndef state():\n return []\n" - "@pytest.mark.parametrize('row', range(4))\n" - "def test_order(state, row):\n" - " assert state == list(range(row))\n state.append(row)\n" - f" with Path('{name}.visits').open('a') as stream:\n" - " stream.write(f'{os.getpid()}:{row}\\n')\n", - encoding="utf-8", - ) - (tmp_path / "pytest.ini").write_text("[pytest]\nmarkers = stage2c_e2e\n") env = {key: value for key, value in os.environ.items() if not key.startswith(("PYTEST", "COVERAGE", "COV_CORE"))} - actual = subprocess.run(args, cwd=tmp_path, env=env, capture_output=True, - text=True, timeout=60, check=False) - assert actual.returncode == 0, actual.stdout + actual.stderr - cases = ET.parse(tmp_path / "stage2c-e2e.xml").findall(".//testcase") - assert len(cases) == 8 - workers = set() - for name in ("first", "second"): - rows = [line.split(":") for line in (tmp_path / f"{name}.visits").read_text().splitlines()] - assert [row for _, row in rows] == ["0", "1", "2", "3"] - assert len({pid for pid, _ in rows}) == 1 - workers.add(rows[0][0]) - assert len(workers) == 2 + env["PYTHONPATH"] = str(WORKFLOW_ROOT) + visited = set() + for shard in (1, 2): + root = tmp_path / f"shard-{shard}" + root.mkdir() + args = shlex.split(command.replace("${{ matrix.shard }}", str(shard))) + args[0] = sys.executable + for name in ("first", "second", "third", "fourth"): + (root / f"test_{name}.py").write_text( + "import os\nfrom pathlib import Path\nimport pytest\n" + "pytestmark = pytest.mark.stage2c_e2e\n" + "@pytest.fixture(scope='module')\ndef state():\n return []\n" + "@pytest.mark.parametrize('row', range(4))\n" + "def test_order(state, row):\n" + " assert state == list(range(row))\n state.append(row)\n" + f" with Path('{name}.visits').open('a') as stream:\n" + " stream.write(f'{os.getpid()}:{row}\\n')\n", encoding="utf-8") + (root / "pytest.ini").write_text("[pytest]\nmarkers = stage2c_e2e\n") + actual = subprocess.run(args, cwd=root, env=env, capture_output=True, + text=True, timeout=60, check=False) + assert actual.returncode == 0, actual.stdout + actual.stderr + assert len(ET.parse(root / "stage2c-e2e.xml").findall(".//testcase")) == 8 + workers = set() + for report in root.glob("*.visits"): + assert report.stem not in visited + visited.add(report.stem) + rows = [line.split(":") for line in report.read_text().splitlines()] + assert [row for _, row in rows] == ["0", "1", "2", "3"] + assert len({pid for pid, _ in rows}) == 1 + workers.add(rows[0][0]) + assert len(workers) == 2 + assert visited == {"first", "second", "third", "fourth"} + + +def test_module_sharding_is_deterministic_and_balances_complete_modules() -> None: + from scripts.ci.module_shard import assign_modules + counts = {"large": 100, "medium": 60, "small": 40, "tiny": 1} + expected = {"large": 1, "medium": 2, "small": 2, "tiny": 1} + assert assign_modules(counts, 2) == expected + assert assign_modules(dict(reversed(list(counts.items()))), 2) == expected + assert assign_modules(counts, 1) == dict.fromkeys(counts, 1) + with pytest.raises(ValueError): + assign_modules(counts, 0) @pytest.mark.parametrize("checks", ["success", "failure", "cancelled", "skipped"]) @@ -100,7 +114,7 @@ def test_required_pytest_check_rejects_incomplete_upstream_jobs( check=False, ) assert (result.returncode == 0) == (checks == shards == "success") - assert "if: always() && needs.changes.outputs.core_tests == 'true'" in WORKFLOW + assert "if: always() && needs.changes.outputs.python_tests == 'true'" in WORKFLOW assert "needs: [changes, checks, test-shard]" in WORKFLOW @@ -110,40 +124,36 @@ def test_merge_gate_runs_on_all_prs_and_checks_every_core_aggregate() -> None: gate = WORKFLOW.split(" merge-gate:", 1)[1] assert "if: always()" in gate assert ( - "needs: [changes, pytest, node-minimum-compatibility, " - "stage2c-correctness-e2e, windows-powershell, impact-shadow]" + "needs: [changes, checks, pytest, node-minimum-compatibility, " + "stage2c-correctness-e2e, windows-powershell, presentation]" ) in gate assert "NEEDS_JSON: ${{ toJSON(needs) }}" in gate - assert "run: python scripts/ci/review_gate.py verify --shadow" in gate + assert "run: python scripts/ci/review_gate.py verify" in gate assert "continue-on-error" not in gate - for name in ("checks", "test-shard", "stage2c-suite", "windows-powershell"): + for name, output in (("checks", "core_tests"), ("test-shard", "python_tests"), ("stage2c-suite", "stage2c_tests"), ("windows-powershell", "python_tests"), ("presentation", "presentation_tests")): job = WORKFLOW.split(f" {name}:\n", 1)[1].split(" steps:", 1)[0] assert "needs: changes" in job - assert "if: needs.changes.outputs.core_tests == 'true'" in job - - -def test_shadow_workflow_keeps_full_coverage_and_requires_exact_artifacts() -> None: - selected = WORKFLOW.split(" impact-tests:\n", 1)[1].split(" impact-shadow:\n", 1)[0] - audit = WORKFLOW.split(" impact-shadow:\n", 1)[1].split(" pytest:\n", 1)[0] - assert "if: needs.changes.outputs.shadow_profile == 'vision'" in selected - assert "impact_shadow.py run --plan impact-plan.json" in selected - assert "--cov" not in selected and "coverage-xml" not in selected - assert "needs: [changes, impact-tests, test-shard]" in audit - assert "if: always() && needs.changes.outputs.shadow_profile == 'vision'" in audit - for artifact in ("ci-impact-plan", "ci-impact-selected", "python-junit-*"): - assert artifact in audit - assert "impact_shadow.py audit --plan impact-plan.json" in audit - assert 'test "$SELECTED_RESULT" = success' in audit - assert 'test "$FULL_RESULT" = success' in audit - assert "continue-on-error" not in selected + audit - assert "--junitxml=junit.xml" in WORKFLOW - assert "name: python-junit-${{ matrix.shard }}" in WORKFLOW - - -def test_two_shards_execute_each_test_once_and_merge_portable_coverage( + assert f"if: needs.changes.outputs.{output} == 'true'" in job + + +def test_presentation_exemption_retains_real_frontend_checks_and_force_full() -> None: + job = WORKFLOW.split(" presentation:\n", 1)[1].split(" merge-gate:\n", 1)[0] + assert "npm run build:chat" in job + assert "npm run smoke:personal-workspace-packaged" in job + assert "status --short --untracked-files=all -- loopx/web/chat" in job + assert "continue-on-error" not in job + assert "labels.*.name, 'ci:full'" in WORKFLOW + assert "labeled, unlabeled" in WORKFLOW + assert "workflow_dispatch:" in WORKFLOW + assert "--force-full" in WORKFLOW + assert "impact-shadow" not in WORKFLOW + assert "matrix:\n shard: [1, 2, 3, 4]" in WORKFLOW + + +def test_four_shards_execute_each_test_once_and_merge_portable_coverage( tmp_path: Path, ) -> None: - # Real pytest-split + xdist + coverage, in two distinct checkout roots. + # Real pytest-split + xdist + coverage, in four distinct checkout roots. # Each shard alone misses a function; their union must cover the whole file. shard_step = WORKFLOW.split("name: Run test shard", 1)[1] template = shard_step.split("run: >-", 1)[1].split(" - name:", 1)[0] @@ -152,7 +162,7 @@ def test_two_shards_execute_each_test_once_and_merge_portable_coverage( if not key.startswith(("COVERAGE", "COV_CORE", "PYTEST")) } seen: list[set[str]] = [] - for shard in (1, 2): + for shard in (1, 2, 3, 4): root = tmp_path / f"checkout-{shard}" root.mkdir() (root / "ci_subject.py").write_text( @@ -161,8 +171,10 @@ def test_two_shards_execute_each_test_once_and_merge_portable_coverage( ) (root / "test_subject.py").write_text( "from ci_subject import first, second\n" - "def test_first():\n assert first() == 1\n" - "def test_second():\n assert second() == 2\n", + "def test_first_one():\n assert first() == 1\n" + "def test_first_two():\n assert first() == 1\n" + "def test_second_one():\n assert second() == 2\n" + "def test_second_two():\n assert second() == 2\n", encoding="utf-8", ) (root / "pyproject.toml").write_text( @@ -188,8 +200,9 @@ def test_two_shards_execute_each_test_once_and_merge_portable_coverage( destination.mkdir(parents=True) (root / ".coverage").rename(destination / ".coverage") - assert seen[0] and seen[1] and seen[0].isdisjoint(seen[1]) - assert seen[0] | seen[1] == {"test_first", "test_second"} + assert all(seen) + assert sum(map(len, seen)) == len(set.union(*seen)) == 4 + assert set.union(*seen) == {"test_first_one", "test_first_two", "test_second_one", "test_second_two"} # Reuse the real aggregate shell commands, with a 100% synthetic oracle. step = WORKFLOW.split("name: Combine complete coverage", 1)[1] script = step.split("run: |", 1)[1].split(" - uses:", 1)[0] @@ -197,7 +210,7 @@ def test_two_shards_execute_each_test_once_and_merge_portable_coverage( script = script.replace("--fail-under=19.6", "--fail-under=100") root = tmp_path / "checkout-1" (tmp_path / "coverage-shards").rename(root / "coverage-shards") - for shard in (1, 2): + for shard in (1, 2, 3, 4): data = root / "coverage-shards" / f"python-coverage-{shard}" / ".coverage" held = data.with_name("held") data.rename(held) @@ -220,6 +233,6 @@ def test_two_shards_execute_each_test_once_and_merge_portable_coverage( capture_output=True, check=False, ) assert missing.returncode != 0 - assert re.search(r"shard: \[1, 2\]", WORKFLOW) + assert re.search(r"shard: \[1, 2, 3, 4\]", WORKFLOW) assert "include-hidden-files: true" in WORKFLOW assert "--cov-fail-under" not in template