diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index cb3a7e1247..4189a98b22 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -2,6 +2,8 @@ name: Python Tests on: pull_request: + types: [opened, synchronize, reopened, ready_for_review, labeled, unlabeled] + workflow_dispatch: push: branches: - main @@ -32,6 +34,10 @@ jobs: timeout-minutes: 3 outputs: core_tests: ${{ steps.classify.outputs.core_tests }} + change_kind: ${{ steps.classify.outputs.change_kind }} + python_tests: ${{ steps.classify.outputs.python_tests }} + stage2c_tests: ${{ steps.classify.outputs.stage2c_tests }} + presentation_tests: ${{ steps.classify.outputs.presentation_tests }} steps: - uses: actions/checkout@v7 with: @@ -40,21 +46,42 @@ jobs: with: python-version: "3.11" - name: Validate merge gate semantics - run: python -m unittest discover -s scripts/ci -p 'test_review_gate.py' + run: python -m unittest discover -s scripts/ci -p 'test_*.py' - name: Classify the exact pull-request change id: classify env: EVENT_NAME: ${{ github.event_name }} BASE_SHA: ${{ github.event.pull_request.base.sha }} HEAD_SHA: ${{ github.event.pull_request.head.sha }} + CHECKOUT_SHA: ${{ github.sha }} + FORCE_FULL: ${{ contains(github.event.pull_request.labels.*.name, 'ci:full') }} shell: bash run: | set -euo pipefail + extra=() + if [[ "$FORCE_FULL" == true ]]; then extra+=(--force-full); fi if [[ "$EVENT_NAME" == pull_request ]]; then - python scripts/ci/review_gate.py classify --base "$BASE_SHA" --head "$HEAD_SHA" >> "$GITHUB_OUTPUT" + python scripts/ci/review_gate.py classify --base "$BASE_SHA" --head "$HEAD_SHA" --plan impact-plan.json "${extra[@]}" >> "$GITHUB_OUTPUT" else - echo 'core_tests=true' >> "$GITHUB_OUTPUT" + python scripts/ci/review_gate.py classify --base "$CHECKOUT_SHA" --head "$CHECKOUT_SHA" --non-pr --plan impact-plan.json >> "$GITHUB_OUTPUT" fi + python - <<'PY' + import json, os + from pathlib import Path + plan = json.loads(Path("impact-plan.json").read_text()) + with open(os.environ["GITHUB_STEP_SUMMARY"], "a") as summary: + summary.write("## CI job exemptions\n\n") + summary.write(f"Change kind: **{plan['change_kind']}**; Python shards: **{plan['python_shards']}**.\n\n") + summary.write(plan['reason'] + ".\n\n") + summary.write(f"Python: {plan['python_tests']}; Stage2c: {plan['stage2c_tests']}; frontend: {plan['presentation_tests']}.\n\n") + summary.write(plan["coverage_scope"] + ". Unknown/mixed changes and main run full. Label ci:full forces full qualification.\n") + PY + - uses: actions/upload-artifact@v7 + with: + name: ci-impact-plan + path: impact-plan.json + if-no-files-found: error + retention-days: 7 checks: needs: changes @@ -193,13 +220,13 @@ jobs: test-shard: needs: changes - if: needs.changes.outputs.core_tests == 'true' + if: needs.changes.outputs.python_tests == 'true' runs-on: ubuntu-latest timeout-minutes: 30 strategy: fail-fast: false matrix: - shard: [1, 2] + shard: [1, 2, 3, 4] steps: - uses: actions/checkout@v7 with: @@ -223,11 +250,20 @@ jobs: # Each runner retains the measured two-worker pool. run: >- python -m pytest -q -n 2 -m "not stage2c_e2e" - --splits 2 --group ${{ matrix.shard }} + --splits 4 --group ${{ matrix.shard }} --splitting-algorithm least_duration --durations=25 --durations-min=1 + --junitxml=junit.xml --cov=loopx --cov-report=term + - name: Upload full-shard outcomes for selection audit + if: always() + uses: actions/upload-artifact@v7 + with: + name: python-junit-${{ matrix.shard }} + path: junit.xml + if-no-files-found: error + retention-days: 7 - name: Upload shard coverage uses: actions/upload-artifact@v7 with: @@ -239,7 +275,7 @@ jobs: pytest: # Keep the required check name; a skipped/failed shard must not turn it green. - if: always() && needs.changes.outputs.core_tests == 'true' + if: always() && needs.changes.outputs.python_tests == 'true' needs: [changes, checks, test-shard] runs-on: ubuntu-latest timeout-minutes: 10 @@ -263,9 +299,13 @@ jobs: path: coverage-shards - name: Combine complete coverage and enforce the existing floor run: | - test -s coverage-shards/python-coverage-1/.coverage - test -s coverage-shards/python-coverage-2/.coverage - python -m coverage combine coverage-shards/python-coverage-1/.coverage coverage-shards/python-coverage-2/.coverage + shards=() + for shard in 1 2 3 4; do + path="coverage-shards/python-coverage-${shard}/.coverage" + test -s "$path" + shards+=("$path") + done + python -m coverage combine "${shards[@]}" python -m coverage report --fail-under=19.6 python -m coverage xml -o coverage.xml - uses: actions/upload-artifact@v7 @@ -283,14 +323,18 @@ jobs: stage2c-suite: needs: changes - if: needs.changes.outputs.core_tests == 'true' - name: stage2c (${{ matrix.suite }}) + if: needs.changes.outputs.stage2c_tests == 'true' + name: stage2c (${{ matrix.suite }} ${{ matrix.shard }}) runs-on: ubuntu-latest timeout-minutes: 30 strategy: fail-fast: false matrix: - suite: [e2e, mutants, installed] + include: + - {suite: e2e, shard: 1} + - {suite: e2e, shard: 2} + - {suite: mutants, shard: 0} + - {suite: installed, shard: 0} steps: - uses: actions/checkout@v7 - uses: actions/setup-python@v6 @@ -332,7 +376,7 @@ jobs: env: LOOPX_SHADOW_COMPARISON_OUTPUT: .local/stage2c-observables # Keep each module's shared workspace and ordered parity rows on one worker. - run: python -m pytest -q -n 4 --dist loadfile -m stage2c_e2e --durations=20 --junitxml=stage2c-e2e.xml + run: python -m pytest -q -n 2 --dist loadfile -m stage2c_e2e -p scripts.ci.module_shard --ci-module-shards 2 --ci-module-shard ${{ matrix.shard }} --durations=20 --junitxml=stage2c-e2e.xml - name: Reject deliberate correctness regressions if: matrix.suite == 'mutants' run: python examples/shared-goal-authority-e2e/mutants.py --output .local/stage2c-mutants @@ -349,7 +393,7 @@ jobs: if: always() uses: actions/upload-artifact@v7 with: - name: stage2c-correctness-evidence-${{ matrix.suite }} + name: stage2c-correctness-evidence-${{ matrix.suite }}-${{ matrix.shard }} include-hidden-files: true if-no-files-found: error path: | @@ -361,7 +405,7 @@ jobs: stage2c-correctness-e2e: # Preserve the public check name and reject failed, cancelled or skipped lanes. - if: always() && needs.changes.outputs.core_tests == 'true' + if: always() && needs.changes.outputs.stage2c_tests == 'true' needs: [changes, stage2c-suite] runs-on: ubuntu-latest timeout-minutes: 2 @@ -373,7 +417,7 @@ jobs: windows-powershell: needs: changes - if: needs.changes.outputs.core_tests == 'true' + if: needs.changes.outputs.python_tests == 'true' runs-on: windows-latest timeout-minutes: 20 steps: @@ -426,10 +470,39 @@ jobs: tests/control_plane_ts/local_authority_provider.test.ts tests/control_plane_ts/sqlite_runtime_admission.test.ts + presentation: + needs: changes + if: needs.changes.outputs.presentation_tests == 'true' + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-python@v6 + with: + python-version: "3.11" + cache: pip + - uses: actions/setup-node@v6 + with: + node-version: "24" + cache: npm + cache-dependency-path: apps/presentation/dashboard/package-lock.json + - run: python -m pip install --disable-pip-version-check -e ".[test]" + - name: Build and test the actual packaged Dashboard + working-directory: apps/presentation/dashboard + run: | + npm ci --ignore-scripts + npm run build:chat + if [[ -n "$(git -C ../../.. status --short --untracked-files=all -- loopx/web/chat)" ]]; then + echo "Packaged Dashboard differs from its source build." >&2 + exit 1 + fi + ./node_modules/.bin/playwright install --with-deps chromium + npm run smoke:personal-workspace-packaged + merge-gate: # Always publish one stable outcome, including documentation-only PRs. if: always() - needs: [changes, pytest, node-minimum-compatibility, stage2c-correctness-e2e, windows-powershell] + needs: [changes, checks, pytest, node-minimum-compatibility, stage2c-correctness-e2e, windows-powershell, presentation] runs-on: ubuntu-latest timeout-minutes: 3 steps: diff --git a/docs/development/ci-impact-selection.md b/docs/development/ci-impact-selection.md new file mode 100644 index 0000000000..61a57d3fe0 --- /dev/null +++ b/docs/development/ci-impact-selection.md @@ -0,0 +1,88 @@ +# CI job exemptions / 按职责免跑重型 CI + +## Policy, not a hand-maintained test selection + +Every PR receives the same stable merge gate. The classifier reads the complete +NUL-delimited Git diff at immutable base/head revisions; a PR title, label or +author description cannot claim that runtime changes are “just UI”. + +| Whole PR | Common TS/lint/contracts | Full Python / Windows | Stage2c | Packaged Dashboard | +| --- | --- | --- | --- | --- | +| Existing Markdown-only documentation exemption | Skip | Skip | Skip | Existing Frontstage workflow | +| Client Dashboard source/assets only, optionally with docs | Run | Skip | Skip | Required build, freshness and browser smoke | +| Backend, prompt, tests, dependency, build, CI policy, mixed or unknown | Run | Run | Run | Also run for CI-policy rehearsal or forced-full UI | +| main push / manual run | Run | Run | Run | Existing surface workflows; CI-policy rehearsal when applicable | + +The presentation boundary is deliberately small: Dashboard `src/`, `public/` +and packaged `loopx/web/chat/`, with explicit client-code/image/font extensions. +Package manifests, Vite/build configuration, native desktop code, backend Python +and arbitrary JSON are not exempt. Both sides of renames are classified; moving +runtime code into a UI directory stays full. Symlinks/type changes cannot qualify. + +The new exemption is available only when the selector, gate and workflow blobs +match the already-reviewed target branch. Changing CI policy cannot exempt its +own PR. Missing policy or uncertain ownership runs full; missing Git revisions +fail classification. The pre-existing documentation exemption remains supported. + +本方案不是为每类 PR 维护一套测试清单，而是明确重型 job 的职责。纯前端变化不需要 +重跑后端持久化与崩溃恢复矩阵，但前端自己的实际构建与浏览器验收成为必需项。 +预算、静态宿主 prompt 和 Python/TS 逻辑暂不享受免跑；它们仍可能改变核心行为。 +新增一种豁免只需审阅其业务边界和保留的验收，不要求列举全部替代测试文件。 + +## Four complete Python shards + +Full Python qualification uses four runners with two xdist workers each: +`--splits 4 --group N --splitting-algorithm least_duration`. It still partitions +the whole collection, excluding only the separately executed Stage2c marker. +No tests are removed. Without timing history the splitter uses equal weights; +four-way parallelism is not a claim of perfect duration balancing. + +The aggregate requires every shard to succeed and all four coverage files to +exist before combining them. The existing full-suite coverage floor remains. +No Python coverage artifact or Sonar run is manufactured when Python is exempt. + +全量 Python 从 2 个分片扩大到 4 个，每片仍为 2 个 worker，不提高单机进程争抢。 +真实 pytest-split/xdist/coverage 回归覆盖分片集合互斥、并集完整、四份报告合并以及 +缺失任意报告时拒绝通过。分片增加会增加安装开销与同时占用的 runner；应看实际 +critical path 和 runner-minutes，而不是宣称“4 片必然快一倍”。 + +## Override and evidence + +Add the **`ci:full`** PR label to force full qualification. Label addition/removal +reruns the workflow. Manually dispatching Python Tests also runs full. The label +can only add checks, never waive them. Main retains full qualification. + +The `ci-impact-plan` artifact and job summary report exact revisions, change kind, +per-job execution flags, reason and coverage scope. The merge gate requires +success for required jobs and an explicit skip for exempt ones; failure, +cancellation, missing outputs, contradictory flags or unexpected skips fail. + +Stage2c retains all correctness cases: its E2E lane uses two runners with two +workers each, while mutants and installed-package lanes remain separate. The +small pytest plugin assigns whole modules using deterministic largest-first +test-count balancing and retains collection order within each module. It does +not split a stateful module across machines or workers. This is not timing-based +optimal scheduling: one very large module can still dominate a shard. + +Stage2c E2E 从单 runner 的 4 个 worker 改为两个 runner 各 2 个 worker；总 worker +数不增加，但不再挤在同一台机器。按完整模块分片，并保留 loadfile 与模块内顺序。 +真实回归以共享状态、顺序敏感的模块验证两路并集完整且互斥，避免盲目按单测试分片。 + +The first implementation retains minimum-Node checks and removes the earlier vision selected-test runner, +selected/full comparison machinery and its extra shadow workload. The prior +shadow results remain historical evidence, not a permanent extra CI obligation. + +## Qualification + +```bash +python -m unittest discover -s scripts/ci -p 'test_*.py' +python -m pytest tests/test_python_ci_workflow.py tests/test_sonarcloud_workflow.py -q +python scripts/ci/review_gate.py classify --base origin/main --head HEAD --plan impact-plan.json +python scripts/ci/review_gate.py classify --base origin/main --head HEAD --force-full --plan impact-plan.json +``` + +Use repository-supported Python/test dependencies. Keep generated plans and +JUnit/coverage artifacts outside tracked source. Hosted CI must qualify the real +workflow after policy changes; unit checks do not prove runner scheduling or +latency. Paid model tests remain release/manual only. No Goal, automation, +authority provider, runtime permission or live state is changed by this policy. diff --git a/docs/development/testing-and-quality.md b/docs/development/testing-and-quality.md index 5dababc6c3..cd7adfc50c 100644 --- a/docs/development/testing-and-quality.md +++ b/docs/development/testing-and-quality.md @@ -77,9 +77,11 @@ golden 来让测试通过。 `python-tests.yml` publishes `merge-gate` for every pull request. Code, workflow, policy and unknown paths require the existing `pytest` aggregate -(including TypeScript checks and both Python shards), Stage 2C correctness +(including TypeScript checks and all four Python shards), Stage 2C correctness aggregate, and Windows tests to succeed. Failed, cancelled, missing or -unexpectedly skipped results cannot pass the gate. +unexpectedly skipped results cannot pass the gate. The client-only exception +below retains common checks and substitutes packaged Dashboard qualification +for unrelated backend jobs. For a change limited to allowlisted root Markdown or `docs/**/*.md`, the classifier explicitly skips the expensive core jobs and the aggregate checks @@ -110,6 +112,14 @@ Changes to the classifier or workflow need both code-path and documentation-only qualification. Keep required check names stable and never require a workflow-level path-filtered check that cannot report on every PR. +The [job exemption policy](ci-impact-selection.md) additionally permits pure +Dashboard-client changes to skip backend Python/Windows and Stage2c, while +requiring common checks and the real packaged Dashboard build/browser smoke. +Mixed, prompt, dependency and unknown changes stay full. Full Python runs four +complete shards and combines all four coverage files. The `ci:full` label forces +full qualification; main stays full. No selected-only report impersonates full +coverage. 新的前端豁免由目标分支已审阅的策略控制；CI 自身变更仍全量验证。 + PRs opened before activation may need a branch update to produce the new required check; an old green suite alone does not supply a missing aggregate. @@ -335,7 +345,7 @@ network latency, provider availability, or a two-hour matrix. 它刻意不包含真实模型调用和 full smoke catalog，因此普通迭代不依赖凭证、网络 时延、模型服务可用性或两小时级测试矩阵。 -The Linux suite uses two hosted runners with two xdist workers each. +The Linux suite uses four hosted runners with two xdist workers each. `pytest-split` partitions the complete collection using `least_duration`; without a timing file, tests have equal weight and alternate between shards. Lint, type checks, and the CLI budget run separately. The required `pytest` @@ -345,18 +355,18 @@ individual shards. Relative coverage paths make reports portable across runners. The reusable Sonar workflow consumes that same run's XML and never reruns pytest or reads cross-run artifacts. Missing Sonar tokens still skip analysis successfully; test jobs receive no Sonar secret. The trigger is the union of -the former Python and Sonar paths, so app-only and Sonar-configuration changes -also run this lane, including on forks without a token. +the former Python and Sonar paths. Client-only PRs use the exemption above; +Sonar-configuration changes remain full, including on forks without a token. -Linux 全套测试分到两台 hosted runner，每台保留两个 xdist worker。`pytest-split` +Linux 全套测试分到四台 hosted runner，每台保留两个 xdist worker。`pytest-split` 按完整 collection 分片；没有历史耗时时，等权测试交替分配。lint、类型检查和 CLI 预算独立执行。必需的 `pytest` 汇总检查会拒绝失败／跳过的分片和缺失的 coverage， 合并后再执行原有 19.6% 门槛；不要求单个分片达到全套覆盖率。coverage 使用相对路径， Sonar 只复用同一次 run 的 XML，不重复测试、不跨 run 取产物。缺少 token 仍成功跳过 -Sonar，测试 job 不接收 Sonar secret。触发范围取原有两套 workflow 的并集，因此仅改 -前端或 Sonar 配置也走此通道，包括没有 token 的 fork。 +Sonar，测试 job 不接收 Sonar secret。触发范围取原有两套 workflow 的并集；纯前端 +PR 使用前述豁免，Sonar 配置变更仍全量运行，包括没有 token 的 fork。 -Reproduce one shard locally with `python -m pytest -q -n 2 --splits 2 --group 1 +Reproduce one shard locally with `python -m pytest -q -n 2 --splits 4 --group 1 --splitting-algorithm least_duration --cov=loopx`. Omit the split arguments to run the complete suite locally. 全量本地测试仍省略分片参数即可。 diff --git a/scripts/ci/impact_plan.py b/scripts/ci/impact_plan.py new file mode 100644 index 0000000000..7ea54b91d6 --- /dev/null +++ b/scripts/ci/impact_plan.py @@ -0,0 +1,108 @@ +"""Classify whole-PR ownership; exempt unrelated heavy jobs, never select tests.""" + +from __future__ import annotations + +from dataclasses import dataclass +import json +import os +from pathlib import Path, PurePosixPath +import subprocess + +SCHEMA = "loopx_ci_job_plan_v1" +ROOT_DOCS = {"README.md", "README.zh-CN.md", "CHANGELOG.md", "CONTRIBUTING.md"} +POLICY_PATHS = ("scripts/ci/impact_plan.py", "scripts/ci/review_gate.py", ".github/workflows/python-tests.yml") +OUTPUTS = ("core_tests", "python_tests", "stage2c_tests", "presentation_tests") +PRESENTATION_ROOTS = ("apps/presentation/dashboard/src/", "apps/presentation/dashboard/public/", "loopx/web/chat/") +PRESENTATION_SUFFIXES = {".ts", ".tsx", ".js", ".mjs", ".css", ".html", ".svg", ".png", ".jpg", ".jpeg", ".webp", ".ico", ".woff", ".woff2"} + + +@dataclass(frozen=True) +class Change: + status: str + path: str + + +def is_document(path: str) -> bool: + return path in ROOT_DOCS or (path.startswith("docs/") and PurePosixPath(path).suffix == ".md") + + +def candidate(changes: list[Change], *, pull_request: bool = True) -> tuple[str, str]: + if not pull_request: + return "full", "main and manual runs retain full qualification" + if not changes: + return "full", "empty or unavailable diff is not an exemption" + for change in changes: + path = PurePosixPath(change.path) + if path.is_absolute() or ".." in path.parts or str(path) != change.path or change.status not in {"A", "M", "D"}: + return "full", "noncanonical paths or type changes require full qualification" + code = [item for item in changes if not is_document(item.path)] + if not code: + return "docs", "documentation only; existing runtime exemption" + if all(item.path.startswith(PRESENTATION_ROOTS) and PurePosixPath(item.path).suffix in PRESENTATION_SUFFIXES for item in code): + return "presentation", "client-only source/assets; retain frontend build/browser and common checks" + return "full", "runtime, prompts, tests, dependencies, build policy or unknown paths may affect backend behavior" + + +def job_flags(kind: str, *, presentation: bool = False) -> dict[str, bool]: + if kind not in {"docs", "presentation", "full"}: + raise ValueError("unknown CI change kind") + return {"core_tests": kind != "docs", "python_tests": kind == "full", + "stage2c_tests": kind == "full", "presentation_tests": kind == "presentation" or presentation} + + +def git(*args: str) -> bytes: + return subprocess.check_output(["git", *args]) + + +def revision(ref: str) -> str: + return git("rev-parse", "--verify", "--end-of-options", f"{ref}^{{commit}}").decode().strip() + + +def diff_changes(base: str, head: str) -> list[Change]: + raw = git("diff", "--name-status", "--no-renames", "-z", base, head, "--").split(b"\0") + if raw[-1] != b"" or (len(raw) - 1) % 2: + raise ValueError("malformed NUL-delimited Git changes") + return [Change(os.fsdecode(raw[i]), os.fsdecode(raw[i + 1])) for i in range(0, len(raw) - 1, 2)] + + +def trusted_policy(base: str, head: str) -> bool: + # A PR cannot introduce its own exemption. No manifest or per-test inventory. + for path in POLICY_PATHS: + try: + old = subprocess.check_output(["git", "rev-parse", f"{base}:{path}"], stderr=subprocess.DEVNULL) + new = subprocess.check_output(["git", "rev-parse", f"{head}:{path}"], stderr=subprocess.DEVNULL) + except subprocess.CalledProcessError: + return False + if old != new: + return False + return True + + +def plan(base: str, head: str, *, pull_request: bool = True, force_full: bool = False) -> dict: + base_sha, head_sha = revision(base), revision(head) + merge_base = git("merge-base", base_sha, head_sha).decode().strip() + changes = diff_changes(merge_base, head_sha) + kind, reason = candidate(changes, pull_request=pull_request) + presentation = kind == "presentation" + policy_change = any(item.path.startswith("scripts/ci/") or item.path == POLICY_PATHS[-1] for item in changes) + if kind != "full": + for item in changes: + for sha in (merge_base, head_sha): + mode = git("ls-tree", "--format=%(objectmode)", sha, "--", item.path).decode().strip() + if mode and mode not in {"100644", "100755"}: + kind, reason = "full", "symlinks and nonregular Git objects cannot grant an exemption" + if force_full: + kind, reason = "full", "explicit force-full override" + elif kind == "presentation" and not trusted_policy(base_sha, head_sha): + kind, reason = "full", "presentation exemption policy is not already trusted on the target branch" + flags = job_flags(kind, presentation=presentation or policy_change) + return {"schema_version": SCHEMA, "base_sha": base_sha, "head_sha": head_sha, + "merge_base_sha": merge_base, "checkout_sha": revision("HEAD"), "change_kind": kind, + "force_full": force_full, "reason": reason, **flags, + "changes": [{"status": item.status, "path": item.path} for item in changes], + "coverage_scope": "full Python suite" if flags["python_tests"] else "no Python coverage claim", + "python_shards": 4 if flags["python_tests"] else 0} + + +def write_plan(packet: dict, output: str) -> None: + Path(output).write_text(json.dumps(packet, ensure_ascii=True, indent=2) + "\n", encoding="utf-8") diff --git a/scripts/ci/module_shard.py b/scripts/ci/module_shard.py new file mode 100644 index 0000000000..92adb06167 --- /dev/null +++ b/scripts/ci/module_shard.py @@ -0,0 +1,40 @@ +"""Shard complete pytest modules; preserve ordered shared-fixture tests.""" +from collections import defaultdict + +import pytest + + +def assign_modules(counts: dict[str, int], shards: int) -> dict[str, int]: + """Deterministic largest-module-first balancing, without timing-cache authority.""" + if shards < 1: + raise ValueError("shards must be positive") + loads = [0] * shards + result = {} + for name, count in sorted(counts.items(), key=lambda item: (-item[1], item[0])): + slot = min(range(shards), key=lambda index: (loads[index], index)) + result[name] = slot + 1 + loads[slot] += count + return result + + +def pytest_addoption(parser): + parser.addoption("--ci-module-shards", type=int, default=1) + parser.addoption("--ci-module-shard", type=int, default=1) + + +@pytest.hookimpl(trylast=True) +def pytest_collection_modifyitems(config, items): + shards = config.getoption("--ci-module-shards") + shard = config.getoption("--ci-module-shard") + if shards < 1 or not 1 <= shard <= shards: + raise pytest.UsageError("invalid module shard coordinates") + counts = defaultdict(int) + for item in items: + counts[item.nodeid.split("::", 1)[0]] += 1 + assigned = assign_modules(counts, shards) + selected, deselected = [], [] + for item in items: + target = selected if assigned[item.nodeid.split("::", 1)[0]] == shard else deselected + target.append(item) + items[:] = selected + config.hook.pytest_deselected(items=deselected) diff --git a/scripts/ci/review_gate.py b/scripts/ci/review_gate.py index 38c36ef8e8..9ca7e5ffbd 100644 --- a/scripts/ci/review_gate.py +++ b/scripts/ci/review_gate.py @@ -1,35 +1,25 @@ -"""Conservative PR classification and fail-closed core CI aggregation.""" +"""Per-job qualification: an intentional exemption is not an accidental skip.""" from __future__ import annotations import argparse import json import os -from pathlib import PurePosixPath -import subprocess +from impact_plan import Change, OUTPUTS, candidate, job_flags, plan, write_plan - -CORE_JOBS = ( - "pytest", - "node-minimum-compatibility", - "stage2c-correctness-e2e", - "windows-powershell", -) -ROOT_DOCS = {"README.md", "README.zh-CN.md", "CHANGELOG.md", "CONTRIBUTING.md"} +JOB_OUTPUT = { + "checks": "core_tests", + "pytest": "python_tests", + "node-minimum-compatibility": "core_tests", + "stage2c-correctness-e2e": "stage2c_tests", + "windows-powershell": "python_tests", + "presentation": "presentation_tests", +} +CORE_JOBS = tuple(JOB_OUTPUT) def requires_core_tests(paths: list[str]) -> bool: - # Unknown paths, executable documentation, policy and runtime prompts run CI. - # Disable rename detection at the caller so both old and new paths count. - if not paths: - return True - return any( - not ( - path in ROOT_DOCS - or (path.startswith("docs/") and PurePosixPath(path).suffix == ".md") - ) - for path in paths - ) + return candidate([Change("M", path) for path in paths])[0] != "docs" def verify(needs: object) -> None: @@ -39,14 +29,17 @@ def verify(needs: object) -> None: if not isinstance(changes, dict) or changes.get("result") != "success": raise ValueError("change classification did not succeed") outputs = changes.get("outputs") - classification = outputs.get("core_tests") if isinstance(outputs, dict) else None - if classification not in ("true", "false"): - raise ValueError("missing or invalid core-test classification") - expected = "success" if classification == "true" else "skipped" - for name in CORE_JOBS: + if not isinstance(outputs, dict) or any(outputs.get(key) not in {"true", "false"} for key in OUTPUTS): + raise ValueError("missing or invalid job classification") + kind = outputs.get("change_kind") + expected = job_flags(kind, presentation=kind == "full" and outputs["presentation_tests"] == "true") + if any(outputs[key] != str(value).lower() for key, value in expected.items()): + raise ValueError("contradictory job exemptions") + for name, output in JOB_OUTPUT.items(): + required = "success" if outputs[output] == "true" else "skipped" job = needs[name] - if not isinstance(job, dict) or job.get("result") != expected: - raise ValueError(f"{name} must be {expected}") + if not isinstance(job, dict) or job.get("result") != required: + raise ValueError(f"{name} must be {required}") def main() -> None: @@ -55,19 +48,20 @@ def main() -> None: classify = sub.add_parser("classify") classify.add_argument("--base", required=True) classify.add_argument("--head", required=True) + classify.add_argument("--plan") + classify.add_argument("--non-pr", action="store_true") + classify.add_argument("--force-full", action="store_true") sub.add_parser("verify") args = parser.parse_args() if args.command == "classify": - # SHAs come through environment variables, never interpolated shell code. - # Git failure propagates; it must never become a documentation-only pass. - base = subprocess.check_output( - ["git", "merge-base", args.base, args.head], text=True - ).strip() - raw = subprocess.check_output( - ["git", "diff", "--name-only", "--no-renames", "-z", base, args.head, "--"] - ) - paths = [os.fsdecode(path) for path in raw.split(b"\0") if path] - print(f"core_tests={str(requires_core_tests(paths)).lower()}") + packet = plan(args.base, args.head, pull_request=not args.non_pr, force_full=args.force_full) + if args.plan: + write_plan(packet, args.plan) + print(f"change_kind={packet['change_kind']}") + for key in OUTPUTS: + print(f"{key}={str(packet[key]).lower()}") + else: + print(f"core_tests={str(packet['core_tests']).lower()}") else: verify(json.loads(os.environ["NEEDS_JSON"])) print("merge-gate: qualification complete") diff --git a/scripts/ci/test_impact_plan.py b/scripts/ci/test_impact_plan.py new file mode 100644 index 0000000000..80c71fc63a --- /dev/null +++ b/scripts/ci/test_impact_plan.py @@ -0,0 +1,99 @@ +"""Whole-PR boundaries, trusted policy and real Git negative controls.""" +from __future__ import annotations +import json +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest +from impact_plan import Change, POLICY_PATHS, candidate + + +class ImpactTests(unittest.TestCase): + def test_presentation_and_docs_union(self): + for status in ("A", "M", "D"): + for path in ("apps/presentation/dashboard/src/App.tsx", "loopx/web/chat/assets/chat.js", + "apps/presentation/dashboard/public/icon.svg"): + self.assertEqual(candidate([Change(status, path), Change("M", "docs/design.md")])[0], "presentation") + + def test_one_shared_unknown_prompt_test_or_dependency_forces_full(self): + for path in ("loopx/cli.py", "loopx/control_plane/goals/vision_checkpoint.ts", + "loopx/claude_goal_mode/commands/loopx.md", "tests/test_ui.py", + "apps/presentation/dashboard/package-lock.json", "apps/presentation/dashboard/vite.config.ts", + "apps/desktop/src/main.rs", "scripts/ci/impact_plan.py", ".github/workflows/python-tests.yml", + "loopx/web/chat/backend.py", "loopx/web/chat/config.json", "unknown", "docs/build.py"): + with self.subTest(path=path): + self.assertEqual(candidate([Change("M", "loopx/web/chat/index.html"), Change("M", path)])[0], "full") + + def test_invalid_empty_non_pr_and_move_out_of_runtime_are_not_exempt(self): + for changes in ([], [Change("T", "docs/a.md")], [Change("M", "docs/../code.md")], + [Change("M", "/docs/a.md")], [Change("M", "docs//a.md")], + [Change("D", "loopx/code.py"), Change("A", "docs/code.md")]): + self.assertEqual(candidate(changes)[0], "full") + self.assertEqual(candidate([Change("M", "docs/a.md")], pull_request=False)[0], "full") + + def test_real_git_policy_force_full_and_type_boundaries(self): + script = str(Path(__file__).with_name("review_gate.py")) + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + env = {k: v for k, v in os.environ.items() if not k.startswith(("PYTEST", "COVERAGE", "COV_CORE"))} + def git(*args): + return subprocess.check_output(["git", *args], cwd=root, env=env, text=True).strip() + def write(path, content): + target = root / path + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(content) + def classify(base, *extra): + result = subprocess.run([sys.executable, script, "classify", "--base", base, "--head", "HEAD", + "--plan", str(root / "plan.json"), *extra], cwd=root, env=env, capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stderr) + return json.loads((root / "plan.json").read_text()), result.stdout + git("init", "-q") + git("config", "user.name", "CI Fixture") + git("config", "user.email", "ci@example.invalid") + git("config", "core.hooksPath", str(root / "no-hooks")) + path = "loopx/web/chat/index.html" + write(path, "old") + for policy in POLICY_PATHS: + write(policy, "reviewed policy") + git("add", ".") + git("commit", "-qm", "base") + base = git("rev-parse", "HEAD") + write(path, "new") + git("commit", "-qam", "presentation") + packet, _ = classify(base) + self.assertEqual(packet["change_kind"], "presentation") + self.assertFalse(packet["python_tests"]) + self.assertFalse(packet["stage2c_tests"]) + self.assertTrue(packet["presentation_tests"]) + self.assertEqual(packet["head_sha"], git("rev-parse", "HEAD")) + for flag in ("--force-full", "--non-pr"): + forced, _ = classify(base, flag) + self.assertTrue(forced["python_tests"]) + self.assertTrue(forced["stage2c_tests"]) + self.assertEqual(forced["python_shards"], 4) + # A policy change cannot authorize its own exemption. + write(POLICY_PATHS[0], "changed policy") + git("commit", "-qam", "policy") + self.assertEqual(classify(base)[0]["change_kind"], "full") + strange = "unknown\npython_tests=false" + write(strange, "fixture") + git("add", strange) + git("commit", "-qm", "unusual path") + packet, output = classify(base) + self.assertNotIn("python_tests=false", output) + self.assertIn(strange, [item["path"] for item in packet["changes"]]) + before_link = git("rev-parse", "HEAD") + (root / "docs").mkdir() + (root / "docs/link.md").symlink_to("../loopx/web/chat/index.html") + git("add", "docs/link.md") + git("commit", "-qm", "symlink") + self.assertEqual(classify(before_link)[0]["change_kind"], "full") + failed = subprocess.run([sys.executable, script, "classify", "--base", "missing", "--head", "HEAD"], + cwd=root, env=env, capture_output=True) + self.assertNotEqual(failed.returncode, 0) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/ci/test_review_gate.py b/scripts/ci/test_review_gate.py index 7b7e9c7d82..0c71c9cbea 100644 --- a/scripts/ci/test_review_gate.py +++ b/scripts/ci/test_review_gate.py @@ -1,131 +1,63 @@ -"""The required gate must not report success for absent or skipped core work.""" - +"""An intentionally exempt job must be skipped; all required jobs must pass.""" from __future__ import annotations - import copy -import os -from pathlib import Path -import subprocess -import sys -import tempfile import unittest - -from review_gate import CORE_JOBS, requires_core_tests, verify +from impact_plan import job_flags +from review_gate import CORE_JOBS, JOB_OUTPUT, requires_core_tests, verify -def needs(core: bool = True) -> dict: - return { - "changes": {"result": "success", "outputs": {"core_tests": str(core).lower()}}, - **{job: {"result": "success" if core else "skipped"} for job in CORE_JOBS}, - } +def needs(kind="full", *, presentation=False): + flags = job_flags(kind, presentation=presentation) + return {"changes": {"result": "success", "outputs": {"change_kind": kind, + **{key: str(value).lower() for key, value in flags.items()}}}, + **{job: {"result": "success" if flags[output] else "skipped"} for job, output in JOB_OUTPUT.items()}} -class ReviewGateTests(unittest.TestCase): - def test_documentation_only(self): +class GateTests(unittest.TestCase): + def test_legal_profiles_and_policy_rehearsal(self): + for kind in ("docs", "presentation", "full"): + verify(needs(kind)) + verify(needs("full", presentation=True)) self.assertFalse(requires_core_tests(["README.md", "docs/guide.md"])) - - def test_unknown_and_executable_paths_require_tests(self): - for path in ( - "loopx/prompt.md", "docs/build.py", "AGENTS.md", ".github/CODEOWNERS", - ".github/workflows/python-tests.yml", "scripts/ci/review_gate.py", - "packages/new-package/index.ts", "new-config.json", "docs/demo.js", - ): - with self.subTest(path=path): - self.assertTrue(requires_core_tests(["docs/guide.md", path])) - - def test_empty_diff_is_not_an_exemption(self): + self.assertTrue(requires_core_tests(["loopx/prompt.md"])) self.assertTrue(requires_core_tests([])) - def test_rename_out_of_code_is_not_an_exemption(self): - self.assertTrue(requires_core_tests(["loopx/code.py", "docs/code.md"])) - - def test_core_success(self): - verify(needs()) - - def test_explicit_documentation_skip(self): - verify(needs(False)) - - def test_every_unsuccessful_core_result_is_rejected(self): - for job in CORE_JOBS: - for result in ("skipped", "failure", "cancelled", "neutral", "", None): - value = needs() - value[job]["result"] = result - with self.subTest(job=job, result=result), self.assertRaises(ValueError): + def test_every_missing_failure_cancel_or_unexpected_skip_is_rejected(self): + for kind in ("docs", "presentation", "full"): + good = needs(kind) + for job in CORE_JOBS: + for state in ("success", "skipped", "failure", "cancelled", "neutral", None): + if state == good[job]["result"]: + continue + value = copy.deepcopy(good) + value[job]["result"] = state + with self.subTest(kind=kind, job=job, state=state), self.assertRaises(ValueError): + verify(value) + for job in ("changes", *CORE_JOBS): + value = copy.deepcopy(good) + del value[job] + with self.assertRaises(ValueError): verify(value) - def test_missing_or_extra_dependencies_are_rejected(self): - for job in ("changes", *CORE_JOBS): - value = needs() - del value[job] - with self.subTest(job=job), self.assertRaises(ValueError): - verify(value) + def test_bad_and_contradictory_classification_cannot_skip_work(self): + for kind in ("docs", "presentation", "full"): + for field in ("change_kind", "core_tests", "python_tests", "stage2c_tests", "presentation_tests"): + for replacement in (None, "", True, "unknown"): + value = needs(kind) + value["changes"]["outputs"][field] = replacement + with self.assertRaises((ValueError, TypeError)): + verify(value) + value = needs("full") + value["changes"]["outputs"]["stage2c_tests"] = "false" with self.assertRaises(ValueError): - verify({**needs(), "extra": {"result": "success"}}) - - def test_invalid_classification_is_rejected(self): - for result in ("failure", "cancelled", "skipped"): + verify(value) + for state in ("failure", "skipped", "cancelled"): value = needs() - value["changes"]["result"] = result + value["changes"]["result"] = state with self.assertRaises(ValueError): verify(value) - for output in ({}, {"core_tests": ""}, {"core_tests": True}, None): - value = needs() - value["changes"]["outputs"] = output - with self.assertRaises(ValueError): - verify(value) - - def test_docs_cannot_hide_a_failed_job(self): - for job in CORE_JOBS: - value = copy.deepcopy(needs(False)) - value[job]["result"] = "failure" - with self.assertRaises(ValueError): - verify(value) - - def test_real_git_diff_handles_code_deletion_and_missing_base(self): - script = str(Path(__file__).with_name("review_gate.py")) - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - # pytest-cov injects subprocess startup variables. The synthetic - # checkout must not contribute temporary source paths to real CI. - env = {key: value for key, value in os.environ.items() - if not key.startswith(("PYTEST", "COVERAGE", "COV_CORE"))} - - def git(*args): - return subprocess.check_output( - ["git", *args], cwd=root, env=env, text=True - ).strip() - - git("init", "-q") - git("config", "user.name", "CI Fixture") - git("config", "user.email", "ci@example.invalid") - git("config", "core.hooksPath", str(root / "no-hooks")) - (root / "docs").mkdir() - (root / "loopx").mkdir() - (root / "docs/readme.md").write_text("baseline\n") - (root / "loopx/code.py").write_text("pass\n") - git("add", "docs/readme.md", "loopx/code.py") - git("commit", "-qm", "baseline") - base = git("rev-parse", "HEAD") - (root / "docs/readme.md").write_text("documentation only\n") - git("commit", "-qam", "docs") - result = subprocess.check_output( - [sys.executable, script, "classify", "--base", base, "--head", "HEAD"], - cwd=root, env=env, text=True, - ) - self.assertEqual(result.strip(), "core_tests=false") - git("mv", "loopx/code.py", "docs/code.md") - git("commit", "-qm", "move code into docs") - result = subprocess.check_output( - [sys.executable, script, "classify", "--base", base, "--head", "HEAD"], - cwd=root, env=env, text=True, - ) - self.assertEqual(result.strip(), "core_tests=true") - failed = subprocess.run( - [sys.executable, script, "classify", "--base", "missing-ref", "--head", "HEAD"], - cwd=root, env=env, capture_output=True, text=True, - ) - self.assertNotEqual(failed.returncode, 0) - self.assertNotIn("core_tests=false", failed.stdout) + with self.assertRaises(ValueError): + verify({**needs(), "extra": {"result": "success"}}) if __name__ == "__main__": diff --git a/tests/test_python_ci_workflow.py b/tests/test_python_ci_workflow.py index bb5b41450a..8b8fea7f9f 100644 --- a/tests/test_python_ci_workflow.py +++ b/tests/test_python_ci_workflow.py @@ -11,9 +11,8 @@ import pytest -WORKFLOW = ( - Path(__file__).resolve().parents[1] / ".github" / "workflows" / "python-tests.yml" -).read_text(encoding="utf-8") +WORKFLOW_ROOT = Path(__file__).resolve().parents[1] +WORKFLOW = (WORKFLOW_ROOT / ".github/workflows/python-tests.yml").read_text(encoding="utf-8") @pytest.mark.parametrize("result", ["success", "failure", "cancelled", "skipped", ""]) @@ -21,7 +20,7 @@ def test_stage2c_gate_requires_all_lanes(result: str) -> None: gate = WORKFLOW.split(" stage2c-correctness-e2e:", 1)[1].split(" windows-powershell:", 1)[0] assert "if: always()" in gate assert "needs: [changes, stage2c-suite]" in gate - assert "if: always() && needs.changes.outputs.core_tests == 'true'" in gate + assert "if: always() && needs.changes.outputs.stage2c_tests == 'true'" in gate assert "STAGE2C_RESULT: ${{ needs.stage2c-suite.result }}" in gate script = gate.split("run: ", 1)[1].strip() actual = subprocess.run( @@ -31,8 +30,9 @@ def test_stage2c_gate_requires_all_lanes(result: str) -> None: assert (actual.returncode == 0) == (result == "success") suite = WORKFLOW.split(" stage2c-suite:", 1)[1].split(" stage2c-correctness-e2e:", 1)[0] assert "fail-fast: false" in suite - assert "suite: [e2e, mutants, installed]" in suite - assert "if: needs.changes.outputs.core_tests == 'true'" in suite.split(" steps:", 1)[0] + for entry in ("{suite: e2e, shard: 1}", "{suite: e2e, shard: 2}", "{suite: mutants, shard: 0}", "{suite: installed, shard: 0}"): + assert entry in suite + assert "if: needs.changes.outputs.stage2c_tests == 'true'" in suite.split(" steps:", 1)[0] steps = {step.splitlines()[0]: step for step in suite.split(" - name: ")[1:]} for name, lane in [ ("Qualify real CLI, mixed writers, process death, and recovery", "e2e"), @@ -45,44 +45,58 @@ def test_stage2c_gate_requires_all_lanes(result: str) -> None: assert "--case" not in steps["Reject deliberate correctness regressions"] artifact = steps["Retain bounded acceptance evidence"] assert "if: always()" in artifact - assert "name: stage2c-correctness-evidence-${{ matrix.suite }}" in artifact + assert "name: stage2c-correctness-evidence-${{ matrix.suite }}-${{ matrix.shard }}" in artifact assert "if-no-files-found: error" in artifact def test_stage2c_workers_preserve_module_state_and_execute_every_row(tmp_path: Path) -> None: step = WORKFLOW.split("name: Qualify real CLI, mixed writers, process death, and recovery", 1)[1] command = step.split("run: ", 1)[1].splitlines()[0] - args = shlex.split(command) - args[0] = sys.executable - # Actual workflow command against two modules with order-sensitive shared state. - # Per-test distribution would break the module fixture's accumulated state. - for name in ("first", "second"): - (tmp_path / f"test_{name}.py").write_text( - "import os\nfrom pathlib import Path\nimport pytest\n" - "pytestmark = pytest.mark.stage2c_e2e\n" - "@pytest.fixture(scope='module')\ndef state():\n return []\n" - "@pytest.mark.parametrize('row', range(4))\n" - "def test_order(state, row):\n" - " assert state == list(range(row))\n state.append(row)\n" - f" with Path('{name}.visits').open('a') as stream:\n" - " stream.write(f'{os.getpid()}:{row}\\n')\n", - encoding="utf-8", - ) - (tmp_path / "pytest.ini").write_text("[pytest]\nmarkers = stage2c_e2e\n") env = {key: value for key, value in os.environ.items() if not key.startswith(("PYTEST", "COVERAGE", "COV_CORE"))} - actual = subprocess.run(args, cwd=tmp_path, env=env, capture_output=True, - text=True, timeout=60, check=False) - assert actual.returncode == 0, actual.stdout + actual.stderr - cases = ET.parse(tmp_path / "stage2c-e2e.xml").findall(".//testcase") - assert len(cases) == 8 - workers = set() - for name in ("first", "second"): - rows = [line.split(":") for line in (tmp_path / f"{name}.visits").read_text().splitlines()] - assert [row for _, row in rows] == ["0", "1", "2", "3"] - assert len({pid for pid, _ in rows}) == 1 - workers.add(rows[0][0]) - assert len(workers) == 2 + env["PYTHONPATH"] = str(WORKFLOW_ROOT) + visited = set() + for shard in (1, 2): + root = tmp_path / f"shard-{shard}" + root.mkdir() + args = shlex.split(command.replace("${{ matrix.shard }}", str(shard))) + args[0] = sys.executable + for name in ("first", "second", "third", "fourth"): + (root / f"test_{name}.py").write_text( + "import os\nfrom pathlib import Path\nimport pytest\n" + "pytestmark = pytest.mark.stage2c_e2e\n" + "@pytest.fixture(scope='module')\ndef state():\n return []\n" + "@pytest.mark.parametrize('row', range(4))\n" + "def test_order(state, row):\n" + " assert state == list(range(row))\n state.append(row)\n" + f" with Path('{name}.visits').open('a') as stream:\n" + " stream.write(f'{os.getpid()}:{row}\\n')\n", encoding="utf-8") + (root / "pytest.ini").write_text("[pytest]\nmarkers = stage2c_e2e\n") + actual = subprocess.run(args, cwd=root, env=env, capture_output=True, + text=True, timeout=60, check=False) + assert actual.returncode == 0, actual.stdout + actual.stderr + assert len(ET.parse(root / "stage2c-e2e.xml").findall(".//testcase")) == 8 + workers = set() + for report in root.glob("*.visits"): + assert report.stem not in visited + visited.add(report.stem) + rows = [line.split(":") for line in report.read_text().splitlines()] + assert [row for _, row in rows] == ["0", "1", "2", "3"] + assert len({pid for pid, _ in rows}) == 1 + workers.add(rows[0][0]) + assert len(workers) == 2 + assert visited == {"first", "second", "third", "fourth"} + + +def test_module_sharding_is_deterministic_and_balances_complete_modules() -> None: + from scripts.ci.module_shard import assign_modules + counts = {"large": 100, "medium": 60, "small": 40, "tiny": 1} + expected = {"large": 1, "medium": 2, "small": 2, "tiny": 1} + assert assign_modules(counts, 2) == expected + assert assign_modules(dict(reversed(list(counts.items()))), 2) == expected + assert assign_modules(counts, 1) == dict.fromkeys(counts, 1) + with pytest.raises(ValueError): + assign_modules(counts, 0) @pytest.mark.parametrize("checks", ["success", "failure", "cancelled", "skipped"]) @@ -100,7 +114,7 @@ def test_required_pytest_check_rejects_incomplete_upstream_jobs( check=False, ) assert (result.returncode == 0) == (checks == shards == "success") - assert "if: always() && needs.changes.outputs.core_tests == 'true'" in WORKFLOW + assert "if: always() && needs.changes.outputs.python_tests == 'true'" in WORKFLOW assert "needs: [changes, checks, test-shard]" in WORKFLOW @@ -110,22 +124,36 @@ def test_merge_gate_runs_on_all_prs_and_checks_every_core_aggregate() -> None: gate = WORKFLOW.split(" merge-gate:", 1)[1] assert "if: always()" in gate assert ( - "needs: [changes, pytest, node-minimum-compatibility, " - "stage2c-correctness-e2e, windows-powershell]" + "needs: [changes, checks, pytest, node-minimum-compatibility, " + "stage2c-correctness-e2e, windows-powershell, presentation]" ) in gate assert "NEEDS_JSON: ${{ toJSON(needs) }}" in gate assert "run: python scripts/ci/review_gate.py verify" in gate assert "continue-on-error" not in gate - for name in ("checks", "test-shard", "stage2c-suite", "windows-powershell"): + for name, output in (("checks", "core_tests"), ("test-shard", "python_tests"), ("stage2c-suite", "stage2c_tests"), ("windows-powershell", "python_tests"), ("presentation", "presentation_tests")): job = WORKFLOW.split(f" {name}:\n", 1)[1].split(" steps:", 1)[0] assert "needs: changes" in job - assert "if: needs.changes.outputs.core_tests == 'true'" in job + assert f"if: needs.changes.outputs.{output} == 'true'" in job + + +def test_presentation_exemption_retains_real_frontend_checks_and_force_full() -> None: + job = WORKFLOW.split(" presentation:\n", 1)[1].split(" merge-gate:\n", 1)[0] + assert "npm run build:chat" in job + assert "npm run smoke:personal-workspace-packaged" in job + assert "status --short --untracked-files=all -- loopx/web/chat" in job + assert "continue-on-error" not in job + assert "labels.*.name, 'ci:full'" in WORKFLOW + assert "labeled, unlabeled" in WORKFLOW + assert "workflow_dispatch:" in WORKFLOW + assert "--force-full" in WORKFLOW + assert "impact-shadow" not in WORKFLOW + assert "matrix:\n shard: [1, 2, 3, 4]" in WORKFLOW -def test_two_shards_execute_each_test_once_and_merge_portable_coverage( +def test_four_shards_execute_each_test_once_and_merge_portable_coverage( tmp_path: Path, ) -> None: - # Real pytest-split + xdist + coverage, in two distinct checkout roots. + # Real pytest-split + xdist + coverage, in four distinct checkout roots. # Each shard alone misses a function; their union must cover the whole file. shard_step = WORKFLOW.split("name: Run test shard", 1)[1] template = shard_step.split("run: >-", 1)[1].split(" - name:", 1)[0] @@ -134,7 +162,7 @@ def test_two_shards_execute_each_test_once_and_merge_portable_coverage( if not key.startswith(("COVERAGE", "COV_CORE", "PYTEST")) } seen: list[set[str]] = [] - for shard in (1, 2): + for shard in (1, 2, 3, 4): root = tmp_path / f"checkout-{shard}" root.mkdir() (root / "ci_subject.py").write_text( @@ -143,8 +171,10 @@ def test_two_shards_execute_each_test_once_and_merge_portable_coverage( ) (root / "test_subject.py").write_text( "from ci_subject import first, second\n" - "def test_first():\n assert first() == 1\n" - "def test_second():\n assert second() == 2\n", + "def test_first_one():\n assert first() == 1\n" + "def test_first_two():\n assert first() == 1\n" + "def test_second_one():\n assert second() == 2\n" + "def test_second_two():\n assert second() == 2\n", encoding="utf-8", ) (root / "pyproject.toml").write_text( @@ -170,8 +200,9 @@ def test_two_shards_execute_each_test_once_and_merge_portable_coverage( destination.mkdir(parents=True) (root / ".coverage").rename(destination / ".coverage") - assert seen[0] and seen[1] and seen[0].isdisjoint(seen[1]) - assert seen[0] | seen[1] == {"test_first", "test_second"} + assert all(seen) + assert sum(map(len, seen)) == len(set.union(*seen)) == 4 + assert set.union(*seen) == {"test_first_one", "test_first_two", "test_second_one", "test_second_two"} # Reuse the real aggregate shell commands, with a 100% synthetic oracle. step = WORKFLOW.split("name: Combine complete coverage", 1)[1] script = step.split("run: |", 1)[1].split(" - uses:", 1)[0] @@ -179,7 +210,7 @@ def test_two_shards_execute_each_test_once_and_merge_portable_coverage( script = script.replace("--fail-under=19.6", "--fail-under=100") root = tmp_path / "checkout-1" (tmp_path / "coverage-shards").rename(root / "coverage-shards") - for shard in (1, 2): + for shard in (1, 2, 3, 4): data = root / "coverage-shards" / f"python-coverage-{shard}" / ".coverage" held = data.with_name("held") data.rename(held) @@ -202,6 +233,6 @@ def test_two_shards_execute_each_test_once_and_merge_portable_coverage( capture_output=True, check=False, ) assert missing.returncode != 0 - assert re.search(r"shard: \[1, 2\]", WORKFLOW) + assert re.search(r"shard: \[1, 2, 3, 4\]", WORKFLOW) assert "include-hidden-files: true" in WORKFLOW assert "--cov-fail-under" not in template