From 9943708e67e0c4bd4bc723c58b6f5f0ca1a618df Mon Sep 17 00:00:00 2001 From: sebi Date: Wed, 16 Sep 2026 08:45:27 -0500 Subject: [PATCH 1/8] ci: one packages job instead of a 32-job matrix; coverage gate reads the test exit code MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit GitHub bills every job rounded up to a whole minute. The per-package matrix (package-list + one job per package) was 33 of the ~40 runner-minutes a push to main cost, for roughly nine minutes of actual work. It is now one `packages` job running `coverage-gate.ts --all`, which already spawns one `bun test` process per package — the isolation the matrix existed for — and now runs them concurrently on every core (--jobs bounds it). Locally --all went 3m38s -> 2m24s; cli is the long pole. Fixed while there: coverage-gate never read `bun test`'s exit code, so a package whose suite FAILED alone still wrote lcov, cleared its bar and passed. Measured with a probe `expect(1).toBe(2)` in packages/money. It is now X_TEST_FAILED naming the failing tests. The CI workflow `x new` writes pays for each commit once: a same-repo branch with an open PR no longer gates the tree twice (push + pull_request; forks still run), a newer push cancels the superseded run off the default branch (default branch keyed by SHA), and Bun's install cache is restored before bin/setup. Blacksmith was considered and not adopted: the repo is public, so GitHub-hosted minutes bill at $0 net, and Blacksmith meters past its free tier. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/ci.yml | 56 ++++--------- CHANGELOG.md | 20 ++++- CLAUDE.md | 3 +- .../cli/src/templates/github/ci.yml.test.ts | 34 ++++++++ packages/cli/src/templates/github/ci.yml.ts | 18 ++++ scripts/coverage-gate.test.ts | 38 ++++++++- scripts/coverage-gate.ts | 82 +++++++++++++++++-- scripts/list-package-dirs.ts | 4 +- 8 files changed, 202 insertions(+), 53 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f125b1a0a..a95ce844e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,11 +3,11 @@ name: ci # The gate. Same steps a contributor runs locally with `bun run verify` — a check that lives only # in CI is a check developers cannot run. Free GitHub-hosted runners only. # -# Six jobs, and each answers a question no other job answers: does the framework ship (`verify`), +# Five jobs, and each answers a question no other job answers: does the framework ship (`verify`), # do the tracked apps still gate (`reference-app-verify`), can a stranger scaffold an app that # gates (`scaffold-smoke`), do the deployable artifacts BUILD and RENDER (`container`), and is each -# package green and above its coverage bar in isolation (`package-list` → `package`). The last two -# each carry their own argument, above the job. There is deliberately no separate `lint`, +# package green and above its coverage bar in isolation (`packages`). The last two each carry their +# own argument, above the job. There is deliberately no separate `lint`, # `typecheck`, `boundaries` or `test` job — those are steps of `x verify`, and running them again # beside the gate is the second path axiom 1 forbids. It also cost what it duplicated: the four # jobs took ~2.2 runner-minutes and, because `verify` waited on them through `needs`, added their @@ -427,12 +427,12 @@ jobs: exit 1 fi - # One job per package: its own tests and its own coverage bar, in isolation. No lint — the - # step beside `test + coverage` below says why, and it is the half of this summary that drifted. + # Every package's own tests and its own coverage bar, each in isolation. No lint — `verify`'s + # `biome check .` is a strict superset of any per-package lint, and a second one cannot fail alone. # # This does NOT duplicate `verify`, and the rule at the top of this file is why it has to be # argued rather than assumed. `verify` answers "does the framework ship" over the whole tree at - # once; these answer two questions it structurally cannot: + # once; this answers two questions it structurally cannot: # # 1. WHICH package is below bar. `x verify`'s `unit` step runs every suite in one process, so a # coverage number over that run is a number for the monorepo, not for a package — and a @@ -442,44 +442,24 @@ jobs: # another package's preload registered something first is invisible. Run alone, it is a # failure — and `registerFrameworkCatalog`/`registerMailCatalog` were exactly that shape. # + # ONE job, not a matrix of one per package, and isolation is not what was traded for it: + # `coverage-gate.ts --all` spawns a separate `bun test` process per package, which is the + # isolation, and runs as many at once as the runner has cores. The matrix was 32 jobs — a + # `package-list` job plus one per package — and GitHub bills each job rounded UP to a whole + # minute, so ~9s of checkout-and-install plus a 5s suite cost a minute 32 times per run: 33 of + # the ~40 runner-minutes one push to `main` cost, measured 2026-09-16, for about nine minutes of + # actual work. `--all` reports every package's findings, never only the first to break. + # # No services, deliberately: `scripts/lib/coverage-pins.ts` explains that the pins are measured # with no Postgres, no Redis and no NATS so the number is identical on a laptop and on a runner. # The live paths are `verify`'s `live` step, which does have them. - package-list: + packages: runs-on: ubuntu-latest - timeout-minutes: 5 - outputs: - packages: ${{ steps.list.outputs.packages }} - steps: - - uses: actions/checkout@v7 - with: - persist-credentials: false - - uses: ./.github/actions/setup - # Derived, never hand-listed — the same rule the publish list follows, and what keeps a new - # package from being silently absent from its own gate. - - id: list - run: | - echo "packages=$(bun run scripts/list-package-dirs.ts --json | jq -c '.data')" >> "$GITHUB_OUTPUT" - - package: - needs: package-list - runs-on: ubuntu-latest - timeout-minutes: 10 - strategy: - # Every package is reported, not just the first to break: a matrix that stops at one failure - # turns a 30-package answer into a one-package answer. - fail-fast: false - matrix: - package: ${{ fromJson(needs.package-list.outputs.packages) }} - name: package (${{ matrix.package }}) + timeout-minutes: 15 steps: - uses: actions/checkout@v7 with: persist-credentials: false - uses: ./.github/actions/setup - # No `lint` step. `bunx biome check packages/` was one, and it is a strict SUBSET of the - # `verify` job's `biome check .` — ~30 matrix steps per push that cannot fail on their own, - # and the second `lint` job the rule at the top of this file forbids. What survives is the - # only thing this matrix answers that the gate cannot: per-package coverage, in isolation. - - name: test + coverage - run: bun run scripts/coverage-gate.ts --package "${{ matrix.package }}" + - name: every package's tests + coverage, each in its own process, all cores + run: bun run scripts/coverage-gate.ts --all diff --git a/CHANGELOG.md b/CHANGELOG.md index 21e652822..e2cb604f7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,25 @@ Semver applies from 1.0.0. A breaking change to a documented API needs a major ## [Unreleased] -Nothing yet. +### Changed + +- **The CI workflow `x new` writes pays for each commit once.** A branch with an open pull request + fired `push` and `pull_request` for the same tree and gated it twice; the `pull_request` run is now + skipped for a branch in the app's own repository and still runs for a fork, which fires no push. A + newer push cancels the run in flight for its branch, except on the default branch, whose group is + keyed by SHA so every commit keeps its verdict. Bun's download cache is restored before + `bin/setup`, keyed on `bun.lock`. An existing app adopts it by copying the `concurrency:` block, + the job's `if:` and the `actions/cache` step from a fresh `x new` — the file is the app's own. + +### Fixed + +- **A package whose tests fail in isolation no longer passes the per-package coverage gate.** + `scripts/coverage-gate.ts` never read `bun test`'s exit code: a failing suite still wrote an lcov + report, cleared its bar, and reported green — measured with a probe `expect(1).toBe(2)` in + `packages/money`. It is now `X_TEST_FAILED`, naming the failing tests. `--all` runs the package + suites concurrently, one process each, on every core (`--jobs ` to bound it). +- **Framework CI: the 32-job per-package matrix is one `packages` job.** GitHub bills each job + rounded up to a whole minute, so the matrix was 33 of the ~40 runner-minutes one push cost. ## 20.1.1 - 2026-09-12 diff --git a/CLAUDE.md b/CLAUDE.md index 6cdb85c0f..a84d1d94b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -409,8 +409,7 @@ Free GitHub Actions runners (`ubuntu-latest`) — never a paid runner. Target un | `reference-app-verify` | both tracked apps' own gate, on its ratchet | | `scaffold-smoke` | `x new` → `bun install` → the documented first run (`x db gen`, `x db migrate`, **every** generator in `GENERATORS`) → the scaffolded app's own `x verify`, outside the checkout | | `container` | `docker/` as a built artifact — every stage of the image, ending in the runtime stage's own `/app/x --version`, plus `helm template` assertions the chart's own values can fail | -| `package-list` | the matrix for the job below, **derived** from `scripts/list-package-dirs.ts` rather than hand-listed | -| `package` | each package tested and covered **alone** — a suite that only passes because another package's preload registered something first is green in `verify` and red here. Its one step is `bun run scripts/coverage-gate.ts --package `; there is deliberately **no `lint` step**, because `bunx biome check packages/` is a strict subset of the `verify` job's `biome check .` and cannot fail on its own | +| `packages` | each package tested and covered **alone** — a suite that only passes because another package's preload registered something first is green in `verify` and red here. Its one step is `bun run scripts/coverage-gate.ts --all`: one `bun test` process per package, as many at once as the runner has cores, and a suite that **fails** is `X_TEST_FAILED` — until 2026-09-16 the exit code was never read, so a package red alone passed on its lcov. **One job, not a matrix, decided 2026-09-16**: GitHub bills each job rounded up to a whole minute, and the 32-job matrix was 33 of the ~40 runner-minutes one push cost for ~9 minutes of work. There is deliberately **no `lint` step**, because a per-package biome check is a strict subset of the `verify` job's `biome check .` and cannot fail on its own | **`container` is the one job with no `./.github/actions/setup`, deliberately.** Nothing in it runs bun, so the composite's frozen install would be pure latency; `docker`, `helm` and `jq` come from the runner image and its first step refuses by name if one stops shipping. Every other job starts with the composite — bun, the install cache, a frozen install. diff --git a/packages/cli/src/templates/github/ci.yml.test.ts b/packages/cli/src/templates/github/ci.yml.test.ts index 3710ea44b..dc9081d89 100644 --- a/packages/cli/src/templates/github/ci.yml.test.ts +++ b/packages/cli/src/templates/github/ci.yml.test.ts @@ -132,3 +132,37 @@ describe('unit · the CI workflow x new writes', () => { expect(workflow()).toContain('timeout-minutes:'); }); }); + +interface Efficient { + readonly concurrency?: { readonly group?: string; readonly 'cancel-in-progress'?: unknown }; + readonly jobs?: Readonly>; +} + +describe('unit · the CI workflow pays for each commit once', () => { + const doc = (): Efficient => YAML.parse(workflow()) as Efficient; + + // Cancelling is safe only off the default branch: there the group is the SHA, so two commits can + // never share one and neither run can cancel the other. + test('a superseded run is cancelled, and a default-branch commit keys its own group', () => { + const concurrency = doc().concurrency; + expect(concurrency?.['cancel-in-progress']).toBe(true); + expect(concurrency?.group).toContain('github.event.repository.default_branch'); + expect(concurrency?.group).toContain('github.sha'); + }); + + // Both halves: dropping the pull_request run for a same-repo branch is the saving, and a fork's + // pull request — which fires no push in this repository — must still be gated. + test('a same-repository pull request is skipped, and a fork pull request still runs', () => { + const gate = doc().jobs?.['check']?.if ?? ''; + expect(gate).toContain("github.event_name == 'push'"); + expect(gate).toContain('github.event.pull_request.head.repo.full_name != github.repository'); + }); + + test('the install cache is keyed on the lockfile, and restored before bin/setup installs', () => { + const cache = steps().findIndex((step) => step.uses?.startsWith('actions/cache@')); + const setup = steps().findIndex((step) => step.run === 'bin/setup'); + expect(cache).toBeGreaterThanOrEqual(0); + expect(setup).toBeGreaterThan(cache); + expect(String(steps()[cache]?.with?.['key'])).toContain("hashFiles('bun.lock')"); + }); +}); diff --git a/packages/cli/src/templates/github/ci.yml.ts b/packages/cli/src/templates/github/ci.yml.ts index ff2793245..d05212ba1 100644 --- a/packages/cli/src/templates/github/ci.yml.ts +++ b/packages/cli/src/templates/github/ci.yml.ts @@ -45,11 +45,22 @@ on: push: pull_request: +# A newer push to a branch makes the run already in flight for it worthless, so it is cancelled — +# except on the default branch, where every commit keeps its own verdict: that group is keyed by +# SHA, so nothing there can supersede anything. +concurrency: + group: ci-\${{ github.ref == format('refs/heads/{0}', github.event.repository.default_branch) && github.sha || github.ref }} + cancel-in-progress: true + permissions: contents: read jobs: check: + # A branch in THIS repository with an open pull request fires both events for one commit, and + # the push run already gates that exact tree — so the pull_request run is skipped, not paid for + # twice. A pull request from a FORK fires no push here, and it still runs. + if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository runs-on: ubuntu-latest # A bound on a HANG, not on cost: a step that never returns holds a runner until GitHub's # six-hour default expires, and the failure is invisible for all six of them. @@ -63,6 +74,13 @@ jobs: # whoever took the app at its word. Never \`latest\`: a Bun minor landing unannounced is a # runtime change nobody chose. bun-version: '${REQUIRED_BUN}' + # Bun's download cache, keyed on the lockfile: \`bin/setup\`'s install then links from disk + # instead of fetching every package again on every run. + - uses: actions/cache@v6 + with: + path: ~/.bun/install/cache + key: bun-\${{ runner.os }}-\${{ hashFiles('bun.lock') }} + restore-keys: bun-\${{ runner.os }}- # Two steps rather than one \`&&\`, so the log names which half failed and times each. - run: bin/setup - run: bin/check diff --git a/scripts/coverage-gate.test.ts b/scripts/coverage-gate.test.ts index 841e6ca6d..9b8507a3d 100644 --- a/scripts/coverage-gate.test.ts +++ b/scripts/coverage-gate.test.ts @@ -2,7 +2,14 @@ // Bun's cross-package dilution, and the ratchet that fails in both directions. import { describe, expect, setDefaultTimeout, test } from 'bun:test'; -import { hasExecutableCode, judge, scopeLcov, unimportedSources } from './coverage-gate'; +import { + concurrency, + hasExecutableCode, + judge, + pool, + scopeLcov, + unimportedSources, +} from './coverage-gate'; import { COVERAGE_TARGET, PIN_SLACK } from './lib/coverage-pins'; import { REPO_SCAN_TIMEOUT_MS, repoRoot } from './lib/run'; @@ -245,3 +252,32 @@ describe('unimportedSources', () => { ); }); }); + +describe('running package suites side by side', () => { + test('--jobs defaults to every core and refuses anything but a positive integer', () => { + expect(concurrency(undefined)).toBe(Math.max(1, navigator.hardwareConcurrency)); + expect(concurrency('3')).toBe(3); + for (const bad of ['0', '-1', '1.5', 'all', '']) expect(concurrency(bad)).toBeUndefined(); + }); + + test('the pool answers in input order, never in finishing order', async () => { + const delays = [30, 0, 20, 10]; + const out = await pool(delays, 4, async (ms) => { + await Bun.sleep(ms); + return ms; + }); + expect(out).toEqual(delays); + }); + + test('the pool never has more than the limit in flight', async () => { + let inFlight = 0; + let peak = 0; + await pool([1, 2, 3, 4, 5, 6, 7], 3, async () => { + inFlight += 1; + peak = Math.max(peak, inFlight); + await Bun.sleep(5); + inFlight -= 1; + }); + expect(peak).toBe(3); + }); +}); diff --git a/scripts/coverage-gate.ts b/scripts/coverage-gate.ts index 04770747b..95e0b654b 100644 --- a/scripts/coverage-gate.ts +++ b/scripts/coverage-gate.ts @@ -8,7 +8,7 @@ // when an unrelated package grows. // // bun run scripts/coverage-gate.ts --package core [--json] -// bun run scripts/coverage-gate.ts --all [--json] +// bun run scripts/coverage-gate.ts --all [--jobs ] [--json] import { readFileSync, rmSync } from 'node:fs'; import { join } from 'node:path'; @@ -324,7 +324,22 @@ async function measure(root: string, pkg: string): Promise { { cwd: root, stdout: 'ignore', stderr: 'pipe' }, ); const stderr = await new Response(proc.stderr).text(); - await proc.exited; + // The exit code is the half of this run that is not coverage, and it was never read: a package + // whose suite FAILED alone still wrote an lcov report, cleared its bar, and passed — so the + // isolation this gate is run per package to prove was proved by nothing. Measured: a probe test + // asserting `1 === 2` in `packages/money` left `--package money` green. + if ((await proc.exited) !== 0) { + const failed = stderr + .split('\n') + .filter((line) => /^\(fail\)|^error:/.test(line.trim())) + .slice(0, 5) + .map((line) => line.trim()); + throw new ScriptError({ + code: 'X_TEST_FAILED', + cause: `bun test packages/${pkg} failed when run alone${failed.length > 0 ? `: ${failed.join('; ')}` : ''}`, + fix: `run bun test packages/${pkg} and fix what it reports — a suite green only beside other packages depends on something another package registered first`, + }); + } const file = Bun.file(join(dir, 'lcov.info')); if (!(await file.exists())) { throw new ScriptError({ @@ -339,6 +354,37 @@ async function measure(root: string, pkg: string): Promise { return reading; } +/** + * How many package suites run at once. Each is already its own `bun test` process — the isolation + * this gate exists for — so running them side by side changes nothing a suite can observe except + * the clock, and serially `--all` was 3m38s on a 12-core box that sat mostly idle. Defaults to + * every core; `--jobs 1` is the serial run. + */ +export function concurrency(flag: string | undefined): number | undefined { + if (flag === undefined) return Math.max(1, navigator.hardwareConcurrency); + const n = Number(flag); + return Number.isInteger(n) && n >= 1 ? n : undefined; +} + +/** Maps `items` through `run` with at most `limit` in flight, answering in input order. */ +export async function pool( + items: readonly T[], + limit: number, + run: (item: T) => Promise, +): Promise { + const out = new Array(items.length); + let next = 0; + const worker = async (): Promise => { + while (next < items.length) { + const index = next; + next += 1; + out[index] = await run(items[index] as T); + } + }; + await Promise.all(Array.from({ length: Math.min(limit, items.length) }, worker)); + return out; +} + async function packagesToGate(root: string, only: string | undefined): Promise { if (only !== undefined) return [only]; const entries = [...new Bun.Glob('packages/*/src').scanSync({ cwd: root, onlyFiles: false })]; @@ -363,17 +409,35 @@ if (import.meta.main) { } const names = await packagesToGate(root, only); - const verdicts: CoverageVerdict[] = []; - const findings: Finding[] = []; - for (const pkg of names) { + const jobs = concurrency(flagString(args, 'jobs')); + if (jobs === undefined) { + report( + { + ok: false, + script: 'coverage-gate', + summary: '--jobs takes a positive integer — the number of package suites run at once', + findings: [], + }, + json, + ); + } + const settled = await pool(names, jobs, async (pkg): Promise => { try { - const verdict = judge(await measure(root, pkg), COVERAGE_PINS[pkg]); - verdicts.push(verdict); - findings.push(...verdict.findings); + return judge(await measure(root, pkg), COVERAGE_PINS[pkg]); } catch (error) { if (!(error instanceof ScriptError)) throw error; - findings.push({ ...error.toFinding(), at: `packages/${pkg}` }); + return { ...error.toFinding(), at: `packages/${pkg}` }; } + }); + // Reported in package order whatever order the pool finished in, so two runs of one tree print + // one report. + const verdicts: CoverageVerdict[] = []; + const findings: Finding[] = []; + for (const outcome of settled) { + if ('reading' in outcome) { + verdicts.push(outcome); + findings.push(...outcome.findings); + } else findings.push(outcome); } const ok = findings.length === 0; diff --git a/scripts/list-package-dirs.ts b/scripts/list-package-dirs.ts index 01e75ab5a..c403ee044 100644 --- a/scripts/list-package-dirs.ts +++ b/scripts/list-package-dirs.ts @@ -1,8 +1,8 @@ #!/usr/bin/env bun -// The package DIRECTORY names, derived from disk — what the per-package CI matrix fans out over. +// The package DIRECTORY names, derived from disk — what the per-package typecheck gate walks. // // Separate from `list-workspaces.ts`, which answers package NAMES and versions for the publish -// list. The matrix needs the directory (`core`, not `@ultimat3/core`), and deriving it here is what +// list. A per-package gate needs the directory (`core`, not `@ultimat3/core`), and deriving it here is what // keeps a newly added package from being silently absent from its own gate. // // bun run scripts/list-package-dirs.ts [--json] From 9b62319d7a00f9633f25d74a62f3b41f1c7cd604 Mon Sep 17 00:00:00 2001 From: sebi Date: Wed, 16 Sep 2026 09:21:36 -0500 Subject: [PATCH 2/8] test(ai): reset the model registry per test, so the unfamilied ladder does not depend on file order Co-Authored-By: Claude Opus 5 (1M context) --- packages/ai/src/openai-models.test.ts | 20 ++++++++++++++++++-- scripts/coverage-gate.ts | 4 ++-- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/packages/ai/src/openai-models.test.ts b/packages/ai/src/openai-models.test.ts index 8e839e646..50e900e12 100644 --- a/packages/ai/src/openai-models.test.ts +++ b/packages/ai/src/openai-models.test.ts @@ -4,8 +4,15 @@ * DELIBERATE absence of the models this package would have had to guess at. */ -import { beforeEach, describe, expect, test } from 'bun:test'; -import { ANTHROPIC_MODEL_IDS, modelIds, modelSpec, moreCapableThan, registerModel } from './models'; +import { afterAll, beforeEach, describe, expect, test } from 'bun:test'; +import { + ANTHROPIC_MODEL_IDS, + modelIds, + modelSpec, + moreCapableThan, + registerModel, + resetModels, +} from './models'; import { OPENAI_MODEL_IDS, registerOpenAiModels } from './openai-models'; import { costOf } from './provider'; @@ -16,7 +23,16 @@ const MTOK = { cacheWriteTokens: 0, }; +// Reset first: the registry is process state, and a model with no family that another file left +// behind sits on the unfamilied ladder this file asserts — green or red by `bun test`'s file order, +// which differs between a laptop and a runner. beforeEach(() => { + resetModels(); + registerOpenAiModels(); +}); + +afterAll(() => { + resetModels(); registerOpenAiModels(); }); diff --git a/scripts/coverage-gate.ts b/scripts/coverage-gate.ts index 95e0b654b..96b245aac 100644 --- a/scripts/coverage-gate.ts +++ b/scripts/coverage-gate.ts @@ -331,8 +331,8 @@ async function measure(root: string, pkg: string): Promise { if ((await proc.exited) !== 0) { const failed = stderr .split('\n') - .filter((line) => /^\(fail\)|^error:/.test(line.trim())) - .slice(0, 5) + .filter((line) => /^\(fail\)|^error:|timed out/.test(line.trim())) + .slice(0, 12) .map((line) => line.trim()); throw new ScriptError({ code: 'X_TEST_FAILED', From 21531f11ec39f64af0aeb033980d210eb23d2904 Mon Sep 17 00:00:00 2001 From: sebi Date: Wed, 16 Sep 2026 09:23:44 -0500 Subject: [PATCH 3/8] ci: DEBUG (temporary) --- .github/workflows/ci.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a95ce844e..2ccf4f17f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -461,5 +461,10 @@ jobs: with: persist-credentials: false - uses: ./.github/actions/setup + - name: DEBUG harness alone + continue-on-error: true + run: | + nproc; bun test packages/testing/src/harness.test.ts 2>&1 | tail -80 + bun test packages/testing 2>&1 | grep -E '^\(fail\)|error|fail$|harness' | head -40 - name: every package's tests + coverage, each in its own process, all cores run: bun run scripts/coverage-gate.ts --all From 4a631d77c562ab9d5c9cdda81b58fde7b4c2f1cb Mon Sep 17 00:00:00 2001 From: sebi Date: Wed, 16 Sep 2026 09:27:46 -0500 Subject: [PATCH 4/8] test(testing): fixture-network restores the seal it found instead of unsealing the process Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/ci.yml | 5 ++--- packages/testing/src/fixture-network.test.ts | 11 +++++++++-- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2ccf4f17f..941bd70f2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -461,10 +461,9 @@ jobs: with: persist-credentials: false - uses: ./.github/actions/setup - - name: DEBUG harness alone + - name: DEBUG cli continue-on-error: true run: | - nproc; bun test packages/testing/src/harness.test.ts 2>&1 | tail -80 - bun test packages/testing 2>&1 | grep -E '^\(fail\)|error|fail$|harness' | head -40 + bun test packages/cli 2>&1 | grep -B40 -A3 "(fail) unit · x dev renders the app routes > a static page answers" | head -120 - name: every package's tests + coverage, each in its own process, all cores run: bun run scripts/coverage-gate.ts --all diff --git a/packages/testing/src/fixture-network.test.ts b/packages/testing/src/fixture-network.test.ts index 8c1c05043..f3df03acb 100644 --- a/packages/testing/src/fixture-network.test.ts +++ b/packages/testing/src/fixture-network.test.ts @@ -11,10 +11,17 @@ import { import { testName } from './test-types'; // The gate is process-global and bun shares one process across files: a test that leaves the -// process offline takes every later file's fetch down with it. +// process offline takes every later file's fetch down with it — and one that leaves it UNSEALED +// hands every later file real egress. So each test puts back the state this file found, which is +// the preload's seal. `unsealNetwork()` here was the second leak: `harness.test.ts` asserts the +// process arrives sealed, and failed whenever bun's file order ran it after this file — a runner +// ordering, never a laptop one. +const SEALED_AT_LOAD = isNetworkSealed(); + afterEach(() => { resetNetwork(); - unsealNetwork(); + if (SEALED_AT_LOAD) sealNetwork(); + else unsealNetwork(); }); const URL_UNDER_TEST = 'https://api.stripe.test/v1/charges'; From b400d73c80869ba5ae11cdf92bde78372c3c354c Mon Sep 17 00:00:00 2001 From: sebi Date: Wed, 16 Sep 2026 09:32:07 -0500 Subject: [PATCH 5/8] ci: DEBUG cli under coverage (temporary) --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 941bd70f2..158993ff7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -464,6 +464,6 @@ jobs: - name: DEBUG cli continue-on-error: true run: | - bun test packages/cli 2>&1 | grep -B40 -A3 "(fail) unit · x dev renders the app routes > a static page answers" | head -120 + bun test --coverage --coverage-reporter=lcov --coverage-dir=/tmp/cov packages/cli 2>&1 | grep -B30 -A2 "(fail) unit · x dev renders" | head -150 - name: every package's tests + coverage, each in its own process, all cores run: bun run scripts/coverage-gate.ts --all From 64e43c4d7e586b87dee5346571fdd9896dc1f31f Mon Sep 17 00:00:00 2001 From: sebi Date: Wed, 16 Sep 2026 09:36:04 -0500 Subject: [PATCH 6/8] test(cli): cmd-dev restores the lifecycle its x dev stop drained Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/ci.yml | 4 ---- packages/cli/src/cmd-dev.test.ts | 6 +++++- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 158993ff7..a95ce844e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -461,9 +461,5 @@ jobs: with: persist-credentials: false - uses: ./.github/actions/setup - - name: DEBUG cli - continue-on-error: true - run: | - bun test --coverage --coverage-reporter=lcov --coverage-dir=/tmp/cov packages/cli 2>&1 | grep -B30 -A2 "(fail) unit · x dev renders" | head -150 - name: every package's tests + coverage, each in its own process, all cores run: bun run scripts/coverage-gate.ts --all diff --git a/packages/cli/src/cmd-dev.test.ts b/packages/cli/src/cmd-dev.test.ts index aa08eff35..d4578bcf6 100644 --- a/packages/cli/src/cmd-dev.test.ts +++ b/packages/cli/src/cmd-dev.test.ts @@ -11,7 +11,7 @@ import { rm } from 'node:fs/promises'; // why: Bun has no recursive remove, only // why: Bun exposes no path-join primitive; Bun.file and import() take one already joined. import { join } from 'node:path'; import { declareTags, invalidateTags, isolateDeclaredTags, tag } from '@ultimat3/cache'; -import { createContext, logger, runWithContext, userActor } from '@ultimat3/core'; +import { createContext, logger, resetLifecycle, runWithContext, userActor } from '@ultimat3/core'; import { statementObserver } from '@ultimat3/db'; import { cspHashSource } from '@ultimat3/http'; import { SyncSocket } from '@ultimat3/realtime/server'; @@ -79,6 +79,10 @@ afterAll(async () => { } finally { resetRegistries(); restoreTags(); + // `stop()` DRAINS core's process-wide lifecycle, and nothing put it back: every later file that + // serves a request answered 503 X_DRAINING. `dev-render.test.ts` failed that way on a runner + // whose file order ran it after this one — never on a laptop that ordered it first. + resetLifecycle(); } }, BOOT_TIMEOUT_MS); From 44a80d000487e02c9f0e32676289bb26cefa1643 Mon Sep 17 00:00:00 2001 From: sebi Date: Wed, 16 Sep 2026 09:43:43 -0500 Subject: [PATCH 7/8] test(cli): keep cmd-dev.test.ts under the 500-line ceiling Co-Authored-By: Claude Opus 5 (1M context) --- packages/cli/src/cmd-dev.test.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/packages/cli/src/cmd-dev.test.ts b/packages/cli/src/cmd-dev.test.ts index d4578bcf6..cd493602e 100644 --- a/packages/cli/src/cmd-dev.test.ts +++ b/packages/cli/src/cmd-dev.test.ts @@ -79,10 +79,7 @@ afterAll(async () => { } finally { resetRegistries(); restoreTags(); - // `stop()` DRAINS core's process-wide lifecycle, and nothing put it back: every later file that - // serves a request answered 503 X_DRAINING. `dev-render.test.ts` failed that way on a runner - // whose file order ran it after this one — never on a laptop that ordered it first. - resetLifecycle(); + resetLifecycle(); // `stop()` drained it: a later file's request would answer 503 X_DRAINING } }, BOOT_TIMEOUT_MS); From b3d3bd1766bb3d4e7989ef62eca74fee789a233c Mon Sep 17 00:00:00 2001 From: sebi Date: Wed, 16 Sep 2026 09:48:56 -0500 Subject: [PATCH 8/8] test(scripts): pin the coverage gate's exit-code verdict; changelog names the three leaks Co-Authored-By: Claude Opus 5 (1M context) --- CHANGELOG.md | 7 +++++- scripts/coverage-gate.test.ts | 31 ++++++++++++++++++++++++++ scripts/coverage-gate.ts | 42 ++++++++++++++++++++++------------- 3 files changed, 63 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e2cb604f7..12881f816 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,7 @@ Semver applies from 1.0.0. A breaking change to a documented API needs a major ### Changed -- **The CI workflow `x new` writes pays for each commit once.** A branch with an open pull request +- **A scaffolded app's CI gates each commit once.** A branch with an open pull request fired `push` and `pull_request` for the same tree and gated it twice; the `pull_request` run is now skipped for a branch in the app's own repository and still runs for a fork, which fires no push. A newer push cancels the run in flight for its branch, except on the default branch, whose group is @@ -25,6 +25,11 @@ Semver applies from 1.0.0. A breaking change to a documented API needs a major report, cleared its bar, and reported green — measured with a probe `expect(1).toBe(2)` in `packages/money`. It is now `X_TEST_FAILED`, naming the failing tests. `--all` runs the package suites concurrently, one process each, on every core (`--jobs ` to bound it). +- **Three framework suites passed only in a lucky file order**, found the moment the coverage gate + read the exit code on a runner: `@ultimat3/ai`'s `openai-models.test.ts` inherited a model + another file registered, `@ultimat3/testing`'s `fixture-network.test.ts` handed later files an + UNSEALED network, and `@ultimat3/cli`'s `cmd-dev.test.ts` left core's lifecycle drained, so every + later request answered 503 `X_DRAINING`. Each file now restores the state it found. - **Framework CI: the 32-job per-package matrix is one `packages` job.** GitHub bills each job rounded up to a whole minute, so the matrix was 33 of the ~40 runner-minutes one push cost. diff --git a/scripts/coverage-gate.test.ts b/scripts/coverage-gate.test.ts index 9b8507a3d..fbea054c3 100644 --- a/scripts/coverage-gate.test.ts +++ b/scripts/coverage-gate.test.ts @@ -8,6 +8,7 @@ import { judge, pool, scopeLcov, + suiteFailure, unimportedSources, } from './coverage-gate'; import { COVERAGE_TARGET, PIN_SLACK } from './lib/coverage-pins'; @@ -281,3 +282,33 @@ describe('running package suites side by side', () => { expect(peak).toBe(3); }); }); + +describe('a suite that fails alone', () => { + // The defect: the exit code was never read, so a red suite with an lcov on disk passed. + test('a non-zero exit is X_TEST_FAILED, naming the failing tests', () => { + const stderr = [ + 'error: expect(received).toBe(expected)', + '(pass) money > adds', + '(fail) money > probe [0.27ms]', + ].join('\n'); + const failure = suiteFailure('money', 1, stderr); + expect(failure?.code).toBe('X_TEST_FAILED'); + expect(failure?.cause).toContain('(fail) money > probe'); + expect(failure?.cause).not.toContain('(pass)'); + expect(failure?.fix).toContain('bun test packages/money'); + }); + + test('a zero exit is no failure, whatever stderr says', () => { + expect(suiteFailure('money', 0, '(fail) not really')).toBeUndefined(); + }); + + test('a hook timeout is named, and a word merely containing the phrase is not', () => { + const cause = suiteFailure( + 'cli', + 1, + 'beforeAll timed out after 5000ms\nuntimed outcomes', + )?.cause; + expect(cause).toContain('beforeAll timed out after 5000ms'); + expect(cause).not.toContain('untimed outcomes'); + }); +}); diff --git a/scripts/coverage-gate.ts b/scripts/coverage-gate.ts index 96b245aac..e2b659252 100644 --- a/scripts/coverage-gate.ts +++ b/scripts/coverage-gate.ts @@ -298,6 +298,30 @@ export function judge(reading: CoverageReading, pin: CoveragePin | undefined): C return { reading, required: pin, findings }; } +/** + * The verdict on a suite's exit code, which is the half of a coverage run that is not coverage and + * was never read: a package whose suite FAILED alone still wrote an lcov report, cleared its bar, + * and passed — so the isolation this gate runs per package to prove was proved by nothing. + * Measured: a probe asserting `1 === 2` in `packages/money` left `--package money` green. + */ +export function suiteFailure( + pkg: string, + exitCode: number, + stderr: string, +): { code: string; cause: string; fix: string } | undefined { + if (exitCode === 0) return undefined; + const failed = stderr + .split('\n') + .map((line) => line.trim()) + .filter((line) => /^\(fail\)|^error:|\btimed out\b/.test(line)) + .slice(0, 12); + return { + code: 'X_TEST_FAILED', + cause: `bun test packages/${pkg} failed when run alone${failed.length > 0 ? `: ${failed.join('; ')}` : ''}`, + fix: `run bun test packages/${pkg} and fix what it reports — a suite green only beside other packages depends on something another package registered first`, + }; +} + /** Runs one package's suite with coverage and reads the report back. */ async function measure(root: string, pkg: string): Promise { const dir = join(root, '.x', 'coverage', pkg); @@ -324,22 +348,8 @@ async function measure(root: string, pkg: string): Promise { { cwd: root, stdout: 'ignore', stderr: 'pipe' }, ); const stderr = await new Response(proc.stderr).text(); - // The exit code is the half of this run that is not coverage, and it was never read: a package - // whose suite FAILED alone still wrote an lcov report, cleared its bar, and passed — so the - // isolation this gate is run per package to prove was proved by nothing. Measured: a probe test - // asserting `1 === 2` in `packages/money` left `--package money` green. - if ((await proc.exited) !== 0) { - const failed = stderr - .split('\n') - .filter((line) => /^\(fail\)|^error:|timed out/.test(line.trim())) - .slice(0, 12) - .map((line) => line.trim()); - throw new ScriptError({ - code: 'X_TEST_FAILED', - cause: `bun test packages/${pkg} failed when run alone${failed.length > 0 ? `: ${failed.join('; ')}` : ''}`, - fix: `run bun test packages/${pkg} and fix what it reports — a suite green only beside other packages depends on something another package registered first`, - }); - } + const failure = suiteFailure(pkg, await proc.exited, stderr); + if (failure !== undefined) throw new ScriptError(failure); const file = Bun.file(join(dir, 'lcov.info')); if (!(await file.exists())) { throw new ScriptError({