From 14b3ea07a36672a9897484095c90807a0ef53d4c Mon Sep 17 00:00:00 2001 From: Ryan Bledsoe Date: Mon, 21 Sep 2026 20:25:11 -0500 Subject: [PATCH 1/3] perf(ci): overlap Sonar setup and balance Electron shards --- .github/workflows/build.yml | 51 ++++---- docs/DEVELOPMENT.md | 22 +++- scripts/ci-optimization-contract.test.mjs | 46 ++++++- scripts/electron-test-durations.json | 61 ++++++++++ scripts/plan-electron-shards.mjs | 111 +++++++++++++++++ scripts/plan-electron-shards.test.mjs | 101 ++++++++++++++++ scripts/run-sonar-ci.mjs | 4 +- scripts/run-sonar-ci.test.mjs | 12 +- scripts/security-workflow-contract.test.mjs | 21 ++-- scripts/wait-for-coverage.mjs | 107 +++++++++++++++++ scripts/wait-for-coverage.test.mjs | 126 ++++++++++++++++++++ 11 files changed, 613 insertions(+), 49 deletions(-) create mode 100644 scripts/electron-test-durations.json create mode 100644 scripts/plan-electron-shards.mjs create mode 100644 scripts/plan-electron-shards.test.mjs create mode 100644 scripts/wait-for-coverage.mjs create mode 100644 scripts/wait-for-coverage.test.mjs diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0b704fda..92911d77 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -281,14 +281,18 @@ jobs: run: npx playwright install-deps chromium # Isolated runners avoid concurrent native rebuilds. Keep one worker per - # shard; fully-parallel distributes individual tests, including large specs. + # shard; duration estimates only assign tests, never select the test inventory. + - name: Balance Electron tests by duration + if: matrix.suite == 'electron' + run: node scripts/plan-electron-shards.mjs ${{ matrix.shard-index }}/${{ matrix.shard-total }} "$RUNNER_TEMP/electron-shard.txt" + - name: Run Electron workflows if: matrix.suite == 'electron' run: | sudo apt-get install --yes dbus-x11 gnome-keyring dbus-run-session -- bash -euo pipefail -c ' openssl rand -hex 32 | gnome-keyring-daemon --unlock --components=secrets - xvfb-run --auto-servernum npm run test:electron -- --fully-parallel --workers=1 --shard=${{ matrix.shard-index }}/${{ matrix.shard-total }} + xvfb-run --auto-servernum npm run test:electron -- --fully-parallel --workers=1 --test-list="$RUNNER_TEMP/electron-shard.txt" ' - name: Run browser workflows @@ -313,7 +317,7 @@ jobs: github.event.pull_request.base.ref == 'main' && github.event.pull_request.head.repo.full_name == github.repository) ) - needs: [provenance, unit-coverage, renderer-coverage] + needs: provenance permissions: actions: read contents: read @@ -322,13 +326,11 @@ jobs: env: PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1 steps: - - name: Require valid provenance or successful coverage + - name: Require valid provenance env: ELIGIBLE: ${{ needs.provenance.outputs.eligible }} PROVENANCE_RESULT: ${{ needs.provenance.result }} REUSE: ${{ needs.provenance.outputs.reuse }} - UNIT_COVERAGE_RESULT: ${{ needs.unit-coverage.result }} - RENDERER_COVERAGE_RESULT: ${{ needs.renderer-coverage.result }} shell: bash run: | if [[ "$PROVENANCE_RESULT" != "success" ]]; then @@ -344,12 +346,6 @@ jobs: exit 1 fi - if [[ "$UNIT_COVERAGE_RESULT" != "success" || - "$RENDERER_COVERAGE_RESULT" != "success" ]]; then - echo "Coverage jobs did not both succeed: unit-coverage=$UNIT_COVERAGE_RESULT renderer-coverage=$RENDERER_COVERAGE_RESULT" - exit 1 - fi - - name: Checkout exact commit uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 with: @@ -379,6 +375,25 @@ jobs: RELAY_SKIP_POCKETBASE_DOWNLOAD: '1' run: npm ci --prefer-offline + - name: Install verified SonarScanner CLI + shell: bash + run: | + archive="$RUNNER_TEMP/sonar-scanner.zip" + curl --fail --silent --show-error --location --proto '=https' --tlsv1.2 \ + --retry 3 --max-time 120 \ + --output "$archive" \ + https://binaries.sonarsource.com/Distribution/sonar-scanner-cli/sonar-scanner-cli-8.1.0.6389-linux-x64.zip + printf '%s %s\n' bb8f709f9cb73352f8d1260a3b3c506c0f41146754bc630762c126d795499d0b "$archive" | shasum -a 256 -c - + unzip -q "$archive" -d "$RUNNER_TEMP/relay-sonar-scanner" + echo "$RUNNER_TEMP/relay-sonar-scanner/sonar-scanner-8.1.0.6389-linux-x64/bin" >> "$GITHUB_PATH" + + - name: Wait for successful coverage + if: needs.provenance.outputs.reuse != 'true' + env: + GH_TOKEN: ${{ github.token }} + EXPECTED_HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + run: node scripts/wait-for-coverage.mjs + - name: Download unit coverage if: needs.provenance.outputs.reuse != 'true' uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 @@ -424,18 +439,6 @@ jobs: if-no-files-found: error retention-days: 1 - - name: Install verified SonarScanner CLI - shell: bash - run: | - archive="$RUNNER_TEMP/sonar-scanner.zip" - curl --fail --silent --show-error --location --proto '=https' --tlsv1.2 \ - --retry 3 --max-time 120 \ - --output "$archive" \ - https://binaries.sonarsource.com/Distribution/sonar-scanner-cli/sonar-scanner-cli-8.1.0.6389-linux-x64.zip - printf '%s %s\n' bb8f709f9cb73352f8d1260a3b3c506c0f41146754bc630762c126d795499d0b "$archive" | shasum -a 256 -c - - unzip -q "$archive" -d "$RUNNER_TEMP/relay-sonar-scanner" - echo "$RUNNER_TEMP/relay-sonar-scanner/sonar-scanner-8.1.0.6389-linux-x64/bin" >> "$GITHUB_PATH" - - name: Run Sonar finding gate shell: bash env: diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md index 9c8f4fee..fe3bb27e 100644 --- a/docs/DEVELOPMENT.md +++ b/docs/DEVELOPMENT.md @@ -243,9 +243,14 @@ The Build workflow owns the full pull-request and `main` verification graph. Its `Build quality gate` fails closed over formatting, linting, type checking, dependency audit, the production build, unit coverage plus cache integration tests, four renderer-coverage shards, and the mandatory `workflow-tests` matrix. Four isolated Electron runners execute -`npm run test:electron -- --fully-parallel --workers=1 --shard=N/4` under Xvfb with an unlocked -ephemeral keyring. Test-level sharding divides large specs across runners while keeping one worker -per runner. A fifth runner executes +`npm run test:electron -- --fully-parallel --workers=1 --test-list=...` under Xvfb with an unlocked +ephemeral keyring. Before execution, `scripts/plan-electron-shards.mjs` discovers the full current +Playwright inventory and assigns every test to exactly one of four shards, balancing the longest +measured tests first using `scripts/electron-test-durations.json`. The timing file records its +source run and only affects assignment: new or renamed tests receive a 30-second estimate, and +removed tests cannot remain in the inventory. Keep one worker per runner. To refresh estimates, +use successful Linux Electron job timings with the same file and full title identifiers; include +setup separately when comparing elapsed job time. A fifth runner executes `npm run test:pocketbase -- verification/offline-replay-real-pb.test.ts verification/dynatrace-pipeline.test.ts`, then `npm run test:web` under Xvfb against Chromium and WebKit. Electron runners install only the Linux libraries they need; the web runner downloads the browsers. Each job uses its own npm install, @@ -254,12 +259,21 @@ The matrix runs on every Build invocation, including when exact-tree reuse succe disabled so every shard reports its result, with uniquely named failure artifacts. Any unsuccessful or missing matrix result blocks the aggregate gate and the Release workflow that waits for it. Those coverage jobs are canonical: Sonar consumes their merged reports instead of rerunning the -same tests. The required `SonarQube quality gate` and `Snyk security gate` names remain stable in +same tests. Sonar checkout, dependency installation, cache restore and scanner setup overlap with +coverage. Before downloading fresh coverage, a bounded five-minute wait requires successful unit +coverage and all four renderer coverage jobs from the exact GitHub run attempt and head SHA. +GitHub's attempt endpoint also includes successful jobs carried forward by partial reruns; failed, +skipped, ambiguous or mismatched jobs cannot authorize analysis. Validated exact-tree reuse still +uses its provenance-checked PR LCOV instead of waiting for intentionally skipped coverage jobs. The required `SonarQube quality gate` and `Snyk security gate` names remain stable in the same workflow. Sonar always runs for the exact final `main` commit, including its reviewed-issue reconciliation; optimization never turns a post-merge branch Sonar scan into a reused PR result. When validated PR Snyk findings are reused, a lightweight main-only monitor still refreshes the canonical Snyk project snapshot before the required Snyk gate succeeds. +Main scanner analysis/upload has a 15-minute deadline; PR scans retain 10 minutes. Both share the +existing 18-minute aggregate deadline across scanning and all subsequent API checks. This allows a +slow main scan to finish without an unnecessary retry while keeping a hard overall bound. + The Sonar wrapper records analysis/upload, server wait, reviewed-issue reconciliation, issue indexing, and quality-gate timings in the GitHub job summary, including failed phases. It also ranks completed sensors reported in the retained normal scanner output. Sensor timings are included in the diff --git a/scripts/ci-optimization-contract.test.mjs b/scripts/ci-optimization-contract.test.mjs index 157bf619..bc2e5e75 100644 --- a/scripts/ci-optimization-contract.test.mjs +++ b/scripts/ci-optimization-contract.test.mjs @@ -4,6 +4,7 @@ import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { describe, expect, it } from 'vitest'; import { parse } from 'yaml'; +import { COVERAGE_JOBS } from './wait-for-coverage.mjs'; import unitConfig from '../vitest.config.ts'; import cacheConfig from '../vitest.cache.config.ts'; import rendererConfig from '../vitest.renderer.config.ts'; @@ -178,13 +179,21 @@ describe('CI optimization contracts', () => { expect(electronDependencies.run).toBe('npx playwright install-deps chromium'); const electron = findStep(workflows, 'Run Electron workflows'); const web = findStep(workflows, 'Run browser workflows'); + const plan = findStep(workflows, 'Balance Electron tests by duration'); + expect(plan.if).toBe("matrix.suite == 'electron'"); + expect(plan.run).toBe( + 'node scripts/plan-electron-shards.mjs ${{ matrix.shard-index }}/${{ matrix.shard-total }} "$RUNNER_TEMP/electron-shard.txt"', + ); + expect(workflows.steps.indexOf(plan)).toBeLessThan(workflows.steps.indexOf(electron)); + expect(plan['continue-on-error']).not.toBe(true); + expect(electron.run).toContain('sudo apt-get install --yes dbus-x11 gnome-keyring'); expect(electron.run).toContain('dbus-run-session -- bash -euo pipefail'); expect(electron.run).toContain( 'openssl rand -hex 32 | gnome-keyring-daemon --unlock --components=secrets', ); expect(electron.run).toContain( - 'xvfb-run --auto-servernum npm run test:electron -- --fully-parallel --workers=1 --shard=${{ matrix.shard-index }}/${{ matrix.shard-total }}', + 'xvfb-run --auto-servernum npm run test:electron -- --fully-parallel --workers=1 --test-list="$RUNNER_TEMP/electron-shard.txt"', ); expect(web.run).toBe('xvfb-run --auto-servernum npm run test:web'); expect(workflows.steps.indexOf(pocketbase)).toBeGreaterThan(workflows.steps.indexOf(install)); @@ -268,7 +277,35 @@ describe('CI optimization contracts', () => { const sonar = build.jobs.sonarqube; expect(sonar.name).toBe('SonarQube quality gate'); - expect(sonar.needs).toEqual(['provenance', 'unit-coverage', 'renderer-coverage']); + expect(sonar.needs).toBe('provenance'); + const wait = findStep(sonar, 'Wait for successful coverage'); + expect(wait).toEqual({ + name: 'Wait for successful coverage', + if: "needs.provenance.outputs.reuse != 'true'", + env: { + GH_TOKEN: '${{ github.token }}', + EXPECTED_HEAD_SHA: '${{ github.event.pull_request.head.sha || github.sha }}', + }, + run: 'node scripts/wait-for-coverage.mjs', + }); + expect(COVERAGE_JOBS).toEqual([ + build.jobs['unit-coverage'].name, + ...rendererCoverage.strategy.matrix['shard-index'].map((index) => + rendererCoverage.name + .replace('${{ matrix.shard-index }}', index) + .replace('${{ matrix.shard-total }}', rendererCoverage.strategy.matrix['shard-total'][0]), + ), + ]); + for (const name of ['Install dependencies', 'Install verified SonarScanner CLI']) { + expect(sonar.steps.indexOf(findStep(sonar, name))).toBeLessThan(sonar.steps.indexOf(wait)); + } + for (const name of [ + 'Download unit coverage', + 'Download renderer coverage shards', + 'Run Sonar finding gate', + ]) { + expect(sonar.steps.indexOf(findStep(sonar, name))).toBeGreaterThan(sonar.steps.indexOf(wait)); + } const merge = findStep(sonar, 'Merge renderer coverage'); expect(merge.if).toBe("needs.provenance.outputs.reuse != 'true'"); expect(merge.run).toContain('--merge-reports'); @@ -413,19 +450,16 @@ describe('CI optimization contracts', () => { } const sonar = build.jobs.sonarqube; - const preflight = findStep(sonar, 'Require valid provenance or successful coverage'); + const preflight = findStep(sonar, 'Require valid provenance'); expect(sonar.if).toContain('always()'); expect(preflight.env).toEqual({ ELIGIBLE: '${{ needs.provenance.outputs.eligible }}', PROVENANCE_RESULT: '${{ needs.provenance.result }}', - RENDERER_COVERAGE_RESULT: '${{ needs.renderer-coverage.result }}', REUSE: '${{ needs.provenance.outputs.reuse }}', - UNIT_COVERAGE_RESULT: '${{ needs.unit-coverage.result }}', }); expect(preflight.run).toContain('[[ "$PROVENANCE_RESULT" != "success" ]]'); expect(preflight.run).toContain('[[ "$REUSE" == "true" ]]'); expect(preflight.run).toContain('[[ "$ELIGIBLE" == "true" ]]'); - expect(preflight.run).toContain('[[ "$UNIT_COVERAGE_RESULT" != "success"'); expect(findStep(sonar, 'Checkout exact commit').with).toEqual({ 'fetch-depth': 0, diff --git a/scripts/electron-test-durations.json b/scripts/electron-test-durations.json new file mode 100644 index 00000000..a9b91126 --- /dev/null +++ b/scripts/electron-test-durations.json @@ -0,0 +1,61 @@ +{ + "source": "https://github.com/CrimsonSoul/Relay/actions/runs/35671309552", + "durations": { + "[] \u203a backup-verification.spec.ts \u203a backup deadline kills synchronous native SQLite before disposing its files": 6.0, + "[] \u203a backup-verification.spec.ts \u203a completed restore exposes original records, IDs, unknown collections and files": 3.1, + "[] \u203a backup-verification.spec.ts \u203a failed restored server startup recovers original running data": 3.2, + "[] \u203a critical-path.spec.ts \u203a Vital Critical Path \u203a Compose bridge action buttons do not overlap on compact desktop widths": 3.5, + "[] \u203a critical-path.spec.ts \u203a Vital Critical Path \u203a Dynatrace Problems demo seed is repeatable, isolated, and renders concise problem details": 11.3, + "[] \u203a critical-path.spec.ts \u203a Vital Critical Path \u203a Dynatrace Problems tab opens without requiring a configured token": 4.8, + "[] \u203a critical-path.spec.ts \u203a Vital Critical Path \u203a Knowledge launches Wiki, Contacts, and Servers in order and retains contextual state": 25.0, + "[] \u203a critical-path.spec.ts \u203a Vital Critical Path \u203a Relay shell and Dynatrace workspace adapt to compact desktop widths": 7.6, + "[] \u203a critical-path.spec.ts \u203a Vital Critical Path \u203a Service Status uses the operational queue layout": 5.9, + "[] \u203a critical-path.spec.ts \u203a Vital Critical Path \u203a Vital 1: App Launch & Compose Tab": 2.3, + "[] \u203a critical-path.spec.ts \u203a Vital Critical Path \u203a Vital 2: Navigation to On-Call & Servers": 8.3, + "[] \u203a critical-path.spec.ts \u203a Vital Critical Path \u203a Vital 3: Data Integrity (Add/Delete Contact)": 27.0, + "[] \u203a critical-path.spec.ts \u203a Vital Critical Path \u203a Vital 4: On-Call Management (Add/Rename/Remove Card)": 23.3, + "[] \u203a critical-path.spec.ts \u203a Vital Critical Path \u203a Vital 5: Composer Workflow (Add, Group, Draft)": 66.0, + "[] \u203a critical-path.spec.ts \u203a Vital Critical Path \u203a Workstation Settings exposes platform-accurate keep-awake protection": 6.1, + "[] \u203a critical-path.spec.ts \u203a Vital Critical Path \u203a ambient delay isolation contract \u203a null delays clear ambient launch configuration": 2.3, + "[] \u203a critical-path.spec.ts \u203a Vital Critical Path \u203a connected client queues ordinary attributed Dynatrace actions offline": 19.4, + "[] \u203a critical-path.spec.ts \u203a Vital Critical Path \u203a delayed startup fixture profile \u203a startup shell appears before workspace readiness and healthy relaunch skips credential repair": 4.9, + "[] \u203a critical-path.spec.ts \u203a Vital Critical Path \u203a desktop isolation contract \u203a keeps the native Electron test window hidden": 2.3, + "[] \u203a critical-path.spec.ts \u203a Vital Critical Path \u203a ordinary Dynatrace actions stay passwordless while historical snapshots remain visible": 21.3, + "[] \u203a critical-path.spec.ts \u203a Vital Critical Path \u203a publisher resumes a Knowledge batch after interruption and publishes for passwordless readers": 84.0, + "[] \u203a critical-path.spec.ts \u203a Vital Critical Path \u203a rename-safe fixture profile contract \u203a fixture configuration remains stable when this test title changes": 3.4, + "[] \u203a critical-path.spec.ts \u203a Vital Critical Path \u203a seeded Knowledge link fixture profile \u203a Knowledge PDF links navigate within Relay without escaping the E2E desktop boundary": 46.8, + "[] \u203a critical-path.spec.ts \u203a Vital Critical Path \u203a seeded Knowledge management fixture profile \u203a Knowledge management document workflow preserves search edit rename and pagination": 31.5, + "[] \u203a critical-path.spec.ts \u203a Vital Critical Path \u203a seeded Knowledge management fixture profile \u203a Knowledge management keeps retained audit records out of the retired navigation": 14.1, + "[] \u203a critical-path.spec.ts \u203a Vital Critical Path \u203a seeded Knowledge management fixture profile \u203a Knowledge management responsive geometry preserves navigation and bottom gutters": 14.4, + "[] \u203a critical-path.spec.ts \u203a Vital Critical Path \u203a seeded Knowledge management fixture profile \u203a Knowledge management trash workflow restores and permanently deletes live documents": 30.5, + "[] \u203a critical-path.spec.ts \u203a Vital Critical Path \u203a seeded Knowledge management fixture profile \u203a delayed Knowledge upload fixture profile \u203a Knowledge management upload workflow preserves transfer publish and replace controls": 60.0, + "[] \u203a critical-path.spec.ts \u203a Vital Critical Path \u203a seeded Knowledge reader fixture profile \u203a Knowledge PDF links survive repeated top-level tab leave and return cycles": 34.7, + "[] \u203a critical-path.spec.ts \u203a Vital Critical Path \u203a seeded Knowledge reader fixture profile \u203a Knowledge PDF links use the compact Wiki Library drawer without losing reader state": 38.3, + "[] \u203a critical-path.spec.ts \u203a Vital Critical Path \u203a seeded Knowledge reader fixture profile \u203a continuous Wiki PDF scrolls, tracks pages, bounds canvases, and retains reader state": 31.6, + "[] \u203a critical-path.spec.ts \u203a Vital Critical Path \u203a seeded Knowledge reader fixture profile \u203a role accounts preserve username sign-in, owner boundaries, passwordless use, and offline reading": 60.0, + "[] \u203a css-visual-contracts.spec.ts \u203a Dynatrace problem details keep long unbroken text inside the narrow detail pane": 0.461, + "[] \u203a css-visual-contracts.spec.ts \u203a Radar keeps the health rail left when wide and stacks without overflow when narrow": 0.47200000000000003, + "[] \u203a css-visual-contracts.spec.ts \u203a Radar status keeps the standard sidebar footprint in full and compact shells": 0.597, + "[] \u203a css-visual-contracts.spec.ts \u203a Windows shell dividers align in full and compact sidebars": 0.446, + "[] \u203a css-visual-contracts.spec.ts \u203a collapsed Wiki reader keeps the PDF viewer full-width at the medium desktop breakpoint": 0.454, + "[] \u203a css-visual-contracts.spec.ts \u203a emitted cascade layers preserve top-level and lazy-feature precedence": 0.498, + "[] \u203a css-visual-contracts.spec.ts \u203a flagged chips retain WCAG text contrast across every Relay accent and opaque background": 0.6, + "[] \u203a css-visual-contracts.spec.ts \u203a release progress stays thin and visible with honest determinate and reduced-motion states": 0.498, + "[] \u203a css-visual-contracts.spec.ts \u203a release update actions stay inside the dialog when restart adds a third button": 0.452, + "[] \u203a css-visual-contracts.spec.ts \u203a stable gutters preserve Relay topology under overlay and classic scrollbar widths": 0.492, + "[] \u203a css-visual-contracts.spec.ts \u203a tab chrome toolbar geometry and Header Search actions stay aligned": 0.659, + "[] \u203a knowledge-pdf-layout.spec.ts \u203a Wiki reader geometry preserves the compact container drawer": 0.46, + "[] \u203a knowledge-pdf-layout.spec.ts \u203a continuous PDF keeps oversized pages reachable and smaller pages centered": 3.0, + "[] \u203a knowledge-pdf-layout.spec.ts \u203a narrow reader controls and fitted page content stay contained and readable": 0.395, + "[] \u203a radar-certificate.spec.ts \u203a Radar shares sign-in cookies and recovers from 401 while scoping its private CA exception": 0.541, + "[] \u203a recovery-runtime-integrity.spec.ts \u203a recovery verifies physical ASAR bytes inside Electron and rejects archive tampering": 0.624, + "[] \u203a sdp-changes.spec.ts \u203a SDP change correlation explains automatic and suggested relationships without provider writes": 31.1, + "[] \u203a service-desk.spec.ts \u203a live ticket shell, detail, major incident confirmation and no demo controls": 120.0, + "[] \u203a service-desk.spec.ts \u203a request history, forwarding, checklists, reminders and bulk reviews work with isolated fixtures": 84.0, + "[] \u203a setup-auth.spec.ts \u203a Setup Screen & Auth Flow \u203a Back button returns to mode selection": 4.6, + "[] \u203a setup-auth.spec.ts \u203a Setup Screen & Auth Flow \u203a Client mode: validates server URL": 4.5, + "[] \u203a setup-auth.spec.ts \u203a Setup Screen & Auth Flow \u203a Server mode: accepts valid config and transitions past setup": 6.7, + "[] \u203a setup-auth.spec.ts \u203a Setup Screen & Auth Flow \u203a Server mode: validates passphrase length": 4.5, + "[] \u203a setup-auth.spec.ts \u203a Setup Screen & Auth Flow \u203a Shows setup screen on first launch": 0.516 + } +} diff --git a/scripts/plan-electron-shards.mjs b/scripts/plan-electron-shards.mjs new file mode 100644 index 00000000..5154589a --- /dev/null +++ b/scripts/plan-electron-shards.mjs @@ -0,0 +1,111 @@ +import { spawnSync } from 'node:child_process'; +import { readFileSync, writeFileSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; + +const root = dirname(dirname(fileURLToPath(import.meta.url))); +const compare = (a, b) => { + if (a === b) return 0; + return a < b ? -1 : 1; +}; + +// Always discover the current suite. Timings affect placement only; a renamed or +// newly added test receives a conservative estimate and still runs exactly once. +export function inventoryFromReport(report) { + if (!Array.isArray(report.suites) || !Array.isArray(report.errors) || report.errors.length) { + throw new Error('Electron test discovery failed.'); + } + const inventory = []; + function visit(suite, titles) { + for (const spec of suite.specs ?? []) { + for (const test of spec.tests ?? []) { + const parts = [spec.file, ...titles, spec.title]; + if ( + parts.some((part) => typeof part !== 'string' || !part.trim() || /[\r\n›]/u.test(part)) || + typeof test.projectName !== 'string' || + /[\r\n›[\]]/u.test(test.projectName) + ) { + throw new Error('Unsupported Electron test-list identifier.'); + } + inventory.push(`[${test.projectName}] › ${parts.join(' › ')}`); + } + } + for (const child of suite.suites ?? []) visit(child, [...titles, child.title]); + } + for (const suite of report.suites) visit(suite, []); + if (!inventory.length || new Set(inventory).size !== inventory.length) { + throw new Error('Electron test inventory must be nonempty and unique.'); + } + // Playwright test lists match title prefixes. Reject overlapping identifiers + // instead of accidentally executing a test in two different shards. + const ordered = [...inventory].sort(compare); + if (ordered.some((id, index) => index > 0 && id.startsWith(`${ordered[index - 1]} › `))) { + throw new Error('Electron test-list identifiers overlap.'); + } + return inventory; +} + +export function balanceTests(inventory, durations, count) { + if ( + !Number.isSafeInteger(count) || + count < 1 || + count > inventory.length || + new Set(inventory).size !== inventory.length + ) { + throw new Error('Invalid Electron shard count or inventory.'); + } + const weighted = inventory + .map((id) => { + const seconds = Object.hasOwn(durations, id) ? durations[id] : 30; + if (!Number.isFinite(seconds) || seconds <= 0) throw new Error(`Invalid duration: ${id}.`); + return { id, seconds }; + }) + .sort((a, b) => b.seconds - a.seconds || compare(a.id, b.id)); + const shards = Array.from({ length: count }, () => ({ tests: [], seconds: 0 })); + for (const test of weighted) { + const shard = shards.reduce((best, candidate) => + candidate.seconds < best.seconds ? candidate : best, + ); + shard.tests.push(test.id); + shard.seconds += test.seconds; + } + for (const shard of shards) shard.tests.sort(compare); + return shards; +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + const [, , selection, output] = process.argv; + const match = /^([1-9]\d*)\/([1-9]\d*)$/u.exec(selection ?? ''); + if (!match || !output || Number(match[1]) > Number(match[2])) { + throw new Error('Usage: node scripts/plan-electron-shards.mjs INDEX/TOTAL OUTPUT'); + } + const result = spawnSync( + process.execPath, + [ + join(root, 'node_modules/@playwright/test/cli.js'), + 'test', + '-c', + 'playwright.electron.config.ts', + '--list', + '--reporter=json', + ], + { cwd: root, encoding: 'utf8', maxBuffer: 16 * 1024 * 1024 }, + ); + if (result.error || result.signal || result.status !== 0) { + throw new Error( + `Electron test discovery failed: ${result.stderr || result.error || result.signal}`, + ); + } + const inventory = inventoryFromReport(JSON.parse(result.stdout)); + const { durations } = JSON.parse( + readFileSync(new URL('./electron-test-durations.json', import.meta.url), 'utf8'), + ); + const shards = balanceTests(inventory, durations, Number(match[2])); + writeFileSync(output, `${shards[Number(match[1]) - 1].tests.join('\n')}\n`); + console.log( + `Discovered ${inventory.length} tests; shard ${selection} selects ${shards[Number(match[1]) - 1].tests.length}.`, + ); + console.log( + `Estimated test seconds per shard: ${shards.map((shard) => Math.round(shard.seconds)).join(', ')} (excludes setup).`, + ); +} diff --git a/scripts/plan-electron-shards.test.mjs b/scripts/plan-electron-shards.test.mjs new file mode 100644 index 00000000..1df72fc9 --- /dev/null +++ b/scripts/plan-electron-shards.test.mjs @@ -0,0 +1,101 @@ +import { describe, expect, it } from 'vitest'; +import { execFileSync } from 'node:child_process'; +import { mkdtempSync, readFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { balanceTests, inventoryFromReport } from './plan-electron-shards.mjs'; + +const spec = (title, projectName = '') => ({ + file: 'example.spec.ts', + title, + tests: [{ projectName }], +}); +const report = (specs, suites = []) => ({ + errors: [], + suites: [{ title: 'example.spec.ts', specs, suites }], +}); + +describe('duration-balanced Electron shards', () => { + it('selects the full real Playwright suite exactly once across all generated lists', () => { + const directory = mkdtempSync(join(tmpdir(), 'relay-shards-')); + const list = (extra = []) => + inventoryFromReport( + JSON.parse( + execFileSync( + process.execPath, + [ + 'node_modules/@playwright/test/cli.js', + 'test', + '-c', + 'playwright.electron.config.ts', + '--list', + '--reporter=json', + ...extra, + ], + { encoding: 'utf8' }, + ), + ), + ); + try { + const full = list(); + const selected = []; + for (let index = 1; index <= 4; index += 1) { + const path = join(directory, `shard-${index}.txt`); + execFileSync(process.execPath, ['scripts/plan-electron-shards.mjs', `${index}/4`, path]); + const actual = list([`--test-list=${path}`]); + expect(actual.sort()).toEqual(readFileSync(path, 'utf8').trim().split('\n').sort()); + selected.push(...actual); + } + expect(selected.sort()).toEqual(full.sort()); + expect(new Set(selected).size).toBe(full.length); + } finally { + rmSync(directory, { recursive: true, force: true }); + } + }, 30_000); + + it('assigns every discovered test exactly once, including new and renamed tests', () => { + const inventory = inventoryFromReport( + report( + [spec('slow'), spec('medium'), spec('new')], + [{ title: 'nested', specs: [spec('renamed')] }], + ), + ); + const shards = balanceTests( + inventory, + { [inventory[0]]: 100, [inventory[1]]: 50, deleted: 900 }, + 2, + ); + expect(shards.flatMap((shard) => shard.tests).sort()).toEqual([...inventory].sort()); + expect(shards.map((shard) => shard.seconds)).toEqual([100, 110]); + expect(inventory[3]).toBe('[] › example.spec.ts › nested › renamed'); + expect( + balanceTests([...inventory].reverse(), { [inventory[0]]: 100, [inventory[1]]: 50 }, 2), + ).toEqual(shards); + }); + + it('keeps projects distinct and never omits skipped tests from discovery', () => { + const input = report([spec('same', 'one'), spec('same', 'two')]); + input.suites[0].specs[0].tests[0].expectedStatus = 'skipped'; + expect(inventoryFromReport(input)).toEqual([ + '[one] › example.spec.ts › same', + '[two] › example.spec.ts › same', + ]); + }); + + it.each([ + { errors: ['load failed'], suites: [] }, + report([]), + report([spec('duplicate'), spec('duplicate')]), + report([spec('title › delimiter')]), + report([spec('multiline\ntitle')]), + report([spec('prefix')], [{ title: 'prefix', specs: [spec('child')] }]), + ])('rejects broken or ambiguous discovery rather than silently dropping tests', (input) => { + expect(() => inventoryFromReport(input)).toThrow(); + }); + + it('rejects invalid shard counts and invalid timing data', () => { + for (const count of [0, 3, 1.5]) expect(() => balanceTests(['a', 'b'], {}, count)).toThrow(); + for (const seconds of [0, -1, NaN, Infinity]) + expect(() => balanceTests(['a'], { a: seconds }, 1)).toThrow(); + }); +}); diff --git a/scripts/run-sonar-ci.mjs b/scripts/run-sonar-ci.mjs index 7b4e3edc..d4d6c1ff 100644 --- a/scripts/run-sonar-ci.mjs +++ b/scripts/run-sonar-ci.mjs @@ -17,6 +17,7 @@ import { runSonarReviewedIssues } from './sonar-reviewed-issues.mjs'; import { writeSonarPerformance } from './sonar-performance.mjs'; const COMMAND_TIMEOUT_MS = 600_000; +const MAIN_COMMAND_TIMEOUT_MS = 900_000; const AGGREGATE_TIMEOUT_MS = 1_080_000; const API_PHASE_TIMEOUT_MS = 300_000; const REQUEST_TIMEOUT_MS = 30_000; @@ -179,8 +180,9 @@ export async function runSonarCi({ const scope = validateConfiguration(argv, env); const deadline = now() + AGGREGATE_TIMEOUT_MS; const scopedArgument = scopeArgument(scope); + const uploadTimeout = 'branch' in scope ? MAIN_COMMAND_TIMEOUT_MS : COMMAND_TIMEOUT_MS; const upload = await measure('Scanner analysis and upload', () => - runCommand(scannerCommand(env, phaseTimeout(deadline, now, 'upload', COMMAND_TIMEOUT_MS))), + runCommand(scannerCommand(env, phaseTimeout(deadline, now, 'upload', uploadTimeout))), ); scannerOutput = upload.output; const uploadOutcome = classifyCommandResult(upload, SONAR_UPLOAD_POLICY); diff --git a/scripts/run-sonar-ci.test.mjs b/scripts/run-sonar-ci.test.mjs index 5590769d..ed0eb36b 100644 --- a/scripts/run-sonar-ci.test.mjs +++ b/scripts/run-sonar-ci.test.mjs @@ -286,16 +286,16 @@ test('uses one aggregate deadline across Sonar upload and API phases', async () now: () => clock, runCommand: async (command) => { calls.push(['upload', command.timeoutMs]); - clock += 600_000; + clock += 900_000; return { code: 0, timedOut: false, output: '' }; }, waitAnalysis: async (options) => { calls.push(['wait', options.timeoutMs]); - clock += 300_000; + clock += 120_000; }, reconcile: async (options) => { calls.push(['reconcile', options.timeoutMs]); - clock += 180_000; + clock += 60_000; }, readIssues: async () => { calls.push(['issues']); @@ -307,9 +307,9 @@ test('uses one aggregate deadline across Sonar upload and API phases', async () assert.equal(result.outcome, SCANNER_OUTCOME.UNAVAILABLE); assert.deepEqual(calls, [ - ['upload', 600_000], - ['wait', 300_000], - ['reconcile', 180_000], + ['upload', 900_000], + ['wait', 180_000], + ['reconcile', 60_000], ]); assert.equal(reports.length, 1); }); diff --git a/scripts/security-workflow-contract.test.mjs b/scripts/security-workflow-contract.test.mjs index c883892b..669fe96c 100644 --- a/scripts/security-workflow-contract.test.mjs +++ b/scripts/security-workflow-contract.test.mjs @@ -95,7 +95,7 @@ test('Sonar consumes unit coverage and all four merged renderer coverage shards' 'retention-days': 1, }, }); - assert.deepEqual(sonar.needs, ['provenance', 'unit-coverage', 'renderer-coverage']); + assert.equal(sonar.needs, 'provenance'); assert.deepEqual(findStep(sonar, 'Download unit coverage').with, { name: 'unit-coverage', path: 'coverage/unit', @@ -139,22 +139,27 @@ test('Sonar runs on the exact commit and fails closed without valid reuse or fre `), ); - const coverageGate = findStep(sonar, 'Require valid provenance or successful coverage'); + const coverageGate = findStep(sonar, 'Require valid provenance'); assert.equal(sonar.steps.indexOf(coverageGate), 0); assert.deepEqual(coverageGate.env, { ELIGIBLE: '${{ needs.provenance.outputs.eligible }}', PROVENANCE_RESULT: '${{ needs.provenance.result }}', - RENDERER_COVERAGE_RESULT: '${{ needs.renderer-coverage.result }}', REUSE: '${{ needs.provenance.outputs.reuse }}', - UNIT_COVERAGE_RESULT: '${{ needs.unit-coverage.result }}', }); assert.match(coverageGate.run, /\$PROVENANCE_RESULT.*success/u); assert.match(coverageGate.run, /\$REUSE.*true/u); assert.match(coverageGate.run, /\$ELIGIBLE.*true/u); - assert.match( - normalizeExpression(coverageGate.run), - /if \[\[ "\$UNIT_COVERAGE_RESULT" != "success" \|\| "\$RENDERER_COVERAGE_RESULT" != "success" \]\]; then/u, - ); + const wait = findStep(sonar, 'Wait for successful coverage'); + assert.equal(wait.if, "needs.provenance.outputs.reuse != 'true'"); + assert.equal(wait.run, 'node scripts/wait-for-coverage.mjs'); + assert.notEqual(wait['continue-on-error'], true); + for (const name of [ + 'Download unit coverage', + 'Download renderer coverage shards', + 'Run Sonar finding gate', + ]) { + assert.ok(sonar.steps.indexOf(wait) < sonar.steps.indexOf(findStep(sonar, name))); + } assert.match(coverageGate.run, /exit 1/u); assert.deepEqual(findStep(sonar, 'Checkout exact commit').with, { 'fetch-depth': 0, diff --git a/scripts/wait-for-coverage.mjs b/scripts/wait-for-coverage.mjs new file mode 100644 index 00000000..f71a57b2 --- /dev/null +++ b/scripts/wait-for-coverage.mjs @@ -0,0 +1,107 @@ +import { pathToFileURL } from 'node:url'; +import { setTimeout as sleep } from 'node:timers/promises'; + +export const COVERAGE_JOBS = Object.freeze([ + 'Unit coverage', + ...[1, 2, 3, 4].map((index) => `Renderer coverage (${index}/4)`), +]); + +function contextFromEnvironment(env) { + const { + GITHUB_REPOSITORY: repository, + GITHUB_RUN_ID: runId, + GITHUB_RUN_ATTEMPT: attempt, + EXPECTED_HEAD_SHA: headSha, + GH_TOKEN: token, + } = env; + if ( + !/^[\w.-]+\/[\w.-]+$/u.test(repository ?? '') || + !/^[1-9]\d*$/u.test(runId ?? '') || + !/^[1-9]\d*$/u.test(attempt ?? '') || + !/^[a-f0-9]{40}$/u.test(headSha ?? '') || + !token + ) { + throw new Error('Coverage wait requires repository, run, attempt, head SHA and token.'); + } + return { repository, runId, attempt, headSha, token }; +} + +async function fetchJobs({ repository, runId, attempt, token }, fetchImpl, now, deadline) { + const jobs = []; + let total; + for (let page = 1; page <= 10; page += 1) { + const remaining = deadline - now(); + if (remaining <= 0) throw new Error('Timed out waiting for successful coverage.'); + const response = await fetchImpl( + `https://api.github.com/repos/${repository}/actions/runs/${runId}/attempts/${attempt}/jobs?per_page=100&page=${page}`, + { + headers: { + Authorization: `Bearer ${token}`, + Accept: 'application/vnd.github+json', + 'X-GitHub-Api-Version': '2022-11-28', + }, + signal: AbortSignal.timeout(Math.max(1, Math.ceil(Math.min(30_000, remaining)))), + redirect: 'error', + }, + ); + if (!response.ok) throw new Error(`Coverage job lookup failed: HTTP ${response.status}.`); + const data = await response.json(); + if ( + !Array.isArray(data.jobs) || + !Number.isSafeInteger(data.total_count) || + data.total_count < 0 + ) { + throw new Error('Malformed coverage job lookup.'); + } + total = data.total_count; + jobs.push(...data.jobs); + if (jobs.length >= total) break; + if (data.jobs.length === 0) throw new Error('Incomplete coverage job lookup.'); + } + if (jobs.length !== total) throw new Error('Incomplete coverage job lookup.'); + return jobs; +} + +function coverageSucceeded(jobs, name, { runId, attempt, headSha }) { + const matches = jobs.filter((job) => job.name === name); + if (matches.length > 1) throw new Error(`Ambiguous coverage job: ${name}.`); + if (matches.length === 0) return false; + const job = matches[0]; + if ( + String(job.run_id) !== runId || + String(job.run_attempt) !== attempt || + job.head_sha !== headSha + ) { + throw new Error(`Coverage job provenance mismatch: ${name}.`); + } + if (job.status !== 'completed') return false; + if (job.conclusion !== 'success') + throw new Error(`Coverage failed: ${name} (${job.conclusion}).`); + return true; +} + +// Query the exact attempt: GitHub includes successful jobs carried forward by a +// partial rerun here, without accepting an older success over a rerun failure. +export async function waitForCoverage({ + env = process.env, + fetchImpl = fetch, + now = () => performance.now(), + wait = sleep, + timeoutMs = 300_000, +} = {}) { + const context = contextFromEnvironment(env); + const deadline = now() + timeoutMs; + while (now() < deadline) { + const jobs = await fetchJobs(context, fetchImpl, now, deadline); + if (now() >= deadline) break; + const results = COVERAGE_JOBS.map((name) => coverageSucceeded(jobs, name, context)); + if (results.every(Boolean)) return; + await wait(Math.max(0, Math.min(5_000, deadline - now()))); + } + throw new Error('Timed out waiting for successful coverage.'); +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + await waitForCoverage(); + console.log('All coverage jobs succeeded for this run, attempt and head commit.'); +} diff --git a/scripts/wait-for-coverage.test.mjs b/scripts/wait-for-coverage.test.mjs new file mode 100644 index 00000000..2c1308f4 --- /dev/null +++ b/scripts/wait-for-coverage.test.mjs @@ -0,0 +1,126 @@ +import { describe, expect, it } from 'vitest'; +import { COVERAGE_JOBS, waitForCoverage } from './wait-for-coverage.mjs'; + +const env = { + GITHUB_REPOSITORY: 'owner/repo', + GITHUB_RUN_ID: '123', + GITHUB_RUN_ATTEMPT: '2', + EXPECTED_HEAD_SHA: 'a'.repeat(40), + GH_TOKEN: 'token-not-for-logs', +}; +const successfulJobs = () => + COVERAGE_JOBS.map((name) => ({ + name, + run_id: 123, + run_attempt: 2, + head_sha: env.EXPECTED_HEAD_SHA, + status: 'completed', + conclusion: 'success', + })); +const response = (jobs, total = jobs.length) => ({ + ok: true, + json: async () => ({ jobs, total_count: total }), +}); + +describe('coverage preparation overlap', () => { + it('waits for every job in the exact attempt before authorizing report download', async () => { + let calls = 0; + let clock = 0; + await waitForCoverage({ + env, + now: () => clock, + wait: async (ms) => { + clock += ms; + }, + fetchImpl: async (url, options) => { + expect(url).toContain('/runs/123/attempts/2/jobs?per_page=100&page=1'); + expect(options.headers.Authorization).toBe(`Bearer ${env.GH_TOKEN}`); + expect(options.redirect).toBe('error'); + const jobs = successfulJobs(); + if (calls++ === 0) { + jobs[4].status = 'in_progress'; + jobs[4].conclusion = null; + } + return response(jobs); + }, + }); + expect(calls).toBe(2); + expect(clock).toBe(5000); + }); + + it.each(['failure', 'cancelled', 'skipped', 'neutral', 'timed_out', null])( + 'rejects completed coverage with conclusion %s', + async (conclusion) => { + const jobs = successfulJobs(); + jobs[0].conclusion = conclusion; + await expect(waitForCoverage({ env, fetchImpl: async () => response(jobs) })).rejects.toThrow( + 'Coverage failed', + ); + }, + ); + + it.each([{ run_id: 124 }, { run_attempt: 1 }, { head_sha: 'b'.repeat(40) }])( + 'rejects mismatched job provenance %j', + async (change) => { + const jobs = successfulJobs(); + Object.assign(jobs[1], change); + await expect(waitForCoverage({ env, fetchImpl: async () => response(jobs) })).rejects.toThrow( + 'provenance mismatch', + ); + }, + ); + + it('does not accept an older successful duplicate over a failed rerun', async () => { + const jobs = successfulJobs(); + jobs.push({ ...jobs[0], conclusion: 'failure' }); + await expect(waitForCoverage({ env, fetchImpl: async () => response(jobs) })).rejects.toThrow( + 'Ambiguous', + ); + }); + + it('times out on missing coverage without treating artifacts as success', async () => { + let clock = 0; + await expect( + waitForCoverage({ + env, + now: () => clock, + wait: async (ms) => { + clock += ms; + }, + timeoutMs: 6000, + fetchImpl: async () => response(successfulJobs().slice(1)), + }), + ).rejects.toThrow('Timed out'); + expect(clock).toBe(6000); + }); + + it('follows pagination before accepting coverage', async () => { + const jobs = successfulJobs(); + await expect( + waitForCoverage({ + env, + fetchImpl: async (url) => + url.endsWith('page=1') ? response(jobs.slice(0, 2), 5) : response(jobs.slice(2), 5), + }), + ).resolves.toBeUndefined(); + }); + + it.each([ + { ok: false, status: 403 }, + { ok: true, json: async () => ({ jobs: [], total_count: 2 }) }, + { ok: true, json: async () => ({ jobs: null }) }, + ])('fails closed on an unavailable or incomplete API response', async (reply) => { + await expect(waitForCoverage({ env, fetchImpl: async () => reply })).rejects.toThrow(); + }); + + it('rejects missing context before sending credentials', async () => { + await expect( + waitForCoverage({ + env: { ...env, GITHUB_RUN_ATTEMPT: '' }, + fetchImpl: async () => { + throw new Error('must not fetch'); + }, + }), + ).rejects.toThrow('requires'); + }); +}); From 80410bef769444e770c34e44c8dcbd8dcf9c623e Mon Sep 17 00:00:00 2001 From: Ryan Bledsoe Date: Mon, 21 Sep 2026 20:35:35 -0500 Subject: [PATCH 2/3] fix(ci): discover Electron shards after building assets --- .github/workflows/build.yml | 6 +--- docs/DEVELOPMENT.md | 6 ++-- scripts/ci-optimization-contract.test.mjs | 16 ++++++---- scripts/plan-electron-shards.mjs | 13 ++++++-- scripts/plan-electron-shards.test.mjs | 31 +++++++++++++++--- scripts/run-electron-tests.mjs | 39 ++++++++++++++++------- 6 files changed, 77 insertions(+), 34 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 92911d77..33afdae7 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -282,17 +282,13 @@ jobs: # Isolated runners avoid concurrent native rebuilds. Keep one worker per # shard; duration estimates only assign tests, never select the test inventory. - - name: Balance Electron tests by duration - if: matrix.suite == 'electron' - run: node scripts/plan-electron-shards.mjs ${{ matrix.shard-index }}/${{ matrix.shard-total }} "$RUNNER_TEMP/electron-shard.txt" - - name: Run Electron workflows if: matrix.suite == 'electron' run: | sudo apt-get install --yes dbus-x11 gnome-keyring dbus-run-session -- bash -euo pipefail -c ' openssl rand -hex 32 | gnome-keyring-daemon --unlock --components=secrets - xvfb-run --auto-servernum npm run test:electron -- --fully-parallel --workers=1 --test-list="$RUNNER_TEMP/electron-shard.txt" + xvfb-run --auto-servernum npm run test:electron -- --fully-parallel --workers=1 --balanced-shard=${{ matrix.shard-index }}/${{ matrix.shard-total }} ' - name: Run browser workflows diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md index fe3bb27e..23eb9103 100644 --- a/docs/DEVELOPMENT.md +++ b/docs/DEVELOPMENT.md @@ -243,9 +243,9 @@ The Build workflow owns the full pull-request and `main` verification graph. Its `Build quality gate` fails closed over formatting, linting, type checking, dependency audit, the production build, unit coverage plus cache integration tests, four renderer-coverage shards, and the mandatory `workflow-tests` matrix. Four isolated Electron runners execute -`npm run test:electron -- --fully-parallel --workers=1 --test-list=...` under Xvfb with an unlocked -ephemeral keyring. Before execution, `scripts/plan-electron-shards.mjs` discovers the full current -Playwright inventory and assigns every test to exactly one of four shards, balancing the longest +`npm run test:electron -- --fully-parallel --workers=1 --balanced-shard=N/4` under Xvfb with an unlocked +ephemeral keyring. After the npm wrapper builds the app, `scripts/plan-electron-shards.mjs` discovers the full current +Playwright inventory (including specs that import emitted CSS) and assigns every test to exactly one of four shards, balancing the longest measured tests first using `scripts/electron-test-durations.json`. The timing file records its source run and only affects assignment: new or renamed tests receive a 30-second estimate, and removed tests cannot remain in the inventory. Keep one worker per runner. To refresh estimates, diff --git a/scripts/ci-optimization-contract.test.mjs b/scripts/ci-optimization-contract.test.mjs index bc2e5e75..f9a91616 100644 --- a/scripts/ci-optimization-contract.test.mjs +++ b/scripts/ci-optimization-contract.test.mjs @@ -179,13 +179,15 @@ describe('CI optimization contracts', () => { expect(electronDependencies.run).toBe('npx playwright install-deps chromium'); const electron = findStep(workflows, 'Run Electron workflows'); const web = findStep(workflows, 'Run browser workflows'); - const plan = findStep(workflows, 'Balance Electron tests by duration'); - expect(plan.if).toBe("matrix.suite == 'electron'"); - expect(plan.run).toBe( - 'node scripts/plan-electron-shards.mjs ${{ matrix.shard-index }}/${{ matrix.shard-total }} "$RUNNER_TEMP/electron-shard.txt"', + const pkg = await readJson('package.json'); + expect(pkg.scripts['test:electron']).toBe( + 'npm run build && node scripts/run-electron-tests.mjs', + ); + const runner = await readProjectFile('scripts/run-electron-tests.mjs'); + expect(runner).toContain('writeElectronShard('); + expect(runner.indexOf('writeElectronShard(balanced')).toBeLessThan( + runner.indexOf('runElectronTests({'), ); - expect(workflows.steps.indexOf(plan)).toBeLessThan(workflows.steps.indexOf(electron)); - expect(plan['continue-on-error']).not.toBe(true); expect(electron.run).toContain('sudo apt-get install --yes dbus-x11 gnome-keyring'); expect(electron.run).toContain('dbus-run-session -- bash -euo pipefail'); @@ -193,7 +195,7 @@ describe('CI optimization contracts', () => { 'openssl rand -hex 32 | gnome-keyring-daemon --unlock --components=secrets', ); expect(electron.run).toContain( - 'xvfb-run --auto-servernum npm run test:electron -- --fully-parallel --workers=1 --test-list="$RUNNER_TEMP/electron-shard.txt"', + 'xvfb-run --auto-servernum npm run test:electron -- --fully-parallel --workers=1 --balanced-shard=${{ matrix.shard-index }}/${{ matrix.shard-total }}', ); expect(web.run).toBe('xvfb-run --auto-servernum npm run test:web'); expect(workflows.steps.indexOf(pocketbase)).toBeGreaterThan(workflows.steps.indexOf(install)); diff --git a/scripts/plan-electron-shards.mjs b/scripts/plan-electron-shards.mjs index 5154589a..af34b9ab 100644 --- a/scripts/plan-electron-shards.mjs +++ b/scripts/plan-electron-shards.mjs @@ -73,8 +73,11 @@ export function balanceTests(inventory, durations, count) { return shards; } -if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { - const [, , selection, output] = process.argv; +export function writeElectronShard( + selection, + output, + configPath = 'playwright.electron.config.ts', +) { const match = /^([1-9]\d*)\/([1-9]\d*)$/u.exec(selection ?? ''); if (!match || !output || Number(match[1]) > Number(match[2])) { throw new Error('Usage: node scripts/plan-electron-shards.mjs INDEX/TOTAL OUTPUT'); @@ -85,7 +88,7 @@ if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) join(root, 'node_modules/@playwright/test/cli.js'), 'test', '-c', - 'playwright.electron.config.ts', + configPath, '--list', '--reporter=json', ], @@ -109,3 +112,7 @@ if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) `Estimated test seconds per shard: ${shards.map((shard) => Math.round(shard.seconds)).join(', ')} (excludes setup).`, ); } + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + writeElectronShard(process.argv[2], process.argv[3]); +} diff --git a/scripts/plan-electron-shards.test.mjs b/scripts/plan-electron-shards.test.mjs index 1df72fc9..3513b871 100644 --- a/scripts/plan-electron-shards.test.mjs +++ b/scripts/plan-electron-shards.test.mjs @@ -1,9 +1,10 @@ import { describe, expect, it } from 'vitest'; import { execFileSync } from 'node:child_process'; -import { mkdtempSync, readFileSync, rmSync } from 'node:fs'; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { balanceTests, inventoryFromReport } from './plan-electron-shards.mjs'; +import { fileURLToPath } from 'node:url'; +import { balanceTests, inventoryFromReport, writeElectronShard } from './plan-electron-shards.mjs'; const spec = (title, projectName = '') => ({ file: 'example.spec.ts', @@ -16,8 +17,27 @@ const report = (specs, suites = []) => ({ }); describe('duration-balanced Electron shards', () => { - it('selects the full real Playwright suite exactly once across all generated lists', () => { + it('selects every test with real Playwright, including projects and skips, without app build artifacts', () => { const directory = mkdtempSync(join(tmpdir(), 'relay-shards-')); + const configPath = join(directory, 'playwright.config.cjs'); + writeFileSync( + configPath, + `module.exports = {testDir: __dirname, testMatch: '**/*.spec.cjs', projects: [{name: 'one'}, {name: 'two'}]};`, + ); + const playwrightModule = fileURLToPath( + new URL('../node_modules/@playwright/test/index.js', import.meta.url), + ); + writeFileSync( + join(directory, 'example.spec.cjs'), + ` + const { test } = require(${JSON.stringify(playwrightModule)}); + test('first', () => {}); + test.skip('skipped', () => {}); + test.describe('nested', () => { + for (let index = 0; index < 4; index++) test('case ' + index, () => {}); + }); + `, + ); const list = (extra = []) => inventoryFromReport( JSON.parse( @@ -27,7 +47,7 @@ describe('duration-balanced Electron shards', () => { 'node_modules/@playwright/test/cli.js', 'test', '-c', - 'playwright.electron.config.ts', + configPath, '--list', '--reporter=json', ...extra, @@ -38,10 +58,11 @@ describe('duration-balanced Electron shards', () => { ); try { const full = list(); + expect(full).toHaveLength(12); const selected = []; for (let index = 1; index <= 4; index += 1) { const path = join(directory, `shard-${index}.txt`); - execFileSync(process.execPath, ['scripts/plan-electron-shards.mjs', `${index}/4`, path]); + writeElectronShard(`${index}/4`, path, configPath); const actual = list([`--test-list=${path}`]); expect(actual.sort()).toEqual(readFileSync(path, 'utf8').trim().split('\n').sort()); selected.push(...actual); diff --git a/scripts/run-electron-tests.mjs b/scripts/run-electron-tests.mjs index 601433d2..b7806cbb 100644 --- a/scripts/run-electron-tests.mjs +++ b/scripts/run-electron-tests.mjs @@ -1,9 +1,11 @@ #!/usr/bin/env node -import { readFileSync } from 'node:fs'; +import { mkdtempSync, readFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; import { dirname, join } from 'node:path'; import { fileURLToPath } from 'node:url'; import { runElectronTests } from './electron-test-runner.mjs'; +import { writeElectronShard } from './plan-electron-shards.mjs'; const root = dirname(dirname(fileURLToPath(import.meta.url))); const electronVersion = JSON.parse( @@ -12,13 +14,28 @@ const electronVersion = JSON.parse( const electronRebuild = join(root, 'node_modules', '@electron', 'rebuild', 'lib', 'cli.js'); const playwright = join(root, 'node_modules', '@playwright', 'test', 'cli.js'); -process.exitCode = runElectronTests({ - electronVersion, - electronRebuildPath: electronRebuild, - playwrightPath: playwright, - npmExecPath: process.env.npm_execpath, - nodePath: process.execPath, - playwrightArgs: process.argv.slice(2), - cwd: root, - env: process.env, -}); +const args = process.argv.slice(2); +const balanced = args.filter((arg) => arg.startsWith('--balanced-shard=')); +if (balanced.length > 1) throw new Error('Specify one balanced Electron shard.'); +const directory = balanced.length ? mkdtempSync(join(tmpdir(), 'relay-electron-shard-')) : null; +try { + const playwrightArgs = args.filter((arg) => !arg.startsWith('--balanced-shard=')); + if (directory) { + // npm run test:electron builds first: discovery imports emitted CSS assets. + const list = join(directory, 'tests.txt'); + writeElectronShard(balanced[0].slice('--balanced-shard='.length), list); + playwrightArgs.push(`--test-list=${list}`); + } + process.exitCode = runElectronTests({ + electronVersion, + electronRebuildPath: electronRebuild, + playwrightPath: playwright, + npmExecPath: process.env.npm_execpath, + nodePath: process.execPath, + playwrightArgs, + cwd: root, + env: process.env, + }); +} finally { + if (directory) rmSync(directory, { recursive: true, force: true }); +} From 06099986b9480c18e948825e562af0451f4e55b1 Mon Sep 17 00:00:00 2001 From: Ryan Bledsoe Date: Mon, 21 Sep 2026 20:55:56 -0500 Subject: [PATCH 3/3] fix(ci): constrain shard output and initialize balancing --- scripts/plan-electron-shards.mjs | 13 +++++-------- scripts/plan-electron-shards.test.mjs | 12 ++++++++++++ 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/scripts/plan-electron-shards.mjs b/scripts/plan-electron-shards.mjs index af34b9ab..110701ad 100644 --- a/scripts/plan-electron-shards.mjs +++ b/scripts/plan-electron-shards.mjs @@ -1,7 +1,7 @@ import { spawnSync } from 'node:child_process'; import { readFileSync, writeFileSync } from 'node:fs'; import { dirname, join } from 'node:path'; -import { fileURLToPath, pathToFileURL } from 'node:url'; +import { fileURLToPath } from 'node:url'; const root = dirname(dirname(fileURLToPath(import.meta.url))); const compare = (a, b) => { @@ -63,8 +63,9 @@ export function balanceTests(inventory, durations, count) { .sort((a, b) => b.seconds - a.seconds || compare(a.id, b.id)); const shards = Array.from({ length: count }, () => ({ tests: [], seconds: 0 })); for (const test of weighted) { - const shard = shards.reduce((best, candidate) => - candidate.seconds < best.seconds ? candidate : best, + const shard = shards.reduce( + (best, candidate) => (candidate.seconds < best.seconds ? candidate : best), + shards[0], ); shard.tests.push(test.id); shard.seconds += test.seconds; @@ -80,7 +81,7 @@ export function writeElectronShard( ) { const match = /^([1-9]\d*)\/([1-9]\d*)$/u.exec(selection ?? ''); if (!match || !output || Number(match[1]) > Number(match[2])) { - throw new Error('Usage: node scripts/plan-electron-shards.mjs INDEX/TOTAL OUTPUT'); + throw new Error('Balanced Electron shard must be INDEX/TOTAL with INDEX <= TOTAL.'); } const result = spawnSync( process.execPath, @@ -112,7 +113,3 @@ export function writeElectronShard( `Estimated test seconds per shard: ${shards.map((shard) => Math.round(shard.seconds)).join(', ')} (excludes setup).`, ); } - -if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { - writeElectronShard(process.argv[2], process.argv[3]); -} diff --git a/scripts/plan-electron-shards.test.mjs b/scripts/plan-electron-shards.test.mjs index 3513b871..a835cb0a 100644 --- a/scripts/plan-electron-shards.test.mjs +++ b/scripts/plan-electron-shards.test.mjs @@ -17,6 +17,18 @@ const report = (specs, suites = []) => ({ }); describe('duration-balanced Electron shards', () => { + it('does not expose file writes through command-line arguments', () => { + const directory = mkdtempSync(join(tmpdir(), 'relay-shard-cli-')); + const file = join(directory, 'existing.txt'); + try { + writeFileSync(file, 'preserve this file'); + execFileSync(process.execPath, ['scripts/plan-electron-shards.mjs', '1/1', file]); + expect(readFileSync(file, 'utf8')).toBe('preserve this file'); + } finally { + rmSync(directory, { recursive: true, force: true }); + } + }); + it('selects every test with real Playwright, including projects and skips, without app build artifacts', () => { const directory = mkdtempSync(join(tmpdir(), 'relay-shards-')); const configPath = join(directory, 'playwright.config.cjs');