diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f5045ffe..9cebb18a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,7 +5,8 @@ on: branches: [main] pull_request: -# Least-privilege default token; no job in this workflow writes to the repo. +# Least-privilege default token; the only write anywhere is the coverage-comment +# job's pull-requests:write, scoped to that job (and it never runs for fork PRs). permissions: contents: read @@ -88,7 +89,25 @@ jobs: run: pnpm typecheck - name: Test if: ${{ !cancelled() }} - run: pnpm test + run: pnpm test -- --coverage + # Coverage ratchet (#93): per-workspace lines/branches may not drop below + # coverage-baseline.json (0.5pp tolerance). The checker self-tests first, + # so the gate is itself gated — same pattern as the docs check below. + - name: Coverage ratchet + if: ${{ !cancelled() }} + run: | + pnpm check:coverage:test + pnpm check:coverage + - name: Upload coverage report + if: ${{ !cancelled() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: coverage-report + retention-days: 14 + path: | + coverage-summary.md + apps/*/coverage/coverage-summary.json + packages/*/coverage/coverage-summary.json # Docs integrity (#102): fail on a dangling docs reference in source or an # unindexed doc. The checker self-tests first, so the gate is itself gated. - name: Docs integrity @@ -96,3 +115,48 @@ jobs: run: | pnpm check:docs:test pnpm check:docs + + # Sticky PR comment with the coverage table (#93). Same-repo PRs only: fork + # PRs get a read-only token no matter what `permissions:` says, so for them + # the table lives in the step summary + artifact instead. `always()` so the + # comment still updates when the ratchet fails; continue-on-error so a + # comment hiccup never gates a merge. + coverage-comment: + needs: check + if: >- + always() && github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name == github.repository + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + pull-requests: write + continue-on-error: true + steps: + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: coverage-report + - name: Sticky coverage comment + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + PR_NUMBER: ${{ github.event.pull_request.number }} + run: | + if [ ! -f coverage-summary.md ]; then + echo "coverage-summary.md missing from artifact; nothing to post" + exit 0 + fi + marker='' + # Two steps, not a pipeline: `gh api | head -1` under the runner's + # default `pipefail` can surface gh's SIGPIPE (exit 141) as failure. + gh api "repos/${GH_REPO}/issues/${PR_NUMBER}/comments" --paginate \ + --jq ".[] | select(.body | startswith(\"${marker}\")) | .id" > comment-ids.txt + existing=$(head -1 comment-ids.txt) + if [ -n "$existing" ]; then + gh api --method PATCH "repos/${GH_REPO}/issues/comments/${existing}" \ + -F body=@coverage-summary.md > /dev/null + echo "updated comment ${existing}" + else + gh api --method POST "repos/${GH_REPO}/issues/${PR_NUMBER}/comments" \ + -F body=@coverage-summary.md > /dev/null + echo "created comment" + fi diff --git a/.gitignore b/.gitignore index d5004100..3f9b69d3 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,8 @@ build/ .react-router/ .turbo/ coverage/ +# rendered by scripts/check-coverage.mjs for the CI artifact/PR comment +/coverage-summary.md # Cloudflare / wrangler .wrangler/ diff --git a/apps/etl/vitest.config.ts b/apps/etl/vitest.config.ts index 0b5a511c..dd6320a9 100644 --- a/apps/etl/vitest.config.ts +++ b/apps/etl/vitest.config.ts @@ -3,6 +3,7 @@ import { readFileSync } from 'node:fs'; import { dirname, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; import { defineConfig } from 'vitest/config'; +import { sharedCoverage } from '../../vitest.shared'; const here = dirname(fileURLToPath(import.meta.url)); @@ -29,7 +30,11 @@ export default defineConfig({ 'cloudflare:workflows': resolve(here, 'src/test/cloudflare-workflows-stub.ts'), }, }, - // The refresh Workflow test runs the full refresh-slice.sql derive against a real SQLite — - // generous headroom for loaded CI runners, same rationale as packages/db/vitest.config.ts. - test: { testTimeout: 120_000 }, + test: { + environment: 'node', + // The refresh Workflow test runs the full refresh-slice.sql derive against a real SQLite — + // generous headroom for loaded CI runners, same rationale as packages/db/vitest.config.ts. + testTimeout: 120_000, + coverage: sharedCoverage(['src/**']), + }, }); diff --git a/apps/web/tsconfig.node.json b/apps/web/tsconfig.node.json index 7bc350d2..4fd2e709 100644 --- a/apps/web/tsconfig.node.json +++ b/apps/web/tsconfig.node.json @@ -1,6 +1,6 @@ { "extends": "./tsconfig.json", - "include": ["vite.config.ts", "vitest.config.ts"], + "include": ["vite.config.ts", "vitest.config.ts", "../../vitest.shared.ts"], "compilerOptions": { "composite": true, "strict": true, diff --git a/apps/web/vitest.config.ts b/apps/web/vitest.config.ts index b569e588..4b7f34fd 100644 --- a/apps/web/vitest.config.ts +++ b/apps/web/vitest.config.ts @@ -1,8 +1,10 @@ import { defineConfig } from 'vitest/config'; +import { sharedCoverage } from '../../vitest.shared'; export default defineConfig({ test: { environment: 'node', include: ['app/**/*.test.ts', 'app/**/*.test.tsx', 'workers/**/*.test.ts'], + coverage: sharedCoverage(['app/**', 'workers/**']), }, }); diff --git a/coverage-baseline.json b/coverage-baseline.json new file mode 100644 index 00000000..5abe85c9 --- /dev/null +++ b/coverage-baseline.json @@ -0,0 +1,29 @@ +{ + "tolerance": 0.5, + "workspaces": { + "apps/etl": { + "lines": 74, + "branches": 58.2 + }, + "apps/web": { + "lines": 89.7, + "branches": 81.8 + }, + "packages/config": { + "lines": 92.8, + "branches": 72.2 + }, + "packages/db": { + "lines": 94.2, + "branches": 79 + }, + "packages/ingest": { + "lines": 85.8, + "branches": 80 + }, + "packages/shared": { + "lines": 95.4, + "branches": 80 + } + } +} diff --git a/docs/review-testing.md b/docs/review-testing.md index a7ac4787..3ab4b106 100644 --- a/docs/review-testing.md +++ b/docs/review-testing.md @@ -29,6 +29,20 @@ - Всеки пакет с тестове закача `"test": "vitest run"` в turbo графа — `apps/etl` (`eop.test.ts`) беше тихо прескачан от `turbo run test`, докато това не се поправи. +## Coverage ratchet + +- Локално: `pnpm test -- --coverage && pnpm check:coverage`. CI пуска същото — покритието се + измерва с `@vitest/coverage-v8` през общия preset (`vitest.shared.ts`), по workspace. +- Ratchet правилото: lines% и branches% на всеки workspace не може да падне под комитнатия + `coverage-baseline.json` с повече от 0.5pp (толерансът поглъща шум от малките пакети с 1–2 + тестови файла). Спад ⇒ червено CI; PR-ът показва таблицата с делтите (step summary + sticky + коментар за същия-repo PR-и, artifact за форкове). +- Покачване с >1pp ⇒ скриптът подканя `node scripts/check-coverage.mjs --update` — прегледайте и + комитнете новия baseline в същия PR. Умишлен, ревюиран спад се изразява със сваляне на числото + в `coverage-baseline.json`, не с изключване на проверката. +- Нов workspace с тестове се добавя и в `coverage-baseline.json` (и получава `vitest.config.ts` + с `sharedCoverage(...)`); `packages/api-contract` е освободен, докато няма тестове. + ## Integrity gate - Пуска се върху обслужвания D1 след `precompute` в `ship-domain.mjs` и след `runSliceDerive()` в diff --git a/package.json b/package.json index c1c75613..17b09cdd 100644 --- a/package.json +++ b/package.json @@ -15,6 +15,8 @@ "lint": "prettier --check .", "check:docs": "node scripts/check-docs.mjs", "check:docs:test": "node --test scripts/check-docs.test.mjs", + "check:coverage": "node scripts/check-coverage.mjs", + "check:coverage:test": "node --test scripts/check-coverage.test.mjs", "format": "prettier --write .", "setup": "node scripts/setup.mjs", "import": "node scripts/import.mjs", @@ -26,6 +28,7 @@ }, "devDependencies": { "@cloudflare/workers-types": "^4.20260521.1", + "@vitest/coverage-v8": "^4.1.7", "@types/node": "^25.9.1", "prettier": "^3.8.3", "turbo": "^2.9.14", diff --git a/packages/config/vitest.config.ts b/packages/config/vitest.config.ts new file mode 100644 index 00000000..53770d78 --- /dev/null +++ b/packages/config/vitest.config.ts @@ -0,0 +1,9 @@ +import { defineConfig } from 'vitest/config'; +import { sharedCoverage } from '../../vitest.shared'; + +export default defineConfig({ + test: { + environment: 'node', + coverage: sharedCoverage(['src/**']), + }, +}); diff --git a/packages/db/vitest.config.ts b/packages/db/vitest.config.ts index 4022fb5a..addec285 100644 --- a/packages/db/vitest.config.ts +++ b/packages/db/vitest.config.ts @@ -1,4 +1,5 @@ import { defineConfig } from 'vitest/config'; +import { sharedCoverage } from '../../vitest.shared'; // The db suite is real-SQLite integration style: tests shell out to the sqlite3 CLI dozens of // times each, and the heaviest (ship-domain) legitimately runs for around a minute. On loaded CI @@ -6,5 +7,9 @@ import { defineConfig } from 'vitest/config'; // refresh-slice EOP derivation test), so give the whole suite generous headroom - correctness // here is asserted by the checks, not by speed. export default defineConfig({ - test: { testTimeout: 120_000 }, + test: { + environment: 'node', + testTimeout: 120_000, + coverage: sharedCoverage(['src/**']), + }, }); diff --git a/packages/ingest/vitest.config.ts b/packages/ingest/vitest.config.ts new file mode 100644 index 00000000..53770d78 --- /dev/null +++ b/packages/ingest/vitest.config.ts @@ -0,0 +1,9 @@ +import { defineConfig } from 'vitest/config'; +import { sharedCoverage } from '../../vitest.shared'; + +export default defineConfig({ + test: { + environment: 'node', + coverage: sharedCoverage(['src/**']), + }, +}); diff --git a/packages/shared/vitest.config.ts b/packages/shared/vitest.config.ts new file mode 100644 index 00000000..53770d78 --- /dev/null +++ b/packages/shared/vitest.config.ts @@ -0,0 +1,9 @@ +import { defineConfig } from 'vitest/config'; +import { sharedCoverage } from '../../vitest.shared'; + +export default defineConfig({ + test: { + environment: 'node', + coverage: sharedCoverage(['src/**']), + }, +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index eb6ef078..12dc4203 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -24,6 +24,9 @@ importers: '@types/node': specifier: ^25.9.1 version: 25.9.1 + '@vitest/coverage-v8': + specifier: ^4.1.7 + version: 4.1.10(vitest@4.1.7) prettier: specifier: ^3.8.3 version: 3.8.3 @@ -35,7 +38,7 @@ importers: version: 6.0.3 vitest: specifier: ^4.1.7 - version: 4.1.7(@opentelemetry/api@1.9.1)(@types/node@25.9.1)(jsdom@29.1.1)(vite@8.0.16(@types/node@25.9.1)(esbuild@0.28.1)(jiti@2.7.0)) + version: 4.1.7(@opentelemetry/api@1.9.1)(@types/node@25.9.1)(@vitest/coverage-v8@4.1.10)(jsdom@29.1.1)(vite@8.0.16(@types/node@25.9.1)(esbuild@0.28.1)(jiti@2.7.0)) wrangler: specifier: ^4.93.1 version: 4.93.1(@cloudflare/workers-types@4.20260521.1) @@ -312,6 +315,10 @@ packages: resolution: {integrity: sha512-4zBIxpPzowiZpusoFkyGVwakdRJUyuH5PxQ/PrqghfdFWWasvnCdPfQXHrenDai+gyLARulZjZowCOj6fjT4pA==} engines: {node: '>=6.9.0'} + '@bcoe/v8-coverage@1.0.2': + resolution: {integrity: sha512-6zABk/ECA/QYSCQ1NGiVwwbQerUCZ+TQbp64Q3AgmfNvurHH0j8TtXa1qbShXA6qqkpAj4V5W8pP6mLe1mcMqA==} + engines: {node: '>=18'} + '@bramus/specificity@2.4.2': resolution: {integrity: sha512-ctxtJ/eA+t+6q2++vj5j7FYX3nRu311q1wfYH3xjlLOsczhlhxAg2FWNUXhpGvAw3BWo1xBcvOV6/YLc2r5FJw==} hasBin: true @@ -1222,6 +1229,15 @@ packages: resolution: {integrity: sha512-UycprH3T6n3jH0k44NHMa7pnFHGu/N05MjojYr+Mc6I7obkoLIJujSWwin1pCvdy/eOxrI/l3uDLQsmcrOb4ug==} engines: {node: '>= 20'} + '@vitest/coverage-v8@4.1.10': + resolution: {integrity: sha512-IM49HmthevbgAO4anp1hwtoT9wYe59w0LR00gr+eagHE+ZJ5lK4sLPeO0ubgoJcwLk6dehU3R24N+FbEEKDc8g==} + peerDependencies: + '@vitest/browser': 4.1.10 + vitest: 4.1.10 + peerDependenciesMeta: + '@vitest/browser': + optional: true + '@vitest/expect@4.1.7': resolution: {integrity: sha512-1R+tw0ortHEbZDGMymm+pN7/AFQ/RkFFdtd7EN+VBpynKmLbP8A3rpEXdshBJ7+8hQ9zBJh/i1s0yKNtxAnU7w==} @@ -1236,6 +1252,9 @@ packages: vite: optional: true + '@vitest/pretty-format@4.1.10': + resolution: {integrity: sha512-W1HsjSH4MXQ9YfmmhLAoIYf1HRfekQCGngeIgcei6MP5QQGWUe0gkopdZQaVCFO+JDJMrAJGwa5pRpNpvy4P8Q==} + '@vitest/pretty-format@4.1.7': resolution: {integrity: sha512-umgCarTOYQWIaDMvGDRZij+6b9oVeLIyJzfN+AS88e0ZOU3QTgNNSTtjQOpcvWr3np1N0j4WgZj+sb3oYBDscw==} @@ -1248,6 +1267,9 @@ packages: '@vitest/spy@4.1.7': resolution: {integrity: sha512-kbkI5LMWakyuTIvs6fUJ5qdIVb1XVKsYJAT4OJ938cHMROYMSfmoQdZy0aaAnjbbc8F61vkoTqz/Az+/HiIu5Q==} + '@vitest/utils@4.1.10': + resolution: {integrity: sha512-fy9am/HWxbaGt/Sawrp90vt6Y6jQwf1RX77cz3uwoJwJVMli/e1IEwRPnMNJ7vKfPTwo0diXifkpPvwH9v7nGA==} + '@vitest/utils@4.1.7': resolution: {integrity: sha512-T532WBu791cBxJlCl6SO+J14l81DQx6uQHm1bQbmCDY7nqlEIgkza/UFnSBNaUtSf41unldDFjdOBYEQC4b5Hw==} @@ -1264,6 +1286,9 @@ packages: resolution: {integrity: sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==} engines: {node: '>=12'} + ast-v8-to-istanbul@1.0.5: + resolution: {integrity: sha512-UPAgKJFSEGMWSDr3LX4tqnAb4f7KGT8O40Tyx8wbYmmZ/yn58lNCm8h3svs3eXgiGd5AXxz8NDOvXWvicq+rJA==} + babel-dead-code-elimination@1.0.12: resolution: {integrity: sha512-GERT7L2TiYcYDtYk1IpD+ASAYXjKbLTDPhBtYj7X1NuRMDTMtAx9kyBenub1Ev41lo91OHCKdmP+egTDmfQ7Ig==} @@ -1415,10 +1440,17 @@ packages: graceful-fs@4.2.11: resolution: {integrity: sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==} + has-flag@4.0.0: + resolution: {integrity: sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==} + engines: {node: '>=8'} + html-encoding-sniffer@6.0.0: resolution: {integrity: sha512-CV9TW3Y3f8/wT0BRFc1/KAVQ3TUHiXmaAb6VW9vtiMFf7SLoMd1PdAc4W3KFOFETBJUb90KatHqlsZMWV+R9Gg==} engines: {node: ^20.19.0 || ^22.12.0 || >=24.0.0} + html-escaper@2.0.2: + resolution: {integrity: sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg==} + is-potential-custom-element-name@1.0.1: resolution: {integrity: sha512-bCYeRA2rVibKZd+s2625gGnGF/t7DSqDs4dP7CrLA1m7jKWz6pps0LpYLJN8Q64HtmPKJ1hrN3nzPNKFEKOUiQ==} @@ -1426,10 +1458,25 @@ packages: resolution: {integrity: sha512-yNeeynhhtIVRBk12tBV4eHNxwB42HzR4Q3Ea7vCOiJhImGaAIdIMrbJtacQlBizGLjUPw+akkFI5Dn9T70XoVQ==} engines: {node: '>=18'} + istanbul-lib-coverage@3.2.2: + resolution: {integrity: sha512-O8dpsF+r0WV/8MNRKfnmrtCWhuKjxrq2w+jpzBL5UZKTi2LeVWnWOmWRxFlesJONmc+wLAGvKQZEOanko0LFTg==} + engines: {node: '>=8'} + + istanbul-lib-report@3.0.1: + resolution: {integrity: sha512-GCfE1mtsHGOELCU8e/Z7YWzpmybrx/+dSTfLrvY8qRmaY6zXTKWn6WQIjaAFw069icm6GVMNkgu0NzI4iPZUNw==} + engines: {node: '>=10'} + + istanbul-reports@3.2.0: + resolution: {integrity: sha512-HGYWWS/ehqTV3xN10i23tkPkpH46MLCIMFNCaaKNavAXTF1RkqxawEPtnjnGZ6XKSInBKkiOA5BKS+aZiY3AvA==} + engines: {node: '>=8'} + jiti@2.7.0: resolution: {integrity: sha512-AC/7JofJvZGrrneWNaEnJeOLUx+JlGt7tNa0wZiRPT4MY1wmfKjt2+6O2p2uz2+skll8OZZmJMNqeke7kKbNgQ==} hasBin: true + js-tokens@10.0.0: + resolution: {integrity: sha512-lM/UBzQmfJRo9ABXbPWemivdCW8V2G8FHaHdypQaIy523snUjog0W71ayWXTjiR+ixeMyVHN2XcpnTd/liPg/Q==} + js-tokens@4.0.0: resolution: {integrity: sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==} @@ -1546,6 +1593,13 @@ packages: magic-string@0.30.21: resolution: {integrity: sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ==} + magicast@0.5.3: + resolution: {integrity: sha512-pVKE4UdSQ7DvHzivsCIFx2BJn1mHG6KsyrFcaxFx6tONdneEuThrDx0Cj3AMg58KyN4pzYT+LHOotxDQDjNvkw==} + + make-dir@4.0.0: + resolution: {integrity: sha512-hXdUTZYIVOt1Ex//jAQi+wTZZpUpwBj/0QsOzqegb3rGMMeJiSEu5xLHnYfBrRV4RH2+OCSOO95Is/7x1WJ4bw==} + engines: {node: '>=10'} + mdn-data@2.27.1: resolution: {integrity: sha512-9Yubnt3e8A0OKwxYSXyhLymGW4sCufcLG6VdiDdUGVkPhpqLxlvP5vl1983gQjJl3tqbrM731mjaZaP68AgosQ==} @@ -1693,6 +1747,10 @@ packages: resolution: {integrity: sha512-SS+jx45GF1QjgEXQx4NJZV9ImqmO2NPz5FNsIHrsDjh2YsHnawpan7SNQ1o8NuhrbHZy9AZhIoCUiCeaW/C80g==} engines: {node: '>=18'} + supports-color@7.2.0: + resolution: {integrity: sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==} + engines: {node: '>=8'} + symbol-tree@3.2.4: resolution: {integrity: sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw==} @@ -2209,6 +2267,8 @@ snapshots: '@babel/helper-string-parser': 7.29.7 '@babel/helper-validator-identifier': 7.29.7 + '@bcoe/v8-coverage@1.0.2': {} + '@bramus/specificity@2.4.2': dependencies: css-tree: 3.2.1 @@ -2833,6 +2893,20 @@ snapshots: '@vercel/oidc@3.2.0': {} + '@vitest/coverage-v8@4.1.10(vitest@4.1.7)': + dependencies: + '@bcoe/v8-coverage': 1.0.2 + '@vitest/utils': 4.1.10 + ast-v8-to-istanbul: 1.0.5 + istanbul-lib-coverage: 3.2.2 + istanbul-lib-report: 3.0.1 + istanbul-reports: 3.2.0 + magicast: 0.5.3 + obug: 2.1.1 + std-env: 4.1.0 + tinyrainbow: 3.1.0 + vitest: 4.1.7(@opentelemetry/api@1.9.1)(@types/node@25.9.1)(@vitest/coverage-v8@4.1.10)(jsdom@29.1.1)(vite@8.0.16(@types/node@25.9.1)(esbuild@0.28.1)(jiti@2.7.0)) + '@vitest/expect@4.1.7': dependencies: '@standard-schema/spec': 1.1.0 @@ -2850,6 +2924,10 @@ snapshots: optionalDependencies: vite: 8.0.16(@types/node@25.9.1)(esbuild@0.28.1)(jiti@2.7.0) + '@vitest/pretty-format@4.1.10': + dependencies: + tinyrainbow: 3.1.0 + '@vitest/pretty-format@4.1.7': dependencies: tinyrainbow: 3.1.0 @@ -2868,6 +2946,12 @@ snapshots: '@vitest/spy@4.1.7': {} + '@vitest/utils@4.1.10': + dependencies: + '@vitest/pretty-format': 4.1.10 + convert-source-map: 2.0.0 + tinyrainbow: 3.1.0 + '@vitest/utils@4.1.7': dependencies: '@vitest/pretty-format': 4.1.7 @@ -2886,6 +2970,12 @@ snapshots: assertion-error@2.0.1: {} + ast-v8-to-istanbul@1.0.5: + dependencies: + '@jridgewell/trace-mapping': 0.3.31 + estree-walker: 3.0.3 + js-tokens: 10.0.0 + babel-dead-code-elimination@1.0.12: dependencies: '@babel/core': 7.29.7 @@ -3022,18 +3112,37 @@ snapshots: graceful-fs@4.2.11: {} + has-flag@4.0.0: {} + html-encoding-sniffer@6.0.0: dependencies: '@exodus/bytes': 1.15.1 transitivePeerDependencies: - '@noble/hashes' + html-escaper@2.0.2: {} + is-potential-custom-element-name@1.0.1: {} isbot@5.1.40: {} + istanbul-lib-coverage@3.2.2: {} + + istanbul-lib-report@3.0.1: + dependencies: + istanbul-lib-coverage: 3.2.2 + make-dir: 4.0.0 + supports-color: 7.2.0 + + istanbul-reports@3.2.0: + dependencies: + html-escaper: 2.0.2 + istanbul-lib-report: 3.0.1 + jiti@2.7.0: {} + js-tokens@10.0.0: {} + js-tokens@4.0.0: {} jsdom@29.1.1: @@ -3131,6 +3240,16 @@ snapshots: dependencies: '@jridgewell/sourcemap-codec': 1.5.5 + magicast@0.5.3: + dependencies: + '@babel/parser': 7.29.7 + '@babel/types': 7.29.7 + source-map-js: 1.2.1 + + make-dir@4.0.0: + dependencies: + semver: 7.8.0 + mdn-data@2.27.1: {} miniflare@4.20260520.0: @@ -3316,6 +3435,10 @@ snapshots: supports-color@10.2.2: {} + supports-color@7.2.0: + dependencies: + has-flag: 4.0.0 + symbol-tree@3.2.4: {} tailwindcss@4.3.0: {} @@ -3449,7 +3572,7 @@ snapshots: fsevents: 2.3.3 jiti: 2.7.0 - vitest@4.1.7(@opentelemetry/api@1.9.1)(@types/node@25.9.1)(jsdom@29.1.1)(vite@8.0.16(@types/node@25.9.1)(esbuild@0.28.1)(jiti@2.7.0)): + vitest@4.1.7(@opentelemetry/api@1.9.1)(@types/node@25.9.1)(@vitest/coverage-v8@4.1.10)(jsdom@29.1.1)(vite@8.0.16(@types/node@25.9.1)(esbuild@0.28.1)(jiti@2.7.0)): dependencies: '@vitest/expect': 4.1.7 '@vitest/mocker': 4.1.7(vite@8.0.16(@types/node@25.9.1)(esbuild@0.28.1)(jiti@2.7.0)) @@ -3474,6 +3597,7 @@ snapshots: optionalDependencies: '@opentelemetry/api': 1.9.1 '@types/node': 25.9.1 + '@vitest/coverage-v8': 4.1.10(vitest@4.1.7) jsdom: 29.1.1 transitivePeerDependencies: - msw diff --git a/scripts/check-coverage.mjs b/scripts/check-coverage.mjs new file mode 100644 index 00000000..9190af50 --- /dev/null +++ b/scripts/check-coverage.mjs @@ -0,0 +1,313 @@ +// Test-coverage ratchet gate (#93). Mirrors the philosophy of +// scripts/check-docs.mjs: don't just print — fail CI on drift. +// +// 1. Every workspace listed in coverage-baseline.json must have a +// coverage/coverage-summary.json (produced by `pnpm test -- --coverage`). +// A missing report is a hard error — it also catches a broken turbo +// cache restore. +// 2. Per-workspace ratchet: lines% and branches% may not drop more than +// `tolerance` percentage points below the committed baseline. A drop +// fails CI; an intentional, reviewed decrease is expressed by lowering +// the baseline in the same PR. +// 3. When coverage rises by more than 1pp the script nudges (without +// failing) to run `node scripts/check-coverage.mjs --update`, which +// rewrites the baseline from the current reports (never run in CI). +// +// Output: a markdown table (per-workspace metrics + delta vs baseline and an +// informational monorepo total computed from summed covered/total counts, not +// averaged percentages). The table goes to stdout, to coverage-summary.md at +// the repo root (git-ignored; picked up by the CI artifact / PR comment job), +// and — when $GITHUB_STEP_SUMMARY is set — to the CI step summary, on pass +// and on fail alike. +// +// The comparison/rendering logic is pure and exercised adversarially by +// scripts/check-coverage.test.mjs; main() wires it to the real repo. +// +// Not to be confused with apps/web/app/lib/coverage.ts — that is domain-level +// *data* coverage, unrelated to test coverage. + +import { readFileSync, writeFileSync, appendFileSync, existsSync, readdirSync } from 'node:fs'; +import { join, dirname, resolve } from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; + +const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..'); +const BASELINE_FILE = 'coverage-baseline.json'; +const METRICS = ['lines', 'branches', 'functions', 'statements']; +// Only lines/branches are ratcheted: functions% swings hard in the tiny +// workspaces (one helper more or less), and statements ≈ lines under v8. +// Both are still measured and shown in the table. +const RATCHETED = ['lines', 'branches']; +const BUMP_NUDGE_PP = 1.0; +// Baseline keys become filesystem paths and markdown content; keep them to +// plain `apps/...`/`packages/...` segments — no `..`, no absolute paths, no +// prototype-shaped keys. +const WORKSPACE_KEY = /^(apps|packages)\/[A-Za-z0-9_-]+$/; + +// ── pure helpers (unit-tested in check-coverage.test.mjs) ────────────────────── + +/** Floor to one decimal, so baseline bumps don't churn on noise digits. */ +export function floor1(pct) { + return Math.floor(pct * 10) / 10; +} + +/** + * Extract { pct, covered, total } per metric from a vitest + * coverage-summary.json `total` block. + */ +export function extractTotals(summaryJson) { + const out = {}; + for (const metric of METRICS) { + const m = summaryJson.total?.[metric]; + if (!m || typeof m.pct !== 'number') { + throw new Error(`coverage summary is missing total.${metric}.pct`); + } + out[metric] = { pct: m.pct, covered: m.covered ?? 0, total: m.total ?? 0 }; + } + return out; +} + +/** + * Compare one workspace's actual percentages against its baseline. + * Returns { failures: string[], bumpable: boolean }. + */ +export function compareWorkspace(name, actual, baseline, tolerance) { + const failures = []; + let bumpable = false; + for (const metric of RATCHETED) { + const base = baseline[metric]; + if (typeof base !== 'number') { + failures.push(`${name}: baseline has no "${metric}" — add it to ${BASELINE_FILE}`); + continue; + } + const pct = actual[metric].pct; + if (pct < base - tolerance) { + failures.push( + `${name}: ${metric} coverage ${pct.toFixed(2)}% dropped below the baseline ` + + `${base}% (tolerance ${tolerance}pp). Add tests for the new/changed code — or, ` + + `if the drop is intentional and reviewed, lower "${metric}" for "${name}" in ${BASELINE_FILE}.`, + ); + } else if (pct > base + BUMP_NUDGE_PP) { + bumpable = true; + } + } + return { failures, bumpable }; +} + +/** Sum covered/total counts across workspaces into a true merged total row. */ +export function mergedTotal(perWorkspace) { + const sums = {}; + for (const metric of METRICS) { + let covered = 0; + let total = 0; + for (const totals of Object.values(perWorkspace)) { + covered += totals[metric].covered; + total += totals[metric].total; + } + sums[metric] = { covered, total, pct: total === 0 ? 100 : (covered / total) * 100 }; + } + return sums; +} + +/** Render the markdown report table. */ +export function renderMarkdown(perWorkspace, baselines, tolerance, failures, bumpable) { + const lines = [ + '', + '### Test coverage', + '', + '| Workspace | Lines | Δ | Branches | Δ | Functions | Statements |', + '| --- | ---: | ---: | ---: | ---: | ---: | ---: |', + ]; + const delta = (pct, base) => + typeof base === 'number' ? `${pct - base >= 0 ? '+' : ''}${(pct - base).toFixed(2)}pp` : '—'; + for (const [name, totals] of Object.entries(perWorkspace)) { + const base = baselines[name] ?? {}; + lines.push( + `| \`${name}\` | ${totals.lines.pct.toFixed(2)}% | ${delta(totals.lines.pct, base.lines)} ` + + `| ${totals.branches.pct.toFixed(2)}% | ${delta(totals.branches.pct, base.branches)} ` + + `| ${totals.functions.pct.toFixed(2)}% | ${totals.statements.pct.toFixed(2)}% |`, + ); + } + const total = mergedTotal(perWorkspace); + lines.push( + `| **Total** _(informational)_ | ${total.lines.pct.toFixed(2)}% | — ` + + `| ${total.branches.pct.toFixed(2)}% | — | ${total.functions.pct.toFixed(2)}% | ${total.statements.pct.toFixed(2)}% |`, + ); + lines.push(''); + if (failures.length > 0) { + lines.push('**❌ Coverage ratchet failed:**', ''); + for (const f of failures) lines.push(`- ${f}`); + } else { + lines.push(`✅ No workspace dropped below its baseline (tolerance ${tolerance}pp).`); + } + if (bumpable) { + lines.push( + '', + '📈 Coverage rose by more than 1pp — run `node scripts/check-coverage.mjs --update` ' + + `locally and commit ${BASELINE_FILE} to ratchet the threshold up.`, + ); + } + lines.push(''); + return lines.join('\n'); +} + +/** Build the updated baseline object from current reports (used by --update). */ +export function updatedBaseline(perWorkspace, tolerance) { + const workspaces = {}; + for (const [name, totals] of Object.entries(perWorkspace)) { + workspaces[name] = {}; + for (const metric of RATCHETED) workspaces[name][metric] = floor1(totals[metric].pct); + } + return { tolerance, workspaces }; +} + +/** + * Fail-closed validation of the baseline against the repo's test-bearing + * workspaces. Errors when the baseline is empty or malformed, when a + * workspace with a `test` script has no baseline entry (a forgotten or + * deleted key would otherwise silently remove enforcement), or when the + * baseline lists a workspace that no longer has tests. + */ +export function validateBaseline(baseline, testWorkspaces) { + const errors = []; + const ws = baseline?.workspaces; + if (ws === null || typeof ws !== 'object' || Array.isArray(ws)) { + return [`${BASELINE_FILE}: "workspaces" must be an object`]; + } + const keys = Object.keys(ws); + if (keys.length === 0) { + return [`${BASELINE_FILE}: "workspaces" is empty — the ratchet would silently pass`]; + } + for (const key of keys) { + if (!WORKSPACE_KEY.test(key)) { + errors.push(`${BASELINE_FILE}: invalid workspace key "${key}"`); + } + } + for (const name of testWorkspaces) { + if (!Object.prototype.hasOwnProperty.call(ws, name)) { + errors.push( + `${name} has a "test" script but no entry in ${BASELINE_FILE} — add one ` + + '(run `node scripts/check-coverage.mjs --update` after `pnpm test -- --coverage`).', + ); + } + } + for (const key of keys) { + if (WORKSPACE_KEY.test(key) && !testWorkspaces.includes(key)) { + errors.push( + `${BASELINE_FILE} lists "${key}" but that workspace has no "test" script — remove the stale entry.`, + ); + } + } + return errors; +} + +/** + * Per-metric changes between two baseline `workspaces` maps (used by --update + * to surface what the rewrite actually did). Decreases are the dangerous + * case: --update recomputes every workspace, so a real regression elsewhere + * would silently ride along with an intended bump. + */ +export function baselineChanges(oldWs, newWs) { + const changes = []; + for (const [name, metrics] of Object.entries(newWs)) { + for (const metric of RATCHETED) { + const from = oldWs?.[name]?.[metric]; + const to = metrics[metric]; + if (typeof from === 'number' && from !== to) { + changes.push({ name, metric, from, to, decreased: to < from }); + } + } + } + return changes; +} + +/** `true` iff this module is the entry point — URL-safe. */ +export function isMain(importMetaUrl, argvPath) { + return Boolean(argvPath) && importMetaUrl === pathToFileURL(argvPath).href; +} + +// ── entry point ──────────────────────────────────────────────────────────────── + +/** Workspaces (as `apps/x` / `packages/y`) whose package.json has a test script. */ +function findTestWorkspaces() { + const found = []; + for (const group of ['apps', 'packages']) { + for (const entry of readdirSync(join(ROOT, group), { withFileTypes: true })) { + if (!entry.isDirectory()) continue; + const pkgPath = join(ROOT, group, entry.name, 'package.json'); + if (!existsSync(pkgPath)) continue; + const pkg = JSON.parse(readFileSync(pkgPath, 'utf8')); + if (pkg.scripts?.test) found.push(`${group}/${entry.name}`); + } + } + return found.sort(); +} + +function main() { + const update = process.argv.includes('--update'); + + const baselinePath = join(ROOT, BASELINE_FILE); + if (!existsSync(baselinePath)) { + console.error(`Missing ${BASELINE_FILE} at the repo root.`); + process.exit(1); + } + const baseline = JSON.parse(readFileSync(baselinePath, 'utf8')); + const tolerance = typeof baseline.tolerance === 'number' ? baseline.tolerance : 0.5; + + // Fail closed before comparing anything: an empty/blanked baseline or a + // missing/stale workspace key must be an error, not a green no-op. + const baselineErrors = validateBaseline(baseline, findTestWorkspaces()); + if (baselineErrors.length > 0) { + for (const e of baselineErrors) console.error(e); + process.exit(1); + } + + const perWorkspace = {}; + const failures = []; + let bumpable = false; + + for (const name of Object.keys(baseline.workspaces)) { + const summaryPath = join(ROOT, name, 'coverage', 'coverage-summary.json'); + if (!existsSync(summaryPath)) { + console.error( + `No coverage report for "${name}" (expected ${name}/coverage/coverage-summary.json). ` + + 'Run `pnpm test -- --coverage` first. If it already ran and failed, check the test ' + + 'output — a crashed vitest process leaves no report.', + ); + process.exit(1); + } + const totals = extractTotals(JSON.parse(readFileSync(summaryPath, 'utf8'))); + perWorkspace[name] = totals; + const result = compareWorkspace(name, totals, baseline.workspaces[name], tolerance); + failures.push(...result.failures); + bumpable ||= result.bumpable; + } + + if (update) { + const next = updatedBaseline(perWorkspace, tolerance); + const changes = baselineChanges(baseline.workspaces, next.workspaces); + for (const c of changes) { + const line = `${c.name}: ${c.metric} ${c.from}% → ${c.to}%`; + if (c.decreased) { + console.error(`⚠ DECREASE ${line} — make sure this drop is intentional before committing.`); + } else { + console.log(` ${line}`); + } + } + writeFileSync(baselinePath, `${JSON.stringify(next, null, 2)}\n`); + console.log(`Wrote ${BASELINE_FILE} from current coverage. Review the diff and commit it.`); + return; + } + + const markdown = renderMarkdown(perWorkspace, baseline.workspaces, tolerance, failures, bumpable); + console.log(markdown); + writeFileSync(join(ROOT, 'coverage-summary.md'), markdown); + if (process.env.GITHUB_STEP_SUMMARY) { + appendFileSync(process.env.GITHUB_STEP_SUMMARY, markdown); + } + + if (failures.length > 0) process.exit(1); +} + +if (isMain(import.meta.url, process.argv[1])) { + main(); +} diff --git a/scripts/check-coverage.test.mjs b/scripts/check-coverage.test.mjs new file mode 100644 index 00000000..5a6993a2 --- /dev/null +++ b/scripts/check-coverage.test.mjs @@ -0,0 +1,213 @@ +// Adversarial unit tests for the pure logic in check-coverage.mjs. Run first +// in CI (check:coverage:test) so the ratchet gate is itself gated — mirroring +// the check-docs.mjs / check-docs.test.mjs convention. +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { + floor1, + extractTotals, + compareWorkspace, + mergedTotal, + renderMarkdown, + updatedBaseline, + validateBaseline, + baselineChanges, +} from './check-coverage.mjs'; + +function metric(pct, covered = 0, total = 0) { + return { pct, covered, total }; +} + +function totals({ lines, branches, functions = 100, statements = 100 }) { + return { + lines: metric(lines, lines, 100), + branches: metric(branches, branches, 100), + functions: metric(functions, functions, 100), + statements: metric(statements, statements, 100), + }; +} + +test('floor1 floors to one decimal (never rounds up past actual coverage)', () => { + assert.equal(floor1(87.6789), 87.6); + assert.equal(floor1(87.09), 87); + assert.equal(floor1(0), 0); +}); + +test('extractTotals reads pct/covered/total per metric', () => { + const summary = { + total: { + lines: { pct: 81.5, covered: 163, total: 200 }, + branches: { pct: 70, covered: 70, total: 100 }, + functions: { pct: 90, covered: 9, total: 10 }, + statements: { pct: 81.5, covered: 163, total: 200 }, + }, + }; + const t = extractTotals(summary); + assert.equal(t.lines.pct, 81.5); + assert.equal(t.branches.covered, 70); +}); + +test('extractTotals rejects a malformed summary', () => { + assert.throws( + () => extractTotals({ total: { lines: { pct: 80 } } }), + /missing total\.branches\.pct/, + ); +}); + +test('drop beyond tolerance fails with an actionable message', () => { + const { failures } = compareWorkspace( + 'packages/db', + totals({ lines: 79.0, branches: 70 }), + { lines: 80, branches: 70 }, + 0.5, + ); + assert.equal(failures.length, 1); + assert.match(failures[0], /packages\/db: lines coverage 79\.00% dropped below/); + assert.match(failures[0], /coverage-baseline\.json/); +}); + +test('drop within tolerance passes', () => { + const { failures } = compareWorkspace( + 'packages/shared', + totals({ lines: 79.6, branches: 69.8 }), + { lines: 80, branches: 70 }, + 0.5, + ); + assert.deepEqual(failures, []); +}); + +test('branches ratchet is enforced independently of lines', () => { + const { failures } = compareWorkspace( + 'apps/web', + totals({ lines: 85, branches: 60 }), + { lines: 80, branches: 70 }, + 0.5, + ); + assert.equal(failures.length, 1); + assert.match(failures[0], /branches coverage 60\.00%/); +}); + +test('rise beyond 1pp flags a baseline bump, without failing', () => { + const { failures, bumpable } = compareWorkspace( + 'apps/etl', + totals({ lines: 82.5, branches: 70 }), + { lines: 80, branches: 70 }, + 0.5, + ); + assert.deepEqual(failures, []); + assert.equal(bumpable, true); +}); + +test('missing baseline metric is reported, not silently skipped', () => { + const { failures } = compareWorkspace( + 'packages/config', + totals({ lines: 80, branches: 70 }), + { lines: 80 }, + 0.5, + ); + assert.equal(failures.length, 1); + assert.match(failures[0], /baseline has no "branches"/); +}); + +test('mergedTotal sums counts instead of averaging percentages', () => { + const merged = mergedTotal({ + big: { ...totals({ lines: 90, branches: 90 }), lines: metric(90, 900, 1000) }, + tiny: { ...totals({ lines: 0, branches: 0 }), lines: metric(0, 0, 10) }, + }); + // 900/1010 ≈ 89.1% — a naive average of 90% and 0% would say 45%. + assert.ok(Math.abs(merged.lines.pct - 89.1) < 0.1); +}); + +test('renderMarkdown carries the sticky marker, deltas, and failure details', () => { + const per = { 'packages/db': totals({ lines: 79, branches: 70 }) }; + const md = renderMarkdown( + per, + { 'packages/db': { lines: 80, branches: 70 } }, + 0.5, + ['packages/db: lines coverage 79.00% dropped below the baseline 80%'], + false, + ); + assert.match(md, //); + assert.match(md, /-1\.00pp/); + assert.match(md, /❌ Coverage ratchet failed/); + assert.match(md, /Total.*informational/); +}); + +test('renderMarkdown on pass shows the tolerance and bump nudge', () => { + const per = { 'packages/db': totals({ lines: 85, branches: 75 }) }; + const md = renderMarkdown(per, { 'packages/db': { lines: 80, branches: 70 } }, 0.5, [], true); + assert.match(md, /✅ No workspace dropped below its baseline \(tolerance 0\.5pp\)/); + assert.match(md, /--update/); +}); + +test('empty workspaces object fails closed, not a green no-op', () => { + const errors = validateBaseline({ workspaces: {} }, ['apps/web']); + assert.equal(errors.length, 1); + assert.match(errors[0], /"workspaces" is empty/); +}); + +test('non-object workspaces (bad merge) fails closed', () => { + for (const ws of [null, undefined, [], 'x']) { + const errors = validateBaseline({ workspaces: ws }, []); + assert.equal(errors.length, 1, `workspaces=${JSON.stringify(ws)}`); + assert.match(errors[0], /must be an object/); + } +}); + +test('test-bearing workspace missing from baseline is an error (deleted-key guard)', () => { + const errors = validateBaseline({ workspaces: { 'apps/web': { lines: 80, branches: 70 } } }, [ + 'apps/web', + 'packages/db', + ]); + assert.equal(errors.length, 1); + assert.match(errors[0], /packages\/db has a "test" script but no entry/); +}); + +test('stale baseline key for a workspace without tests is an error', () => { + const errors = validateBaseline({ workspaces: { 'apps/web': {}, 'packages/gone': {} } }, [ + 'apps/web', + ]); + assert.equal(errors.length, 1); + assert.match(errors[0], /"packages\/gone".*no "test" script/); +}); + +test('traversal and prototype-shaped baseline keys are rejected', () => { + for (const key of ['../../etc', 'apps/../secret', '__proto__', '/abs', 'apps/a/b']) { + const errors = validateBaseline({ workspaces: { [key]: {}, 'apps/web': {} } }, ['apps/web']); + assert.ok( + errors.some((e) => e.includes('invalid workspace key')), + `key ${key} should be rejected`, + ); + } +}); + +test('valid baseline matching the test workspaces passes validation', () => { + const errors = validateBaseline({ workspaces: { 'apps/web': { lines: 80, branches: 70 } } }, [ + 'apps/web', + ]); + assert.deepEqual(errors, []); +}); + +test('baselineChanges flags decreases so --update cannot hide a regression', () => { + const changes = baselineChanges( + { 'apps/web': { lines: 89.3, branches: 81.3 }, 'packages/db': { lines: 82, branches: 65.5 } }, + { 'apps/web': { lines: 90.1, branches: 81.3 }, 'packages/db': { lines: 79.2, branches: 65.5 } }, + ); + assert.deepEqual(changes, [ + { name: 'apps/web', metric: 'lines', from: 89.3, to: 90.1, decreased: false }, + { name: 'packages/db', metric: 'lines', from: 82, to: 79.2, decreased: true }, + ]); +}); + +test('baselineChanges is silent for a brand-new workspace (no old entry)', () => { + const changes = baselineChanges({}, { 'apps/new': { lines: 50, branches: 40 } }); + assert.deepEqual(changes, []); +}); + +test('updatedBaseline floors current values and keeps tolerance', () => { + const next = updatedBaseline({ 'apps/web': totals({ lines: 87.68, branches: 74.99 }) }, 0.5); + assert.deepEqual(next, { + tolerance: 0.5, + workspaces: { 'apps/web': { lines: 87.6, branches: 74.9 } }, + }); +}); diff --git a/turbo.json b/turbo.json index a7ebd3ce..1491bd6d 100644 --- a/turbo.json +++ b/turbo.json @@ -10,7 +10,8 @@ "dependsOn": ["^build"] }, "test": { - "dependsOn": ["^build"] + "dependsOn": ["^build"], + "outputs": ["coverage/**"] }, "dev": { "cache": false, diff --git a/vitest.shared.ts b/vitest.shared.ts new file mode 100644 index 00000000..d5102bae --- /dev/null +++ b/vitest.shared.ts @@ -0,0 +1,25 @@ +// Shared test-coverage preset (#93). Every test-bearing workspace's +// vitest.config.ts passes its source globs through sharedCoverage() so the +// provider, reporters and output location stay identical across the monorepo — +// scripts/check-coverage.mjs depends on each workspace emitting +// coverage/coverage-summary.json in this exact shape. +import type { ViteUserConfig } from 'vitest/config'; + +export function sharedCoverage(include: string[]): NonNullable['coverage'] { + return { + provider: 'v8', + // Explicit include: without it the v8 provider only reports files loaded + // by tests, so a new untested module would be invisible to the ratchet. + include, + exclude: [ + '**/*.test.*', + '**/*.spec.*', + '**/node_modules/**', + '**/dist/**', + '**/build/**', + '**/.react-router/**', + ], + reporter: ['text', 'json-summary'], + reportsDirectory: './coverage', + }; +}