From a790e6c71ff1690b3260fe440891fa47f70cb8e1 Mon Sep 17 00:00:00 2001 From: ivndev001 Date: Fri, 11 Sep 2026 08:47:04 +0000 Subject: [PATCH 1/3] fix(ci): retry the chromium dependency install and raise the job budget The apt fetch behind `playwright install(-deps) chromium` is what makes CI flaky: the Playwright cache covers only the browser binary, so the system libraries install through apt on every run and a transient mirror failure fails the job (8 of the last 15 runs red; run 34577257568 died fetching fonts-ipafont-gothic with exit 100, another was cancelled at the ceiling after 10m17s). Wrap the install in a bounded retry: 3 attempts, `apt-get update` plus a short backoff between them, and a loud failure with the tool's own output after the last. Raise the 2-vCPU job's timeout from 10 to 20 minutes so a slow-but-healthy run is not read as a failure. Co-Authored-By: Claude Code --- .github/workflows/ci.yml | 43 ++++++++++++++++++++++++++++++++++------ 1 file changed, 37 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 71a8fda..49f949b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,7 +14,10 @@ jobs: name: bun (lint + typecheck + test) # Blacksmith 2 vCPU runner — fast on small matrices, plenty for this suite. runs-on: blacksmith-2vcpu-ubuntu-2204 - timeout-minutes: 10 + # 10 was too tight for a 2-vCPU runner doing an apt install + build + + # typecheck + test: one healthy-but-slow run was cancelled at 10m17s + # (run 34577257568). 20 fits a full failed-then-retried apt cycle too. + timeout-minutes: 20 steps: - uses: actions/checkout@v4 @@ -49,13 +52,41 @@ jobs: key: ${{ runner.os }}-playwright-${{ hashFiles('bun.lock') }} restore-keys: ${{ runner.os }}-playwright- + # The cache above covers only the browser binary. System libraries install + # through apt on EVERY run — cache hit or not — and that apt fetch is the + # flaky part (2026-09-11: fonts-ipafont-gothic at ~150 kB/s, exit 100 + # "Unable to fetch some archives"; 8 of the last 15 runs failed). So retry + # a bounded 3x, refreshing the package index between attempts (a stale + # index is the usual cause of "Unable to fetch some archives"). The final + # attempt still fails loudly — nothing is redirected, the tool's own + # output above is the error. - name: install chromium run: | - if [ "${{ steps.playwright-cache.outputs.cache-hit }}" = "true" ]; then - bunx playwright install-deps chromium - else - bunx playwright install --with-deps chromium - fi + install_chromium() { + if [ "${{ steps.playwright-cache.outputs.cache-hit }}" = "true" ]; then + bunx playwright install-deps chromium + else + bunx playwright install --with-deps chromium + fi + } + + for attempt in 1 2 3; do + if install_chromium; then + echo "chromium + deps installed (attempt $attempt/3)" + exit 0 + fi + echo "::warning::chromium install attempt $attempt/3 failed" + if [ "$attempt" -lt 3 ]; then + echo "refreshing apt package index, retrying in $((attempt * 20))s" + # a failed index refresh must not kill the step; the retried + # install decides + sudo apt-get update || true + sleep $((attempt * 20)) + fi + done + + echo "::error::chromium install failed after 3 attempts — see apt output above" + exit 1 # bun test runs test files in parallel by default. The low-level BrowserAdapter # integration suite flakes under the runner's parallel 2-vCPU load (its own From cec57eceff410946213157b2f66814f25ad3c226 Mon Sep 17 00:00:00 2001 From: ivndev001 Date: Fri, 11 Sep 2026 09:20:37 +0000 Subject: [PATCH 2/3] fix(ci): bound each install attempt and drop the apt dependency install The retry loop could not fire: a failed apt attempt crawled through per-mirror connection timeouts for 11m before dying, and the 20m job ceiling killed attempt 2 mid-crawl. Each attempt is now wrapped in timeout --kill-after=30 300 (exit 124/137 = budget exceeded, treated as a failed attempt), so three bounded attempts plus backoff fit inside timeout-minutes: 20. apt is gone from both cache paths: the killed run's apt plan listed all 14 Chromium runtime libraries as upgrades (already present on the stock image) and only font packages as new, and no CI assertion reads rendered output. The remaining install pulls the browser binary over HTTPS. A launch check step gates the no-apt claim at runtime and fails loudly with the loader error if it is ever wrong. Co-Authored-By: Claude Code --- .github/workflows/ci.yml | 72 ++++++++++++++++++++++++---------------- 1 file changed, 44 insertions(+), 28 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 49f949b..b833a63 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,9 +14,10 @@ jobs: name: bun (lint + typecheck + test) # Blacksmith 2 vCPU runner — fast on small matrices, plenty for this suite. runs-on: blacksmith-2vcpu-ubuntu-2204 - # 10 was too tight for a 2-vCPU runner doing an apt install + build + - # typecheck + test: one healthy-but-slow run was cancelled at 10m17s - # (run 34577257568). 20 fits a full failed-then-retried apt cycle too. + # 10 was too tight (run 34577257568 was cancelled at 10m17s mid-install). + # 20 now exists for the retry's sake, not to accommodate a hang: three + # bounded attempts (3 × 300s) + backoff (20s + 40s) + ~30s for the rest of + # the job ≈ 16.5m, so every attempt gets its full budget inside the ceiling. timeout-minutes: 20 steps: - uses: actions/checkout@v4 @@ -45,49 +46,64 @@ jobs: # managed browser (keyed on bun.lock so a playwright-core bump invalidates it) # so it isn't re-downloaded each run; the adapter prefers managed Chromium. - name: cache Playwright Chromium - id: playwright-cache uses: actions/cache@v4 with: path: ~/.cache/ms-playwright key: ${{ runner.os }}-playwright-${{ hashFiles('bun.lock') }} restore-keys: ${{ runner.os }}-playwright- - # The cache above covers only the browser binary. System libraries install - # through apt on EVERY run — cache hit or not — and that apt fetch is the - # flaky part (2026-09-11: fonts-ipafont-gothic at ~150 kB/s, exit 100 - # "Unable to fetch some archives"; 8 of the last 15 runs failed). So retry - # a bounded 3x, refreshing the package index between attempts (a stale - # index is the usual cause of "Unable to fetch some archives"). The final - # attempt still fails loudly — nothing is redirected, the tool's own - # output above is the error. + # No apt on any path. `playwright install chromium` fetches the browser + # binary from Playwright's CDN over HTTPS, or no-ops when the cache above + # restored it — it never touches the Ubuntu mirrors, whose port-80 egress + # from these runners is what actually flakes (run 34580929047: one apt + # attempt crawled 11m before failing, the second was still crawling when + # the 20m ceiling killed the job). + # The system libraries `install-deps` would add are already on the stock + # image: apt's own plan in that run listed all 14 runtime libs (libnss3, + # libgbm1, libasound2, libcups2, …) as UPGRADES, not new installs, and the + # only NEW packages were fonts — which affect glyph rendering in + # screenshots, and no CI assertion reads rendered output. The launch check + # in the next step gates that claim at runtime. + # Each attempt is bounded: `timeout` kills a wedged or crawling attempt so + # the retry actually fires. Exit 124 (or 137 after the KILL fallback) means + # the budget was exceeded — logged as such and treated as a failed attempt. + # 300s dwarfs the ~19s a healthy install took while leaving 3 attempts + + # backoff inside timeout-minutes. - name: install chromium run: | - install_chromium() { - if [ "${{ steps.playwright-cache.outputs.cache-hit }}" = "true" ]; then - bunx playwright install-deps chromium - else - bunx playwright install --with-deps chromium - fi - } - for attempt in 1 2 3; do - if install_chromium; then - echo "chromium + deps installed (attempt $attempt/3)" + timeout --kill-after=30 300 bunx playwright install chromium + rc=$? + if [ "$rc" -eq 0 ]; then + echo "chromium installed (attempt $attempt/3)" exit 0 fi - echo "::warning::chromium install attempt $attempt/3 failed" + if [ "$rc" -eq 124 ] || [ "$rc" -eq 137 ]; then + echo "::warning::attempt $attempt/3 exceeded its 300s budget and was killed (exit $rc)" + else + echo "::warning::chromium install attempt $attempt/3 failed (exit $rc)" + fi if [ "$attempt" -lt 3 ]; then - echo "refreshing apt package index, retrying in $((attempt * 20))s" - # a failed index refresh must not kill the step; the retried - # install decides - sudo apt-get update || true sleep $((attempt * 20)) fi done - echo "::error::chromium install failed after 3 attempts — see apt output above" + echo "::error::chromium install failed after 3 attempts — see output above" exit 1 + # Runtime gate on dropping apt above: prove the stock image's libraries + # really do launch this Chromium. If one is ever missing, this fails HERE + # with the loader/launch error, not as a mystery failure inside `bun test`. + # Not retried — a missing library is not transient. + - name: verify chromium launches + run: | + bun -e '(async () => { + const { chromium } = await import("playwright-core"); + const browser = await chromium.launch({ headless: true }); + console.log("chromium " + browser.version() + " launched headless, no apt deps"); + await browser.close(); + })().catch((err) => { console.error(err); process.exit(1); })' + # bun test runs test files in parallel by default. The low-level BrowserAdapter # integration suite flakes under the runner's parallel 2-vCPU load (its own # Bun.serve fixture refuses connections) — skip it here; the e2e suite covers From d54a1f032942fcef637c7b698779d1b885353a56 Mon Sep 17 00:00:00 2001 From: ivndev001 Date: Fri, 11 Sep 2026 09:28:27 +0000 Subject: [PATCH 3/3] fix(ci): pin the browser CLI to the repo's playwright-core and rekey the cache The launch gate added in the previous commit caught run 34583721201: the cached browser set had chromium-1228 but not chromium_headless_shell-1228, which every headless launch resolves to, including the suite's launchPersistentContext. bunx playwright had exited 0 in 0.7s without fetching it - bunx resolves an unpinned playwright version, not the repo's. Install through node_modules/playwright-core/cli.js so the revision matches the dependency exactly, and bump the cache key to v2: the v1 entry is immutable and incomplete, and a new key lets the first run save a complete set while the prefix restore-key still falls back to the old entry. Co-Authored-By: Claude Code --- .github/workflows/ci.yml | 36 +++++++++++++++++++++++------------- 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b833a63..608a4f2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -45,34 +45,41 @@ jobs: # Browser integration tests drive a real Chromium. Cache the Playwright- # managed browser (keyed on bun.lock so a playwright-core bump invalidates it) # so it isn't re-downloaded each run; the adapter prefers managed Chromium. + # v2 because the v1 entry is incomplete and immutable: it holds + # chromium-1228 but not chromium_headless_shell-1228, which every headless + # launch needs (run 34583721201). v2 lets the first run save a complete + # set; the prefix restore-key still falls back to v1. - name: cache Playwright Chromium uses: actions/cache@v4 with: path: ~/.cache/ms-playwright - key: ${{ runner.os }}-playwright-${{ hashFiles('bun.lock') }} + key: ${{ runner.os }}-playwright-v2-${{ hashFiles('bun.lock') }} restore-keys: ${{ runner.os }}-playwright- - # No apt on any path. `playwright install chromium` fetches the browser - # binary from Playwright's CDN over HTTPS, or no-ops when the cache above - # restored it — it never touches the Ubuntu mirrors, whose port-80 egress - # from these runners is what actually flakes (run 34580929047: one apt - # attempt crawled 11m before failing, the second was still crawling when - # the 20m ceiling killed the job). + # No apt on any path. The install fetches the browser binaries from + # Playwright's CDN over HTTPS, or fills in what the cache above is missing + # — it never touches the Ubuntu mirrors, whose port-80 egress from these + # runners is what actually flakes (run 34580929047: one apt attempt + # crawled 11m before failing, the second was still crawling when the 20m + # ceiling killed the job). # The system libraries `install-deps` would add are already on the stock # image: apt's own plan in that run listed all 14 runtime libs (libnss3, # libgbm1, libasound2, libcups2, …) as UPGRADES, not new installs, and the # only NEW packages were fonts — which affect glyph rendering in # screenshots, and no CI assertion reads rendered output. The launch check # in the next step gates that claim at runtime. + # The CLI is the repo-pinned playwright-core's own — NOT `bunx playwright`, + # which resolves an unpinned version: run 34583721201 had it exit 0 in + # 0.7s without fetching the headless shell the pinned version requires. # Each attempt is bounded: `timeout` kills a wedged or crawling attempt so # the retry actually fires. Exit 124 (or 137 after the KILL fallback) means # the budget was exceeded — logged as such and treated as a failed attempt. # 300s dwarfs the ~19s a healthy install took while leaving 3 attempts + - # backoff inside timeout-minutes. + # backoff inside timeout-minutes (a cold full fetch is ~300 MB). - name: install chromium run: | for attempt in 1 2 3; do - timeout --kill-after=30 300 bunx playwright install chromium + timeout --kill-after=30 300 node node_modules/playwright-core/cli.js install chromium rc=$? if [ "$rc" -eq 0 ]; then echo "chromium installed (attempt $attempt/3)" @@ -91,10 +98,13 @@ jobs: echo "::error::chromium install failed after 3 attempts — see output above" exit 1 - # Runtime gate on dropping apt above: prove the stock image's libraries - # really do launch this Chromium. If one is ever missing, this fails HERE - # with the loader/launch error, not as a mystery failure inside `bun test`. - # Not retried — a missing library is not transient. + # Runtime gate: prove the launch surface the suite needs is actually + # there — both the system libraries (the no-apt claim above) and the + # browser set (this is what caught run 34583721201: the cached browser + # lacked chrome-headless-shell, which headless launches — the suite's + # launchPersistentContext included — resolve to). If anything is missing, + # this fails HERE with the error naming it, not as a mystery inside + # `bun test`. Not retried: a missing binary or library is not transient. - name: verify chromium launches run: | bun -e '(async () => {