From 70c6f36c9b7fe43b1c1cb691eb9193b64e54cfd4 Mon Sep 17 00:00:00 2001 From: randrini Date: Wed, 7 Oct 2026 22:04:41 +0200 Subject: [PATCH 1/9] ci: cache the full-tree lint and give its DB tests a database The "Go lint (router recovery, full tree)" job was the CI critical path at ~19.6 min. Three changes cut it down. It never restored golangci-lint's analysis cache, so `make lint-router-recovery` re-analyzed the whole tree from scratch every run (~14 min). The go-lint job runs the identical command over a restored cache in under a minute, so this job now restores that cache too, including the entries go-lint writes. It also carried three Postgres tests with no database service and no SILO_TEST_DATABASE_URL, so every one skipped while still paying its compile time on the critical path. They move to a new parallel go-integration job with a pgvector service and a migration step, so they actually run. The job now uses the repo's composite setup-go action, which restores a per-commit build cache, instead of a bare setup-go that restored only its own stale go.sum-keyed cache. --- .github/workflows/ci.yml | 104 ++++++++++++++++++++++++++++----------- 1 file changed, 76 insertions(+), 28 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9908d192b2..cbb3ab094d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -289,6 +289,57 @@ jobs: - name: Test run: make test-go + # DB-backed tests that need the full pgvector schema migrated before they + # run. They used to sit in the full-tree lint job, which provides no Postgres + # service and no SILO_TEST_DATABASE_URL: every one skipped while still paying + # its compile time on the critical path. They run here instead, in their own + # database job, in parallel with the rest of the gate. + go-integration: + name: Go integration + runs-on: ubuntu-latest + services: + integration-postgres: + # pgvector, not stock postgres: the migrations and the taste-cluster + # tests need the vector extension. + image: pgvector/pgvector:pg18 + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: silo_test + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U postgres -d silo_test" + --health-interval 5s + --health-timeout 5s + --health-retries 10 + env: + SILO_TEST_DATABASE_URL: postgres://postgres:postgres@localhost:5432/silo_test?sslmode=disable + steps: + - name: Checkout + uses: actions/checkout@v5 + with: + persist-credentials: false + + - name: Set up Go + uses: ./.github/actions/setup-go + with: + cache-name: integration + + # The tests below read the migrated base schema (media_items, admin_jobs, + # ...), so the database is migrated once before they run. + - name: Migrate the test database + run: DATABASE_URL="$SILO_TEST_DATABASE_URL" SECRET_KEY="$(openssl rand -base64 48)" go run ./cmd/silo/ --migrate-only + + - name: Test image cache cleanup resumption against Postgres + run: go test -race -count=1 -run '^(TestImageCacheCleanup|TestRequeuedImageCacheCleanup|TestJobOutcomeIsRecorded)' ./internal/adminjob/ + + - name: Test taste cluster replacement against Postgres + run: go test -race -count=1 -run '^TestUpsertTasteClusters.*Postgres$' ./internal/recommendations/ + + - name: Test Jellyfin detail preferences against Postgres + run: go test -race -count=1 -run '^TestCompatDetailServiceResolvesViewerSubtitleDefaultsPostgres$' ./cmd/silo/ + lint: name: Go lint (changed lines) runs-on: ubuntu-latest @@ -363,9 +414,9 @@ jobs: # The inventory's lint rule (recovering a router from a handler) has to hold # over the whole tree, not only the lines a branch touched: an old line - # folded in by a rebase is as much of a hole as a new one. It re-analyzes - # the tree from scratch, so it runs as its own lane rather than behind the - # changed-lines lint. + # folded in by a rebase is as much of a hole as a new one. It runs as its own + # lane rather than behind the changed-lines lint; the build and analysis + # caches below keep the full-tree pass cheap. lint-router-recovery: name: Go lint (router recovery, full tree) runs-on: ubuntu-latest @@ -376,24 +427,15 @@ jobs: fetch-depth: 0 persist-credentials: false - - name: Install libvips - run: | - test -f /etc/apt/sources.list.d/ubuntu.sources - apt_options=( - -o Dir::Etc::sourcelist=/etc/apt/sources.list.d/ubuntu.sources - -o Dir::Etc::sourceparts=- - ) - sudo apt-get "${apt_options[@]}" update - sudo apt-get "${apt_options[@]}" install -y --no-install-recommends libvips-dev - + # The composite action installs libvips, sets up the Go toolchain, stubs + # the embedded frontend bundle, and restores a per-commit build and module + # cache. That last part is what pre-warms the build cache the full-tree + # type-check reuses; a bare setup-go here restored only its own stale + # go.sum-keyed cache instead. - name: Set up Go - uses: actions/setup-go@v6 + uses: ./.github/actions/setup-go with: - go-version-file: go.mod - cache: true - - - name: Stub the embedded frontend bundle - run: make embed-stub + cache-name: lint-router-recovery # The toolchain the binary was built with matters: a golangci-lint # binary refuses a host Go newer than the one it was built against, so @@ -419,18 +461,24 @@ jobs: - name: Put golangci-lint on PATH run: echo "$HOME/bin" >> "$GITHUB_PATH" + # golangci-lint caches per-package analysis and facts. Without restoring + # that cache this job re-analyzed the whole tree from scratch every run + # (~14 min), while the go-lint job runs the identical command over a + # restored cache in under a minute. The cache is saved under a + # job-specific key and restored from the shared prefix, so it also picks + # up the cache the go-lint job writes. + - name: Restore the golangci-lint analysis cache + uses: actions/cache@v4 + with: + path: ~/.cache/golangci-lint + key: golangci-lint-cache-router-${{ runner.os }}-${{ env.GOLANGCI_LINT_VERSION }}-${{ github.sha }} + restore-keys: | + golangci-lint-cache-router-${{ runner.os }}-${{ env.GOLANGCI_LINT_VERSION }}- + golangci-lint-cache-${{ runner.os }}-${{ env.GOLANGCI_LINT_VERSION }}- + - name: Lint router recovery full-tree run: make lint-router-recovery - - name: Test taste cluster replacement against Postgres - run: go test -race -count=1 -run '^TestUpsertTasteClusters.*Postgres$' ./internal/recommendations/ - - - name: Test Jellyfin detail preferences against Postgres - run: go test -race -count=1 -run '^TestCompatDetailServiceResolvesViewerSubtitleDefaultsPostgres$' ./cmd/silo/ - - - name: Test image cache cleanup resumption against Postgres - run: go test -race -count=1 -run '^(TestImageCacheCleanup|TestRequeuedImageCacheCleanup|TestJobOutcomeIsRecorded)' ./internal/adminjob/ - web: name: Web runs-on: ubuntu-latest From dbd7cfa6d48d037420ae22a212aeb808c43fe634 Mon Sep 17 00:00:00 2001 From: randrini Date: Wed, 7 Oct 2026 22:21:37 +0200 Subject: [PATCH 2/9] ci: refresh the Go job split comment for the integration lane --- .github/workflows/ci.yml | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cbb3ab094d..7de21b3939 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,13 +29,14 @@ env: permissions: contents: read -# The Go-side work is split across three parallel jobs: build+verify+test, -# lint of the lines a branch touched, and the full-tree router-recovery lint. -# They recompile the same tree in different ways and each dominated the old -# single-job run, so running them side by side costs one tree build instead -# of three in sequence. Every check runs unchanged; only the scheduling -# differs. docker.yml triggers on this workflow's success for main/apiv2, so -# the workflow name and its overall success semantics must not change. +# The Go-side work is split across parallel jobs: build+verify+test, lint of +# the lines a branch touched, the full-tree router-recovery lint, and the +# DB-backed integration tests. They recompile the same tree in different ways +# and each dominated the old single-job run, so running them side by side +# costs one tree build instead of several in sequence. Every check runs +# unchanged; only the scheduling differs. docker.yml triggers on this +# workflow's success for main/apiv2, so the workflow name and its overall +# success semantics must not change. jobs: # The Go gate is three jobs so that lint, the contract checks, and the test # suite run side by side; each one alone is shorter than the serial job they From a4d221e778819b084b18a3ff920b807df6c74180 Mon Sep 17 00:00:00 2001 From: randrini Date: Wed, 7 Oct 2026 22:58:52 +0200 Subject: [PATCH 3/9] ci: cache libvips packages so the Go jobs skip apt on warm runners Every GitHub-hosted Go job installs libvips-dev from apt: an apt-get update plus a ~108 MB dependency tree, about three minutes. It also occasionally stalls on a slow mirror; one run sat on that step for over twenty minutes. The composite setup-go action now caches the downloaded .deb files per runner image. A warm cache installs them offline with --no-download, so there is no apt-get update and no downloads. The key is the runner image, not the commit, and the cache is saved only on a push to main, where PRs can restore it. The private runner image still short-circuits on pkg-config. The changed-lines lint job stops hand-rolling its own apt install and uses the composite action, so it gets the same cached packages and build cache. --- .github/actions/setup-go/action.yml | 45 +++++++++++++++++++++++++++++ .github/workflows/ci.yml | 25 ++++------------ 2 files changed, 51 insertions(+), 19 deletions(-) diff --git a/.github/actions/setup-go/action.yml b/.github/actions/setup-go/action.yml index 2cc3431b3b..6b5afa3ef5 100644 --- a/.github/actions/setup-go/action.yml +++ b/.github/actions/setup-go/action.yml @@ -17,6 +17,25 @@ runs: # github.com/h2non/bimg binds libvips through cgo and pkg-config, so # nothing under ./... compiles without the headers. The Dockerfile # installs the same package in its build stage. + # + # On a GitHub-hosted runner the .deb files are cached per runner image, so + # a warm cache installs them offline: no apt-get update and no downloads. + # The private runner image already carries the headers, so pkg-config + # short-circuits there. + - name: Read the libvips cache key + if: runner.environment == 'github-hosted' + id: libvips-cache-key + shell: bash + run: echo "key=libvips-dev-${ImageOS:?}-${ImageVersion:?}-${RUNNER_ARCH}" >> "$GITHUB_OUTPUT" + + - name: Restore the libvips packages + if: runner.environment == 'github-hosted' + id: libvips-cache + uses: actions/cache/restore@v4 + with: + path: ${{ runner.temp }}/libvips-debs + key: ${{ steps.libvips-cache-key.outputs.key }} + - name: Install libvips shell: bash run: | @@ -30,9 +49,35 @@ runs: -o Dir::Etc::sourcelist=/etc/apt/sources.list.d/ubuntu.sources -o Dir::Etc::sourceparts=- ) + if [ "$RUNNER_ENVIRONMENT" = github-hosted ]; then + mkdir -p "$RUNNER_TEMP/libvips-debs" + apt_options+=(-o "Dir::Cache::archives=$RUNNER_TEMP/libvips-debs") + # The restore has no fallback keys, so packages are present only when + # they were saved on this runner image. apt then installs them as + # local files without updating its package lists; if it cannot, the + # update and install below run as usual. + debs=("$RUNNER_TEMP"/libvips-debs/*.deb) + if [ -e "${debs[0]}" ]; then + echo "installing libvips from the cache" + if sudo apt-get "${apt_options[@]}" install -y --no-install-recommends --no-download "${debs[@]}"; then + exit 0 + fi + fi + fi + echo "downloading libvips" sudo apt-get "${apt_options[@]}" update sudo apt-get "${apt_options[@]}" install -y --no-install-recommends libvips-dev + # Saved only on a push to main, where the cache is shared with PRs; the + # key is the runner image, not the commit, so one entry serves every run + # on that image. + - name: Save the libvips packages + if: runner.environment == 'github-hosted' && github.event_name != 'pull_request' && steps.libvips-cache.outputs.cache-hit != 'true' + uses: actions/cache/save@v4 + with: + path: ${{ runner.temp }}/libvips-debs + key: ${{ steps.libvips-cache.outputs.cache-primary-key }} + # setup-go's own cache is keyed on go.sum alone and written only on a # miss, so between dependency bumps every run restored a build cache # from weeks ago and recompiled the tree. The cache below is keyed on the diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7de21b3939..b716d07288 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -353,27 +353,14 @@ jobs: fetch-depth: 0 persist-credentials: false - # The lint jobs typecheck the whole tree, so cgo packages need libvips - # headers even though neither lint step links anything. - - name: Install libvips - run: | - test -f /etc/apt/sources.list.d/ubuntu.sources - apt_options=( - -o Dir::Etc::sourcelist=/etc/apt/sources.list.d/ubuntu.sources - -o Dir::Etc::sourceparts=- - ) - sudo apt-get "${apt_options[@]}" update - sudo apt-get "${apt_options[@]}" install -y --no-install-recommends libvips-dev - + # The composite action installs libvips (cached per runner image), sets + # up the Go toolchain, restores the per-commit build cache, and stubs the + # embedded frontend bundle. The lint jobs typecheck the whole tree, so + # cgo packages need libvips headers even though neither lint step links. - name: Set up Go - uses: actions/setup-go@v6 + uses: ./.github/actions/setup-go with: - go-version-file: go.mod - cache: true - - # cmd/silo embeds the built frontend; the stub satisfies the compiler. - - name: Stub the embedded frontend bundle - run: make embed-stub + cache-name: lint # The toolchain the binary was built with matters: a golangci-lint # binary refuses a host Go newer than the one it was built against, so From 9d0f0caa8543a385ea20f300511e13e898f28d08 Mon Sep 17 00:00:00 2001 From: randrini Date: Wed, 7 Oct 2026 23:40:04 +0200 Subject: [PATCH 4/9] ci: stop PR runs from filling the cache quota The composite action restored the build and module cache and then saved a per-PR copy: 0.6-1.4 GB per lane, per run. Two commits of one PR left about 10 GB of PR-scoped caches, filling GitHub's 10 GB budget. GitHub evicted the small caches first, including the golangci-lint analysis cache, so the lint lanes ran cold: router-recovery took about 9 minutes in both jobs even with go-lint's build cache warm. PR runs now restore the build and module cache without saving it; only a push to main saves. The golangci-lint analysis caches follow the same rule, so every PR restores one shared cache instead of saving a copy no other PR can read. A new prune job on main keeps only the newest cache per lane, bounding main's own per-commit saves. --- .github/actions/setup-go/action.yml | 19 ++++++++- .github/workflows/ci.yml | 65 +++++++++++++++++++++++++++-- 2 files changed, 80 insertions(+), 4 deletions(-) diff --git a/.github/actions/setup-go/action.yml b/.github/actions/setup-go/action.yml index 6b5afa3ef5..692d72176f 100644 --- a/.github/actions/setup-go/action.yml +++ b/.github/actions/setup-go/action.yml @@ -94,8 +94,25 @@ runs: shell: bash run: echo "version=$(go env GOVERSION)" >> "$GITHUB_OUTPUT" + # A PR run restores the build and module cache main last saved but does not + # save one. Each lane's cache is 0.6-1.4 GB, and saving a per-PR copy on + # every run filled GitHub's 10 GB budget, which then evicted the small + # caches (the golangci-lint analysis cache) that make the lint lanes fast. + # Only a push to main saves, and the prune job keeps the newest per lane. - name: Restore the Go build and module cache - if: runner.environment == 'github-hosted' + if: runner.environment == 'github-hosted' && github.event_name == 'pull_request' + uses: actions/cache/restore@v4 + with: + path: | + ~/.cache/go-build + ~/go/pkg/mod + key: go-${{ inputs.cache-name }}-${{ runner.os }}-${{ steps.go.outputs.version }}-${{ hashFiles('go.sum') }}-${{ github.sha }} + restore-keys: | + go-${{ inputs.cache-name }}-${{ runner.os }}-${{ steps.go.outputs.version }}-${{ hashFiles('go.sum') }}- + go-${{ inputs.cache-name }}-${{ runner.os }}-${{ steps.go.outputs.version }}- + + - name: Restore and save the Go build and module cache + if: runner.environment == 'github-hosted' && github.event_name != 'pull_request' uses: actions/cache@v4 with: path: | diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b716d07288..4d51f7d10b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,6 +38,43 @@ permissions: # workflow's success for main/apiv2, so the workflow name and its overall # success semantics must not change. jobs: + # Old per-commit caches pile up until GitHub's 10 GB budget evicts whatever it + # wants, which was taking the small golangci-lint analysis caches with it and + # leaving the lint lanes cold. Keep only the newest cache per lane on this ref. + # Runs on a push to main, before the other jobs save this push's entries. + prune-go-caches: + name: Prune old Go caches + if: github.ref_name == github.event.repository.default_branch && github.event_name != 'pull_request' + runs-on: ubuntu-latest + permissions: + actions: write + contents: read + steps: + - name: Keep the newest cache for each lane + shell: bash + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + run: | + # Paginate so pruning covers every cache on this ref, even above the + # CLI's default list limit. + gh api --paginate -X GET "repos/$GH_REPO/actions/caches" \ + -f ref="$GITHUB_REF" -f per_page=100 \ + --jq '.actions_caches[] | {id, key, ref, createdAt: .created_at}' > "$RUNNER_TEMP/go-caches.json" + cache_ids="$(jq -sr --arg ref "$GITHUB_REF" ' + map(select(.ref == $ref) | . + { + prefix: (.key | capture("^(?go-lint-router-recovery-|go-lint-|go-verify-|go-test-|go-integration-|golangci-lint-cache-router-|golangci-lint-cache-|libvips-dev-)").prefix) + }) + | group_by(.prefix)[] + | sort_by(.createdAt, .id) | reverse + | .[1:][] | .id + ' "$RUNNER_TEMP/go-caches.json")" + while IFS= read -r cache_id; do + [ -n "$cache_id" ] || continue + echo "Deleting Go cache: $cache_id" + gh cache delete "$cache_id" + done <<< "$cache_ids" + # The Go gate is three jobs so that lint, the contract checks, and the test # suite run side by side; each one alone is shorter than the serial job they # replace. They share the composite setup action under .github/actions. @@ -110,7 +147,19 @@ jobs: if: steps.golangci-lint-binary.outputs.cache-hit != 'true' run: go install "github.com/golangci/golangci-lint/v2/cmd/golangci-lint@${GOLANGCI_LINT_VERSION}" + # Restored by every run; saved only on a push to main, so all PRs share + # one analysis cache instead of each saving a copy no other PR can read. - name: Restore the golangci-lint analysis cache + if: github.event_name == 'pull_request' + uses: actions/cache/restore@v4 + with: + path: ~/.cache/golangci-lint + key: golangci-lint-cache-${{ runner.os }}-${{ env.GOLANGCI_LINT_VERSION }}-${{ github.sha }} + restore-keys: | + golangci-lint-cache-${{ runner.os }}-${{ env.GOLANGCI_LINT_VERSION }}- + + - name: Restore and save the golangci-lint analysis cache + if: github.event_name != 'pull_request' uses: actions/cache@v4 with: path: ~/.cache/golangci-lint @@ -452,10 +501,20 @@ jobs: # golangci-lint caches per-package analysis and facts. Without restoring # that cache this job re-analyzed the whole tree from scratch every run # (~14 min), while the go-lint job runs the identical command over a - # restored cache in under a minute. The cache is saved under a - # job-specific key and restored from the shared prefix, so it also picks - # up the cache the go-lint job writes. + # restored cache in under a minute. Restored from the shared prefix, so + # it also picks up what go-lint writes; saved only on a push to main. - name: Restore the golangci-lint analysis cache + if: github.event_name == 'pull_request' + uses: actions/cache/restore@v4 + with: + path: ~/.cache/golangci-lint + key: golangci-lint-cache-router-${{ runner.os }}-${{ env.GOLANGCI_LINT_VERSION }}-${{ github.sha }} + restore-keys: | + golangci-lint-cache-router-${{ runner.os }}-${{ env.GOLANGCI_LINT_VERSION }}- + golangci-lint-cache-${{ runner.os }}-${{ env.GOLANGCI_LINT_VERSION }}- + + - name: Restore and save the golangci-lint analysis cache + if: github.event_name != 'pull_request' uses: actions/cache@v4 with: path: ~/.cache/golangci-lint From 0039f9eb1db3e1e88bf60e58cad69928e4141e94 Mon Sep 17 00:00:00 2001 From: randrini Date: Thu, 8 Oct 2026 00:05:53 +0200 Subject: [PATCH 5/9] ci: avoid the apt-get update that stalls the libvips install The composite's libvips install ran apt-get update and then install. On one run the update stalled fetching package indexes from the mirrors for 22 minutes (azure.archive.ubuntu.com was ignored and archive.ubuntu.com was slow), wedging the job. The runner image ships package lists, so try the install first and only run the update if it cannot resolve. Add Acquire timeouts and retries so a stalled mirror fails fast instead of holding the job open, and skip the translation indexes. --- .github/actions/setup-go/action.yml | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/.github/actions/setup-go/action.yml b/.github/actions/setup-go/action.yml index 692d72176f..58e4c9a24c 100644 --- a/.github/actions/setup-go/action.yml +++ b/.github/actions/setup-go/action.yml @@ -44,18 +44,25 @@ runs: exit 0 fi # libvips and its dependencies use the runner's official Ubuntu sources. + # The Acquire options bound a stalled mirror: without them apt-get + # update can sit on a slow connection for the whole job (seen at 22 + # minutes). Languages=none skips the translation indexes. test -f /etc/apt/sources.list.d/ubuntu.sources apt_options=( -o Dir::Etc::sourcelist=/etc/apt/sources.list.d/ubuntu.sources -o Dir::Etc::sourceparts=- + -o Acquire::Retries=3 + -o Acquire::http::Timeout=30 + -o Acquire::https::Timeout=30 + -o Acquire::ForceIPv4=true + -o Acquire::Languages=none ) if [ "$RUNNER_ENVIRONMENT" = github-hosted ]; then mkdir -p "$RUNNER_TEMP/libvips-debs" apt_options+=(-o "Dir::Cache::archives=$RUNNER_TEMP/libvips-debs") # The restore has no fallback keys, so packages are present only when # they were saved on this runner image. apt then installs them as - # local files without updating its package lists; if it cannot, the - # update and install below run as usual. + # local files without touching the network. debs=("$RUNNER_TEMP"/libvips-debs/*.deb) if [ -e "${debs[0]}" ]; then echo "installing libvips from the cache" @@ -64,7 +71,14 @@ runs: fi fi fi - echo "downloading libvips" + # The runner image ships package lists, so try the install before + # paying for an index update. This is the step that stalled; a stale or + # missing list falls through to the update below. + echo "installing libvips" + if sudo apt-get "${apt_options[@]}" install -y --no-install-recommends libvips-dev; then + exit 0 + fi + echo "updating package lists and retrying" sudo apt-get "${apt_options[@]}" update sudo apt-get "${apt_options[@]}" install -y --no-install-recommends libvips-dev From 817d67a60ae13f37de5ed23dbec41a194c806ca8 Mon Sep 17 00:00:00 2001 From: randrini Date: Thu, 8 Oct 2026 00:39:43 +0200 Subject: [PATCH 6/9] ci: guard the prune filter against foreign cache keys The prune jq used capture() without a guard, so a cache whose key matched no lane (golangci-lint-binary-, setup-go-, node-cache-, docker) aborted the whole map and the job pruned nothing. Skip non-lane keys before the capture. Delete through the REST API instead of `gh cache delete`, whose bare-number argument has parsed as a key on some gh versions. Refresh the stale "three jobs" comment now that the gate has four lanes. --- .github/workflows/ci.yml | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4d51f7d10b..b9ba4404e2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -61,10 +61,15 @@ jobs: gh api --paginate -X GET "repos/$GH_REPO/actions/caches" \ -f ref="$GITHUB_REF" -f per_page=100 \ --jq '.actions_caches[] | {id, key, ref, createdAt: .created_at}' > "$RUNNER_TEMP/go-caches.json" - cache_ids="$(jq -sr --arg ref "$GITHUB_REF" ' - map(select(.ref == $ref) | . + { - prefix: (.key | capture("^(?go-lint-router-recovery-|go-lint-|go-verify-|go-test-|go-integration-|golangci-lint-cache-router-|golangci-lint-cache-|libvips-dev-)").prefix) - }) + # Only lane keys are grouped; any other cache (golangci-lint-binary-, + # setup-go-, node-cache-, docker) is skipped, so one foreign key + # cannot abort the capture and silently skip every lane. + cache_ids="$(jq -sr --arg ref "$GITHUB_REF" --arg re "^(?go-lint-router-recovery-|go-lint-|go-verify-|go-test-|go-integration-|golangci-lint-cache-router-|golangci-lint-cache-|libvips-dev-)" ' + map( + select(.ref == $ref) + | select(.key | test($re)) + | . + { prefix: (.key | capture($re).prefix) } + ) | group_by(.prefix)[] | sort_by(.createdAt, .id) | reverse | .[1:][] | .id @@ -72,12 +77,13 @@ jobs: while IFS= read -r cache_id; do [ -n "$cache_id" ] || continue echo "Deleting Go cache: $cache_id" - gh cache delete "$cache_id" + gh api -X DELETE "repos/$GH_REPO/actions/caches/$cache_id" done <<< "$cache_ids" - # The Go gate is three jobs so that lint, the contract checks, and the test - # suite run side by side; each one alone is shorter than the serial job they - # replace. They share the composite setup action under .github/actions. + # The Go gate runs as four parallel lanes — the lint lanes, the contract + # checks, the test suite, and the DB-backed integration tests — so each one + # alone is shorter than the serial job they replace. They share the composite + # setup action under .github/actions. go-lint: name: Go lint runs-on: ubuntu-latest From e446239eb357d1df5048fea125d81a23fb4ccd31 Mon Sep 17 00:00:00 2001 From: randrini Date: Thu, 8 Oct 2026 01:08:24 +0200 Subject: [PATCH 7/9] ci: bound the libvips apt install and let the deb cache warm Two cold-path stalls hit the libvips install: apt-get update sat on a slow mirror for 22 minutes, and an install-first attempt against a mirror returning 502s spent 16 minutes retrying per package (Acquire::Retries=3 multiplied the per-package retries). Drop the install-first shortcut, let the update refresh the runner's mirror selection, and bound the fetches with Retries=1 and 20s timeouts. The .deb cache used a restore-only step on PRs, so a PR could never warm it and every run paid the flaky apt path. Use a single actions/cache step: a run that restores the entry does not re-save, so it only writes when the shared entry is missing, which lets a PR warm it before main ever runs. --- .github/actions/setup-go/action.yml | 38 +++++++++-------------------- 1 file changed, 12 insertions(+), 26 deletions(-) diff --git a/.github/actions/setup-go/action.yml b/.github/actions/setup-go/action.yml index 58e4c9a24c..7108dc121b 100644 --- a/.github/actions/setup-go/action.yml +++ b/.github/actions/setup-go/action.yml @@ -30,8 +30,7 @@ runs: - name: Restore the libvips packages if: runner.environment == 'github-hosted' - id: libvips-cache - uses: actions/cache/restore@v4 + uses: actions/cache@v4 with: path: ${{ runner.temp }}/libvips-debs key: ${{ steps.libvips-cache-key.outputs.key }} @@ -44,16 +43,19 @@ runs: exit 0 fi # libvips and its dependencies use the runner's official Ubuntu sources. - # The Acquire options bound a stalled mirror: without them apt-get - # update can sit on a slow connection for the whole job (seen at 22 - # minutes). Languages=none skips the translation indexes. + # The Acquire options bound a stalled or failing mirror: without them + # apt-get update once sat on a slow connection for 22 minutes, and with + # three retries an install against a mirror returning 502s spent 16 + # minutes retrying per package. Languages=none skips the translation + # indexes. The update below refreshes the runner's mirror selection, so + # the install that follows uses whatever mirror is healthy now. test -f /etc/apt/sources.list.d/ubuntu.sources apt_options=( -o Dir::Etc::sourcelist=/etc/apt/sources.list.d/ubuntu.sources -o Dir::Etc::sourceparts=- - -o Acquire::Retries=3 - -o Acquire::http::Timeout=30 - -o Acquire::https::Timeout=30 + -o Acquire::Retries=1 + -o Acquire::http::Timeout=20 + -o Acquire::https::Timeout=20 -o Acquire::ForceIPv4=true -o Acquire::Languages=none ) @@ -71,27 +73,11 @@ runs: fi fi fi - # The runner image ships package lists, so try the install before - # paying for an index update. This is the step that stalled; a stale or - # missing list falls through to the update below. - echo "installing libvips" - if sudo apt-get "${apt_options[@]}" install -y --no-install-recommends libvips-dev; then - exit 0 - fi - echo "updating package lists and retrying" + echo "updating package lists" sudo apt-get "${apt_options[@]}" update + echo "installing libvips" sudo apt-get "${apt_options[@]}" install -y --no-install-recommends libvips-dev - # Saved only on a push to main, where the cache is shared with PRs; the - # key is the runner image, not the commit, so one entry serves every run - # on that image. - - name: Save the libvips packages - if: runner.environment == 'github-hosted' && github.event_name != 'pull_request' && steps.libvips-cache.outputs.cache-hit != 'true' - uses: actions/cache/save@v4 - with: - path: ${{ runner.temp }}/libvips-debs - key: ${{ steps.libvips-cache.outputs.cache-primary-key }} - # setup-go's own cache is keyed on go.sum alone and written only on a # miss, so between dependency bumps every run restored a build cache # from weeks ago and recompiled the tree. The cache below is keyed on the From 740c9d372b4174aa2096bf40de123155acdcc4af Mon Sep 17 00:00:00 2001 From: randrini Date: Thu, 8 Oct 2026 01:18:05 +0200 Subject: [PATCH 8/9] ci: fix the libvips deb cache save The deb cache never saved, so every run started cold and paid the flaky apt path. sudo apt-get creates lock/ and partial/ in the archive dir as root, and the cache step runs as the runner user, so tar failed with "Cannot open: Permission denied" and the save was skipped. chown the archive dir back to the runner user on exit, including the early cache-hit exit, so the save succeeds and later runs install offline. --- .github/actions/setup-go/action.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/actions/setup-go/action.yml b/.github/actions/setup-go/action.yml index 7108dc121b..5b1e6cb97b 100644 --- a/.github/actions/setup-go/action.yml +++ b/.github/actions/setup-go/action.yml @@ -62,6 +62,11 @@ runs: if [ "$RUNNER_ENVIRONMENT" = github-hosted ]; then mkdir -p "$RUNNER_TEMP/libvips-debs" apt_options+=(-o "Dir::Cache::archives=$RUNNER_TEMP/libvips-debs") + # sudo apt-get creates lock/ and partial/ in the archive dir as root, + # and the cache step then cannot read them, so the save failed and the + # cache stayed empty on every run. Hand the dir back before the step + # (and any early exit) finishes. + trap 'sudo chown -R "$(id -u):$(id -g)" "$RUNNER_TEMP/libvips-debs" 2>/dev/null || true' EXIT # The restore has no fallback keys, so packages are present only when # they were saved on this runner image. apt then installs them as # local files without touching the network. From 2afd8200e6edae00f96caa2aeee22456cd733798 Mon Sep 17 00:00:00 2001 From: randrini Date: Thu, 8 Oct 2026 01:28:47 +0200 Subject: [PATCH 9/9] ci: restore the reviewed libvips cache policy Address the re-review: - The .deb cache goes back to restore-only on PRs and save-on-main, so a cold key cannot be raced into a duplicate save by every job in a PR run. Now that the save bug is fixed, main warms it and PRs restore it. - The composite header now says the warm path skips apt while the cold path runs the bounded update and install. - The chown trap emits a warning instead of swallowing a failure, so a broken save is visible rather than silently re-emptying the cache. --- .github/actions/setup-go/action.yml | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/.github/actions/setup-go/action.yml b/.github/actions/setup-go/action.yml index 5b1e6cb97b..c4662fd4a3 100644 --- a/.github/actions/setup-go/action.yml +++ b/.github/actions/setup-go/action.yml @@ -18,10 +18,11 @@ runs: # nothing under ./... compiles without the headers. The Dockerfile # installs the same package in its build stage. # - # On a GitHub-hosted runner the .deb files are cached per runner image, so - # a warm cache installs them offline: no apt-get update and no downloads. - # The private runner image already carries the headers, so pkg-config - # short-circuits there. + # On a GitHub-hosted runner the .deb files are cached per runner image: a + # warm cache installs them offline with no apt-get update and no downloads, + # while a cold cache runs the bounded update and install below. The private + # runner image already carries the headers, so pkg-config short-circuits + # there. - name: Read the libvips cache key if: runner.environment == 'github-hosted' id: libvips-cache-key @@ -30,7 +31,8 @@ runs: - name: Restore the libvips packages if: runner.environment == 'github-hosted' - uses: actions/cache@v4 + id: libvips-cache + uses: actions/cache/restore@v4 with: path: ${{ runner.temp }}/libvips-debs key: ${{ steps.libvips-cache-key.outputs.key }} @@ -66,7 +68,7 @@ runs: # and the cache step then cannot read them, so the save failed and the # cache stayed empty on every run. Hand the dir back before the step # (and any early exit) finishes. - trap 'sudo chown -R "$(id -u):$(id -g)" "$RUNNER_TEMP/libvips-debs" 2>/dev/null || true' EXIT + trap 'sudo chown -R "$(id -u):$(id -g)" "$RUNNER_TEMP/libvips-debs" 2>/dev/null || echo "::warning::could not chown the libvips deb cache; the save may be skipped"' EXIT # The restore has no fallback keys, so packages are present only when # they were saved on this runner image. apt then installs them as # local files without touching the network. @@ -83,6 +85,16 @@ runs: echo "installing libvips" sudo apt-get "${apt_options[@]}" install -y --no-install-recommends libvips-dev + # Saved only on a push to main, where the key (the runner image, not the + # commit) is shared with every PR. PR runs restore only, so a cold key + # cannot be raced into a duplicate save by every job in the run. + - name: Save the libvips packages + if: runner.environment == 'github-hosted' && github.event_name != 'pull_request' && steps.libvips-cache.outputs.cache-hit != 'true' + uses: actions/cache/save@v4 + with: + path: ${{ runner.temp }}/libvips-debs + key: ${{ steps.libvips-cache.outputs.cache-primary-key }} + # setup-go's own cache is keyed on go.sum alone and written only on a # miss, so between dependency bumps every run restored a build cache # from weeks ago and recompiled the tree. The cache below is keyed on the