diff --git a/.github/actions/setup-go/action.yml b/.github/actions/setup-go/action.yml index 2cc3431b3b..c4662fd4a3 100644 --- a/.github/actions/setup-go/action.yml +++ b/.github/actions/setup-go/action.yml @@ -17,6 +17,26 @@ runs: # github.com/h2non/bimg binds libvips through cgo and pkg-config, so # nothing under ./... compiles without the headers. The Dockerfile # installs the same package in its build stage. + # + # On a GitHub-hosted runner the .deb files are cached per runner image: a + # warm cache installs them offline with no apt-get update and no downloads, + # while a cold cache runs the bounded update and install below. The private + # runner image already carries the headers, so pkg-config short-circuits + # there. + - name: Read the libvips cache key + if: runner.environment == 'github-hosted' + id: libvips-cache-key + shell: bash + run: echo "key=libvips-dev-${ImageOS:?}-${ImageVersion:?}-${RUNNER_ARCH}" >> "$GITHUB_OUTPUT" + + - name: Restore the libvips packages + if: runner.environment == 'github-hosted' + id: libvips-cache + uses: actions/cache/restore@v4 + with: + path: ${{ runner.temp }}/libvips-debs + key: ${{ steps.libvips-cache-key.outputs.key }} + - name: Install libvips shell: bash run: | @@ -25,14 +45,56 @@ runs: exit 0 fi # libvips and its dependencies use the runner's official Ubuntu sources. + # The Acquire options bound a stalled or failing mirror: without them + # apt-get update once sat on a slow connection for 22 minutes, and with + # three retries an install against a mirror returning 502s spent 16 + # minutes retrying per package. Languages=none skips the translation + # indexes. The update below refreshes the runner's mirror selection, so + # the install that follows uses whatever mirror is healthy now. test -f /etc/apt/sources.list.d/ubuntu.sources apt_options=( -o Dir::Etc::sourcelist=/etc/apt/sources.list.d/ubuntu.sources -o Dir::Etc::sourceparts=- + -o Acquire::Retries=1 + -o Acquire::http::Timeout=20 + -o Acquire::https::Timeout=20 + -o Acquire::ForceIPv4=true + -o Acquire::Languages=none ) + if [ "$RUNNER_ENVIRONMENT" = github-hosted ]; then + mkdir -p "$RUNNER_TEMP/libvips-debs" + apt_options+=(-o "Dir::Cache::archives=$RUNNER_TEMP/libvips-debs") + # sudo apt-get creates lock/ and partial/ in the archive dir as root, + # and the cache step then cannot read them, so the save failed and the + # cache stayed empty on every run. Hand the dir back before the step + # (and any early exit) finishes. + trap 'sudo chown -R "$(id -u):$(id -g)" "$RUNNER_TEMP/libvips-debs" 2>/dev/null || echo "::warning::could not chown the libvips deb cache; the save may be skipped"' EXIT + # The restore has no fallback keys, so packages are present only when + # they were saved on this runner image. apt then installs them as + # local files without touching the network. + debs=("$RUNNER_TEMP"/libvips-debs/*.deb) + if [ -e "${debs[0]}" ]; then + echo "installing libvips from the cache" + if sudo apt-get "${apt_options[@]}" install -y --no-install-recommends --no-download "${debs[@]}"; then + exit 0 + fi + fi + fi + echo "updating package lists" sudo apt-get "${apt_options[@]}" update + echo "installing libvips" sudo apt-get "${apt_options[@]}" install -y --no-install-recommends libvips-dev + # Saved only on a push to main, where the key (the runner image, not the + # commit) is shared with every PR. PR runs restore only, so a cold key + # cannot be raced into a duplicate save by every job in the run. + - name: Save the libvips packages + if: runner.environment == 'github-hosted' && github.event_name != 'pull_request' && steps.libvips-cache.outputs.cache-hit != 'true' + uses: actions/cache/save@v4 + with: + path: ${{ runner.temp }}/libvips-debs + key: ${{ steps.libvips-cache.outputs.cache-primary-key }} + # setup-go's own cache is keyed on go.sum alone and written only on a # miss, so between dependency bumps every run restored a build cache # from weeks ago and recompiled the tree. The cache below is keyed on the @@ -49,8 +111,25 @@ runs: shell: bash run: echo "version=$(go env GOVERSION)" >> "$GITHUB_OUTPUT" + # A PR run restores the build and module cache main last saved but does not + # save one. Each lane's cache is 0.6-1.4 GB, and saving a per-PR copy on + # every run filled GitHub's 10 GB budget, which then evicted the small + # caches (the golangci-lint analysis cache) that make the lint lanes fast. + # Only a push to main saves, and the prune job keeps the newest per lane. - name: Restore the Go build and module cache - if: runner.environment == 'github-hosted' + if: runner.environment == 'github-hosted' && github.event_name == 'pull_request' + uses: actions/cache/restore@v4 + with: + path: | + ~/.cache/go-build + ~/go/pkg/mod + key: go-${{ inputs.cache-name }}-${{ runner.os }}-${{ steps.go.outputs.version }}-${{ hashFiles('go.sum') }}-${{ github.sha }} + restore-keys: | + go-${{ inputs.cache-name }}-${{ runner.os }}-${{ steps.go.outputs.version }}-${{ hashFiles('go.sum') }}- + go-${{ inputs.cache-name }}-${{ runner.os }}-${{ steps.go.outputs.version }}- + + - name: Restore and save the Go build and module cache + if: runner.environment == 'github-hosted' && github.event_name != 'pull_request' uses: actions/cache@v4 with: path: | diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9908d192b2..b9ba4404e2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,17 +29,61 @@ env: permissions: contents: read -# The Go-side work is split across three parallel jobs: build+verify+test, -# lint of the lines a branch touched, and the full-tree router-recovery lint. -# They recompile the same tree in different ways and each dominated the old -# single-job run, so running them side by side costs one tree build instead -# of three in sequence. Every check runs unchanged; only the scheduling -# differs. docker.yml triggers on this workflow's success for main/apiv2, so -# the workflow name and its overall success semantics must not change. +# The Go-side work is split across parallel jobs: build+verify+test, lint of +# the lines a branch touched, the full-tree router-recovery lint, and the +# DB-backed integration tests. They recompile the same tree in different ways +# and each dominated the old single-job run, so running them side by side +# costs one tree build instead of several in sequence. Every check runs +# unchanged; only the scheduling differs. docker.yml triggers on this +# workflow's success for main/apiv2, so the workflow name and its overall +# success semantics must not change. jobs: - # The Go gate is three jobs so that lint, the contract checks, and the test - # suite run side by side; each one alone is shorter than the serial job they - # replace. They share the composite setup action under .github/actions. + # Old per-commit caches pile up until GitHub's 10 GB budget evicts whatever it + # wants, which was taking the small golangci-lint analysis caches with it and + # leaving the lint lanes cold. Keep only the newest cache per lane on this ref. + # Runs on a push to main, before the other jobs save this push's entries. + prune-go-caches: + name: Prune old Go caches + if: github.ref_name == github.event.repository.default_branch && github.event_name != 'pull_request' + runs-on: ubuntu-latest + permissions: + actions: write + contents: read + steps: + - name: Keep the newest cache for each lane + shell: bash + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + run: | + # Paginate so pruning covers every cache on this ref, even above the + # CLI's default list limit. + gh api --paginate -X GET "repos/$GH_REPO/actions/caches" \ + -f ref="$GITHUB_REF" -f per_page=100 \ + --jq '.actions_caches[] | {id, key, ref, createdAt: .created_at}' > "$RUNNER_TEMP/go-caches.json" + # Only lane keys are grouped; any other cache (golangci-lint-binary-, + # setup-go-, node-cache-, docker) is skipped, so one foreign key + # cannot abort the capture and silently skip every lane. + cache_ids="$(jq -sr --arg ref "$GITHUB_REF" --arg re "^(?go-lint-router-recovery-|go-lint-|go-verify-|go-test-|go-integration-|golangci-lint-cache-router-|golangci-lint-cache-|libvips-dev-)" ' + map( + select(.ref == $ref) + | select(.key | test($re)) + | . + { prefix: (.key | capture($re).prefix) } + ) + | group_by(.prefix)[] + | sort_by(.createdAt, .id) | reverse + | .[1:][] | .id + ' "$RUNNER_TEMP/go-caches.json")" + while IFS= read -r cache_id; do + [ -n "$cache_id" ] || continue + echo "Deleting Go cache: $cache_id" + gh api -X DELETE "repos/$GH_REPO/actions/caches/$cache_id" + done <<< "$cache_ids" + + # The Go gate runs as four parallel lanes — the lint lanes, the contract + # checks, the test suite, and the DB-backed integration tests — so each one + # alone is shorter than the serial job they replace. They share the composite + # setup action under .github/actions. go-lint: name: Go lint runs-on: ubuntu-latest @@ -109,7 +153,19 @@ jobs: if: steps.golangci-lint-binary.outputs.cache-hit != 'true' run: go install "github.com/golangci/golangci-lint/v2/cmd/golangci-lint@${GOLANGCI_LINT_VERSION}" + # Restored by every run; saved only on a push to main, so all PRs share + # one analysis cache instead of each saving a copy no other PR can read. - name: Restore the golangci-lint analysis cache + if: github.event_name == 'pull_request' + uses: actions/cache/restore@v4 + with: + path: ~/.cache/golangci-lint + key: golangci-lint-cache-${{ runner.os }}-${{ env.GOLANGCI_LINT_VERSION }}-${{ github.sha }} + restore-keys: | + golangci-lint-cache-${{ runner.os }}-${{ env.GOLANGCI_LINT_VERSION }}- + + - name: Restore and save the golangci-lint analysis cache + if: github.event_name != 'pull_request' uses: actions/cache@v4 with: path: ~/.cache/golangci-lint @@ -289,6 +345,57 @@ jobs: - name: Test run: make test-go + # DB-backed tests that need the full pgvector schema migrated before they + # run. They used to sit in the full-tree lint job, which provides no Postgres + # service and no SILO_TEST_DATABASE_URL: every one skipped while still paying + # its compile time on the critical path. They run here instead, in their own + # database job, in parallel with the rest of the gate. + go-integration: + name: Go integration + runs-on: ubuntu-latest + services: + integration-postgres: + # pgvector, not stock postgres: the migrations and the taste-cluster + # tests need the vector extension. + image: pgvector/pgvector:pg18 + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: silo_test + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U postgres -d silo_test" + --health-interval 5s + --health-timeout 5s + --health-retries 10 + env: + SILO_TEST_DATABASE_URL: postgres://postgres:postgres@localhost:5432/silo_test?sslmode=disable + steps: + - name: Checkout + uses: actions/checkout@v5 + with: + persist-credentials: false + + - name: Set up Go + uses: ./.github/actions/setup-go + with: + cache-name: integration + + # The tests below read the migrated base schema (media_items, admin_jobs, + # ...), so the database is migrated once before they run. + - name: Migrate the test database + run: DATABASE_URL="$SILO_TEST_DATABASE_URL" SECRET_KEY="$(openssl rand -base64 48)" go run ./cmd/silo/ --migrate-only + + - name: Test image cache cleanup resumption against Postgres + run: go test -race -count=1 -run '^(TestImageCacheCleanup|TestRequeuedImageCacheCleanup|TestJobOutcomeIsRecorded)' ./internal/adminjob/ + + - name: Test taste cluster replacement against Postgres + run: go test -race -count=1 -run '^TestUpsertTasteClusters.*Postgres$' ./internal/recommendations/ + + - name: Test Jellyfin detail preferences against Postgres + run: go test -race -count=1 -run '^TestCompatDetailServiceResolvesViewerSubtitleDefaultsPostgres$' ./cmd/silo/ + lint: name: Go lint (changed lines) runs-on: ubuntu-latest @@ -301,27 +408,14 @@ jobs: fetch-depth: 0 persist-credentials: false - # The lint jobs typecheck the whole tree, so cgo packages need libvips - # headers even though neither lint step links anything. - - name: Install libvips - run: | - test -f /etc/apt/sources.list.d/ubuntu.sources - apt_options=( - -o Dir::Etc::sourcelist=/etc/apt/sources.list.d/ubuntu.sources - -o Dir::Etc::sourceparts=- - ) - sudo apt-get "${apt_options[@]}" update - sudo apt-get "${apt_options[@]}" install -y --no-install-recommends libvips-dev - + # The composite action installs libvips (cached per runner image), sets + # up the Go toolchain, restores the per-commit build cache, and stubs the + # embedded frontend bundle. The lint jobs typecheck the whole tree, so + # cgo packages need libvips headers even though neither lint step links. - name: Set up Go - uses: actions/setup-go@v6 + uses: ./.github/actions/setup-go with: - go-version-file: go.mod - cache: true - - # cmd/silo embeds the built frontend; the stub satisfies the compiler. - - name: Stub the embedded frontend bundle - run: make embed-stub + cache-name: lint # The toolchain the binary was built with matters: a golangci-lint # binary refuses a host Go newer than the one it was built against, so @@ -363,9 +457,9 @@ jobs: # The inventory's lint rule (recovering a router from a handler) has to hold # over the whole tree, not only the lines a branch touched: an old line - # folded in by a rebase is as much of a hole as a new one. It re-analyzes - # the tree from scratch, so it runs as its own lane rather than behind the - # changed-lines lint. + # folded in by a rebase is as much of a hole as a new one. It runs as its own + # lane rather than behind the changed-lines lint; the build and analysis + # caches below keep the full-tree pass cheap. lint-router-recovery: name: Go lint (router recovery, full tree) runs-on: ubuntu-latest @@ -376,24 +470,15 @@ jobs: fetch-depth: 0 persist-credentials: false - - name: Install libvips - run: | - test -f /etc/apt/sources.list.d/ubuntu.sources - apt_options=( - -o Dir::Etc::sourcelist=/etc/apt/sources.list.d/ubuntu.sources - -o Dir::Etc::sourceparts=- - ) - sudo apt-get "${apt_options[@]}" update - sudo apt-get "${apt_options[@]}" install -y --no-install-recommends libvips-dev - + # The composite action installs libvips, sets up the Go toolchain, stubs + # the embedded frontend bundle, and restores a per-commit build and module + # cache. That last part is what pre-warms the build cache the full-tree + # type-check reuses; a bare setup-go here restored only its own stale + # go.sum-keyed cache instead. - name: Set up Go - uses: actions/setup-go@v6 + uses: ./.github/actions/setup-go with: - go-version-file: go.mod - cache: true - - - name: Stub the embedded frontend bundle - run: make embed-stub + cache-name: lint-router-recovery # The toolchain the binary was built with matters: a golangci-lint # binary refuses a host Go newer than the one it was built against, so @@ -419,17 +504,33 @@ jobs: - name: Put golangci-lint on PATH run: echo "$HOME/bin" >> "$GITHUB_PATH" - - name: Lint router recovery full-tree - run: make lint-router-recovery - - - name: Test taste cluster replacement against Postgres - run: go test -race -count=1 -run '^TestUpsertTasteClusters.*Postgres$' ./internal/recommendations/ + # golangci-lint caches per-package analysis and facts. Without restoring + # that cache this job re-analyzed the whole tree from scratch every run + # (~14 min), while the go-lint job runs the identical command over a + # restored cache in under a minute. Restored from the shared prefix, so + # it also picks up what go-lint writes; saved only on a push to main. + - name: Restore the golangci-lint analysis cache + if: github.event_name == 'pull_request' + uses: actions/cache/restore@v4 + with: + path: ~/.cache/golangci-lint + key: golangci-lint-cache-router-${{ runner.os }}-${{ env.GOLANGCI_LINT_VERSION }}-${{ github.sha }} + restore-keys: | + golangci-lint-cache-router-${{ runner.os }}-${{ env.GOLANGCI_LINT_VERSION }}- + golangci-lint-cache-${{ runner.os }}-${{ env.GOLANGCI_LINT_VERSION }}- - - name: Test Jellyfin detail preferences against Postgres - run: go test -race -count=1 -run '^TestCompatDetailServiceResolvesViewerSubtitleDefaultsPostgres$' ./cmd/silo/ + - name: Restore and save the golangci-lint analysis cache + if: github.event_name != 'pull_request' + uses: actions/cache@v4 + with: + path: ~/.cache/golangci-lint + key: golangci-lint-cache-router-${{ runner.os }}-${{ env.GOLANGCI_LINT_VERSION }}-${{ github.sha }} + restore-keys: | + golangci-lint-cache-router-${{ runner.os }}-${{ env.GOLANGCI_LINT_VERSION }}- + golangci-lint-cache-${{ runner.os }}-${{ env.GOLANGCI_LINT_VERSION }}- - - name: Test image cache cleanup resumption against Postgres - run: go test -race -count=1 -run '^(TestImageCacheCleanup|TestRequeuedImageCacheCleanup|TestJobOutcomeIsRecorded)' ./internal/adminjob/ + - name: Lint router recovery full-tree + run: make lint-router-recovery web: name: Web