From 7a7436c2c8deef20fe3ef0fe149521a7a6901594 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ver=C3=B3nica=20L=C3=B3pez?= Date: Thu, 17 Sep 2026 08:38:31 -0600 Subject: [PATCH 1/2] ci(images): promote canary-validated runtime defaults by digest MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Update the six runtime defaults and promotion evidence atomically through one persistent PR after a successful scheduled canary. - Keep failed promotions retryable and separate from compatibility reporting, rejecting stale revisions and duplicate image updates. - Run promotion PR e2e against committed defaults. Signed-off-by: Verónica López --- .github/workflows/_reusable-e2e.yaml | 35 +-- .github/workflows/nightly-compatibility.yaml | 170 ++++++++++--- .github/workflows/pull-request.yaml | 25 ++ docs/development/runtime-image-promotion.md | 77 ++++++ scripts/nightly-compatibility.test.js | 158 ++++++++++++ scripts/promote-runtime-images.js | 226 +++++++++++++++++ scripts/promote-runtime-images.test.js | 243 +++++++++++++++++++ test/e2e/framework/image_overrides.go | 39 ++- test/e2e/framework/image_overrides_test.go | 44 +++- 9 files changed, 922 insertions(+), 95 deletions(-) create mode 100644 docs/development/runtime-image-promotion.md create mode 100644 scripts/nightly-compatibility.test.js create mode 100644 scripts/promote-runtime-images.js create mode 100644 scripts/promote-runtime-images.test.js diff --git a/.github/workflows/_reusable-e2e.yaml b/.github/workflows/_reusable-e2e.yaml index 87220df4..c531d984 100644 --- a/.github/workflows/_reusable-e2e.yaml +++ b/.github/workflows/_reusable-e2e.yaml @@ -171,39 +171,10 @@ jobs: run: make container - name: Pull e2e images - env: - POSTGRES_IMAGE: ${{ inputs.postgres-image }} - MULTIADMIN_IMAGE: ${{ inputs.multiadmin-image }} - MULTIADMIN_WEB_IMAGE: ${{ inputs.multiadmin-web-image }} - MULTIORCH_IMAGE: ${{ inputs.multiorch-image }} - MULTIPOOLER_IMAGE: ${{ inputs.multipooler-image }} - MULTIGATEWAY_IMAGE: ${{ inputs.multigateway-image }} run: | - images=("gcr.io/etcd-development/etcd:v3.6.7") - - # One upstream multigres image supplies all four Go components. - if [ -z "$MULTIADMIN_IMAGE" ] || - [ -z "$MULTIORCH_IMAGE" ] || - [ -z "$MULTIPOOLER_IMAGE" ] || - [ -z "$MULTIGATEWAY_IMAGE" ]; then - images+=("ghcr.io/multigres/multigres:main") - fi - [ -n "$POSTGRES_IMAGE" ] || - images+=("ghcr.io/multigres/pgctld:main") - [ -n "$MULTIADMIN_WEB_IMAGE" ] || - images+=("ghcr.io/multigres/multiadmin-web:main") - - for image in \ - "$POSTGRES_IMAGE" \ - "$MULTIADMIN_IMAGE" \ - "$MULTIADMIN_WEB_IMAGE" \ - "$MULTIORCH_IMAGE" \ - "$MULTIPOOLER_IMAGE" \ - "$MULTIGATEWAY_IMAGE"; do - [ -z "$image" ] || images+=("$image") - done - - E2E_IMAGES="$(printf '%s\n' "${images[@]}" | sort -u | tr '\n' ' ')" \ + # Source overrides have already been applied. Pull exactly what the + # operator will compile, including digest-pinned promotion defaults. + E2E_IMAGES="$(sed -n 's/.*= "\(.*\)"$/\1/p' api/v1alpha1/image_defaults.go | sort -u | tr '\n' ' ')" \ make pull-e2e-images - name: Compute test packages diff --git a/.github/workflows/nightly-compatibility.yaml b/.github/workflows/nightly-compatibility.yaml index 099e443d..93e1b1fe 100644 --- a/.github/workflows/nightly-compatibility.yaml +++ b/.github/workflows/nightly-compatibility.yaml @@ -3,7 +3,8 @@ name: Nightly upstream compatibility on: # Poll once after the upstream 05:00 UTC build. The run selects only a fully # successful nightly build; an in-progress build is never consumed. A SHA -# already recorded green is skipped, while failures can be retried manually. +# is skipped only after its promotion PR was successfully created or updated. +# Manual runs remain diagnostic and never promote or checkpoint a revision. schedule: - cron: "0 9 * * *" workflow_dispatch: @@ -36,6 +37,9 @@ jobs: sha: ${{ steps.resolve.outputs.sha }} short-sha: ${{ steps.resolve.outputs.short_sha }} operator-ref: ${{ steps.resolve.outputs.operator_ref }} + operator-sha: ${{ steps.resolve.outputs.operator_sha }} + nightly-run-id: ${{ steps.resolve.outputs.nightly_run_id }} + nightly-run-attempt: ${{ steps.resolve.outputs.nightly_run_attempt }} last-green-sha: ${{ steps.last-green.outputs.sha }} should-run: ${{ steps.gate.outputs.should_run }} multigres-image: ${{ steps.resolve.outputs.multigres_image }} @@ -47,7 +51,7 @@ jobs: uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: script: | - const prefix = 'nightly-compatibility-green-'; + const prefix = 'nightly-compatibility-promoted-'; const runs = await github.paginate( github.rest.actions.listWorkflowRuns, { @@ -59,7 +63,8 @@ jobs: }, ); const canonicalRuns = runs - .filter((run) => run.event === 'schedule') + .filter((run) => run.event === 'schedule' && run.head_branch === 'main' && + run.repository.full_name === 'multigres/multigres-operator') .sort((a, b) => new Date(b.created_at) - new Date(a.created_at)); for (const run of canonicalRuns) { const { data } = await github.rest.actions.listWorkflowRunArtifacts({ @@ -87,16 +92,18 @@ jobs: INPUT_OPERATOR_REF: ${{ inputs.operator-ref }} run: | sha="$INPUT_SHA" + # Resolve one completed scheduled build, keeping its identity alongside + # the source SHA. Manual SHA overrides still cannot enter promotion. + nightly='{}' if [ -z "$sha" ]; then - sha="$( + nightly="$( curl --fail --retry 3 --silent --show-error \ -H "Accept: application/vnd.github+json" \ -H "X-GitHub-Api-Version: 2022-11-28" \ - "https://api.github.com/repos/multigres/multigres/actions/workflows/nightly-build.yml/runs?branch=main&status=success&per_page=1" | - jq -r '.workflow_runs[0].head_sha // empty' + "https://api.github.com/repos/multigres/multigres/actions/workflows/nightly-build.yml/runs?branch=main&event=schedule&status=success&per_page=1" | + jq -ec '.workflow_runs[0] | select(.conclusion == "success" and .head_branch == "main" and .event == "schedule")' )" - [ -n "$sha" ] || - { echo "::error::No successful upstream nightly build found"; exit 1; } + sha="$(jq -er '.head_sha' <<< "$nightly")" fi [[ "$sha" =~ ^[0-9a-f]{40}$ ]] || @@ -106,10 +113,25 @@ jobs: [[ "$operator_ref" =~ ^[A-Za-z0-9._/@+-]+$ ]] || { echo "::error::Invalid operator ref '$operator_ref'"; exit 1; } + if [ "$GITHUB_EVENT_NAME" = "schedule" ]; then + operator_sha="$GITHUB_SHA" + else + operator_sha="$( + curl --fail --retry 3 --silent --show-error \ + "https://api.github.com/repos/$GITHUB_REPOSITORY/commits/$operator_ref" | + jq -er '.sha' + )" + fi + [[ "$operator_sha" =~ ^[0-9a-f]{40}$ ]] || + { echo "::error::Expected a full operator SHA"; exit 1; } + { echo "sha=$sha" echo "short_sha=$short_sha" echo "operator_ref=$operator_ref" + echo "operator_sha=$operator_sha" + echo "nightly_run_id=$(jq -r '.id // empty' <<< "$nightly")" + echo "nightly_run_attempt=$(jq -r '.run_attempt // empty' <<< "$nightly")" echo "multigres_image=ghcr.io/multigres/multigres:nightly-sha-$sha" echo "pgctld_image=ghcr.io/multigres/pgctld:nightly-sha-$sha" echo "multiadmin_web_image=ghcr.io/multigres/multiadmin-web:nightly-sha-$sha" @@ -197,16 +219,33 @@ jobs: docker buildx imagetools inspect "$image" \ --format '{{json .Manifest}}' | jq -er '.digest | select(test("^sha256:[0-9a-f]{64}$"))' - )" + )" || return 1 echo "${image%:*}@$digest" } + multigres_image="$(resolve_digest "$MULTIGRES_IMAGE")" + pgctld_image="$(resolve_digest "$PGCTLD_IMAGE")" + multiadmin_web_image="$(resolve_digest "$MULTIADMIN_WEB_IMAGE")" { - echo "multigres_image=$(resolve_digest "$MULTIGRES_IMAGE")" - echo "pgctld_image=$(resolve_digest "$PGCTLD_IMAGE")" - echo "multiadmin_web_image=$(resolve_digest "$MULTIADMIN_WEB_IMAGE")" + echo "multigres_image=$multigres_image" + echo "pgctld_image=$pgctld_image" + echo "multiadmin_web_image=$multiadmin_web_image" } >> "$GITHUB_OUTPUT" + - name: Verify both supported architectures + env: + MULTIGRES_IMAGE: ${{ steps.images.outputs.multigres_image }} + PGCTLD_IMAGE: ${{ steps.images.outputs.pgctld_image }} + MULTIADMIN_WEB_IMAGE: ${{ steps.images.outputs.multiadmin_web_image }} + run: | + for image in "$MULTIGRES_IMAGE" "$PGCTLD_IMAGE" "$MULTIADMIN_WEB_IMAGE"; do + docker buildx imagetools inspect "$image" --raw | + jq -e ' + [.manifests[] | select(.platform.os == "linux") | .platform.architecture] | + (index("amd64") != null) and (index("arm64") != null) + ' > /dev/null + done + - name: Verify nightly image provenance env: GH_TOKEN: ${{ github.token }} @@ -257,7 +296,7 @@ jobs: contents: read uses: ./.github/workflows/_reusable-e2e.yaml with: - ref: ${{ needs.resolve.outputs.operator-ref }} + ref: ${{ needs.resolve.outputs.operator-sha }} timeout-minutes: 30 postgres-image: ${{ needs.preflight.outputs.pgctld-image }} multiadmin-image: ${{ needs.preflight.outputs.multigres-image }} @@ -303,7 +342,10 @@ jobs: process.env.E2E_RESULT === 'success'; const canonicalRun = process.env.OPERATOR_REF === 'main' && - process.env.EVENT_NAME === 'schedule'; + process.env.EVENT_NAME === 'schedule' && + context.ref === 'refs/heads/main' && + context.repo.owner === 'multigres' && + context.repo.repo === 'multigres-operator'; const failureStage = process.env.RESOLVE_RESULT !== 'success' ? 'SHA resolution/validation' : process.env.PREFLIGHT_RESULT !== 'success' ? 'nightly image preflight' : @@ -422,32 +464,6 @@ jobs: }); } - - name: Write green state - if: >- - needs.resolve.result == 'success' && - needs.preflight.result == 'success' && - needs.e2e.result == 'success' && - needs.resolve.outputs.operator-ref == 'main' && - github.event_name == 'schedule' - env: - SHA: ${{ needs.resolve.outputs.sha }} - run: | - mkdir -p /tmp/nightly-compatibility - echo "$SHA" > /tmp/nightly-compatibility/upstream-sha - - - name: Preserve green state - if: >- - needs.resolve.result == 'success' && - needs.preflight.result == 'success' && - needs.e2e.result == 'success' && - needs.resolve.outputs.operator-ref == 'main' && - github.event_name == 'schedule' - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: nightly-compatibility-green-${{ needs.resolve.outputs.sha }} - path: /tmp/nightly-compatibility/upstream-sha - retention-days: 90 - - name: Fail when canary failed if: >- always() && @@ -461,3 +477,77 @@ jobs: run: | echo "::error::Canary results: resolve=$RESOLVE_RESULT preflight=$PREFLIGHT_RESULT e2e=$E2E_RESULT" exit 1 + + promote: + name: Promote verified runtime defaults + needs: [resolve, preflight, e2e] + if: >- + github.repository == 'multigres/multigres-operator' && + github.event_name == 'schedule' && + github.ref == 'refs/heads/main' && + needs.resolve.outputs.operator-ref == 'main' && + needs.resolve.outputs.should-run == 'true' && + needs.preflight.result == 'success' && + needs.e2e.result == 'success' + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: write + pull-requests: write + steps: + - name: Check out tested operator revision + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ needs.resolve.outputs.operator-sha }} + persist-credentials: false + + # An App token triggers PR checks; PRs authored with GITHUB_TOKEN do not. + - name: Generate promotion App token + id: token + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + with: + app-id: ${{ vars.MULTIGRES_BOT_APP_ID }} + private-key: ${{ secrets.MULTIGRES_BOT_APP_PRIVATE_KEY }} + owner: multigres + repositories: multigres-operator + permission-contents: write + permission-pull-requests: write + + - name: Create or update promotion PR + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + UPSTREAM_SHA: ${{ needs.resolve.outputs.sha }} + OPERATOR_SHA: ${{ needs.resolve.outputs.operator-sha }} + NIGHTLY_RUN_ID: ${{ needs.resolve.outputs.nightly-run-id }} + NIGHTLY_RUN_ATTEMPT: ${{ needs.resolve.outputs.nightly-run-attempt }} + MULTIGRES_IMAGE: ${{ needs.preflight.outputs.multigres-image }} + PGCTLD_IMAGE: ${{ needs.preflight.outputs.pgctld-image }} + MULTIADMIN_WEB_IMAGE: ${{ needs.preflight.outputs.multiadmin-web-image }} + with: + github-token: ${{ steps.token.outputs.token }} + script: | + const { promote, recordFromEnv } = require('./scripts/promote-runtime-images.js'); + const result = await promote({ + github, + context: { ...context, repo: context.repo, runAttempt: process.env.GITHUB_RUN_ATTEMPT }, + record: recordFromEnv(process.env), + }); + core.info(JSON.stringify(result)); + + # Only successful PR publication (including a verified duplicate) makes a + # revision handled. A failed promotion stays red and is retried next time. + - name: Write promoted state + env: + SHA: ${{ needs.resolve.outputs.sha }} + run: | + mkdir -p /tmp/nightly-compatibility + echo "$SHA" > /tmp/nightly-compatibility/upstream-sha + + - name: Preserve promoted state + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: nightly-compatibility-promoted-${{ needs.resolve.outputs.sha }} + path: /tmp/nightly-compatibility/upstream-sha + retention-days: 90 + overwrite: true # A rerun may replace this run's existing checkpoint. + if-no-files-found: error diff --git a/.github/workflows/pull-request.yaml b/.github/workflows/pull-request.yaml index 37486d5c..e2b2935a 100644 --- a/.github/workflows/pull-request.yaml +++ b/.github/workflows/pull-request.yaml @@ -33,3 +33,28 @@ jobs: uses: ./.github/workflows/_reusable-test-coverage.yaml with: COVERAGE_THRESHOLD: 70 + + promotion-tests: + name: Image promotion regression tests + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - run: node --test scripts/promote-runtime-images.test.js scripts/nightly-compatibility.test.js + - name: Validate committed promotion record + if: github.head_ref == 'chore/promote-runtime-images' + run: node scripts/promote-runtime-images.js --check + + promotion-e2e: + name: Promotion committed defaults + if: github.head_ref == 'chore/promote-runtime-images' + needs: promotion-tests + permissions: + contents: read + uses: ./.github/workflows/_reusable-e2e.yaml + with: + ref: ${{ github.event.pull_request.head.sha }} + timeout-minutes: 30 diff --git a/docs/development/runtime-image-promotion.md b/docs/development/runtime-image-promotion.md new file mode 100644 index 00000000..f4886dca --- /dev/null +++ b/docs/development/runtime-image-promotion.md @@ -0,0 +1,77 @@ +# Runtime image promotion + +The nightly compatibility workflow promotes runtime defaults only after a +scheduled canary on `multigres/multigres-operator`'s `main` branch passes. Proto +dependency sync does not change these defaults. Manual canaries are diagnostic; +they cannot publish a promotion PR or mark an upstream revision handled. + +## Evidence and image mapping + +Resolution selects a completed, successful scheduled upstream nightly build and +freezes the operator commit being tested. Preflight resolves immutable image +digests, checks for Linux amd64 and arm64 manifests, and verifies GitHub build +attestations against the upstream source SHA, `refs/heads/main`, the nightly +workflow identity, and the expected image repository. E2E then tests those exact +digests against the frozen operator commit. + +After a successful canary, the promotion job commits +`config/runtime-image-promotion.json` with schema version 1, `upstream_sha`, +`operator_sha`, `nightly_run` and `canary_run` (IDs, attempts, and URLs), and an +`images` map containing the three fully qualified digest references. This file is +created by the first successful promotion; it is not seeded with untested images. + +The same Git commit updates these constants in `api/v1alpha1/image_defaults.go`: + +| Image | Defaults | +| --- | --- | +| `pgctld` | `DefaultPostgresImage` | +| `multigres` | `DefaultMultiadminImage`, `DefaultMultiorchImage`, `DefaultMultipoolerImage`, `DefaultMultigatewayImage` | +| `multiadmin-web` | `DefaultMultiadminWebImage` | + +Etcd and Postgres exporter defaults are preserved. The PR describes the current +main and proposed images and links both build and test evidence. + +## PR lifecycle and retries + +Promotion maintains one open PR from `chore/promote-runtime-images` to `main`. +It uses the existing `MULTIGRES_BOT_APP_ID` and +`MULTIGRES_BOT_APP_PRIVATE_KEY`, scoped to contents and pull requests on the +operator repository. The App token ensures branch pushes and PR creation trigger +CI. Promotion PR CI validates the record and runs e2e against the committed PR +head without runtime image overrides. Review the PR and require those checks +before merging; promotion never merges automatically. + +The promotion script checks both main's record and any pending branch record for +stale revisions. Before the first record exists, it also compares the nightly +against the source revisions in the existing pinned image tags. It refuses to publish if operator main moved since the canary, +the upstream revision is stale/divergent, or an already recorded source revision +has a different digest set. Newer green sets replace all images and evidence in +one tree/ref update, retaining branch ancestry without a force push. Reprocessing +an identical set leaves its commit and original evidence unchanged. + +Compatibility reporting and promotion are sibling terminal jobs. Reporting has +only `issues: write`; promotion has only `contents: write` and +`pull-requests: write`. A green canary can close the compatibility incident even +if promotion fails. Promotion errors fail the workflow without opening a +compatibility incident. + +Only successful PR publication (or verification that the same set is already +published/merged) writes the `nightly-compatibility-promoted-` checkpoint. +Resolution reads these checkpoints only from successful scheduled runs on main; +older green-only artifacts do not count. A failed promotion remains eligible for +the next scheduled run. A failed PR API call may leave the complete candidate +commit on the promotion branch; retry repairs PR publication before writing the +checkpoint. No partially updated image set can become visible on the branch. + +## Local validation + +```sh +node --test scripts/promote-runtime-images.test.js scripts/nightly-compatibility.test.js +go test -tags=e2e ./test/e2e/framework -count=1 +actionlint .github/workflows/nightly-compatibility.yaml .github/workflows/pull-request.yaml .github/workflows/_reusable-e2e.yaml +``` + +On a generated promotion branch, also run +`node scripts/promote-runtime-images.js --check` to check the record against all +six committed constants. Full canary and promotion e2e execution takes place in +GitHub Actions with registry access and a disposable Kind cluster. diff --git a/scripts/nightly-compatibility.test.js b/scripts/nightly-compatibility.test.js new file mode 100644 index 00000000..dcdce652 --- /dev/null +++ b/scripts/nightly-compatibility.test.js @@ -0,0 +1,158 @@ +const { test } = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { spawnSync } = require('node:child_process'); + +const workflow = fs.readFileSync('.github/workflows/nightly-compatibility.yaml', 'utf8'); + +// Execute the workflow's actual shell snippets with registry/API stand-ins. +function step(name, key = 'run', indent = 8) { + const block = workflow.split(` - name: ${name}\n`)[1]?.split(/\n - |\n [a-z]/)[0]; + assert.ok(block, `Missing step ${name}`); + const code = block.split(`${' '.repeat(indent)}${key}: |\n`)[1]; + assert.ok(code, `Missing ${key} in ${name}`); + return code.split('\n').map((line) => line.slice(indent + 2)).join('\n'); +} + +function shell(t, code, env = {}) { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'image-promotion-')); + t.after(() => fs.rmSync(dir, { recursive: true, force: true })); + fs.writeFileSync(path.join(dir, 'docker'), `#!/bin/bash +if [ "$4" = "$FAIL_IMAGE" ]; then exit 1; fi +printf '%s\\n' "$DOCKER_JSON" +`, { mode: 0o755 }); + fs.writeFileSync(path.join(dir, 'curl'), `#!/bin/bash +case "$*" in + */commits/*) printf '%s\\n' "$OPERATOR_JSON" ;; + *) printf '%s\\n' "$NIGHTLY_JSON" ;; +esac +`, { mode: 0o755 }); + fs.writeFileSync(path.join(dir, 'gh'), `#!/bin/bash +printf '%s\\n' "$*" >> "$MOCK_LOG" +if [ "$FAIL_VERIFY" = "true" ]; then exit 1; fi +name="\${3#oci://}" +name="\${name%@*}" +if [ "$WRONG_SUBJECT" = "true" ]; then name=wrong; fi +printf '[{"verificationResult":{"statement":{"subject":[{"name":"%s"}]}}}]' "$name" +`, { mode: 0o755 }); + const output = path.join(dir, 'output'); + const log = path.join(dir, 'log'); + fs.writeFileSync(output, ''); + fs.writeFileSync(log, ''); + const result = spawnSync('bash', ['--noprofile', '--norc', '-eo', 'pipefail', '-c', code], { + encoding: 'utf8', env: { ...process.env, PATH: `${dir}:${process.env.PATH}`, GITHUB_OUTPUT: output, MOCK_LOG: log, ...env }, + }); + return { ...result, output: fs.readFileSync(output, 'utf8'), log: fs.readFileSync(log, 'utf8') }; +} + +const images = { + MULTIGRES_IMAGE: `ghcr.io/multigres/multigres@sha256:${'a'.repeat(64)}`, + PGCTLD_IMAGE: `ghcr.io/multigres/pgctld@sha256:${'b'.repeat(64)}`, + MULTIADMIN_WEB_IMAGE: `ghcr.io/multigres/multiadmin-web@sha256:${'c'.repeat(64)}`, +}; + +test('scheduled resolution freezes the operator SHA and captures the nightly run attempt', (t) => { + const result = shell(t, step('Resolve upstream SHA'), { + INPUT_SHA: '', INPUT_OPERATOR_REF: '', GITHUB_EVENT_NAME: 'schedule', GITHUB_SHA: 'a'.repeat(40), + NIGHTLY_JSON: JSON.stringify({ workflow_runs: [{ head_sha: 'b'.repeat(40), id: 123, run_attempt: 2, conclusion: 'success', head_branch: 'main', event: 'schedule' }] }), + }); + assert.equal(result.status, 0, result.stderr); + assert.ok(result.output.includes(`operator_sha=${'a'.repeat(40)}\n`)); + assert.ok(result.output.includes(`sha=${'b'.repeat(40)}\n`)); + assert.ok(result.output.includes('nightly_run_id=123\nnightly_run_attempt=2\n')); +}); + +test('manual SHA diagnostics do not depend on an available nightly run', (t) => { + const result = shell(t, step('Resolve upstream SHA'), { + INPUT_SHA: 'b'.repeat(40), INPUT_OPERATOR_REF: 'feature/skew', GITHUB_EVENT_NAME: 'workflow_dispatch', + GITHUB_REPOSITORY: 'multigres/multigres-operator', NIGHTLY_JSON: '{}', + OPERATOR_JSON: JSON.stringify({ sha: 'c'.repeat(40) }), + }); + assert.equal(result.status, 0, result.stderr); + assert.ok(result.output.includes(`operator_sha=${'c'.repeat(40)}\n`)); +}); + +for (const invalid of [{}, { head_branch: 'other' }, { conclusion: 'failure' }, { event: 'workflow_dispatch' }]) { + test(`rejects invalid upstream nightly selection: ${JSON.stringify(invalid)}`, (t) => { + const candidate = Object.keys(invalid).length ? { head_sha: 'b'.repeat(40), conclusion: 'success', head_branch: 'main', event: 'schedule', ...invalid } : undefined; + const result = shell(t, step('Resolve upstream SHA'), { + INPUT_SHA: '', NIGHTLY_JSON: JSON.stringify({ workflow_runs: candidate ? [candidate] : [] }), + }); + assert.notEqual(result.status, 0); + assert.equal(result.output, ''); + }); +} + +for (const failed of Object.values(images)) { + test(`digest resolution failure is not swallowed: ${failed.split('@')[0]}`, (t) => { + const result = shell(t, step('Resolve immutable nightly image digests'), { + ...images, FAIL_IMAGE: failed, DOCKER_JSON: JSON.stringify({ digest: `sha256:${'a'.repeat(64)}` }), + }); + assert.notEqual(result.status, 0); + assert.equal(result.output, ''); + }); +} + +for (const platforms of [['amd64', 'arm64'], ['amd64'], ['arm64'], []]) { + test(`architecture preflight with ${platforms.join('/') || 'no platforms'}`, (t) => { + const result = shell(t, step('Verify both supported architectures'), { + ...images, DOCKER_JSON: JSON.stringify({ manifests: platforms.map((architecture) => ({ platform: { os: 'linux', architecture } })) }), + }); + assert.equal(result.status === 0, platforms.length === 2, result.stderr); + }); +} + +test('provenance checks every digest against source revision and trusted workflow', (t) => { + const result = shell(t, step('Verify nightly image provenance'), { ...images, UPSTREAM_SHA: 'd'.repeat(40) }); + assert.equal(result.status, 0, result.stderr); + for (const image of Object.values(images)) assert.ok(result.log.includes(`oci://${image}`)); + for (const line of result.log.trim().split('\n')) { + assert.ok(line.includes('--signer-workflow multigres/multigres/.github/workflows/nightly-build.yml')); + assert.ok(line.includes('--source-ref refs/heads/main')); + assert.ok(line.includes(`--source-digest ${'d'.repeat(40)}`)); + assert.ok(line.includes('--deny-self-hosted-runners')); + } +}); + +for (const flags of [{ FAIL_VERIFY: 'true' }, { WRONG_SUBJECT: 'true' }]) { + test(`provenance fails closed: ${Object.keys(flags)[0]}`, (t) => { + assert.notEqual(shell(t, step('Verify nightly image provenance'), { ...images, ...flags }).status, 0); + }); +} + +test('a green canary closes the incident independently of a failed promotion', async (t) => { + const env = { RESOLVE_RESULT: 'success', PREFLIGHT_RESULT: 'success', E2E_RESULT: 'success', PROMOTION_RESULT: 'failure', EVENT_NAME: 'schedule', OPERATOR_REF: 'main', UPSTREAM_SHA: 'a'.repeat(40) }; + for (const [key, value] of Object.entries(env)) { + const original = process.env[key]; + process.env[key] = value; + t.after(() => { if (original === undefined) delete process.env[key]; else process.env[key] = original; }); + } + const calls = []; + const issues = { + getLabel: async () => {}, listForRepo: async () => {}, + createComment: async () => {}, + update: async (args) => calls.push(args), + create: async () => assert.fail('Promotion failure must not create a compatibility incident'), + }; + const github = { rest: { issues }, paginate: async () => [{ number: 574, body: '' }] }; + const context = { repo: { owner: 'multigres', repo: 'multigres-operator' }, ref: 'refs/heads/main', runId: 123 }; + const AsyncFunction = Object.getPrototypeOf(async function () {}).constructor; + await new AsyncFunction('github', 'context', 'core', step('Update tracking issue', 'script', 10))(github, context, {}); + assert.equal(calls.length, 1); + assert.equal(calls[0].state, 'closed'); +}); + +test('report and promotion have separate permissions and only promotion writes handled state', () => { + const report = workflow.split('\n report:\n')[1].split('\n promote:\n')[0]; + const promotion = workflow.split('\n promote:\n')[1]; + assert.match(report, /needs: \[resolve, preflight, e2e\]/); + assert.match(promotion, /needs: \[resolve, preflight, e2e\]/); + assert.match(report, /permissions:\n issues: write[^\n]*\n steps:/); + assert.match(promotion, /permissions:\n contents: write\n pull-requests: write\n steps:/); + assert.ok(!report.includes('upload-artifact')); + assert.ok(!workflow.includes('nightly-compatibility-green-')); + assert.ok(promotion.indexOf('Create or update promotion PR') < promotion.indexOf('Write promoted state')); + assert.ok(!promotion.includes('continue-on-error')); +}); diff --git a/scripts/promote-runtime-images.js b/scripts/promote-runtime-images.js new file mode 100644 index 00000000..e1241403 --- /dev/null +++ b/scripts/promote-runtime-images.js @@ -0,0 +1,226 @@ +// The workflow supplies only digests that passed provenance verification and e2e. +// Use one Git tree and one ref update so the record and all defaults move together. +const fs = require('node:fs'); +const assert = require('node:assert/strict'); + +const BRANCH = 'chore/promote-runtime-images'; +const DEFAULTS = 'api/v1alpha1/image_defaults.go'; +const RECORD = 'config/runtime-image-promotion.json'; +const COMPONENTS = { + DefaultPostgresImage: 'pgctld', + DefaultMultiadminImage: 'multigres', + DefaultMultiadminWebImage: 'multiadmin-web', + DefaultMultiorchImage: 'multigres', + DefaultMultipoolerImage: 'multigres', + DefaultMultigatewayImage: 'multigres', +}; + +function validateRecord(record) { + assert.equal(record.schema_version, 1); + for (const sha of [record.upstream_sha, record.operator_sha]) { + assert.match(sha, /^[0-9a-f]{40}$/); + } + for (const [name, repo] of [['nightly_run', 'multigres'], ['canary_run', 'multigres-operator']]) { + const run = record[name]; + assert.match(String(run.id), /^[1-9][0-9]*$/); + assert.match(String(run.attempt), /^[1-9][0-9]*$/); + assert.equal(run.url, `https://github.com/multigres/${repo}/actions/runs/${run.id}/attempts/${run.attempt}`); + } + assert.deepEqual(Object.keys(record.images).sort(), ['multiadmin-web', 'multigres', 'pgctld']); + for (const [name, image] of Object.entries(record.images)) { + assert.match(image, new RegExp(`^ghcr\\.io/multigres/${name}@sha256:[0-9a-f]{64}$`)); + } +} + +function readDefaults(source) { + return Object.fromEntries(Object.keys(COMPONENTS).map((name) => { + const matches = [...source.matchAll(new RegExp(`^(\\s*${name}\\s*=\\s*")([^"\\n]+)(")$`, 'gm'))]; + assert.equal(matches.length, 1, `Expected exactly one ${name}`); + return [name, matches[0][2]]; + })); +} + +function validateDefaults(source, record) { + validateRecord(record); + const defaults = readDefaults(source); + for (const [name, component] of Object.entries(COMPONENTS)) { + assert.equal(defaults[name], record.images[component], `${name} differs from promotion record`); + } +} + +function replaceDefaults(source, record) { + validateRecord(record); + readDefaults(source); + for (const [name, component] of Object.entries(COMPONENTS)) { + source = source.replace(new RegExp(`^(\\s*${name}\\s*=\\s*")[^"\\n]+(")$`, 'm'), + `$1${record.images[component]}$2`); + } + validateDefaults(source, record); + return source; +} + +function sameImages(a, b) { + return Object.keys(a.images).every((name) => a.images[name] === b.images[name]); +} + +function pullRequestBody(previous, record) { + const rows = Object.entries(COMPONENTS).map(([name, component]) => + `| ${name} | \`${previous[name]}\` | \`${record.images[component]}\` |`).join('\n'); + return `Promote the runtime images verified by the scheduled compatibility canary. + +- Upstream revision: [${record.upstream_sha}](https://github.com/multigres/multigres/commit/${record.upstream_sha}) +- Tested operator revision: [${record.operator_sha}](https://github.com/multigres/multigres-operator/commit/${record.operator_sha}) +- [Upstream nightly](${record.nightly_run.url}) · [Green canary](${record.canary_run.url}) + +| Default | Current main | Proposed | +| --- | --- | --- | +${rows} + +All three digests passed source-revision and build-provenance checks and contain linux/amd64 and linux/arm64 images. The promotion record and six defaults are committed atomically. Etcd and Postgres exporter defaults are unchanged. + +The promotion e2e check builds this PR's committed defaults without image overrides. Maintainer review and passing checks are required before merge. +`; +} + +async function promote({ github, context, record }) { + assert.equal(context.repo.owner, 'multigres'); + assert.equal(context.repo.repo, 'multigres-operator'); + assert.equal(context.eventName, 'schedule'); + assert.equal(context.ref, 'refs/heads/main'); + validateRecord(record); + assert.equal(record.operator_sha, context.sha); + assert.equal(String(record.canary_run.id), String(context.runId)); + assert.equal(String(record.canary_run.attempt), String(context.runAttempt)); + + const repo = context.repo; + const api = github.rest; + async function optional(read) { + try { return (await read()).data; } catch (error) { + if (error.status === 404) return null; + throw error; + } + } + async function file(path, ref) { + const data = await optional(() => api.repos.getContent({ ...repo, path, ref })); + if (!data) return null; + assert.equal(data.type, 'file'); + return Buffer.from(data.content, 'base64').toString('utf8'); + } + async function state(ref) { + const source = await file(DEFAULTS, ref); + assert.ok(source, `Missing defaults at ${ref}`); + const json = await file(RECORD, ref); + const prior = json ? JSON.parse(json) : null; + if (prior) validateDefaults(source, prior); + return { source, record: prior }; + } + async function compare(base, head) { + const { data } = await api.repos.compareCommits({ owner: 'multigres', repo: 'multigres', base, head }); + return data.status; + } + async function assertCurrentMain() { + const { data } = await api.git.getRef({ ...repo, ref: 'heads/main' }); + assert.equal(data.object.sha, record.operator_sha, 'Operator main changed since the canary; retry on current main'); + return data.object.sha; + } + + const mainSha = await assertCurrentMain(); + assert.ok(['behind', 'identical'].includes(await compare('main', record.upstream_sha)), 'Upstream SHA is not on main'); + const main = await state(mainSha); + if (!main.record) { + // Bootstrap from the source-pinned defaults that predate promotion records. + // An older successful nightly must not roll those defaults back either. + const revisions = new Set(Object.values(readDefaults(main.source)).map((image) => { + const match = image.match(/:(?:nightly-)?sha-([0-9a-f]{7,40})$/); + assert.ok(match, 'Defaults need a promotion record or source-pinned tags'); + return match[1]; + })); + for (const revision of revisions) { + assert.ok(['ahead', 'identical'].includes(await compare(revision, record.upstream_sha)), + 'Upstream revision is older than the initial runtime defaults'); + } + } + const branch = await optional(() => api.git.getRef({ ...repo, ref: `heads/${BRANCH}` })); + const pending = branch ? await state(branch.object.sha) : null; + if (pending) assert.ok(pending.record, 'Promotion branch is missing its record'); + // Consult both main and the branch, including a branch left by a failed PR API + // call. Artifact retention or an old workflow rerun must never allow rollback. + for (const prior of [main.record, pending?.record].filter(Boolean)) { + const relation = await compare(prior.upstream_sha, record.upstream_sha); + assert.ok(['ahead', 'identical'].includes(relation), 'Stale or divergent upstream revision'); + if (relation === 'identical') assert.ok(sameImages(prior, record), 'Immutable image set changed for the same revision'); + } + + if (main.record && sameImages(main.record, record)) { + return { changed: false, merged: true }; + } + + let head = branch?.object.sha; + let proposed = record; + const unchanged = pending?.record && sameImages(pending.record, record); + if (unchanged) { + // Keep the original evidence for this digest set, but still repair a missing + // or failed PR update before the workflow may checkpoint the revision. + proposed = pending.record; + } else { + const source = replaceDefaults(main.source, record); + const { data: base } = await api.git.getCommit({ ...repo, commit_sha: mainSha }); + const { data: tree } = await api.git.createTree({ + ...repo, base_tree: base.tree.sha, + tree: [ + { path: DEFAULTS, mode: '100644', type: 'blob', content: source }, + { path: RECORD, mode: '100644', type: 'blob', content: `${JSON.stringify(record, null, 2)}\n` }, + ], + }); + const { data: commit } = await api.git.createCommit({ + ...repo, message: `chore(images): promote canary-validated ${record.upstream_sha.slice(0, 7)}`, + tree: tree.sha, + // Retain branch ancestry for a non-forced update and include current main + // so the PR diff contains only the promotion even when main has advanced. + parents: [...new Set([head, mainSha].filter(Boolean))], + }); + await assertCurrentMain(); + if (branch) { + await api.git.updateRef({ ...repo, ref: `heads/${BRANCH}`, sha: commit.sha, force: false }); + } else { + await api.git.createRef({ ...repo, ref: `refs/heads/${BRANCH}`, sha: commit.sha }); + } + head = commit.sha; + } + + const prs = await github.paginate(api.pulls.list, { ...repo, state: 'open', head: `${repo.owner}:${BRANCH}`, base: 'main', per_page: 100 }); + assert.ok(prs.length <= 1, 'Multiple open promotion PRs'); + const fields = { + ...repo, + title: `chore(images): promote canary-validated ${proposed.upstream_sha.slice(0, 7)}`, + body: pullRequestBody(readDefaults(main.source), proposed), + }; + let pr = prs[0]; + if (!pr) { + ({ data: pr } = await api.pulls.create({ ...fields, head: BRANCH, base: 'main' })); + } else if (pr.title !== fields.title || pr.body !== fields.body) { + ({ data: pr } = await api.pulls.update({ ...fields, pull_number: pr.number })); + } + assert.equal(pr.head.sha, head, 'Promotion PR head changed concurrently'); + assert.equal(pr.state, 'open'); + return { changed: !unchanged, url: pr.html_url, head }; +} + +function recordFromEnv(env) { + const run = (repo, id, attempt) => ({ id, attempt, url: `https://github.com/multigres/${repo}/actions/runs/${id}/attempts/${attempt}` }); + return { + schema_version: 1, + upstream_sha: env.UPSTREAM_SHA, + operator_sha: env.OPERATOR_SHA, + nightly_run: run('multigres', env.NIGHTLY_RUN_ID, env.NIGHTLY_RUN_ATTEMPT), + canary_run: run('multigres-operator', env.GITHUB_RUN_ID, env.GITHUB_RUN_ATTEMPT), + images: { multigres: env.MULTIGRES_IMAGE, pgctld: env.PGCTLD_IMAGE, 'multiadmin-web': env.MULTIADMIN_WEB_IMAGE }, + }; +} + +module.exports = { promote, recordFromEnv, validateRecord, validateDefaults, replaceDefaults, readDefaults, BRANCH, DEFAULTS, RECORD }; + +if (require.main === module) { + assert.equal(process.argv[2], '--check', 'Usage: node scripts/promote-runtime-images.js --check'); + validateDefaults(fs.readFileSync(DEFAULTS, 'utf8'), JSON.parse(fs.readFileSync(RECORD, 'utf8'))); +} diff --git a/scripts/promote-runtime-images.test.js b/scripts/promote-runtime-images.test.js new file mode 100644 index 00000000..34575590 --- /dev/null +++ b/scripts/promote-runtime-images.test.js @@ -0,0 +1,243 @@ +const { test } = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const { + promote, recordFromEnv, validateDefaults, replaceDefaults, readDefaults, + DEFAULTS, RECORD, BRANCH, +} = require('./promote-runtime-images'); + +// Model the pre-promotion state even when these tests run on a generated PR +// whose checked-out defaults have already been pinned by digest. +let source = fs.readFileSync(DEFAULTS, 'utf8'); +for (const image of Object.values(readDefaults(source))) { + source = source.replaceAll(image, `${image.split(/[:@]/)[0]}:sha-1111111`); +} +const sha = (digit) => digit.repeat(40); +const image = (name, digit) => `ghcr.io/multigres/${name}@sha256:${digit.repeat(64)}`; +function record(digit = '2') { + return recordFromEnv({ + UPSTREAM_SHA: sha(digit), OPERATOR_SHA: sha('a'), + NIGHTLY_RUN_ID: '100', NIGHTLY_RUN_ATTEMPT: '2', + GITHUB_RUN_ID: '200', GITHUB_RUN_ATTEMPT: '1', + MULTIGRES_IMAGE: image('multigres', digit), PGCTLD_IMAGE: image('pgctld', digit), + MULTIADMIN_WEB_IMAGE: image('multiadmin-web', digit), + }); +} + +// Model the Git database separately from refs: a failed ref update leaves the +// committed pair unreachable, whereas a failed PR call leaves a repairable branch. +function fixture({ mainRecord, pendingRecord, existingPR = false } = {}) { + const calls = []; + const files = (r) => ({ + [DEFAULTS]: r ? replaceDefaults(source, r) : source, + ...(r ? { [RECORD]: JSON.stringify(r) } : {}), + 'unrelated.txt': 'keep current main', + }); + const states = new Map([[sha('a'), files(mainRecord)]]); + let main = sha('a'); + let branch; + if (pendingRecord) { branch = sha('b'); states.set(branch, files(pendingRecord)); } + let pr = existingPR ? { number: 17, title: 'old title', body: 'old body', state: 'open', head: { sha: branch } } : null; + let nextTree; + let writes = 0; + const failures = {}; + const missing = () => { throw Object.assign(new Error('Not found'), { status: 404 }); }; + const method = (name, impl) => async (args) => { + calls.push({ name, args }); + if (failures[name]) throw new Error(`injected ${name} failure`); + return { data: impl(args) }; + }; + const rest = { + repos: { + getContent: method('getContent', ({ path, ref }) => { + const content = states.get(ref)?.[path]; + return content === undefined ? missing() : { type: 'file', content: Buffer.from(content).toString('base64') }; + }), + compareCommits: method('compareCommits', ({ base, head }) => ({ status: + base === 'main' ? 'behind' : base === head ? 'identical' : base < head ? 'ahead' : 'behind', + })), + }, + git: { + getRef: method('getRef', ({ ref }) => { + const value = ref === 'heads/main' ? main : branch; + return value ? { object: { sha: value } } : missing(); + }), + getCommit: method('getCommit', ({ commit_sha }) => ({ tree: { sha: commit_sha } })), + createTree: method('createTree', ({ base_tree, tree }) => { + nextTree = { ...states.get(base_tree), ...Object.fromEntries(tree.map((entry) => [entry.path, entry.content])) }; + return { sha: 'tree' }; + }), + createCommit: method('createCommit', () => { + const commit = `commit-${++writes}`; + states.set(commit, nextTree); + return { sha: commit }; + }), + createRef: method('createRef', ({ sha: value }) => { + assert.equal(branch, undefined); + branch = value; + }), + updateRef: method('updateRef', ({ sha: value, force }) => { + assert.equal(force, false); + branch = value; + if (pr) pr.head.sha = value; + }), + }, + pulls: { + list: async () => {}, + create: method('createPR', (args) => { + pr = { ...args, number: 17, state: 'open', html_url: 'https://github.com/multigres/multigres-operator/pull/17', head: { sha: branch } }; + return pr; + }), + update: method('updatePR', (args) => { + Object.assign(pr, args); + return pr; + }), + }, + }; + return { + github: { rest, paginate: async () => pr ? [pr] : [] }, + context: { repo: { owner: 'multigres', repo: 'multigres-operator' }, eventName: 'schedule', ref: 'refs/heads/main', sha: sha('a'), runId: 200, runAttempt: 1 }, + record: record(), calls, failures, + get branchFiles() { return states.get(branch); }, + get branch() { return branch; }, + setMain(value) { main = value; }, + }; +} + +test('promotes the complete verified set and evidence in one commit and one PR', async () => { + const f = fixture(); + const result = await promote(f); + assert.equal(result.changed, true); + const committed = JSON.parse(f.branchFiles[RECORD]); + assert.deepEqual(committed, f.record); + validateDefaults(f.branchFiles[DEFAULTS], committed); + assert.equal(f.branchFiles['unrelated.txt'], 'keep current main'); + for (const name of ['DefaultEtcdImage', 'DefaultPostgresExporterImage']) { + const pattern = new RegExp(`${name} = "[^"]+"`); + assert.equal(f.branchFiles[DEFAULTS].match(pattern)[0], source.match(pattern)[0]); + } + const trees = f.calls.filter((call) => call.name === 'createTree'); + assert.equal(trees.length, 1); + assert.deepEqual(trees[0].args.tree.map((entry) => entry.path), [DEFAULTS, RECORD]); + assert.equal(f.calls.filter((call) => call.name === 'createRef').length, 1); + const pr = f.calls.find((call) => call.name === 'createPR').args; + assert.equal(pr.head, BRANCH); + assert.equal(pr.base, 'main'); + for (const old of Object.values(readDefaults(source))) assert.ok(pr.body.includes(old)); + for (const next of Object.values(committed.images)) assert.ok(pr.body.includes(next)); + assert.ok(pr.body.includes(committed.nightly_run.url)); + assert.ok(pr.body.includes(committed.canary_run.url)); +}); + +test('reprocessing the same digest set is a no-op, including evidence and PR metadata', async () => { + const f = fixture(); + await promote(f); + const files = structuredClone(f.branchFiles); + f.calls.length = 0; + f.context.runId = 201; + f.record.canary_run = { id: '201', attempt: '1', url: 'https://github.com/multigres/multigres-operator/actions/runs/201/attempts/1' }; + assert.equal((await promote(f)).changed, false); + assert.deepEqual(f.branchFiles, files); + assert.ok(!f.calls.some((call) => /^(create|update)/.test(call.name))); +}); + +test('a merged digest set is a successful no-op', async () => { + const f = fixture({ mainRecord: record() }); + assert.deepEqual(await promote(f), { changed: false, merged: true }); + assert.ok(!f.calls.some((call) => /^(create|update)/.test(call.name))); +}); + +test('a newer green set updates the existing PR atomically', async () => { + const f = fixture({ pendingRecord: record('1'), existingPR: true }); + await promote(f); + validateDefaults(f.branchFiles[DEFAULTS], f.record); + assert.ok(!f.calls.some((call) => call.name === 'createPR')); + assert.equal(f.calls.find((call) => call.name === 'updatePR').args.pull_number, 17); + assert.deepEqual(f.calls.find((call) => call.name === 'createCommit').args.parents, [sha('b'), sha('a')]); +}); + +for (const operation of ['createPR', 'updatePR']) { + test(`failed ${operation} rejects and retry repairs the PR without another commit`, async () => { + const f = fixture(operation === 'updatePR' ? { pendingRecord: record('1'), existingPR: true } : {}); + f.failures[operation] = true; + await assert.rejects(promote(f), /injected/); + validateDefaults(f.branchFiles[DEFAULTS], f.record); + f.failures[operation] = false; + f.calls.length = 0; + assert.equal((await promote(f)).changed, false); + assert.ok(!f.calls.some((call) => call.name === 'createCommit')); + assert.ok(f.calls.some((call) => call.name === operation)); + }); +} + +test('failed ref update cannot publish part of an image set', async () => { + const f = fixture({ pendingRecord: record('1'), existingPR: true }); + const before = structuredClone(f.branchFiles); + f.failures.updateRef = true; + await assert.rejects(promote(f), /injected/); + assert.deepEqual(f.branchFiles, before); + assert.ok(!f.calls.some((call) => call.name === 'updatePR')); +}); + +for (const [name, change] of [ + ['manual run', (f) => { f.context.eventName = 'workflow_dispatch'; }], + ['fork', (f) => { f.context.repo.owner = 'fork'; }], + ['non-main ref', (f) => { f.context.ref = 'refs/heads/test'; }], + ['untested operator', (f) => { f.record.operator_sha = sha('c'); }], + ['unrelated canary', (f) => { f.context.runId = 300; }], + ['wrong attempt', (f) => { f.context.runAttempt = 2; }], + ['mutable image', (f) => { f.record.images.multigres = 'ghcr.io/multigres/multigres:main'; }], + ['wrong registry', (f) => { f.record.images.pgctld = image('pgctld', '2').replace('ghcr.io', 'evil.io'); }], + ['missing image', (f) => { delete f.record.images['multiadmin-web']; }], + ['invalid nightly evidence', (f) => { f.record.nightly_run.id = ''; }], + ['main advanced during e2e', (f) => { f.setMain(sha('c')); }], +]) { + test(`rejects ${name} before writing`, async () => { + const f = fixture(); + change(f); + await assert.rejects(promote(f)); + assert.ok(!f.calls.some((call) => /^(create|update)/.test(call.name))); + }); +} + +for (const location of ['mainRecord', 'pendingRecord']) { + test(`rejects a revision older than ${location}, even without a checkpoint artifact`, async () => { + const f = fixture({ [location]: record('3') }); + await assert.rejects(promote(f), /Stale/); + assert.ok(!f.calls.some((call) => /^(create|update)/.test(call.name))); + }); +} + +test('rejects digest drift at an already recorded upstream revision', async () => { + const f = fixture({ pendingRecord: record() }); + f.record.images.pgctld = image('pgctld', '3'); + await assert.rejects(promote(f), /Immutable image set changed/); +}); + +test('fails closed when a constant is missing, repeated, or disagrees with the record', () => { + assert.throws(() => replaceDefaults(source.replace('DefaultPostgresImage =', 'RenamedImage ='), record())); + assert.throws(() => replaceDefaults(`${source}\nDefaultPostgresImage = "duplicate"`, record())); + const updated = replaceDefaults(source, record()); + assert.throws(() => validateDefaults(updated.replace(image('pgctld', '2'), image('pgctld', '3')), record())); +}); + +test('operator main moving just before publication leaves refs untouched', async () => { + const f = fixture(); + const createCommit = f.github.rest.git.createCommit; + f.github.rest.git.createCommit = async (args) => { + const result = await createCommit(args); + f.setMain(sha('c')); + return result; + }; + await assert.rejects(promote(f), /Operator main changed/); + assert.equal(f.branch, undefined); +}); + +test('first promotion rejects a nightly older than the existing source-pinned defaults', async () => { + const f = fixture(); + const compare = f.github.rest.repos.compareCommits; + f.github.rest.repos.compareCommits = (args) => args.base === '1111111' + ? Promise.resolve({ data: { status: 'behind' } }) : compare(args); + await assert.rejects(promote(f), /older than the initial runtime defaults/); + assert.ok(!f.calls.some((call) => /^(create|update)/.test(call.name))); +}); diff --git a/test/e2e/framework/image_overrides.go b/test/e2e/framework/image_overrides.go index 94c15203..1ddade4b 100644 --- a/test/e2e/framework/image_overrides.go +++ b/test/e2e/framework/image_overrides.go @@ -6,7 +6,6 @@ import ( "os" multigresv1alpha1 "github.com/multigres/multigres-operator/api/v1alpha1" - "github.com/multigres/multigres-operator/pkg/testutil" ) const ( @@ -65,30 +64,26 @@ func applyImageOverrides(cluster *multigresv1alpha1.MultigresCluster) { // references are removed before loading them into Kind. func runtimeImages() []string { overrides := imageOverridesFromEnv() - images := []string{testutil.MultigresImages[3]} // etcd is not overridable. - - if overrides.postgres == "" { - images = append(images, testutil.MultigresImages[1]) - } else { - images = append(images, overrides.postgres) - } - if overrides.multiadminWeb == "" { - images = append(images, testutil.MultigresImages[2]) - } else { - images = append(images, overrides.multiadminWeb) + images := []string{ + multigresv1alpha1.DefaultEtcdImage, + multigresv1alpha1.DefaultPostgresExporterImage, } - - goComponentOverrides := []string{ - overrides.multiadmin, - overrides.multiorch, - overrides.multipooler, - overrides.multigateway, + components := []struct { + override string + fallback string + }{ + {overrides.postgres, multigresv1alpha1.DefaultPostgresImage}, + {overrides.multiadminWeb, multigresv1alpha1.DefaultMultiadminWebImage}, + {overrides.multiadmin, multigresv1alpha1.DefaultMultiadminImage}, + {overrides.multiorch, multigresv1alpha1.DefaultMultiorchImage}, + {overrides.multipooler, multigresv1alpha1.DefaultMultipoolerImage}, + {overrides.multigateway, multigresv1alpha1.DefaultMultigatewayImage}, } - for _, image := range goComponentOverrides { - if image == "" { - images = append(images, testutil.MultigresImages[0]) + for _, component := range components { + if component.override == "" { + images = append(images, component.fallback) } else { - images = append(images, image) + images = append(images, component.override) } } diff --git a/test/e2e/framework/image_overrides_test.go b/test/e2e/framework/image_overrides_test.go index a7498b79..6f852061 100644 --- a/test/e2e/framework/image_overrides_test.go +++ b/test/e2e/framework/image_overrides_test.go @@ -4,6 +4,7 @@ package framework import ( "slices" + "strings" "testing" multigresv1alpha1 "github.com/multigres/multigres-operator/api/v1alpha1" @@ -38,7 +39,7 @@ func TestRuntimeImagesUsesOverridesAndDeduplicates(t *testing.T) { if got := count(images, multigresNightly); got != 1 { t.Fatalf("nightly multigres image occurs %d times in %v", got, images) } - if slices.Contains(images, "ghcr.io/multigres/multigres:main") { + if slices.Contains(images, multigresv1alpha1.DefaultMultiadminImage) { t.Fatalf("default multigres image retained despite complete override: %v", images) } } @@ -68,3 +69,44 @@ func count(values []string, target string) int { } return total } + +func TestRuntimeImagesUsesCommittedDefaults(t *testing.T) { + for _, key := range []string{ + postgresImageEnv, multiadminImageEnv, multiadminWebImageEnv, + multiorchImageEnv, multipoolerImageEnv, multigatewayImageEnv, + } { + t.Setenv(key, "") + } + for _, tc := range []struct { + name string + postgres string + }{ + {name: "all committed defaults"}, + {name: "partial override", postgres: "example.test/pgctld@sha256:" + strings.Repeat("a", 64)}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Setenv(postgresImageEnv, tc.postgres) + wantPostgres := multigresv1alpha1.DefaultPostgresImage + if tc.postgres != "" { + wantPostgres = tc.postgres + } + want := []string{ + wantPostgres, + multigresv1alpha1.DefaultMultiadminImage, + multigresv1alpha1.DefaultMultiadminWebImage, + multigresv1alpha1.DefaultMultiorchImage, + multigresv1alpha1.DefaultMultipoolerImage, + multigresv1alpha1.DefaultMultigatewayImage, + multigresv1alpha1.DefaultEtcdImage, + multigresv1alpha1.DefaultPostgresExporterImage, + } + slices.Sort(want) + want = slices.Compact(want) + got := runtimeImages() + slices.Sort(got) + if !slices.Equal(got, want) { + t.Fatalf("runtimeImages() = %v, want %v", got, want) + } + }) + } +} From e9f5e575dfe93fa5f1e52d7073de8f19a746b42f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ver=C3=B3nica=20L=C3=B3pez?= Date: Thu, 17 Sep 2026 08:45:41 -0600 Subject: [PATCH 2/2] fix(e2e): align image pulls with current and older loaders MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Derive local e2e pulls from committed defaults, including the Postgres exporter, while preserving explicit image lists. - Include the checked-out legacy image list for older frameworks and cover clean-cache loading with empty and partial overrides. Signed-off-by: Verónica López --- .github/workflows/_reusable-e2e.yaml | 16 ++- .github/workflows/pull-request.yaml | 2 +- Makefile | 5 +- docs/development/runtime-image-promotion.md | 12 +- scripts/e2e-images.test.js | 117 ++++++++++++++++++++ 5 files changed, 143 insertions(+), 9 deletions(-) create mode 100644 scripts/e2e-images.test.js diff --git a/.github/workflows/_reusable-e2e.yaml b/.github/workflows/_reusable-e2e.yaml index c531d984..755ef08e 100644 --- a/.github/workflows/_reusable-e2e.yaml +++ b/.github/workflows/_reusable-e2e.yaml @@ -172,9 +172,19 @@ jobs: - name: Pull e2e images run: | - # Source overrides have already been applied. Pull exactly what the - # operator will compile, including digest-pinned promotion defaults. - E2E_IMAGES="$(sed -n 's/.*= "\(.*\)"$/\1/p' api/v1alpha1/image_defaults.go | sort -u | tr '\n' ' ')" \ + # Source overrides have already been applied. Always pull the compiled + # defaults, including digest-pinned promotion images and the exporter. + required_images() { + sed -n 's/.*= "\(.*\)"$/\1/p' api/v1alpha1/image_defaults.go + + # Older checked-out frameworks still load this separate image list + # for unset overrides. Read it from that ref, not from this workflow. + if grep -rq 'testutil.MultigresImages' test/e2e/framework; then + sed -n '/^var MultigresImages = \[\]string{/,/^}/s/^[[:space:]]*"\([^"]*\)".*/\1/p' \ + pkg/testutil/e2e.go + fi + } + E2E_IMAGES="$(required_images | sort -u | tr '\n' ' ')" \ make pull-e2e-images - name: Compute test packages diff --git a/.github/workflows/pull-request.yaml b/.github/workflows/pull-request.yaml index e2b2935a..f787acdb 100644 --- a/.github/workflows/pull-request.yaml +++ b/.github/workflows/pull-request.yaml @@ -43,7 +43,7 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - run: node --test scripts/promote-runtime-images.test.js scripts/nightly-compatibility.test.js + - run: node --test scripts/promote-runtime-images.test.js scripts/nightly-compatibility.test.js scripts/e2e-images.test.js - name: Validate committed promotion record if: github.head_ref == 'chore/promote-runtime-images' run: node scripts/promote-runtime-images.js --check diff --git a/Makefile b/Makefile index 6a7fb8b3..7566d217 100644 --- a/Makefile +++ b/Makefile @@ -23,8 +23,9 @@ OBSERVER_IMG ?= $(if $(wildcard $(IMG_TAG_FILE)),$(IMG_PREFIX)/multigres-observe print-img: ## Print the full operator container image reference @echo $(IMG) -# Images required by MultigresCluster pods (must match pkg/testutil/e2e.go MultigresImages) -E2E_IMAGES ?= ghcr.io/multigres/multigres:main ghcr.io/multigres/pgctld:main ghcr.io/multigres/multiadmin-web:main gcr.io/etcd-development/etcd:v3.6.7 +# Match the compiled defaults used by test/e2e/framework, including the exporter. +# MULTIGRES_IMAGES is derived from image_defaults.go below. +E2E_IMAGES ?= $(MULTIGRES_IMAGES) .PHONY: pull-e2e-images pull-e2e-images: ## Pull container images needed by e2e tests diff --git a/docs/development/runtime-image-promotion.md b/docs/development/runtime-image-promotion.md index f4886dca..fe62afce 100644 --- a/docs/development/runtime-image-promotion.md +++ b/docs/development/runtime-image-promotion.md @@ -38,8 +38,14 @@ It uses the existing `MULTIGRES_BOT_APP_ID` and `MULTIGRES_BOT_APP_PRIVATE_KEY`, scoped to contents and pull requests on the operator repository. The App token ensures branch pushes and PR creation trigger CI. Promotion PR CI validates the record and runs e2e against the committed PR -head without runtime image overrides. Review the PR and require those checks -before merging; promotion never merges automatically. +head without runtime image overrides. Local `make pull-e2e-images` uses the same +compiled defaults, including the Postgres exporter. When the reusable workflow +checks out an older framework that still loads `testutil.MultigresImages`, it +also pulls that revision's legacy list so empty and partial overrides remain +usable. Current frameworks pull only the compiled set. + +Review the PR and require those checks before merging; promotion never merges +automatically. The promotion script checks both main's record and any pending branch record for stale revisions. Before the first record exists, it also compares the nightly @@ -66,7 +72,7 @@ checkpoint. No partially updated image set can become visible on the branch. ## Local validation ```sh -node --test scripts/promote-runtime-images.test.js scripts/nightly-compatibility.test.js +node --test scripts/promote-runtime-images.test.js scripts/nightly-compatibility.test.js scripts/e2e-images.test.js go test -tags=e2e ./test/e2e/framework -count=1 actionlint .github/workflows/nightly-compatibility.yaml .github/workflows/pull-request.yaml .github/workflows/_reusable-e2e.yaml ``` diff --git a/scripts/e2e-images.test.js b/scripts/e2e-images.test.js new file mode 100644 index 00000000..a35104dc --- /dev/null +++ b/scripts/e2e-images.test.js @@ -0,0 +1,117 @@ +const { test } = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { spawnSync } = require('node:child_process'); + +const makefile = fs.readFileSync('Makefile', 'utf8'); +const workflow = fs.readFileSync('.github/workflows/_reusable-e2e.yaml', 'utf8'); +const pullStep = workflow.split(' - name: Pull e2e images\n')[1] + .split('\n - name:')[0].split(' run: |\n')[1] + .split('\n').map((line) => line.slice(10)).join('\n'); +const defaults = { + DefaultPostgresImage: 'ghcr.io/multigres/pgctld:sha-1111111', + DefaultMultiadminImage: 'ghcr.io/multigres/multigres:sha-2222222', + DefaultMultiadminWebImage: 'ghcr.io/multigres/multiadmin-web:sha-3333333', + DefaultMultiorchImage: 'ghcr.io/multigres/multigres:sha-2222222', + DefaultMultipoolerImage: 'ghcr.io/multigres/multigres:sha-2222222', + DefaultMultigatewayImage: 'ghcr.io/multigres/multigres:sha-2222222', + DefaultEtcdImage: 'gcr.io/etcd-development/etcd:v3.6.7', + DefaultPostgresExporterImage: 'quay.io/prometheuscommunity/postgres-exporter:v0.20.1', +}; +const legacy = [ + 'ghcr.io/multigres/multigres:main', + 'ghcr.io/multigres/pgctld:main', + 'ghcr.io/multigres/multiadmin-web:main', + defaults.DefaultEtcdImage, +]; + +function fixture(t, { oldFramework = false, images = defaults } = {}) { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'e2e-images-')); + t.after(() => fs.rmSync(dir, { recursive: true, force: true })); + for (const name of ['api/v1alpha1', 'pkg/testutil', 'test/e2e/framework', 'bin']) { + fs.mkdirSync(path.join(dir, name), { recursive: true }); + } + // Historical Makefiles defaulted to four mutable images. The current workflow + // must supply its complete pull list even when using that older Makefile. + fs.writeFileSync(path.join(dir, 'Makefile'), oldFramework + ? makefile.replace(/^E2E_IMAGES \?=.*$/m, `E2E_IMAGES ?= ${legacy.join(' ')}`) + : makefile); + fs.writeFileSync(path.join(dir, 'api/v1alpha1/image_defaults.go'), + `package v1alpha1\nconst (\n${Object.entries(images).map(([name, image]) => `\t${name} = "${image}"`).join('\n')}\n)\n`); + // Both generations contain this legacy variable, but only the older framework + // uses it. Presence of the declaration alone must not add :main to new runs. + fs.writeFileSync(path.join(dir, 'pkg/testutil/e2e.go'), + `package testutil\nvar MultigresImages = []string{\n${legacy.map((image) => `\t"${image}",`).join('\n')}\n}\n`); + fs.writeFileSync(path.join(dir, 'test/e2e/framework/image_overrides.go'), oldFramework + ? 'package framework\nvar images = testutil.MultigresImages\n' + : fs.readFileSync('test/e2e/framework/image_overrides.go', 'utf8')); + const cache = path.join(dir, 'docker-cache'); + fs.writeFileSync(cache, ''); + fs.writeFileSync(path.join(dir, 'bin/docker'), `#!/bin/sh +case "$1" in + pull) printf '%s\\n' "$2" >> "$MOCK_CACHE" ;; + save) grep -Fx -- "$2" "$MOCK_CACHE" > /dev/null ;; + *) exit 1 ;; +esac +`, { mode: 0o755 }); + fs.writeFileSync(path.join(dir, 'bin/go'), '#!/bin/sh\nexit 0\n', { mode: 0o755 }); + const env = { ...process.env, PATH: `${path.join(dir, 'bin')}:${process.env.PATH}`, MOCK_CACHE: cache }; + delete env.E2E_IMAGES; + delete env.MULTIGRES_IMAGES; + function run(command, args) { + const result = spawnSync(command, args, { cwd: dir, env, encoding: 'utf8' }); + assert.equal(result.status, 0, result.stderr || result.stdout); + } + return { + run, + pulled: () => fs.readFileSync(cache, 'utf8').trim().split('\n').filter(Boolean).sort(), + checkLoadable: (images) => { + for (const image of images) { + if (!image.includes('@')) run('docker', ['save', image]); + } + }, + }; +} + +const unique = (images) => [...new Set(images)].sort(); + +test('local make pulls every committed default into an empty Docker cache', (t) => { + const f = fixture(t); + f.run('make', ['-s', 'pull-e2e-images']); + assert.deepEqual(f.pulled(), unique(Object.values(defaults))); + f.checkLoadable(Object.values(defaults)); +}); + +test('local make retains an explicit E2E_IMAGES override', (t) => { + const f = fixture(t); + f.run('make', ['-s', 'pull-e2e-images', 'E2E_IMAGES=example.test/custom:tag']); + assert.deepEqual(f.pulled(), ['example.test/custom:tag']); +}); + +for (const oldFramework of [false, true]) { + for (const overrides of ['empty', 'partial', 'complete']) { + test(`workflow pulls images needed by ${oldFramework ? 'older' : 'current'} refs with ${overrides} overrides`, (t) => { + // The workflow applies overrides to image_defaults.go before pulling. + const images = { ...defaults }; + if (overrides !== 'empty') images.DefaultPostgresImage = 'example.test/pgctld:custom'; + if (overrides === 'complete') { + for (const name of ['DefaultMultiadminImage', 'DefaultMultiorchImage', 'DefaultMultipoolerImage', 'DefaultMultigatewayImage']) { + images[name] = `ghcr.io/multigres/multigres@sha256:${'a'.repeat(64)}`; + } + images.DefaultMultiadminWebImage = 'example.test/multiadmin-web:custom'; + } + const f = fixture(t, { oldFramework, images }); + f.run('bash', ['--noprofile', '--norc', '-eo', 'pipefail', '-c', pullStep]); + const required = oldFramework ? [defaults.DefaultEtcdImage, + overrides === 'empty' ? legacy[1] : images.DefaultPostgresImage, + overrides === 'complete' ? images.DefaultMultiadminWebImage : legacy[2], + overrides === 'complete' ? images.DefaultMultiadminImage : legacy[0], + ] : Object.values(images); + f.checkLoadable(required); + for (const image of Object.values(images)) assert.ok(f.pulled().includes(image)); + if (!oldFramework) assert.deepEqual(f.pulled(), unique(Object.values(images))); + }); + } +}