diff --git a/.github/workflows/_reusable-e2e.yaml b/.github/workflows/_reusable-e2e.yaml index 87220df4..755ef08e 100644 --- a/.github/workflows/_reusable-e2e.yaml +++ b/.github/workflows/_reusable-e2e.yaml @@ -171,39 +171,20 @@ jobs: run: make container - name: Pull e2e images - env: - POSTGRES_IMAGE: ${{ inputs.postgres-image }} - MULTIADMIN_IMAGE: ${{ inputs.multiadmin-image }} - MULTIADMIN_WEB_IMAGE: ${{ inputs.multiadmin-web-image }} - MULTIORCH_IMAGE: ${{ inputs.multiorch-image }} - MULTIPOOLER_IMAGE: ${{ inputs.multipooler-image }} - MULTIGATEWAY_IMAGE: ${{ inputs.multigateway-image }} run: | - images=("gcr.io/etcd-development/etcd:v3.6.7") - - # One upstream multigres image supplies all four Go components. - if [ -z "$MULTIADMIN_IMAGE" ] || - [ -z "$MULTIORCH_IMAGE" ] || - [ -z "$MULTIPOOLER_IMAGE" ] || - [ -z "$MULTIGATEWAY_IMAGE" ]; then - images+=("ghcr.io/multigres/multigres:main") - fi - [ -n "$POSTGRES_IMAGE" ] || - images+=("ghcr.io/multigres/pgctld:main") - [ -n "$MULTIADMIN_WEB_IMAGE" ] || - images+=("ghcr.io/multigres/multiadmin-web:main") + # Source overrides have already been applied. Always pull the compiled + # defaults, including digest-pinned promotion images and the exporter. + required_images() { + sed -n 's/.*= "\(.*\)"$/\1/p' api/v1alpha1/image_defaults.go - for image in \ - "$POSTGRES_IMAGE" \ - "$MULTIADMIN_IMAGE" \ - "$MULTIADMIN_WEB_IMAGE" \ - "$MULTIORCH_IMAGE" \ - "$MULTIPOOLER_IMAGE" \ - "$MULTIGATEWAY_IMAGE"; do - [ -z "$image" ] || images+=("$image") - done - - E2E_IMAGES="$(printf '%s\n' "${images[@]}" | sort -u | tr '\n' ' ')" \ + # Older checked-out frameworks still load this separate image list + # for unset overrides. Read it from that ref, not from this workflow. + if grep -rq 'testutil.MultigresImages' test/e2e/framework; then + sed -n '/^var MultigresImages = \[\]string{/,/^}/s/^[[:space:]]*"\([^"]*\)".*/\1/p' \ + pkg/testutil/e2e.go + fi + } + E2E_IMAGES="$(required_images | sort -u | tr '\n' ' ')" \ make pull-e2e-images - name: Compute test packages diff --git a/.github/workflows/nightly-compatibility.yaml b/.github/workflows/nightly-compatibility.yaml index 099e443d..93e1b1fe 100644 --- a/.github/workflows/nightly-compatibility.yaml +++ b/.github/workflows/nightly-compatibility.yaml @@ -3,7 +3,8 @@ name: Nightly upstream compatibility on: # Poll once after the upstream 05:00 UTC build. The run selects only a fully # successful nightly build; an in-progress build is never consumed. A SHA -# already recorded green is skipped, while failures can be retried manually. +# is skipped only after its promotion PR was successfully created or updated. +# Manual runs remain diagnostic and never promote or checkpoint a revision. schedule: - cron: "0 9 * * *" workflow_dispatch: @@ -36,6 +37,9 @@ jobs: sha: ${{ steps.resolve.outputs.sha }} short-sha: ${{ steps.resolve.outputs.short_sha }} operator-ref: ${{ steps.resolve.outputs.operator_ref }} + operator-sha: ${{ steps.resolve.outputs.operator_sha }} + nightly-run-id: ${{ steps.resolve.outputs.nightly_run_id }} + nightly-run-attempt: ${{ steps.resolve.outputs.nightly_run_attempt }} last-green-sha: ${{ steps.last-green.outputs.sha }} should-run: ${{ steps.gate.outputs.should_run }} multigres-image: ${{ steps.resolve.outputs.multigres_image }} @@ -47,7 +51,7 @@ jobs: uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: script: | - const prefix = 'nightly-compatibility-green-'; + const prefix = 'nightly-compatibility-promoted-'; const runs = await github.paginate( github.rest.actions.listWorkflowRuns, { @@ -59,7 +63,8 @@ jobs: }, ); const canonicalRuns = runs - .filter((run) => run.event === 'schedule') + .filter((run) => run.event === 'schedule' && run.head_branch === 'main' && + run.repository.full_name === 'multigres/multigres-operator') .sort((a, b) => new Date(b.created_at) - new Date(a.created_at)); for (const run of canonicalRuns) { const { data } = await github.rest.actions.listWorkflowRunArtifacts({ @@ -87,16 +92,18 @@ jobs: INPUT_OPERATOR_REF: ${{ inputs.operator-ref }} run: | sha="$INPUT_SHA" + # Resolve one completed scheduled build, keeping its identity alongside + # the source SHA. Manual SHA overrides still cannot enter promotion. + nightly='{}' if [ -z "$sha" ]; then - sha="$( + nightly="$( curl --fail --retry 3 --silent --show-error \ -H "Accept: application/vnd.github+json" \ -H "X-GitHub-Api-Version: 2022-11-28" \ - "https://api.github.com/repos/multigres/multigres/actions/workflows/nightly-build.yml/runs?branch=main&status=success&per_page=1" | - jq -r '.workflow_runs[0].head_sha // empty' + "https://api.github.com/repos/multigres/multigres/actions/workflows/nightly-build.yml/runs?branch=main&event=schedule&status=success&per_page=1" | + jq -ec '.workflow_runs[0] | select(.conclusion == "success" and .head_branch == "main" and .event == "schedule")' )" - [ -n "$sha" ] || - { echo "::error::No successful upstream nightly build found"; exit 1; } + sha="$(jq -er '.head_sha' <<< "$nightly")" fi [[ "$sha" =~ ^[0-9a-f]{40}$ ]] || @@ -106,10 +113,25 @@ jobs: [[ "$operator_ref" =~ ^[A-Za-z0-9._/@+-]+$ ]] || { echo "::error::Invalid operator ref '$operator_ref'"; exit 1; } + if [ "$GITHUB_EVENT_NAME" = "schedule" ]; then + operator_sha="$GITHUB_SHA" + else + operator_sha="$( + curl --fail --retry 3 --silent --show-error \ + "https://api.github.com/repos/$GITHUB_REPOSITORY/commits/$operator_ref" | + jq -er '.sha' + )" + fi + [[ "$operator_sha" =~ ^[0-9a-f]{40}$ ]] || + { echo "::error::Expected a full operator SHA"; exit 1; } + { echo "sha=$sha" echo "short_sha=$short_sha" echo "operator_ref=$operator_ref" + echo "operator_sha=$operator_sha" + echo "nightly_run_id=$(jq -r '.id // empty' <<< "$nightly")" + echo "nightly_run_attempt=$(jq -r '.run_attempt // empty' <<< "$nightly")" echo "multigres_image=ghcr.io/multigres/multigres:nightly-sha-$sha" echo "pgctld_image=ghcr.io/multigres/pgctld:nightly-sha-$sha" echo "multiadmin_web_image=ghcr.io/multigres/multiadmin-web:nightly-sha-$sha" @@ -197,16 +219,33 @@ jobs: docker buildx imagetools inspect "$image" \ --format '{{json .Manifest}}' | jq -er '.digest | select(test("^sha256:[0-9a-f]{64}$"))' - )" + )" || return 1 echo "${image%:*}@$digest" } + multigres_image="$(resolve_digest "$MULTIGRES_IMAGE")" + pgctld_image="$(resolve_digest "$PGCTLD_IMAGE")" + multiadmin_web_image="$(resolve_digest "$MULTIADMIN_WEB_IMAGE")" { - echo "multigres_image=$(resolve_digest "$MULTIGRES_IMAGE")" - echo "pgctld_image=$(resolve_digest "$PGCTLD_IMAGE")" - echo "multiadmin_web_image=$(resolve_digest "$MULTIADMIN_WEB_IMAGE")" + echo "multigres_image=$multigres_image" + echo "pgctld_image=$pgctld_image" + echo "multiadmin_web_image=$multiadmin_web_image" } >> "$GITHUB_OUTPUT" + - name: Verify both supported architectures + env: + MULTIGRES_IMAGE: ${{ steps.images.outputs.multigres_image }} + PGCTLD_IMAGE: ${{ steps.images.outputs.pgctld_image }} + MULTIADMIN_WEB_IMAGE: ${{ steps.images.outputs.multiadmin_web_image }} + run: | + for image in "$MULTIGRES_IMAGE" "$PGCTLD_IMAGE" "$MULTIADMIN_WEB_IMAGE"; do + docker buildx imagetools inspect "$image" --raw | + jq -e ' + [.manifests[] | select(.platform.os == "linux") | .platform.architecture] | + (index("amd64") != null) and (index("arm64") != null) + ' > /dev/null + done + - name: Verify nightly image provenance env: GH_TOKEN: ${{ github.token }} @@ -257,7 +296,7 @@ jobs: contents: read uses: ./.github/workflows/_reusable-e2e.yaml with: - ref: ${{ needs.resolve.outputs.operator-ref }} + ref: ${{ needs.resolve.outputs.operator-sha }} timeout-minutes: 30 postgres-image: ${{ needs.preflight.outputs.pgctld-image }} multiadmin-image: ${{ needs.preflight.outputs.multigres-image }} @@ -303,7 +342,10 @@ jobs: process.env.E2E_RESULT === 'success'; const canonicalRun = process.env.OPERATOR_REF === 'main' && - process.env.EVENT_NAME === 'schedule'; + process.env.EVENT_NAME === 'schedule' && + context.ref === 'refs/heads/main' && + context.repo.owner === 'multigres' && + context.repo.repo === 'multigres-operator'; const failureStage = process.env.RESOLVE_RESULT !== 'success' ? 'SHA resolution/validation' : process.env.PREFLIGHT_RESULT !== 'success' ? 'nightly image preflight' : @@ -422,32 +464,6 @@ jobs: }); } - - name: Write green state - if: >- - needs.resolve.result == 'success' && - needs.preflight.result == 'success' && - needs.e2e.result == 'success' && - needs.resolve.outputs.operator-ref == 'main' && - github.event_name == 'schedule' - env: - SHA: ${{ needs.resolve.outputs.sha }} - run: | - mkdir -p /tmp/nightly-compatibility - echo "$SHA" > /tmp/nightly-compatibility/upstream-sha - - - name: Preserve green state - if: >- - needs.resolve.result == 'success' && - needs.preflight.result == 'success' && - needs.e2e.result == 'success' && - needs.resolve.outputs.operator-ref == 'main' && - github.event_name == 'schedule' - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: nightly-compatibility-green-${{ needs.resolve.outputs.sha }} - path: /tmp/nightly-compatibility/upstream-sha - retention-days: 90 - - name: Fail when canary failed if: >- always() && @@ -461,3 +477,77 @@ jobs: run: | echo "::error::Canary results: resolve=$RESOLVE_RESULT preflight=$PREFLIGHT_RESULT e2e=$E2E_RESULT" exit 1 + + promote: + name: Promote verified runtime defaults + needs: [resolve, preflight, e2e] + if: >- + github.repository == 'multigres/multigres-operator' && + github.event_name == 'schedule' && + github.ref == 'refs/heads/main' && + needs.resolve.outputs.operator-ref == 'main' && + needs.resolve.outputs.should-run == 'true' && + needs.preflight.result == 'success' && + needs.e2e.result == 'success' + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: write + pull-requests: write + steps: + - name: Check out tested operator revision + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ needs.resolve.outputs.operator-sha }} + persist-credentials: false + + # An App token triggers PR checks; PRs authored with GITHUB_TOKEN do not. + - name: Generate promotion App token + id: token + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + with: + app-id: ${{ vars.MULTIGRES_BOT_APP_ID }} + private-key: ${{ secrets.MULTIGRES_BOT_APP_PRIVATE_KEY }} + owner: multigres + repositories: multigres-operator + permission-contents: write + permission-pull-requests: write + + - name: Create or update promotion PR + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + UPSTREAM_SHA: ${{ needs.resolve.outputs.sha }} + OPERATOR_SHA: ${{ needs.resolve.outputs.operator-sha }} + NIGHTLY_RUN_ID: ${{ needs.resolve.outputs.nightly-run-id }} + NIGHTLY_RUN_ATTEMPT: ${{ needs.resolve.outputs.nightly-run-attempt }} + MULTIGRES_IMAGE: ${{ needs.preflight.outputs.multigres-image }} + PGCTLD_IMAGE: ${{ needs.preflight.outputs.pgctld-image }} + MULTIADMIN_WEB_IMAGE: ${{ needs.preflight.outputs.multiadmin-web-image }} + with: + github-token: ${{ steps.token.outputs.token }} + script: | + const { promote, recordFromEnv } = require('./scripts/promote-runtime-images.js'); + const result = await promote({ + github, + context: { ...context, repo: context.repo, runAttempt: process.env.GITHUB_RUN_ATTEMPT }, + record: recordFromEnv(process.env), + }); + core.info(JSON.stringify(result)); + + # Only successful PR publication (including a verified duplicate) makes a + # revision handled. A failed promotion stays red and is retried next time. + - name: Write promoted state + env: + SHA: ${{ needs.resolve.outputs.sha }} + run: | + mkdir -p /tmp/nightly-compatibility + echo "$SHA" > /tmp/nightly-compatibility/upstream-sha + + - name: Preserve promoted state + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: nightly-compatibility-promoted-${{ needs.resolve.outputs.sha }} + path: /tmp/nightly-compatibility/upstream-sha + retention-days: 90 + overwrite: true # A rerun may replace this run's existing checkpoint. + if-no-files-found: error diff --git a/.github/workflows/pull-request.yaml b/.github/workflows/pull-request.yaml index 37486d5c..f787acdb 100644 --- a/.github/workflows/pull-request.yaml +++ b/.github/workflows/pull-request.yaml @@ -33,3 +33,28 @@ jobs: uses: ./.github/workflows/_reusable-test-coverage.yaml with: COVERAGE_THRESHOLD: 70 + + promotion-tests: + name: Image promotion regression tests + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - run: node --test scripts/promote-runtime-images.test.js scripts/nightly-compatibility.test.js scripts/e2e-images.test.js + - name: Validate committed promotion record + if: github.head_ref == 'chore/promote-runtime-images' + run: node scripts/promote-runtime-images.js --check + + promotion-e2e: + name: Promotion committed defaults + if: github.head_ref == 'chore/promote-runtime-images' + needs: promotion-tests + permissions: + contents: read + uses: ./.github/workflows/_reusable-e2e.yaml + with: + ref: ${{ github.event.pull_request.head.sha }} + timeout-minutes: 30 diff --git a/Makefile b/Makefile index 6a7fb8b3..7566d217 100644 --- a/Makefile +++ b/Makefile @@ -23,8 +23,9 @@ OBSERVER_IMG ?= $(if $(wildcard $(IMG_TAG_FILE)),$(IMG_PREFIX)/multigres-observe print-img: ## Print the full operator container image reference @echo $(IMG) -# Images required by MultigresCluster pods (must match pkg/testutil/e2e.go MultigresImages) -E2E_IMAGES ?= ghcr.io/multigres/multigres:main ghcr.io/multigres/pgctld:main ghcr.io/multigres/multiadmin-web:main gcr.io/etcd-development/etcd:v3.6.7 +# Match the compiled defaults used by test/e2e/framework, including the exporter. +# MULTIGRES_IMAGES is derived from image_defaults.go below. +E2E_IMAGES ?= $(MULTIGRES_IMAGES) .PHONY: pull-e2e-images pull-e2e-images: ## Pull container images needed by e2e tests diff --git a/docs/development/runtime-image-promotion.md b/docs/development/runtime-image-promotion.md new file mode 100644 index 00000000..fe62afce --- /dev/null +++ b/docs/development/runtime-image-promotion.md @@ -0,0 +1,83 @@ +# Runtime image promotion + +The nightly compatibility workflow promotes runtime defaults only after a +scheduled canary on `multigres/multigres-operator`'s `main` branch passes. Proto +dependency sync does not change these defaults. Manual canaries are diagnostic; +they cannot publish a promotion PR or mark an upstream revision handled. + +## Evidence and image mapping + +Resolution selects a completed, successful scheduled upstream nightly build and +freezes the operator commit being tested. Preflight resolves immutable image +digests, checks for Linux amd64 and arm64 manifests, and verifies GitHub build +attestations against the upstream source SHA, `refs/heads/main`, the nightly +workflow identity, and the expected image repository. E2E then tests those exact +digests against the frozen operator commit. + +After a successful canary, the promotion job commits +`config/runtime-image-promotion.json` with schema version 1, `upstream_sha`, +`operator_sha`, `nightly_run` and `canary_run` (IDs, attempts, and URLs), and an +`images` map containing the three fully qualified digest references. This file is +created by the first successful promotion; it is not seeded with untested images. + +The same Git commit updates these constants in `api/v1alpha1/image_defaults.go`: + +| Image | Defaults | +| --- | --- | +| `pgctld` | `DefaultPostgresImage` | +| `multigres` | `DefaultMultiadminImage`, `DefaultMultiorchImage`, `DefaultMultipoolerImage`, `DefaultMultigatewayImage` | +| `multiadmin-web` | `DefaultMultiadminWebImage` | + +Etcd and Postgres exporter defaults are preserved. The PR describes the current +main and proposed images and links both build and test evidence. + +## PR lifecycle and retries + +Promotion maintains one open PR from `chore/promote-runtime-images` to `main`. +It uses the existing `MULTIGRES_BOT_APP_ID` and +`MULTIGRES_BOT_APP_PRIVATE_KEY`, scoped to contents and pull requests on the +operator repository. The App token ensures branch pushes and PR creation trigger +CI. Promotion PR CI validates the record and runs e2e against the committed PR +head without runtime image overrides. Local `make pull-e2e-images` uses the same +compiled defaults, including the Postgres exporter. When the reusable workflow +checks out an older framework that still loads `testutil.MultigresImages`, it +also pulls that revision's legacy list so empty and partial overrides remain +usable. Current frameworks pull only the compiled set. + +Review the PR and require those checks before merging; promotion never merges +automatically. + +The promotion script checks both main's record and any pending branch record for +stale revisions. Before the first record exists, it also compares the nightly +against the source revisions in the existing pinned image tags. It refuses to publish if operator main moved since the canary, +the upstream revision is stale/divergent, or an already recorded source revision +has a different digest set. Newer green sets replace all images and evidence in +one tree/ref update, retaining branch ancestry without a force push. Reprocessing +an identical set leaves its commit and original evidence unchanged. + +Compatibility reporting and promotion are sibling terminal jobs. Reporting has +only `issues: write`; promotion has only `contents: write` and +`pull-requests: write`. A green canary can close the compatibility incident even +if promotion fails. Promotion errors fail the workflow without opening a +compatibility incident. + +Only successful PR publication (or verification that the same set is already +published/merged) writes the `nightly-compatibility-promoted-` checkpoint. +Resolution reads these checkpoints only from successful scheduled runs on main; +older green-only artifacts do not count. A failed promotion remains eligible for +the next scheduled run. A failed PR API call may leave the complete candidate +commit on the promotion branch; retry repairs PR publication before writing the +checkpoint. No partially updated image set can become visible on the branch. + +## Local validation + +```sh +node --test scripts/promote-runtime-images.test.js scripts/nightly-compatibility.test.js scripts/e2e-images.test.js +go test -tags=e2e ./test/e2e/framework -count=1 +actionlint .github/workflows/nightly-compatibility.yaml .github/workflows/pull-request.yaml .github/workflows/_reusable-e2e.yaml +``` + +On a generated promotion branch, also run +`node scripts/promote-runtime-images.js --check` to check the record against all +six committed constants. Full canary and promotion e2e execution takes place in +GitHub Actions with registry access and a disposable Kind cluster. diff --git a/scripts/e2e-images.test.js b/scripts/e2e-images.test.js new file mode 100644 index 00000000..a35104dc --- /dev/null +++ b/scripts/e2e-images.test.js @@ -0,0 +1,117 @@ +const { test } = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { spawnSync } = require('node:child_process'); + +const makefile = fs.readFileSync('Makefile', 'utf8'); +const workflow = fs.readFileSync('.github/workflows/_reusable-e2e.yaml', 'utf8'); +const pullStep = workflow.split(' - name: Pull e2e images\n')[1] + .split('\n - name:')[0].split(' run: |\n')[1] + .split('\n').map((line) => line.slice(10)).join('\n'); +const defaults = { + DefaultPostgresImage: 'ghcr.io/multigres/pgctld:sha-1111111', + DefaultMultiadminImage: 'ghcr.io/multigres/multigres:sha-2222222', + DefaultMultiadminWebImage: 'ghcr.io/multigres/multiadmin-web:sha-3333333', + DefaultMultiorchImage: 'ghcr.io/multigres/multigres:sha-2222222', + DefaultMultipoolerImage: 'ghcr.io/multigres/multigres:sha-2222222', + DefaultMultigatewayImage: 'ghcr.io/multigres/multigres:sha-2222222', + DefaultEtcdImage: 'gcr.io/etcd-development/etcd:v3.6.7', + DefaultPostgresExporterImage: 'quay.io/prometheuscommunity/postgres-exporter:v0.20.1', +}; +const legacy = [ + 'ghcr.io/multigres/multigres:main', + 'ghcr.io/multigres/pgctld:main', + 'ghcr.io/multigres/multiadmin-web:main', + defaults.DefaultEtcdImage, +]; + +function fixture(t, { oldFramework = false, images = defaults } = {}) { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'e2e-images-')); + t.after(() => fs.rmSync(dir, { recursive: true, force: true })); + for (const name of ['api/v1alpha1', 'pkg/testutil', 'test/e2e/framework', 'bin']) { + fs.mkdirSync(path.join(dir, name), { recursive: true }); + } + // Historical Makefiles defaulted to four mutable images. The current workflow + // must supply its complete pull list even when using that older Makefile. + fs.writeFileSync(path.join(dir, 'Makefile'), oldFramework + ? makefile.replace(/^E2E_IMAGES \?=.*$/m, `E2E_IMAGES ?= ${legacy.join(' ')}`) + : makefile); + fs.writeFileSync(path.join(dir, 'api/v1alpha1/image_defaults.go'), + `package v1alpha1\nconst (\n${Object.entries(images).map(([name, image]) => `\t${name} = "${image}"`).join('\n')}\n)\n`); + // Both generations contain this legacy variable, but only the older framework + // uses it. Presence of the declaration alone must not add :main to new runs. + fs.writeFileSync(path.join(dir, 'pkg/testutil/e2e.go'), + `package testutil\nvar MultigresImages = []string{\n${legacy.map((image) => `\t"${image}",`).join('\n')}\n}\n`); + fs.writeFileSync(path.join(dir, 'test/e2e/framework/image_overrides.go'), oldFramework + ? 'package framework\nvar images = testutil.MultigresImages\n' + : fs.readFileSync('test/e2e/framework/image_overrides.go', 'utf8')); + const cache = path.join(dir, 'docker-cache'); + fs.writeFileSync(cache, ''); + fs.writeFileSync(path.join(dir, 'bin/docker'), `#!/bin/sh +case "$1" in + pull) printf '%s\\n' "$2" >> "$MOCK_CACHE" ;; + save) grep -Fx -- "$2" "$MOCK_CACHE" > /dev/null ;; + *) exit 1 ;; +esac +`, { mode: 0o755 }); + fs.writeFileSync(path.join(dir, 'bin/go'), '#!/bin/sh\nexit 0\n', { mode: 0o755 }); + const env = { ...process.env, PATH: `${path.join(dir, 'bin')}:${process.env.PATH}`, MOCK_CACHE: cache }; + delete env.E2E_IMAGES; + delete env.MULTIGRES_IMAGES; + function run(command, args) { + const result = spawnSync(command, args, { cwd: dir, env, encoding: 'utf8' }); + assert.equal(result.status, 0, result.stderr || result.stdout); + } + return { + run, + pulled: () => fs.readFileSync(cache, 'utf8').trim().split('\n').filter(Boolean).sort(), + checkLoadable: (images) => { + for (const image of images) { + if (!image.includes('@')) run('docker', ['save', image]); + } + }, + }; +} + +const unique = (images) => [...new Set(images)].sort(); + +test('local make pulls every committed default into an empty Docker cache', (t) => { + const f = fixture(t); + f.run('make', ['-s', 'pull-e2e-images']); + assert.deepEqual(f.pulled(), unique(Object.values(defaults))); + f.checkLoadable(Object.values(defaults)); +}); + +test('local make retains an explicit E2E_IMAGES override', (t) => { + const f = fixture(t); + f.run('make', ['-s', 'pull-e2e-images', 'E2E_IMAGES=example.test/custom:tag']); + assert.deepEqual(f.pulled(), ['example.test/custom:tag']); +}); + +for (const oldFramework of [false, true]) { + for (const overrides of ['empty', 'partial', 'complete']) { + test(`workflow pulls images needed by ${oldFramework ? 'older' : 'current'} refs with ${overrides} overrides`, (t) => { + // The workflow applies overrides to image_defaults.go before pulling. + const images = { ...defaults }; + if (overrides !== 'empty') images.DefaultPostgresImage = 'example.test/pgctld:custom'; + if (overrides === 'complete') { + for (const name of ['DefaultMultiadminImage', 'DefaultMultiorchImage', 'DefaultMultipoolerImage', 'DefaultMultigatewayImage']) { + images[name] = `ghcr.io/multigres/multigres@sha256:${'a'.repeat(64)}`; + } + images.DefaultMultiadminWebImage = 'example.test/multiadmin-web:custom'; + } + const f = fixture(t, { oldFramework, images }); + f.run('bash', ['--noprofile', '--norc', '-eo', 'pipefail', '-c', pullStep]); + const required = oldFramework ? [defaults.DefaultEtcdImage, + overrides === 'empty' ? legacy[1] : images.DefaultPostgresImage, + overrides === 'complete' ? images.DefaultMultiadminWebImage : legacy[2], + overrides === 'complete' ? images.DefaultMultiadminImage : legacy[0], + ] : Object.values(images); + f.checkLoadable(required); + for (const image of Object.values(images)) assert.ok(f.pulled().includes(image)); + if (!oldFramework) assert.deepEqual(f.pulled(), unique(Object.values(images))); + }); + } +} diff --git a/scripts/nightly-compatibility.test.js b/scripts/nightly-compatibility.test.js new file mode 100644 index 00000000..dcdce652 --- /dev/null +++ b/scripts/nightly-compatibility.test.js @@ -0,0 +1,158 @@ +const { test } = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { spawnSync } = require('node:child_process'); + +const workflow = fs.readFileSync('.github/workflows/nightly-compatibility.yaml', 'utf8'); + +// Execute the workflow's actual shell snippets with registry/API stand-ins. +function step(name, key = 'run', indent = 8) { + const block = workflow.split(` - name: ${name}\n`)[1]?.split(/\n - |\n [a-z]/)[0]; + assert.ok(block, `Missing step ${name}`); + const code = block.split(`${' '.repeat(indent)}${key}: |\n`)[1]; + assert.ok(code, `Missing ${key} in ${name}`); + return code.split('\n').map((line) => line.slice(indent + 2)).join('\n'); +} + +function shell(t, code, env = {}) { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'image-promotion-')); + t.after(() => fs.rmSync(dir, { recursive: true, force: true })); + fs.writeFileSync(path.join(dir, 'docker'), `#!/bin/bash +if [ "$4" = "$FAIL_IMAGE" ]; then exit 1; fi +printf '%s\\n' "$DOCKER_JSON" +`, { mode: 0o755 }); + fs.writeFileSync(path.join(dir, 'curl'), `#!/bin/bash +case "$*" in + */commits/*) printf '%s\\n' "$OPERATOR_JSON" ;; + *) printf '%s\\n' "$NIGHTLY_JSON" ;; +esac +`, { mode: 0o755 }); + fs.writeFileSync(path.join(dir, 'gh'), `#!/bin/bash +printf '%s\\n' "$*" >> "$MOCK_LOG" +if [ "$FAIL_VERIFY" = "true" ]; then exit 1; fi +name="\${3#oci://}" +name="\${name%@*}" +if [ "$WRONG_SUBJECT" = "true" ]; then name=wrong; fi +printf '[{"verificationResult":{"statement":{"subject":[{"name":"%s"}]}}}]' "$name" +`, { mode: 0o755 }); + const output = path.join(dir, 'output'); + const log = path.join(dir, 'log'); + fs.writeFileSync(output, ''); + fs.writeFileSync(log, ''); + const result = spawnSync('bash', ['--noprofile', '--norc', '-eo', 'pipefail', '-c', code], { + encoding: 'utf8', env: { ...process.env, PATH: `${dir}:${process.env.PATH}`, GITHUB_OUTPUT: output, MOCK_LOG: log, ...env }, + }); + return { ...result, output: fs.readFileSync(output, 'utf8'), log: fs.readFileSync(log, 'utf8') }; +} + +const images = { + MULTIGRES_IMAGE: `ghcr.io/multigres/multigres@sha256:${'a'.repeat(64)}`, + PGCTLD_IMAGE: `ghcr.io/multigres/pgctld@sha256:${'b'.repeat(64)}`, + MULTIADMIN_WEB_IMAGE: `ghcr.io/multigres/multiadmin-web@sha256:${'c'.repeat(64)}`, +}; + +test('scheduled resolution freezes the operator SHA and captures the nightly run attempt', (t) => { + const result = shell(t, step('Resolve upstream SHA'), { + INPUT_SHA: '', INPUT_OPERATOR_REF: '', GITHUB_EVENT_NAME: 'schedule', GITHUB_SHA: 'a'.repeat(40), + NIGHTLY_JSON: JSON.stringify({ workflow_runs: [{ head_sha: 'b'.repeat(40), id: 123, run_attempt: 2, conclusion: 'success', head_branch: 'main', event: 'schedule' }] }), + }); + assert.equal(result.status, 0, result.stderr); + assert.ok(result.output.includes(`operator_sha=${'a'.repeat(40)}\n`)); + assert.ok(result.output.includes(`sha=${'b'.repeat(40)}\n`)); + assert.ok(result.output.includes('nightly_run_id=123\nnightly_run_attempt=2\n')); +}); + +test('manual SHA diagnostics do not depend on an available nightly run', (t) => { + const result = shell(t, step('Resolve upstream SHA'), { + INPUT_SHA: 'b'.repeat(40), INPUT_OPERATOR_REF: 'feature/skew', GITHUB_EVENT_NAME: 'workflow_dispatch', + GITHUB_REPOSITORY: 'multigres/multigres-operator', NIGHTLY_JSON: '{}', + OPERATOR_JSON: JSON.stringify({ sha: 'c'.repeat(40) }), + }); + assert.equal(result.status, 0, result.stderr); + assert.ok(result.output.includes(`operator_sha=${'c'.repeat(40)}\n`)); +}); + +for (const invalid of [{}, { head_branch: 'other' }, { conclusion: 'failure' }, { event: 'workflow_dispatch' }]) { + test(`rejects invalid upstream nightly selection: ${JSON.stringify(invalid)}`, (t) => { + const candidate = Object.keys(invalid).length ? { head_sha: 'b'.repeat(40), conclusion: 'success', head_branch: 'main', event: 'schedule', ...invalid } : undefined; + const result = shell(t, step('Resolve upstream SHA'), { + INPUT_SHA: '', NIGHTLY_JSON: JSON.stringify({ workflow_runs: candidate ? [candidate] : [] }), + }); + assert.notEqual(result.status, 0); + assert.equal(result.output, ''); + }); +} + +for (const failed of Object.values(images)) { + test(`digest resolution failure is not swallowed: ${failed.split('@')[0]}`, (t) => { + const result = shell(t, step('Resolve immutable nightly image digests'), { + ...images, FAIL_IMAGE: failed, DOCKER_JSON: JSON.stringify({ digest: `sha256:${'a'.repeat(64)}` }), + }); + assert.notEqual(result.status, 0); + assert.equal(result.output, ''); + }); +} + +for (const platforms of [['amd64', 'arm64'], ['amd64'], ['arm64'], []]) { + test(`architecture preflight with ${platforms.join('/') || 'no platforms'}`, (t) => { + const result = shell(t, step('Verify both supported architectures'), { + ...images, DOCKER_JSON: JSON.stringify({ manifests: platforms.map((architecture) => ({ platform: { os: 'linux', architecture } })) }), + }); + assert.equal(result.status === 0, platforms.length === 2, result.stderr); + }); +} + +test('provenance checks every digest against source revision and trusted workflow', (t) => { + const result = shell(t, step('Verify nightly image provenance'), { ...images, UPSTREAM_SHA: 'd'.repeat(40) }); + assert.equal(result.status, 0, result.stderr); + for (const image of Object.values(images)) assert.ok(result.log.includes(`oci://${image}`)); + for (const line of result.log.trim().split('\n')) { + assert.ok(line.includes('--signer-workflow multigres/multigres/.github/workflows/nightly-build.yml')); + assert.ok(line.includes('--source-ref refs/heads/main')); + assert.ok(line.includes(`--source-digest ${'d'.repeat(40)}`)); + assert.ok(line.includes('--deny-self-hosted-runners')); + } +}); + +for (const flags of [{ FAIL_VERIFY: 'true' }, { WRONG_SUBJECT: 'true' }]) { + test(`provenance fails closed: ${Object.keys(flags)[0]}`, (t) => { + assert.notEqual(shell(t, step('Verify nightly image provenance'), { ...images, ...flags }).status, 0); + }); +} + +test('a green canary closes the incident independently of a failed promotion', async (t) => { + const env = { RESOLVE_RESULT: 'success', PREFLIGHT_RESULT: 'success', E2E_RESULT: 'success', PROMOTION_RESULT: 'failure', EVENT_NAME: 'schedule', OPERATOR_REF: 'main', UPSTREAM_SHA: 'a'.repeat(40) }; + for (const [key, value] of Object.entries(env)) { + const original = process.env[key]; + process.env[key] = value; + t.after(() => { if (original === undefined) delete process.env[key]; else process.env[key] = original; }); + } + const calls = []; + const issues = { + getLabel: async () => {}, listForRepo: async () => {}, + createComment: async () => {}, + update: async (args) => calls.push(args), + create: async () => assert.fail('Promotion failure must not create a compatibility incident'), + }; + const github = { rest: { issues }, paginate: async () => [{ number: 574, body: '' }] }; + const context = { repo: { owner: 'multigres', repo: 'multigres-operator' }, ref: 'refs/heads/main', runId: 123 }; + const AsyncFunction = Object.getPrototypeOf(async function () {}).constructor; + await new AsyncFunction('github', 'context', 'core', step('Update tracking issue', 'script', 10))(github, context, {}); + assert.equal(calls.length, 1); + assert.equal(calls[0].state, 'closed'); +}); + +test('report and promotion have separate permissions and only promotion writes handled state', () => { + const report = workflow.split('\n report:\n')[1].split('\n promote:\n')[0]; + const promotion = workflow.split('\n promote:\n')[1]; + assert.match(report, /needs: \[resolve, preflight, e2e\]/); + assert.match(promotion, /needs: \[resolve, preflight, e2e\]/); + assert.match(report, /permissions:\n issues: write[^\n]*\n steps:/); + assert.match(promotion, /permissions:\n contents: write\n pull-requests: write\n steps:/); + assert.ok(!report.includes('upload-artifact')); + assert.ok(!workflow.includes('nightly-compatibility-green-')); + assert.ok(promotion.indexOf('Create or update promotion PR') < promotion.indexOf('Write promoted state')); + assert.ok(!promotion.includes('continue-on-error')); +}); diff --git a/scripts/promote-runtime-images.js b/scripts/promote-runtime-images.js new file mode 100644 index 00000000..e1241403 --- /dev/null +++ b/scripts/promote-runtime-images.js @@ -0,0 +1,226 @@ +// The workflow supplies only digests that passed provenance verification and e2e. +// Use one Git tree and one ref update so the record and all defaults move together. +const fs = require('node:fs'); +const assert = require('node:assert/strict'); + +const BRANCH = 'chore/promote-runtime-images'; +const DEFAULTS = 'api/v1alpha1/image_defaults.go'; +const RECORD = 'config/runtime-image-promotion.json'; +const COMPONENTS = { + DefaultPostgresImage: 'pgctld', + DefaultMultiadminImage: 'multigres', + DefaultMultiadminWebImage: 'multiadmin-web', + DefaultMultiorchImage: 'multigres', + DefaultMultipoolerImage: 'multigres', + DefaultMultigatewayImage: 'multigres', +}; + +function validateRecord(record) { + assert.equal(record.schema_version, 1); + for (const sha of [record.upstream_sha, record.operator_sha]) { + assert.match(sha, /^[0-9a-f]{40}$/); + } + for (const [name, repo] of [['nightly_run', 'multigres'], ['canary_run', 'multigres-operator']]) { + const run = record[name]; + assert.match(String(run.id), /^[1-9][0-9]*$/); + assert.match(String(run.attempt), /^[1-9][0-9]*$/); + assert.equal(run.url, `https://github.com/multigres/${repo}/actions/runs/${run.id}/attempts/${run.attempt}`); + } + assert.deepEqual(Object.keys(record.images).sort(), ['multiadmin-web', 'multigres', 'pgctld']); + for (const [name, image] of Object.entries(record.images)) { + assert.match(image, new RegExp(`^ghcr\\.io/multigres/${name}@sha256:[0-9a-f]{64}$`)); + } +} + +function readDefaults(source) { + return Object.fromEntries(Object.keys(COMPONENTS).map((name) => { + const matches = [...source.matchAll(new RegExp(`^(\\s*${name}\\s*=\\s*")([^"\\n]+)(")$`, 'gm'))]; + assert.equal(matches.length, 1, `Expected exactly one ${name}`); + return [name, matches[0][2]]; + })); +} + +function validateDefaults(source, record) { + validateRecord(record); + const defaults = readDefaults(source); + for (const [name, component] of Object.entries(COMPONENTS)) { + assert.equal(defaults[name], record.images[component], `${name} differs from promotion record`); + } +} + +function replaceDefaults(source, record) { + validateRecord(record); + readDefaults(source); + for (const [name, component] of Object.entries(COMPONENTS)) { + source = source.replace(new RegExp(`^(\\s*${name}\\s*=\\s*")[^"\\n]+(")$`, 'm'), + `$1${record.images[component]}$2`); + } + validateDefaults(source, record); + return source; +} + +function sameImages(a, b) { + return Object.keys(a.images).every((name) => a.images[name] === b.images[name]); +} + +function pullRequestBody(previous, record) { + const rows = Object.entries(COMPONENTS).map(([name, component]) => + `| ${name} | \`${previous[name]}\` | \`${record.images[component]}\` |`).join('\n'); + return `Promote the runtime images verified by the scheduled compatibility canary. + +- Upstream revision: [${record.upstream_sha}](https://github.com/multigres/multigres/commit/${record.upstream_sha}) +- Tested operator revision: [${record.operator_sha}](https://github.com/multigres/multigres-operator/commit/${record.operator_sha}) +- [Upstream nightly](${record.nightly_run.url}) ยท [Green canary](${record.canary_run.url}) + +| Default | Current main | Proposed | +| --- | --- | --- | +${rows} + +All three digests passed source-revision and build-provenance checks and contain linux/amd64 and linux/arm64 images. The promotion record and six defaults are committed atomically. Etcd and Postgres exporter defaults are unchanged. + +The promotion e2e check builds this PR's committed defaults without image overrides. Maintainer review and passing checks are required before merge. +`; +} + +async function promote({ github, context, record }) { + assert.equal(context.repo.owner, 'multigres'); + assert.equal(context.repo.repo, 'multigres-operator'); + assert.equal(context.eventName, 'schedule'); + assert.equal(context.ref, 'refs/heads/main'); + validateRecord(record); + assert.equal(record.operator_sha, context.sha); + assert.equal(String(record.canary_run.id), String(context.runId)); + assert.equal(String(record.canary_run.attempt), String(context.runAttempt)); + + const repo = context.repo; + const api = github.rest; + async function optional(read) { + try { return (await read()).data; } catch (error) { + if (error.status === 404) return null; + throw error; + } + } + async function file(path, ref) { + const data = await optional(() => api.repos.getContent({ ...repo, path, ref })); + if (!data) return null; + assert.equal(data.type, 'file'); + return Buffer.from(data.content, 'base64').toString('utf8'); + } + async function state(ref) { + const source = await file(DEFAULTS, ref); + assert.ok(source, `Missing defaults at ${ref}`); + const json = await file(RECORD, ref); + const prior = json ? JSON.parse(json) : null; + if (prior) validateDefaults(source, prior); + return { source, record: prior }; + } + async function compare(base, head) { + const { data } = await api.repos.compareCommits({ owner: 'multigres', repo: 'multigres', base, head }); + return data.status; + } + async function assertCurrentMain() { + const { data } = await api.git.getRef({ ...repo, ref: 'heads/main' }); + assert.equal(data.object.sha, record.operator_sha, 'Operator main changed since the canary; retry on current main'); + return data.object.sha; + } + + const mainSha = await assertCurrentMain(); + assert.ok(['behind', 'identical'].includes(await compare('main', record.upstream_sha)), 'Upstream SHA is not on main'); + const main = await state(mainSha); + if (!main.record) { + // Bootstrap from the source-pinned defaults that predate promotion records. + // An older successful nightly must not roll those defaults back either. + const revisions = new Set(Object.values(readDefaults(main.source)).map((image) => { + const match = image.match(/:(?:nightly-)?sha-([0-9a-f]{7,40})$/); + assert.ok(match, 'Defaults need a promotion record or source-pinned tags'); + return match[1]; + })); + for (const revision of revisions) { + assert.ok(['ahead', 'identical'].includes(await compare(revision, record.upstream_sha)), + 'Upstream revision is older than the initial runtime defaults'); + } + } + const branch = await optional(() => api.git.getRef({ ...repo, ref: `heads/${BRANCH}` })); + const pending = branch ? await state(branch.object.sha) : null; + if (pending) assert.ok(pending.record, 'Promotion branch is missing its record'); + // Consult both main and the branch, including a branch left by a failed PR API + // call. Artifact retention or an old workflow rerun must never allow rollback. + for (const prior of [main.record, pending?.record].filter(Boolean)) { + const relation = await compare(prior.upstream_sha, record.upstream_sha); + assert.ok(['ahead', 'identical'].includes(relation), 'Stale or divergent upstream revision'); + if (relation === 'identical') assert.ok(sameImages(prior, record), 'Immutable image set changed for the same revision'); + } + + if (main.record && sameImages(main.record, record)) { + return { changed: false, merged: true }; + } + + let head = branch?.object.sha; + let proposed = record; + const unchanged = pending?.record && sameImages(pending.record, record); + if (unchanged) { + // Keep the original evidence for this digest set, but still repair a missing + // or failed PR update before the workflow may checkpoint the revision. + proposed = pending.record; + } else { + const source = replaceDefaults(main.source, record); + const { data: base } = await api.git.getCommit({ ...repo, commit_sha: mainSha }); + const { data: tree } = await api.git.createTree({ + ...repo, base_tree: base.tree.sha, + tree: [ + { path: DEFAULTS, mode: '100644', type: 'blob', content: source }, + { path: RECORD, mode: '100644', type: 'blob', content: `${JSON.stringify(record, null, 2)}\n` }, + ], + }); + const { data: commit } = await api.git.createCommit({ + ...repo, message: `chore(images): promote canary-validated ${record.upstream_sha.slice(0, 7)}`, + tree: tree.sha, + // Retain branch ancestry for a non-forced update and include current main + // so the PR diff contains only the promotion even when main has advanced. + parents: [...new Set([head, mainSha].filter(Boolean))], + }); + await assertCurrentMain(); + if (branch) { + await api.git.updateRef({ ...repo, ref: `heads/${BRANCH}`, sha: commit.sha, force: false }); + } else { + await api.git.createRef({ ...repo, ref: `refs/heads/${BRANCH}`, sha: commit.sha }); + } + head = commit.sha; + } + + const prs = await github.paginate(api.pulls.list, { ...repo, state: 'open', head: `${repo.owner}:${BRANCH}`, base: 'main', per_page: 100 }); + assert.ok(prs.length <= 1, 'Multiple open promotion PRs'); + const fields = { + ...repo, + title: `chore(images): promote canary-validated ${proposed.upstream_sha.slice(0, 7)}`, + body: pullRequestBody(readDefaults(main.source), proposed), + }; + let pr = prs[0]; + if (!pr) { + ({ data: pr } = await api.pulls.create({ ...fields, head: BRANCH, base: 'main' })); + } else if (pr.title !== fields.title || pr.body !== fields.body) { + ({ data: pr } = await api.pulls.update({ ...fields, pull_number: pr.number })); + } + assert.equal(pr.head.sha, head, 'Promotion PR head changed concurrently'); + assert.equal(pr.state, 'open'); + return { changed: !unchanged, url: pr.html_url, head }; +} + +function recordFromEnv(env) { + const run = (repo, id, attempt) => ({ id, attempt, url: `https://github.com/multigres/${repo}/actions/runs/${id}/attempts/${attempt}` }); + return { + schema_version: 1, + upstream_sha: env.UPSTREAM_SHA, + operator_sha: env.OPERATOR_SHA, + nightly_run: run('multigres', env.NIGHTLY_RUN_ID, env.NIGHTLY_RUN_ATTEMPT), + canary_run: run('multigres-operator', env.GITHUB_RUN_ID, env.GITHUB_RUN_ATTEMPT), + images: { multigres: env.MULTIGRES_IMAGE, pgctld: env.PGCTLD_IMAGE, 'multiadmin-web': env.MULTIADMIN_WEB_IMAGE }, + }; +} + +module.exports = { promote, recordFromEnv, validateRecord, validateDefaults, replaceDefaults, readDefaults, BRANCH, DEFAULTS, RECORD }; + +if (require.main === module) { + assert.equal(process.argv[2], '--check', 'Usage: node scripts/promote-runtime-images.js --check'); + validateDefaults(fs.readFileSync(DEFAULTS, 'utf8'), JSON.parse(fs.readFileSync(RECORD, 'utf8'))); +} diff --git a/scripts/promote-runtime-images.test.js b/scripts/promote-runtime-images.test.js new file mode 100644 index 00000000..34575590 --- /dev/null +++ b/scripts/promote-runtime-images.test.js @@ -0,0 +1,243 @@ +const { test } = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const { + promote, recordFromEnv, validateDefaults, replaceDefaults, readDefaults, + DEFAULTS, RECORD, BRANCH, +} = require('./promote-runtime-images'); + +// Model the pre-promotion state even when these tests run on a generated PR +// whose checked-out defaults have already been pinned by digest. +let source = fs.readFileSync(DEFAULTS, 'utf8'); +for (const image of Object.values(readDefaults(source))) { + source = source.replaceAll(image, `${image.split(/[:@]/)[0]}:sha-1111111`); +} +const sha = (digit) => digit.repeat(40); +const image = (name, digit) => `ghcr.io/multigres/${name}@sha256:${digit.repeat(64)}`; +function record(digit = '2') { + return recordFromEnv({ + UPSTREAM_SHA: sha(digit), OPERATOR_SHA: sha('a'), + NIGHTLY_RUN_ID: '100', NIGHTLY_RUN_ATTEMPT: '2', + GITHUB_RUN_ID: '200', GITHUB_RUN_ATTEMPT: '1', + MULTIGRES_IMAGE: image('multigres', digit), PGCTLD_IMAGE: image('pgctld', digit), + MULTIADMIN_WEB_IMAGE: image('multiadmin-web', digit), + }); +} + +// Model the Git database separately from refs: a failed ref update leaves the +// committed pair unreachable, whereas a failed PR call leaves a repairable branch. +function fixture({ mainRecord, pendingRecord, existingPR = false } = {}) { + const calls = []; + const files = (r) => ({ + [DEFAULTS]: r ? replaceDefaults(source, r) : source, + ...(r ? { [RECORD]: JSON.stringify(r) } : {}), + 'unrelated.txt': 'keep current main', + }); + const states = new Map([[sha('a'), files(mainRecord)]]); + let main = sha('a'); + let branch; + if (pendingRecord) { branch = sha('b'); states.set(branch, files(pendingRecord)); } + let pr = existingPR ? { number: 17, title: 'old title', body: 'old body', state: 'open', head: { sha: branch } } : null; + let nextTree; + let writes = 0; + const failures = {}; + const missing = () => { throw Object.assign(new Error('Not found'), { status: 404 }); }; + const method = (name, impl) => async (args) => { + calls.push({ name, args }); + if (failures[name]) throw new Error(`injected ${name} failure`); + return { data: impl(args) }; + }; + const rest = { + repos: { + getContent: method('getContent', ({ path, ref }) => { + const content = states.get(ref)?.[path]; + return content === undefined ? missing() : { type: 'file', content: Buffer.from(content).toString('base64') }; + }), + compareCommits: method('compareCommits', ({ base, head }) => ({ status: + base === 'main' ? 'behind' : base === head ? 'identical' : base < head ? 'ahead' : 'behind', + })), + }, + git: { + getRef: method('getRef', ({ ref }) => { + const value = ref === 'heads/main' ? main : branch; + return value ? { object: { sha: value } } : missing(); + }), + getCommit: method('getCommit', ({ commit_sha }) => ({ tree: { sha: commit_sha } })), + createTree: method('createTree', ({ base_tree, tree }) => { + nextTree = { ...states.get(base_tree), ...Object.fromEntries(tree.map((entry) => [entry.path, entry.content])) }; + return { sha: 'tree' }; + }), + createCommit: method('createCommit', () => { + const commit = `commit-${++writes}`; + states.set(commit, nextTree); + return { sha: commit }; + }), + createRef: method('createRef', ({ sha: value }) => { + assert.equal(branch, undefined); + branch = value; + }), + updateRef: method('updateRef', ({ sha: value, force }) => { + assert.equal(force, false); + branch = value; + if (pr) pr.head.sha = value; + }), + }, + pulls: { + list: async () => {}, + create: method('createPR', (args) => { + pr = { ...args, number: 17, state: 'open', html_url: 'https://github.com/multigres/multigres-operator/pull/17', head: { sha: branch } }; + return pr; + }), + update: method('updatePR', (args) => { + Object.assign(pr, args); + return pr; + }), + }, + }; + return { + github: { rest, paginate: async () => pr ? [pr] : [] }, + context: { repo: { owner: 'multigres', repo: 'multigres-operator' }, eventName: 'schedule', ref: 'refs/heads/main', sha: sha('a'), runId: 200, runAttempt: 1 }, + record: record(), calls, failures, + get branchFiles() { return states.get(branch); }, + get branch() { return branch; }, + setMain(value) { main = value; }, + }; +} + +test('promotes the complete verified set and evidence in one commit and one PR', async () => { + const f = fixture(); + const result = await promote(f); + assert.equal(result.changed, true); + const committed = JSON.parse(f.branchFiles[RECORD]); + assert.deepEqual(committed, f.record); + validateDefaults(f.branchFiles[DEFAULTS], committed); + assert.equal(f.branchFiles['unrelated.txt'], 'keep current main'); + for (const name of ['DefaultEtcdImage', 'DefaultPostgresExporterImage']) { + const pattern = new RegExp(`${name} = "[^"]+"`); + assert.equal(f.branchFiles[DEFAULTS].match(pattern)[0], source.match(pattern)[0]); + } + const trees = f.calls.filter((call) => call.name === 'createTree'); + assert.equal(trees.length, 1); + assert.deepEqual(trees[0].args.tree.map((entry) => entry.path), [DEFAULTS, RECORD]); + assert.equal(f.calls.filter((call) => call.name === 'createRef').length, 1); + const pr = f.calls.find((call) => call.name === 'createPR').args; + assert.equal(pr.head, BRANCH); + assert.equal(pr.base, 'main'); + for (const old of Object.values(readDefaults(source))) assert.ok(pr.body.includes(old)); + for (const next of Object.values(committed.images)) assert.ok(pr.body.includes(next)); + assert.ok(pr.body.includes(committed.nightly_run.url)); + assert.ok(pr.body.includes(committed.canary_run.url)); +}); + +test('reprocessing the same digest set is a no-op, including evidence and PR metadata', async () => { + const f = fixture(); + await promote(f); + const files = structuredClone(f.branchFiles); + f.calls.length = 0; + f.context.runId = 201; + f.record.canary_run = { id: '201', attempt: '1', url: 'https://github.com/multigres/multigres-operator/actions/runs/201/attempts/1' }; + assert.equal((await promote(f)).changed, false); + assert.deepEqual(f.branchFiles, files); + assert.ok(!f.calls.some((call) => /^(create|update)/.test(call.name))); +}); + +test('a merged digest set is a successful no-op', async () => { + const f = fixture({ mainRecord: record() }); + assert.deepEqual(await promote(f), { changed: false, merged: true }); + assert.ok(!f.calls.some((call) => /^(create|update)/.test(call.name))); +}); + +test('a newer green set updates the existing PR atomically', async () => { + const f = fixture({ pendingRecord: record('1'), existingPR: true }); + await promote(f); + validateDefaults(f.branchFiles[DEFAULTS], f.record); + assert.ok(!f.calls.some((call) => call.name === 'createPR')); + assert.equal(f.calls.find((call) => call.name === 'updatePR').args.pull_number, 17); + assert.deepEqual(f.calls.find((call) => call.name === 'createCommit').args.parents, [sha('b'), sha('a')]); +}); + +for (const operation of ['createPR', 'updatePR']) { + test(`failed ${operation} rejects and retry repairs the PR without another commit`, async () => { + const f = fixture(operation === 'updatePR' ? { pendingRecord: record('1'), existingPR: true } : {}); + f.failures[operation] = true; + await assert.rejects(promote(f), /injected/); + validateDefaults(f.branchFiles[DEFAULTS], f.record); + f.failures[operation] = false; + f.calls.length = 0; + assert.equal((await promote(f)).changed, false); + assert.ok(!f.calls.some((call) => call.name === 'createCommit')); + assert.ok(f.calls.some((call) => call.name === operation)); + }); +} + +test('failed ref update cannot publish part of an image set', async () => { + const f = fixture({ pendingRecord: record('1'), existingPR: true }); + const before = structuredClone(f.branchFiles); + f.failures.updateRef = true; + await assert.rejects(promote(f), /injected/); + assert.deepEqual(f.branchFiles, before); + assert.ok(!f.calls.some((call) => call.name === 'updatePR')); +}); + +for (const [name, change] of [ + ['manual run', (f) => { f.context.eventName = 'workflow_dispatch'; }], + ['fork', (f) => { f.context.repo.owner = 'fork'; }], + ['non-main ref', (f) => { f.context.ref = 'refs/heads/test'; }], + ['untested operator', (f) => { f.record.operator_sha = sha('c'); }], + ['unrelated canary', (f) => { f.context.runId = 300; }], + ['wrong attempt', (f) => { f.context.runAttempt = 2; }], + ['mutable image', (f) => { f.record.images.multigres = 'ghcr.io/multigres/multigres:main'; }], + ['wrong registry', (f) => { f.record.images.pgctld = image('pgctld', '2').replace('ghcr.io', 'evil.io'); }], + ['missing image', (f) => { delete f.record.images['multiadmin-web']; }], + ['invalid nightly evidence', (f) => { f.record.nightly_run.id = ''; }], + ['main advanced during e2e', (f) => { f.setMain(sha('c')); }], +]) { + test(`rejects ${name} before writing`, async () => { + const f = fixture(); + change(f); + await assert.rejects(promote(f)); + assert.ok(!f.calls.some((call) => /^(create|update)/.test(call.name))); + }); +} + +for (const location of ['mainRecord', 'pendingRecord']) { + test(`rejects a revision older than ${location}, even without a checkpoint artifact`, async () => { + const f = fixture({ [location]: record('3') }); + await assert.rejects(promote(f), /Stale/); + assert.ok(!f.calls.some((call) => /^(create|update)/.test(call.name))); + }); +} + +test('rejects digest drift at an already recorded upstream revision', async () => { + const f = fixture({ pendingRecord: record() }); + f.record.images.pgctld = image('pgctld', '3'); + await assert.rejects(promote(f), /Immutable image set changed/); +}); + +test('fails closed when a constant is missing, repeated, or disagrees with the record', () => { + assert.throws(() => replaceDefaults(source.replace('DefaultPostgresImage =', 'RenamedImage ='), record())); + assert.throws(() => replaceDefaults(`${source}\nDefaultPostgresImage = "duplicate"`, record())); + const updated = replaceDefaults(source, record()); + assert.throws(() => validateDefaults(updated.replace(image('pgctld', '2'), image('pgctld', '3')), record())); +}); + +test('operator main moving just before publication leaves refs untouched', async () => { + const f = fixture(); + const createCommit = f.github.rest.git.createCommit; + f.github.rest.git.createCommit = async (args) => { + const result = await createCommit(args); + f.setMain(sha('c')); + return result; + }; + await assert.rejects(promote(f), /Operator main changed/); + assert.equal(f.branch, undefined); +}); + +test('first promotion rejects a nightly older than the existing source-pinned defaults', async () => { + const f = fixture(); + const compare = f.github.rest.repos.compareCommits; + f.github.rest.repos.compareCommits = (args) => args.base === '1111111' + ? Promise.resolve({ data: { status: 'behind' } }) : compare(args); + await assert.rejects(promote(f), /older than the initial runtime defaults/); + assert.ok(!f.calls.some((call) => /^(create|update)/.test(call.name))); +}); diff --git a/test/e2e/framework/image_overrides.go b/test/e2e/framework/image_overrides.go index 94c15203..1ddade4b 100644 --- a/test/e2e/framework/image_overrides.go +++ b/test/e2e/framework/image_overrides.go @@ -6,7 +6,6 @@ import ( "os" multigresv1alpha1 "github.com/multigres/multigres-operator/api/v1alpha1" - "github.com/multigres/multigres-operator/pkg/testutil" ) const ( @@ -65,30 +64,26 @@ func applyImageOverrides(cluster *multigresv1alpha1.MultigresCluster) { // references are removed before loading them into Kind. func runtimeImages() []string { overrides := imageOverridesFromEnv() - images := []string{testutil.MultigresImages[3]} // etcd is not overridable. - - if overrides.postgres == "" { - images = append(images, testutil.MultigresImages[1]) - } else { - images = append(images, overrides.postgres) - } - if overrides.multiadminWeb == "" { - images = append(images, testutil.MultigresImages[2]) - } else { - images = append(images, overrides.multiadminWeb) + images := []string{ + multigresv1alpha1.DefaultEtcdImage, + multigresv1alpha1.DefaultPostgresExporterImage, } - - goComponentOverrides := []string{ - overrides.multiadmin, - overrides.multiorch, - overrides.multipooler, - overrides.multigateway, + components := []struct { + override string + fallback string + }{ + {overrides.postgres, multigresv1alpha1.DefaultPostgresImage}, + {overrides.multiadminWeb, multigresv1alpha1.DefaultMultiadminWebImage}, + {overrides.multiadmin, multigresv1alpha1.DefaultMultiadminImage}, + {overrides.multiorch, multigresv1alpha1.DefaultMultiorchImage}, + {overrides.multipooler, multigresv1alpha1.DefaultMultipoolerImage}, + {overrides.multigateway, multigresv1alpha1.DefaultMultigatewayImage}, } - for _, image := range goComponentOverrides { - if image == "" { - images = append(images, testutil.MultigresImages[0]) + for _, component := range components { + if component.override == "" { + images = append(images, component.fallback) } else { - images = append(images, image) + images = append(images, component.override) } } diff --git a/test/e2e/framework/image_overrides_test.go b/test/e2e/framework/image_overrides_test.go index a7498b79..6f852061 100644 --- a/test/e2e/framework/image_overrides_test.go +++ b/test/e2e/framework/image_overrides_test.go @@ -4,6 +4,7 @@ package framework import ( "slices" + "strings" "testing" multigresv1alpha1 "github.com/multigres/multigres-operator/api/v1alpha1" @@ -38,7 +39,7 @@ func TestRuntimeImagesUsesOverridesAndDeduplicates(t *testing.T) { if got := count(images, multigresNightly); got != 1 { t.Fatalf("nightly multigres image occurs %d times in %v", got, images) } - if slices.Contains(images, "ghcr.io/multigres/multigres:main") { + if slices.Contains(images, multigresv1alpha1.DefaultMultiadminImage) { t.Fatalf("default multigres image retained despite complete override: %v", images) } } @@ -68,3 +69,44 @@ func count(values []string, target string) int { } return total } + +func TestRuntimeImagesUsesCommittedDefaults(t *testing.T) { + for _, key := range []string{ + postgresImageEnv, multiadminImageEnv, multiadminWebImageEnv, + multiorchImageEnv, multipoolerImageEnv, multigatewayImageEnv, + } { + t.Setenv(key, "") + } + for _, tc := range []struct { + name string + postgres string + }{ + {name: "all committed defaults"}, + {name: "partial override", postgres: "example.test/pgctld@sha256:" + strings.Repeat("a", 64)}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Setenv(postgresImageEnv, tc.postgres) + wantPostgres := multigresv1alpha1.DefaultPostgresImage + if tc.postgres != "" { + wantPostgres = tc.postgres + } + want := []string{ + wantPostgres, + multigresv1alpha1.DefaultMultiadminImage, + multigresv1alpha1.DefaultMultiadminWebImage, + multigresv1alpha1.DefaultMultiorchImage, + multigresv1alpha1.DefaultMultipoolerImage, + multigresv1alpha1.DefaultMultigatewayImage, + multigresv1alpha1.DefaultEtcdImage, + multigresv1alpha1.DefaultPostgresExporterImage, + } + slices.Sort(want) + want = slices.Compact(want) + got := runtimeImages() + slices.Sort(got) + if !slices.Equal(got, want) { + t.Fatalf("runtimeImages() = %v, want %v", got, want) + } + }) + } +}