From 569b856aaeb825a71b47c3f53c0817ac7bd6579e Mon Sep 17 00:00:00 2001 From: Richard Pierre Date: Sat, 5 Sep 2026 22:02:23 -0400 Subject: [PATCH 1/2] fix(ci): add a scheduled sweep so merge-queue PRs cannot strand The event-driven path alone is not sufficient on a merge-queue repo. When a check_suite completes GitHub has often not recomputed mergeStateStatus, so the PR still reads BLOCKED; the workflow logged 'waiting for a later check_suite event' and exited. When that was the last such event nothing ever retried and the PR sat open while reading CLEAN. Pin PR #148 stranded for four days, which silently froze the deploy chain: build-and-deploy only classifies mode=webhook for a pin commit, so the redeploy never ran and the host kept the old image. Adds two defences: retry in-run while BLOCKED with checks still PENDING, and a 15-minute scheduled sweep that enqueues any open mergeable unqueued PR. --- .github/workflows/automerge.yml | 76 +++++++++++++++++++++++++++------ 1 file changed, 63 insertions(+), 13 deletions(-) diff --git a/.github/workflows/automerge.yml b/.github/workflows/automerge.yml index 0f0ba6e..d23253a 100644 --- a/.github/workflows/automerge.yml +++ b/.github/workflows/automerge.yml @@ -6,8 +6,24 @@ name: automerge # protection is satisfied. # Merge-queue repos: GitHub auto-merge does NOT reliably add a PR to the queue, # so once the PR is mergeable we call the enqueuePullRequest GraphQL mutation -# explicitly. It fires on check completion (not just PR open) so the PR is -# actually ready when we try. +# explicitly. +# +# WHY THE SCHEDULED SWEEP EXISTS (2026-09-06): +# The event-driven path alone is not sufficient on a merge-queue repo. When a +# check_suite completes, GitHub has often not yet recomputed mergeStateStatus, +# so the PR still reads BLOCKED. The old workflow logged "waiting for a later +# check_suite event" and exited. If that was the LAST check_suite event for the +# PR, nothing ever retried and the PR sat open forever while reading CLEAN. +# Pin PR #148 stranded that way for four days, which silently froze the +# GitOps deploy chain: build-and-deploy.yml only classifies mode=webhook for a +# "chore: pin muninn ->" head commit, so the Portainer redeploy never ran and +# the host kept serving the previous image with no alert anywhere. +# +# Two defences, both needed: +# 1. Retry in-run while the PR is BLOCKED but its checks are still PENDING. +# 2. A scheduled sweep that enqueues ANY open, mergeable, unqueued PR. This +# is the convergence guarantee: it is independent of event delivery, so a +# dropped or badly-timed event can no longer strand a PR indefinitely. # # Branch protection (required reviews/checks) still fully applies either way. @@ -18,13 +34,19 @@ on: types: [submitted] check_suite: types: [completed] + schedule: + # Every 15 minutes. Backstop only; the event path handles the happy case. + - cron: '*/15 * * * *' + workflow_dispatch: permissions: contents: write pull-requests: write concurrency: - group: automerge-${{ github.event.pull_request.number || github.event.check_suite.id }} + # Non-PR events (schedule/dispatch) have no pull_request payload, so fall + # back to run_id rather than collapsing every sweep into one group. + group: automerge-${{ github.event.pull_request.number || github.event.check_suite.id || github.run_id }} cancel-in-progress: false jobs: @@ -35,6 +57,7 @@ jobs: with: script: | const {owner, repo} = context.repo; + const isSweep = context.eventName === 'schedule' || context.eventName === 'workflow_dispatch'; // Collect candidate PR numbers from whatever event fired. let numbers = []; @@ -42,6 +65,12 @@ jobs: numbers = [context.payload.pull_request.number]; } else if (context.eventName === 'check_suite') { numbers = (context.payload.check_suite.pull_requests || []).map(p => p.number); + } else if (isSweep) { + const open = await github.paginate(github.rest.pulls.list, { + owner, repo, state: 'open', per_page: 100, + }); + numbers = open.map(p => p.number); + core.info(`sweep: ${numbers.length} open PR(s)`); } if (numbers.length === 0) { core.info('no candidate PRs'); return; } @@ -50,6 +79,7 @@ jobs: mergeQueue{ id } pullRequest(number:$n){ id number isDraft state merged isInMergeQueue mergeStateStatus + commits(last:1){ nodes{ commit{ statusCheckRollup{ state } } } } } } }`; @@ -57,16 +87,33 @@ jobs: const AUTOMERGE = `mutation($id:ID!){ enablePullRequestAutoMerge(input:{pullRequestId:$id, mergeMethod:SQUASH}){ pullRequest{ number } } }`; const sleep = ms => new Promise(r => setTimeout(r, ms)); + const rollupOf = pr => { + const n = pr.commits && pr.commits.nodes && pr.commits.nodes[0]; + return (n && n.commit.statusCheckRollup && n.commit.statusCheckRollup.state) || null; + }; + + // On the sweep we do a single pass per PR: the next sweep is only + // 15 minutes away, so burning runner minutes waiting is pointless. + // On an event we wait, because we are racing GitHub's own + // mergeStateStatus recomputation and want to land it in this run. + const attempts = isSweep ? 1 : 12; + let stuck = 0; + for (const n of numbers) { - // mergeStateStatus is computed async; retry briefly on UNKNOWN. - let data; - for (let i = 0; i < 6; i++) { - data = await github.graphql(Q, {o: owner, r: repo, n}); - if (data.repository.pullRequest.mergeStateStatus !== 'UNKNOWN') break; - await sleep(5000); + let pr, hasQueue; + for (let i = 0; i < attempts; i++) { + const data = await github.graphql(Q, {o: owner, r: repo, n}); + pr = data.repository.pullRequest; + hasQueue = !!data.repository.mergeQueue; + if (!pr) break; + const st = pr.mergeStateStatus; + const rollup = rollupOf(pr); + // Terminal states: stop waiting. + if (st !== 'UNKNOWN' && st !== 'BLOCKED') break; + if (st === 'BLOCKED' && rollup && rollup !== 'PENDING') break; + if (i < attempts - 1) await sleep(10000); } - const pr = data.repository.pullRequest; - const hasQueue = !!data.repository.mergeQueue; + if (!pr || pr.state !== 'OPEN' || pr.merged) { core.info(`#${n}: not an open PR`); continue; } if (pr.isDraft) { core.info(`#${n}: draft, skipping`); continue; } if (pr.isInMergeQueue) { core.info(`#${n}: already in merge queue`); continue; } @@ -74,12 +121,15 @@ jobs: if (hasQueue) { if (pr.mergeStateStatus === 'CLEAN') { try { await github.graphql(ENQUEUE, {id: pr.id}); core.info(`#${n}: enqueued`); } - catch (e) { core.info(`#${n}: enqueue not accepted yet (${e.message})`); } + catch (e) { core.info(`#${n}: enqueue not accepted (${e.message})`); stuck++; } } else { - core.info(`#${n}: mergeStateStatus=${pr.mergeStateStatus}, waiting for a later check_suite event`); + // Not an error: could be failing checks, conflicts, or a + // required review. The sweep will re-evaluate in 15 minutes. + core.info(`#${n}: mergeStateStatus=${pr.mergeStateStatus} rollup=${rollupOf(pr)}, leaving for the next sweep`); } } else { try { await github.graphql(AUTOMERGE, {id: pr.id}); core.info(`#${n}: auto-merge enabled`); } catch (e) { core.info(`#${n}: auto-merge enable skipped (${e.message})`); } } } + if (stuck > 0) core.warning(`${stuck} PR(s) were CLEAN but the enqueue call was rejected`); From db9ee9d1174fcb559bbaefdd5820bc7311ae346d Mon Sep 17 00:00:00 2001 From: Richard Pierre Date: Sat, 5 Sep 2026 22:06:25 -0400 Subject: [PATCH 2/2] fix(ci): adopt canonical automerge and add a scheduled sweep Rebases muninn's automerge.yml onto the homelab-stacks canonical version and adds the sweep on top. Two pre-existing standard violations are fixed by the rebase: - github-actions-monorepo-standard.md requires automerge.yml to mint the deploy-bot App token and pass it to the step calling enqueuePullRequest. Muninn ran the mutation under the default GITHUB_TOKEN. - The UNSTABLE self-check handling from homelab-stacks #406 was missing, so a fast-CI PR could strand on this job's own in-progress automerge check. The new part is the scheduled sweep. The event path alone cannot guarantee convergence on a merge-queue repo: when a check_suite completes GitHub has often not recomputed mergeStateStatus, the PR reads BLOCKED, and if that was the last event nothing retried. Pin PR #148 stranded four days that way and silently froze the deploy chain. --- .github/workflows/automerge.yml | 141 ++++++++++++++++++++------------ 1 file changed, 90 insertions(+), 51 deletions(-) diff --git a/.github/workflows/automerge.yml b/.github/workflows/automerge.yml index d23253a..01c7d2c 100644 --- a/.github/workflows/automerge.yml +++ b/.github/workflows/automerge.yml @@ -6,26 +6,27 @@ name: automerge # protection is satisfied. # Merge-queue repos: GitHub auto-merge does NOT reliably add a PR to the queue, # so once the PR is mergeable we call the enqueuePullRequest GraphQL mutation -# explicitly. +# explicitly. It fires on check completion (not just PR open) so the PR is +# actually ready when we try. +# +# Branch protection (required reviews/checks) still fully applies either way. # # WHY THE SCHEDULED SWEEP EXISTS (2026-09-06): -# The event-driven path alone is not sufficient on a merge-queue repo. When a +# On a merge-queue repo the event-driven path alone is not sufficient. When a # check_suite completes, GitHub has often not yet recomputed mergeStateStatus, -# so the PR still reads BLOCKED. The old workflow logged "waiting for a later -# check_suite event" and exited. If that was the LAST check_suite event for the -# PR, nothing ever retried and the PR sat open forever while reading CLEAN. -# Pin PR #148 stranded that way for four days, which silently froze the -# GitOps deploy chain: build-and-deploy.yml only classifies mode=webhook for a -# "chore: pin muninn ->" head commit, so the Portainer redeploy never ran and -# the host kept serving the previous image with no alert anywhere. +# so the PR still reads BLOCKED and this job exits without enqueuing. If that +# was the LAST check_suite event for the PR, nothing ever retried: the PR then +# went CLEAN and sat open forever. Pin PR #148 stranded that way for four days, +# which silently froze the GitOps deploy chain, because build-and-deploy.yml +# only classifies mode=webhook for a "chore: pin muninn ->" head commit, so the +# Portainer redeploy never ran and the host kept serving the previous image with +# no alert anywhere. Reproduced live on #150. # -# Two defences, both needed: -# 1. Retry in-run while the PR is BLOCKED but its checks are still PENDING. -# 2. A scheduled sweep that enqueues ANY open, mergeable, unqueued PR. This -# is the convergence guarantee: it is independent of event delivery, so a -# dropped or badly-timed event can no longer strand a PR indefinitely. -# -# Branch protection (required reviews/checks) still fully applies either way. +# The sweep is the convergence guarantee: it re-evaluates every open PR on a +# fixed cadence, independent of event delivery, so a dropped or badly-timed +# event can no longer strand a PR indefinitely. Muninn is currently the only +# Emkraan repo with a merge-queue ruleset, which is why the explicit +# enqueuePullRequest call is load-bearing here and nowhere else. on: pull_request: @@ -35,7 +36,7 @@ on: check_suite: types: [completed] schedule: - # Every 15 minutes. Backstop only; the event path handles the happy case. + # Backstop only; the event path still handles the happy case immediately. - cron: '*/15 * * * *' workflow_dispatch: @@ -44,8 +45,8 @@ permissions: pull-requests: write concurrency: - # Non-PR events (schedule/dispatch) have no pull_request payload, so fall - # back to run_id rather than collapsing every sweep into one group. + # Non-PR events (schedule/dispatch) carry no pull_request payload, so fall + # back to run_id rather than collapsing every sweep into a single group. group: automerge-${{ github.event.pull_request.number || github.event.check_suite.id || github.run_id }} cancel-in-progress: false @@ -53,8 +54,19 @@ jobs: automerge: runs-on: ubuntu-latest steps: - - uses: actions/github-script@v7 + # A PR merged under the default GITHUB_TOKEN (github-actions[bot]) does not + # trigger push-triggered deploy-*.yml (GitHub loop-prevention), so a + # bot-merged stack change lands on main but never redeploys. Merge under the + # emkraan-deploy-bot App token instead; an App-token merge DOES trigger those + # workflows. Per github-actions-monorepo-standard.md. See #291. + - uses: actions/create-github-app-token@v3 + id: app-token with: + app-id: ${{ vars.DEPLOY_APP_ID }} + private-key: ${{ secrets.DEPLOY_APP_PRIVATE_KEY }} + - uses: actions/github-script@v9 + with: + github-token: ${{ steps.app-token.outputs.token }} script: | const {owner, repo} = context.repo; const isSweep = context.eventName === 'schedule' || context.eventName === 'workflow_dispatch'; @@ -66,6 +78,7 @@ jobs: } else if (context.eventName === 'check_suite') { numbers = (context.payload.check_suite.pull_requests || []).map(p => p.number); } else if (isSweep) { + // The backstop: consider every open PR, not just one an event named. const open = await github.paginate(github.rest.pulls.list, { owner, repo, state: 'open', per_page: 100, }); @@ -78,58 +91,84 @@ jobs: repository(owner:$o,name:$r){ mergeQueue{ id } pullRequest(number:$n){ - id number isDraft state merged isInMergeQueue mergeStateStatus - commits(last:1){ nodes{ commit{ statusCheckRollup{ state } } } } + id number isDraft state merged isInMergeQueue mergeStateStatus mergeable headRefOid } } }`; const ENQUEUE = `mutation($id:ID!){ enqueuePullRequest(input:{pullRequestId:$id}){ mergeQueueEntry{ position } } }`; const AUTOMERGE = `mutation($id:ID!){ enablePullRequestAutoMerge(input:{pullRequestId:$id, mergeMethod:SQUASH}){ pullRequest{ number } } }`; + const MERGE = `mutation($id:ID!,$oid:GitObjectID!){ mergePullRequest(input:{pullRequestId:$id, mergeMethod:SQUASH, expectedHeadOid:$oid}){ pullRequest{ number merged } } }`; const sleep = ms => new Promise(r => setTimeout(r, ms)); - const rollupOf = pr => { - const n = pr.commits && pr.commits.nodes && pr.commits.nodes[0]; - return (n && n.commit.statusCheckRollup && n.commit.statusCheckRollup.state) || null; - }; - - // On the sweep we do a single pass per PR: the next sweep is only - // 15 minutes away, so burning runner minutes waiting is pointless. - // On an event we wait, because we are racing GitHub's own - // mergeStateStatus recomputation and want to land it in this run. - const attempts = isSweep ? 1 : 12; - let stuck = 0; + // A PR is ready to merge/enqueue when mergeStateStatus is CLEAN, OR UNSTABLE + // with mergeable === MERGEABLE. UNSTABLE means every REQUIRED check has + // already passed and only a non-required context is failing or still pending + // (a failing/pending REQUIRED check yields BLOCKED, not UNSTABLE) - so branch + // protection is satisfied. Treating UNSTABLE as ready is what unstrands + // fast-CI repos: by the time this job evaluates, the only non-success context + // is often its OWN in-progress `automerge` check, which pins the commit to + // UNSTABLE, blocks the direct merge, and makes enablePullRequestAutoMerge + // fail with "unstable status", leaving the PR clean-but-open with no event to + // re-fire it (#404/#405). We do NOT inspect check runs (the App token cannot + // read statusCheckRollup): the merge/enqueue mutation itself enforces branch + // protection with expectedHeadOid, so a genuinely-unsatisfied required check + // still cannot slip through, and non-required checks are advisory (the same + // semantics as GitHub's native auto-merge). BLOCKED/DIRTY/UNKNOWN are not ready. + const isReady = (pr) => + pr.mergeStateStatus === 'CLEAN' || + (pr.mergeStateStatus === 'UNSTABLE' && pr.mergeable === 'MERGEABLE'); for (const n of numbers) { - let pr, hasQueue; + // mergeStateStatus is computed async; retry briefly on UNKNOWN. + // One pass is enough on the sweep: the next one is 15 minutes away, + // so waiting here would only burn runner minutes. + const attempts = isSweep ? 1 : 6; + let data; for (let i = 0; i < attempts; i++) { - const data = await github.graphql(Q, {o: owner, r: repo, n}); - pr = data.repository.pullRequest; - hasQueue = !!data.repository.mergeQueue; - if (!pr) break; - const st = pr.mergeStateStatus; - const rollup = rollupOf(pr); - // Terminal states: stop waiting. - if (st !== 'UNKNOWN' && st !== 'BLOCKED') break; - if (st === 'BLOCKED' && rollup && rollup !== 'PENDING') break; - if (i < attempts - 1) await sleep(10000); + data = await github.graphql(Q, {o: owner, r: repo, n}); + if (data.repository.pullRequest.mergeStateStatus !== 'UNKNOWN') break; + if (i < attempts - 1) await sleep(5000); } - + const pr = data.repository.pullRequest; + const hasQueue = !!data.repository.mergeQueue; if (!pr || pr.state !== 'OPEN' || pr.merged) { core.info(`#${n}: not an open PR`); continue; } if (pr.isDraft) { core.info(`#${n}: draft, skipping`); continue; } if (pr.isInMergeQueue) { core.info(`#${n}: already in merge queue`); continue; } + const ready = isReady(pr); if (hasQueue) { - if (pr.mergeStateStatus === 'CLEAN') { + if (ready) { try { await github.graphql(ENQUEUE, {id: pr.id}); core.info(`#${n}: enqueued`); } - catch (e) { core.info(`#${n}: enqueue not accepted (${e.message})`); stuck++; } + catch (e) { core.info(`#${n}: enqueue not accepted yet (${e.message})`); } } else { - // Not an error: could be failing checks, conflicts, or a - // required review. The sweep will re-evaluate in 15 minutes. - core.info(`#${n}: mergeStateStatus=${pr.mergeStateStatus} rollup=${rollupOf(pr)}, leaving for the next sweep`); + // Not an error: failing checks, a conflict, or a required review. + // No longer relies on a later event arriving; the sweep re-evaluates + // this PR within 15 minutes, so it cannot strand. + core.info(`#${n}: mergeStateStatus=${pr.mergeStateStatus}, leaving for the next sweep`); } } else { + // Non-merge-queue repo. Prefer a SYNCHRONOUS merge under the App + // token when the PR is ready. GitHub's native auto-merge is completed + // asynchronously by GitHub itself and has two failure modes this avoids: + // it can strand a PR "clean but never merged" with no event left to + // re-fire it (#314), and its merge completion hits the no-cascade rule + // so push:[main] deploy workflows never run for that commit (#315). A + // direct App-token merge is deterministic and lands a normal push that + // DOES cascade to the deploy workflows. + if (ready) { + try { + await github.graphql(MERGE, {id: pr.id, oid: pr.headRefOid}); + core.info(`#${n}: merged (app-token squash)`); + continue; + } catch (e) { + core.info(`#${n}: direct merge not accepted (${e.message}); falling back to auto-merge`); + } + } + // Not ready yet, or the direct merge raced a new head: enable native + // auto-merge so GitHub merges once branch protection is satisfied. A + // later check_suite/review event also re-runs this job and will take + // the direct-merge path above once the PR is ready. try { await github.graphql(AUTOMERGE, {id: pr.id}); core.info(`#${n}: auto-merge enabled`); } catch (e) { core.info(`#${n}: auto-merge enable skipped (${e.message})`); } } } - if (stuck > 0) core.warning(`${stuck} PR(s) were CLEAN but the enqueue call was rejected`);