From 708a57954e1b5969f8bb81c266b8eeb57027683e Mon Sep 17 00:00:00 2001 From: Jason Odoom Date: Tue, 9 Jun 2026 05:46:52 +0000 Subject: [PATCH] ci(eval): persist reports via auto-merge PR, not direct push main is protected (requires the test check), so the eval bots could not push reports directly: the run did all the work, then died at GH006 protected-branch-update-failed. Push the generated report to a dated bot/ branch and open a PR that auto-merges once ci.yml passes. Checkout uses RELEASE_PAT rather than GITHUB_TOKEN so the PR actually triggers the test check; PRs opened by GITHUB_TOKEN do not start workflow runs, which would leave the required check pending and auto-merge stuck. --- .github/workflows/monthly-vision-eval.yml | 29 +++++++++++++++++++---- .github/workflows/weekly-eval.yml | 29 +++++++++++++++++++---- 2 files changed, 48 insertions(+), 10 deletions(-) diff --git a/.github/workflows/monthly-vision-eval.yml b/.github/workflows/monthly-vision-eval.yml index 6b38e03..ae399ad 100644 --- a/.github/workflows/monthly-vision-eval.yml +++ b/.github/workflows/monthly-vision-eval.yml @@ -55,6 +55,7 @@ on: permissions: contents: write + pull-requests: write # open the auto-merge report PR jobs: vision-eval: @@ -63,6 +64,11 @@ jobs: steps: - name: Checkout uses: actions/checkout@v6 + with: + # RELEASE_PAT (not GITHUB_TOKEN) so the report PR triggers + # ci.yml's `test` check; GITHUB_TOKEN-opened PRs don't start + # workflow runs, leaving the required check pending forever. + token: ${{ secrets.RELEASE_PAT }} - name: Setup Node (pinned) uses: actions/setup-node@v6 @@ -158,23 +164,36 @@ jobs: --critic anthropic \ --report "docs/evals/monthly/${{ env.REPORT_DATE }}-image.md" - - name: Commit all three reports + - name: Open report PR (auto-merge) # Smoke runs prove integration; their numbers are not # publishable (n=2 is well under any reportable threshold) so - # the smoke path skips the commit entirely. The reports still + # the smoke path skips the PR entirely. The reports still # exist in the workflow's working tree if you need to inspect # them via the Actions UI. + # + # main is protected (requires the `test` check), so push the + # reports to a dated branch and open a PR that auto-merges + # once ci.yml goes green. if: inputs.smoke_mode != true + env: + GH_TOKEN: ${{ secrets.RELEASE_PAT }} run: | git config user.email "41898282+github-actions[bot]@users.noreply.github.com" git config user.name "github-actions[bot]" git add docs/evals/monthly/ if git diff --cached --quiet; then echo "No report changes to commit." - else - git commit -m "docs(evals): monthly source + vision + image-gen run ${{ env.REPORT_DATE }}" -m "Automated monthly eval across all three verticals: text source linter, text rendered-pixel vision critic, and image-generation vision critic. See the three files for per-cell numbers." - git push + exit 0 fi + BRANCH="bot/monthly-vision-eval-${REPORT_DATE}" + git checkout -b "$BRANCH" + git commit -m "docs(evals): monthly source + vision + image-gen run ${REPORT_DATE}" -m "Automated monthly eval across all three verticals: text source linter, text rendered-pixel vision critic, and image-generation vision critic. See the three files for per-cell numbers." + git push -u origin "$BRANCH" --force-with-lease + gh pr create --base main --head "$BRANCH" \ + --title "docs(evals): monthly run ${REPORT_DATE}" \ + --body "Automated monthly eval report (source + vision + image-gen). Merges automatically once CI passes." \ + || echo "PR already exists for $BRANCH" + gh pr merge "$BRANCH" --auto --squash --delete-branch - name: Smoke-mode summary if: inputs.smoke_mode == true diff --git a/.github/workflows/weekly-eval.yml b/.github/workflows/weekly-eval.yml index 2893d77..02879e3 100644 --- a/.github/workflows/weekly-eval.yml +++ b/.github/workflows/weekly-eval.yml @@ -40,8 +40,9 @@ on: default: "" permissions: - contents: write # commit the report + contents: write # push the report branch issues: write # open a drift issue if the run moves outside tolerance + pull-requests: write # open the auto-merge report PR jobs: eval: @@ -50,6 +51,12 @@ jobs: steps: - name: Checkout uses: actions/checkout@v6 + with: + # RELEASE_PAT (not GITHUB_TOKEN) so the report PR triggers + # ci.yml's `test` check. PRs opened by GITHUB_TOKEN don't + # start workflow runs, which would leave the required check + # forever pending and auto-merge stuck. + token: ${{ secrets.RELEASE_PAT }} - name: Setup Node uses: actions/setup-node@v6 @@ -125,17 +132,29 @@ jobs: echo "drift=stable" >> "$GITHUB_OUTPUT" fi - - name: Commit weekly report + - name: Open report PR (auto-merge) + env: + GH_TOKEN: ${{ secrets.RELEASE_PAT }} run: | + # main is protected (requires the `test` check), so the bot + # can't push reports directly. Instead push to a dated branch + # and open a PR that auto-merges once ci.yml goes green. git config user.email "noreply@adastracomputing.com" git config user.name "ahd-weekly-eval" git add docs/evals/weekly/ if git diff --cached --quiet; then echo "No report changes to commit." - else - git commit -m "docs(evals): weekly run ${{ env.REPORT_DATE }}" -m "Automated weekly eval against Cloudflare Workers AI OSS roster. See the report for per-cell numbers." - git push + exit 0 fi + BRANCH="bot/weekly-eval-${REPORT_DATE}" + git checkout -b "$BRANCH" + git commit -m "docs(evals): weekly run ${REPORT_DATE}" -m "Automated weekly eval against Cloudflare Workers AI OSS roster. See the report for per-cell numbers." + git push -u origin "$BRANCH" --force-with-lease + gh pr create --base main --head "$BRANCH" \ + --title "docs(evals): weekly run ${REPORT_DATE}" \ + --body "Automated weekly eval report. Merges automatically once CI passes." \ + || echo "PR already exists for $BRANCH" + gh pr merge "$BRANCH" --auto --squash --delete-branch - name: Open drift issue if flagged if: steps.drift.outputs.drift == 'flagged'