From f554b8dee66f8eec5cf2a27a41763731ec6c9e3d Mon Sep 17 00:00:00 2001 From: w4ffl35 <25737761+w4ffl35@users.noreply.github.com> Date: Wed, 16 Sep 2026 16:14:43 -0600 Subject: [PATCH 1/3] Review pull requests and work labelled issues without a person Matches what the dashboard repository now has, so a ticket in either place starts the same way. One workflow turns an `agent`-labelled issue into a pull request; the other reviews every pull request. Neither may merge -- required checks decide that, which is the only reason letting a model write code here is reasonable. Both prompts point at this repository's own rules and at the two failures that hurt most from here: a version change that leaves compatibility.json behind, which ships a mismatched engine to the dashboard and the desktop application, and a protocol or published-surface change nothing downstream was updated for. Dependabot is grouped per ecosystem, with torch on its own because it decides what the engine can run and how large every artifact is. The workflow files parse. Neither has run: both need secrets that are not set on this repository yet. --- .github/dependabot.yml | 34 +++++++++++++ .github/workflows/agent-issue.yml | 79 ++++++++++++++++++++++++++++++ .github/workflows/agent-review.yml | 68 +++++++++++++++++++++++++ 3 files changed, 181 insertions(+) create mode 100644 .github/dependabot.yml create mode 100644 .github/workflows/agent-issue.yml create mode 100644 .github/workflows/agent-review.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..d37d95f --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,34 @@ +version: 2 + +# Grouped so a week's updates arrive as one reviewable change per ecosystem +# rather than one per dependency. The packages under `packages/` share a lock +# discipline, so pip updates are grouped across the repository. +updates: + - package-ecosystem: pip + directory: / + schedule: + interval: weekly + open-pull-requests-limit: 3 + groups: + # Torch decides what the engine can run and how large every artifact + # is, so it lands on its own rather than inside a bulk update. + torch: + patterns: + - torch + - torchvision + python: + patterns: + - "*" + exclude-patterns: + - torch + - torchvision + + - package-ecosystem: github-actions + directory: / + schedule: + interval: weekly + open-pull-requests-limit: 3 + groups: + actions: + patterns: + - "*" diff --git a/.github/workflows/agent-issue.yml b/.github/workflows/agent-issue.yml new file mode 100644 index 0000000..d626e33 --- /dev/null +++ b/.github/workflows/agent-issue.yml @@ -0,0 +1,79 @@ +name: Work an issue + +# Turns a labelled issue into a pull request without a person opening a +# session. Label an issue `agent` (or comment `/agent`) and the work starts. +# +# Two secrets are required and the workflow fails loudly without them: +# +# - `ANTHROPIC_API_KEY`, which pays for the model. +# - `AUTOMATION_TOKEN`, a PAT or GitHub App token. The pull request must be +# opened with it, because GitHub does not start workflow runs for events +# raised by the built-in `GITHUB_TOKEN`: a pull request opened with that +# token sits with no checks, forever, and can never satisfy auto-merge. +# +# The agent may not merge. It opens a pull request, and the required checks +# decide -- which is the only reason it is safe to let it write code at all. + +on: + issues: + types: [labeled] + issue_comment: + types: [created] + +permissions: + contents: read + +jobs: + work: + # `issue_comment` fires for pull requests too; `.issue.pull_request` is + # absent on a real issue, which is how the two are told apart. + if: > + (github.event_name == 'issues' && + github.event.label.name == 'agent') || + (github.event_name == 'issue_comment' && + !github.event.issue.pull_request && + startsWith(github.event.comment.body, '/agent')) + runs-on: [self-hosted, linux, x64, spikeforge-ci] + timeout-minutes: 60 + steps: + - name: Check the secrets exist + shell: bash + run: | + missing="" + [ -z "${{ secrets.ANTHROPIC_API_KEY }}" ] && missing="$missing ANTHROPIC_API_KEY" + [ -z "${{ secrets.AUTOMATION_TOKEN }}" ] && missing="$missing AUTOMATION_TOKEN" + if [ -n "$missing" ]; then + echo "::error::missing secrets:$missing" + exit 1 + fi + + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + token: ${{ secrets.AUTOMATION_TOKEN }} + + - uses: anthropics/claude-code-action@v1 + with: + anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }} + github_token: ${{ secrets.AUTOMATION_TOKEN }} + prompt: | + Work issue #${{ github.event.issue.number }} in this repository. + + Read the repository's own guidance first -- `rules.md`, and + `AGENTS.md` or `CLAUDE.md` if present. It is binding, including + the file and function length limits. + + This repository is the engine that five published distributions + and two applications build against, so before you finish: + - The test suite must pass. + - If you changed a version, update `compatibility.json` to match. + The dashboard pins both its end-to-end suite and the engine + inside the desktop application to that file, so leaving it + behind ships a mismatched build. + - If you changed the wire protocol or a published surface, say so + explicitly in the pull request body and name what outside this + repository has to change with it. + + Open a pull request against main describing what you changed and + exactly what you ran, including anything you skipped. Do not claim + tests you did not run. Do not merge it. diff --git a/.github/workflows/agent-review.yml b/.github/workflows/agent-review.yml new file mode 100644 index 0000000..4e0ebcb --- /dev/null +++ b/.github/workflows/agent-review.yml @@ -0,0 +1,68 @@ +name: Review a pull request + +# Reviews every pull request. With nobody reading the diff, this is the only +# thing that looks at a change as a change rather than as a test result. +# +# It posts a review. It does not approve, block, or merge: required checks +# decide that. A reviewer that could also approve its own work would add +# nothing to a pipeline where the author is a model too. +# +# Needs `ANTHROPIC_API_KEY`. Without it the job says so and stops, rather than +# failing every pull request in the repository. + +on: + pull_request: + types: [opened, synchronize, reopened] + +permissions: + contents: read + pull-requests: write + +jobs: + review: + if: ${{ !github.event.pull_request.draft }} + runs-on: [self-hosted, linux, x64, spikeforge-ci] + timeout-minutes: 30 + steps: + - name: Skip without a key + id: guard + shell: bash + run: | + if [ -z "${{ secrets.ANTHROPIC_API_KEY }}" ]; then + echo "ANTHROPIC_API_KEY is not set; no review will be posted." + echo "skip=true" >> "$GITHUB_OUTPUT" + fi + + - uses: actions/checkout@v4 + if: steps.guard.outputs.skip != 'true' + with: + fetch-depth: 0 + + - uses: anthropics/claude-code-action@v1 + if: steps.guard.outputs.skip != 'true' + with: + anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }} + github_token: ${{ github.token }} + prompt: | + Review pull request #${{ github.event.pull_request.number }}. + + Read the repository's own guidance first -- `rules.md`, and + `AGENTS.md` or `CLAUDE.md` if present. They are binding here. + + Report only defects you can point at in the diff. Say plainly + when you find none; a review that invents work to look useful is + worse than a short one. Rank what you do find. + + This repository is the engine five distributions are published + from, and a dashboard and a desktop application build against it. + Weigh accordingly: + - A change to the wire protocol, or to a published surface, that + nothing outside this repository has been updated for. + - `compatibility.json` left behind by a version change. It is the + authority the dashboard pins its end-to-end suite and its + packaged engine to, so a stale entry ships a mismatched build. + - A test that passes while the behaviour it names is broken. + - Public-facing copy asserting something the code does not do. + + Post one review comment. Do not approve and do not request + changes; the required checks decide whether this merges. From 9659ae830508fbdc4c3a8dea344f8f2d01cd333f Mon Sep 17 00:00:00 2001 From: w4ffl35 <25737761+w4ffl35@users.noreply.github.com> Date: Wed, 16 Sep 2026 16:32:04 -0600 Subject: [PATCH 2/3] Stop CI failing on a pip cache that cannot find the repository Every job here sets up Python with `cache: pip`, and on this fleet that step fails outright: "No file ... matched to [**/requirements.txt or **/pyproject.toml], make sure you have checked out the target repository". All seven pyproject.toml files are present in `packages/` at that exact path -- checked on the runner's own workspace while a job sat failed. The runners keep `_work` on another drive behind a symlink, and the action's dependency-file glob does not see through it. It has taken out `blocked-deps`, `client`, `docs` and all five `extras` jobs, and it took out the dashboard's release workflow earlier today for the same reason. It is not reliably reproducible -- an earlier run on main globbed fine -- which makes it worse, not better: a pipeline that merges on green will see spurious red it cannot distinguish from a real failure. Removing it costs nothing. A self-hosted runner keeps `~/.cache/pip` between jobs by itself, so the action was caching a cache. --- .github/workflows/ci.yml | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e5c8e22..d85b9cd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,10 +21,16 @@ jobs: steps: - uses: actions/checkout@v4 + # No `cache: pip` anywhere in this file. This fleet's runners keep + # `_work` on another drive behind a symlink, and setup-python's + # dependency-file glob does not see through it: it reports that no + # pyproject.toml exists and fails the job before anything is built, + # while all seven of them sit in `packages/`. A self-hosted runner also + # keeps `~/.cache/pip` between jobs on its own, so the action's cache + # was buying nothing here even when it worked. - uses: actions/setup-python@v5 with: python-version: "3.12" - cache: pip - name: Install the core dev extra run: | @@ -56,7 +62,6 @@ jobs: - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - cache: pip - name: Install torch CPU wheels run: | @@ -89,7 +94,6 @@ jobs: - uses: actions/setup-python@v5 with: python-version: "3.12" - cache: pip - name: Install torch CPU wheels run: | @@ -141,7 +145,6 @@ jobs: - uses: actions/setup-python@v5 with: python-version: "3.12" - cache: pip - name: Install torch CPU wheels run: | @@ -182,7 +185,6 @@ jobs: - uses: actions/setup-python@v5 with: python-version: "3.12" - cache: pip - name: Install torch CPU wheels run: | @@ -243,7 +245,6 @@ jobs: - uses: actions/setup-python@v5 with: python-version: "3.12" - cache: pip - name: Install torch CPU wheels run: | @@ -310,7 +311,6 @@ jobs: - uses: actions/setup-python@v5 with: python-version: "3.12" - cache: pip - name: Install docs dependencies run: | @@ -337,7 +337,6 @@ jobs: - uses: actions/setup-python@v5 with: python-version: "3.12" - cache: pip - name: Install the renderer PyPI itself uses run: | @@ -405,7 +404,6 @@ jobs: - uses: actions/setup-python@v5 with: python-version: "3.12" - cache: pip - name: Build the core wheel with no extras run: | From b4de5dd338e35826ecf8cb2155147a15d52be553 Mon Sep 17 00:00:00 2001 From: w4ffl35 <25737761+w4ffl35@users.noreply.github.com> Date: Wed, 16 Sep 2026 18:22:52 -0600 Subject: [PATCH 3/3] Reach the model through an OpenAI-compatible endpoint, not Anthropic The two workflows called anthropics/claude-code-action and wanted an ANTHROPIC_API_KEY. This repository is not going to have one, so both now speak plain chat-completions: the review posts through scripts/llm_review.mjs, and the issue worker drives aider with OPENAI_API_BASE pointed wherever LLM_BASE_URL says. OpenRouter and DeepInfra both serve that protocol, so the provider is two variables. LLM_MODEL has no default on purpose. A guessed slug fails at the provider with an opaque error on somebody else's pull request, which is worse than refusing to start. The issue body reaches the agent through a file rather than a shell argument. It is arbitrary text from whoever opened the issue, and interpolating it into a command would let its contents decide what runs. Also raises the subprocess timeout in test_rebuild_is_bit_exact_in_a_ subprocess from 180s to 600s. It timed out today on a runner sharing this workstation with two other jobs, and reported it as a bit-exactness failure that had not happened -- red that looks exactly like the defect the test is named after. Run here: the 9 llm_review tests, and both workflow files parse. The workflows themselves have not run; they need secrets that are not set. --- .github/workflows/agent-issue.yml | 121 ++++++++++++++----- .github/workflows/agent-review.yml | 63 +++++----- scripts/llm_review.mjs | 186 +++++++++++++++++++++++++++++ scripts/llm_review.test.mjs | 106 ++++++++++++++++ tests/test_serving_bundle.py | 7 +- 5 files changed, 413 insertions(+), 70 deletions(-) create mode 100644 scripts/llm_review.mjs create mode 100644 scripts/llm_review.test.mjs diff --git a/.github/workflows/agent-issue.yml b/.github/workflows/agent-issue.yml index d626e33..40fd74a 100644 --- a/.github/workflows/agent-issue.yml +++ b/.github/workflows/agent-issue.yml @@ -1,18 +1,23 @@ name: Work an issue # Turns a labelled issue into a pull request without a person opening a -# session. Label an issue `agent` (or comment `/agent`) and the work starts. +# session. Label an issue `agent`, or comment `/agent`, and the work starts. # -# Two secrets are required and the workflow fails loudly without them: +# The model is reached through aider, which speaks to any OpenAI-compatible +# endpoint, so the provider is configuration rather than code: # -# - `ANTHROPIC_API_KEY`, which pays for the model. -# - `AUTOMATION_TOKEN`, a PAT or GitHub App token. The pull request must be -# opened with it, because GitHub does not start workflow runs for events -# raised by the built-in `GITHUB_TOKEN`: a pull request opened with that -# token sits with no checks, forever, and can never satisfy auto-merge. +# secret LLM_API_KEY the provider key +# secret AUTOMATION_TOKEN a PAT or App token +# variable LLM_BASE_URL e.g. https://openrouter.ai/api/v1 +# variable LLM_MODEL the provider's exact model slug # -# The agent may not merge. It opens a pull request, and the required checks -# decide -- which is the only reason it is safe to let it write code at all. +# `AUTOMATION_TOKEN` is not optional. GitHub does not start workflow runs for +# events raised by the built-in `GITHUB_TOKEN`, so a pull request opened with +# it sits with no checks and can never satisfy auto-merge. +# +# The agent cannot merge, cannot approve, and cannot push to main: `main` +# requires a pull request and its checks. That is the whole reason it is +# reasonable to let a model write here unattended. on: issues: @@ -22,6 +27,7 @@ on: permissions: contents: read + issues: write jobs: work: @@ -36,14 +42,16 @@ jobs: runs-on: [self-hosted, linux, x64, spikeforge-ci] timeout-minutes: 60 steps: - - name: Check the secrets exist + - name: Check the configuration exists shell: bash run: | missing="" - [ -z "${{ secrets.ANTHROPIC_API_KEY }}" ] && missing="$missing ANTHROPIC_API_KEY" + [ -z "${{ secrets.LLM_API_KEY }}" ] && missing="$missing LLM_API_KEY" [ -z "${{ secrets.AUTOMATION_TOKEN }}" ] && missing="$missing AUTOMATION_TOKEN" + [ -z "${{ vars.LLM_BASE_URL }}" ] && missing="$missing LLM_BASE_URL" + [ -z "${{ vars.LLM_MODEL }}" ] && missing="$missing LLM_MODEL" if [ -n "$missing" ]; then - echo "::error::missing secrets:$missing" + echo "::error::missing configuration:$missing" exit 1 fi @@ -52,28 +60,75 @@ jobs: fetch-depth: 0 token: ${{ secrets.AUTOMATION_TOKEN }} - - uses: anthropics/claude-code-action@v1 + - uses: actions/setup-python@v5 with: - anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }} - github_token: ${{ secrets.AUTOMATION_TOKEN }} - prompt: | - Work issue #${{ github.event.issue.number }} in this repository. + python-version: "3.12" + + - name: Install the agent + run: python -m pip install --upgrade aider-chat - Read the repository's own guidance first -- `rules.md`, and - `AGENTS.md` or `CLAUDE.md` if present. It is binding, including - the file and function length limits. + - name: Write the instructions + shell: bash + env: + ISSUE_TITLE: ${{ github.event.issue.title }} + ISSUE_BODY: ${{ github.event.issue.body }} + run: | + # Through a file, not an argument: an issue body is arbitrary text + # and interpolating it into a shell command would let its contents + # decide what runs. + { + echo "Work this issue." + echo + echo "Title: $ISSUE_TITLE" + echo + echo "$ISSUE_BODY" + echo + echo "Read this repository's own guidance first -- rules.md, and" + echo "AGENTS.md or CLAUDE.md if present. It is binding." + echo + echo "Five published distributions and two applications build" + echo "against this repository. If you change a version, update" + echo "compatibility.json to match: the dashboard pins both its" + echo "end-to-end suite and the engine inside the desktop" + echo "application to that file, so leaving it behind ships a" + echo "mismatched build." + } > /tmp/agent-instructions.txt + cat /tmp/agent-instructions.txt - This repository is the engine that five published distributions - and two applications build against, so before you finish: - - The test suite must pass. - - If you changed a version, update `compatibility.json` to match. - The dashboard pins both its end-to-end suite and the engine - inside the desktop application to that file, so leaving it - behind ships a mismatched build. - - If you changed the wire protocol or a published surface, say so - explicitly in the pull request body and name what outside this - repository has to change with it. + - name: Make the change + env: + OPENAI_API_KEY: ${{ secrets.LLM_API_KEY }} + OPENAI_API_BASE: ${{ vars.LLM_BASE_URL }} + run: | + set -euo pipefail + branch="agent/issue-${{ github.event.issue.number }}" + git config user.name "spikeforge-agent" + git config user.email "contact@capsizegames.com" + git checkout -b "$branch" + aider --yes --no-analytics --no-gitignore \ + --model "openai/${{ vars.LLM_MODEL }}" \ + --message-file /tmp/agent-instructions.txt + if git diff --quiet HEAD; then + echo "::error::the agent produced no change" + exit 1 + fi + + - name: Check its work + run: | + python -m pip install -e ".[dev]" || python -m pip install -e . + python -m pytest -x -q + + - name: Open the pull request + env: + GH_TOKEN: ${{ secrets.AUTOMATION_TOKEN }} + run: | + set -euo pipefail + branch="agent/issue-${{ github.event.issue.number }}" + git push --set-upstream origin "$branch" + gh pr create --base main --head "$branch" \ + --title "${{ github.event.issue.title }}" \ + --body "Closes #${{ github.event.issue.number }}. - Open a pull request against main describing what you changed and - exactly what you ran, including anything you skipped. Do not claim - tests you did not run. Do not merge it. + Written by \`${{ vars.LLM_MODEL }}\` from the issue text. Nobody has + read this. The test suite passed on the runner before it was opened; + the required checks decide whether it merges." diff --git a/.github/workflows/agent-review.yml b/.github/workflows/agent-review.yml index 4e0ebcb..ae7a3f2 100644 --- a/.github/workflows/agent-review.yml +++ b/.github/workflows/agent-review.yml @@ -3,12 +3,16 @@ name: Review a pull request # Reviews every pull request. With nobody reading the diff, this is the only # thing that looks at a change as a change rather than as a test result. # -# It posts a review. It does not approve, block, or merge: required checks -# decide that. A reviewer that could also approve its own work would add -# nothing to a pipeline where the author is a model too. +# It talks to any OpenAI-compatible chat-completions endpoint -- OpenRouter, +# DeepInfra, or anything else speaking that protocol -- so the provider and +# model are configuration, not code: # -# Needs `ANTHROPIC_API_KEY`. Without it the job says so and stops, rather than -# failing every pull request in the repository. +# secret LLM_API_KEY the provider key +# variable LLM_BASE_URL e.g. https://openrouter.ai/api/v1 +# variable LLM_MODEL the provider's exact model slug +# +# It posts a comment. It cannot approve, block, or merge: the required checks +# decide that. on: pull_request: @@ -22,47 +26,34 @@ jobs: review: if: ${{ !github.event.pull_request.draft }} runs-on: [self-hosted, linux, x64, spikeforge-ci] - timeout-minutes: 30 + timeout-minutes: 20 steps: - - name: Skip without a key + - name: Skip without a provider id: guard shell: bash run: | - if [ -z "${{ secrets.ANTHROPIC_API_KEY }}" ]; then - echo "ANTHROPIC_API_KEY is not set; no review will be posted." + if [ -z "${{ secrets.LLM_API_KEY }}" ] \ + || [ -z "${{ vars.LLM_BASE_URL }}" ] \ + || [ -z "${{ vars.LLM_MODEL }}" ]; then + echo "LLM_API_KEY, LLM_BASE_URL or LLM_MODEL is unset;" + echo "no review will be posted." echo "skip=true" >> "$GITHUB_OUTPUT" fi - uses: actions/checkout@v4 if: steps.guard.outputs.skip != 'true' - with: - fetch-depth: 0 - - uses: anthropics/claude-code-action@v1 + - uses: actions/setup-node@v4 if: steps.guard.outputs.skip != 'true' with: - anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }} - github_token: ${{ github.token }} - prompt: | - Review pull request #${{ github.event.pull_request.number }}. - - Read the repository's own guidance first -- `rules.md`, and - `AGENTS.md` or `CLAUDE.md` if present. They are binding here. + node-version: "22" - Report only defects you can point at in the diff. Say plainly - when you find none; a review that invents work to look useful is - worse than a short one. Rank what you do find. - - This repository is the engine five distributions are published - from, and a dashboard and a desktop application build against it. - Weigh accordingly: - - A change to the wire protocol, or to a published surface, that - nothing outside this repository has been updated for. - - `compatibility.json` left behind by a version change. It is the - authority the dashboard pins its end-to-end suite and its - packaged engine to, so a stale entry ships a mismatched build. - - A test that passes while the behaviour it names is broken. - - Public-facing copy asserting something the code does not do. - - Post one review comment. Do not approve and do not request - changes; the required checks decide whether this merges. + - name: Post the review + if: steps.guard.outputs.skip != 'true' + env: + LLM_API_KEY: ${{ secrets.LLM_API_KEY }} + LLM_BASE_URL: ${{ vars.LLM_BASE_URL }} + LLM_MODEL: ${{ vars.LLM_MODEL }} + GH_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ github.event.pull_request.number }} + run: node scripts/llm_review.mjs diff --git a/scripts/llm_review.mjs b/scripts/llm_review.mjs new file mode 100644 index 0000000..3256fdc --- /dev/null +++ b/scripts/llm_review.mjs @@ -0,0 +1,186 @@ +#!/usr/bin/env node +/** + * Review a pull request with an OpenAI-compatible model and post the result. + * + * Written against the chat-completions shape rather than any one vendor's SDK, + * so OpenRouter, DeepInfra, or anything else speaking that protocol works by + * changing two variables. Nothing here is specific to a model. + * + * Configuration, all from the environment: + * + * LLM_API_KEY the provider key (secret) + * LLM_BASE_URL e.g. https://openrouter.ai/api/v1 + * LLM_MODEL the provider's exact model slug + * GH_TOKEN token used to read the diff and post the comment + * GITHUB_REPOSITORY, PR_NUMBER + * + * The model slug is deliberately not defaulted. A wrong default fails at the + * provider with an opaque error on somebody else's pull request, and guessing + * one is worse than refusing to start. + */ + +import { readFileSync } from "node:fs"; + +/** Diffs beyond this are truncated; large ones exhaust the context window. */ +const MAX_DIFF_BYTES = 120_000; + +const REQUIRED = ["LLM_API_KEY", "LLM_BASE_URL", "LLM_MODEL", "GH_TOKEN"]; + +export function missingConfig(env) { + return REQUIRED.filter((name) => !env[name]); +} + +/** + * Cut a diff down to a size the model can read, at a file boundary. + * + * Truncating mid-hunk hands the model a fragment it may read as the whole + * change, so the cut lands on the last `diff --git` before the limit and the + * omission is stated in the text. + */ +export function truncateDiff(diff, limit = MAX_DIFF_BYTES) { + if (diff.length <= limit) return { diff, truncated: false }; + const head = diff.slice(0, limit); + const lastFile = head.lastIndexOf("\ndiff --git "); + const cut = lastFile > 0 ? head.slice(0, lastFile) : head; + return { diff: cut, truncated: true }; +} + +/** The instructions the model is judged against. */ +export function buildPrompt({ number, title, body, diff, truncated, guide }) { + const omitted = truncated + ? "\n\nThis diff was truncated. Review only what is shown, and say so." + : ""; + return [ + `Review pull request #${number}: ${title}`, + body ? `\nIts description:\n${body}` : "", + guide ? `\nThe repository's own guidance is binding:\n${guide}` : "", + "\nReport only defects you can point at in the diff. Say plainly when", + "you find none; a review that invents work to look useful is worse than", + "a short one. Rank what you do find, most serious first.", + "\nWeigh hardest the things this repository has shipped broken:", + "- a new file under desktop/ missing from the files: list in", + " electron-builder.yml, which silently will not be packaged;", + "- a check that passes while the product is broken, such as asserting a", + " server answered rather than that the application works;", + "- a startup path that can leave the application running with no window;", + "- public-facing copy stating something the code does not do;", + "- the hard limits in the guidance: file and function length, `any`,", + " non-null assertions, suppression comments, dead code.", + "\nWrite GitHub-flavoured markdown. Do not approve or request changes;", + "the required checks decide whether this merges.", + `\nThe diff:\n\n${diff}${omitted}`, + ].join("\n"); +} + +/** Pull the assistant's text out of a chat-completions response. */ +export function extractMessage(payload) { + const text = payload?.choices?.[0]?.message?.content; + if (typeof text !== "string" || text.trim() === "") { + throw new Error( + `the model returned no message: ${JSON.stringify(payload).slice(0, 400)}`, + ); + } + return text.trim(); +} + +async function github(path, { token, accept, method, body }) { + const response = await fetch(`https://api.github.com${path}`, { + method: method ?? "GET", + headers: { + Accept: accept ?? "application/vnd.github+json", + Authorization: `Bearer ${token}`, + "Content-Type": "application/json", + }, + body: body ? JSON.stringify(body) : undefined, + }); + if (!response.ok) { + throw new Error( + `GitHub ${method ?? "GET"} ${path}: ${response.status} ` + + `${await response.text()}`, + ); + } + return accept?.includes("diff") ? response.text() : response.json(); +} + +async function complete({ baseUrl, key, model, prompt }) { + const response = await fetch(`${baseUrl.replace(/\/$/, "")}/chat/completions`, { + method: "POST", + headers: { + Authorization: `Bearer ${key}`, + "Content-Type": "application/json", + }, + body: JSON.stringify({ + model, + messages: [{ role: "user", content: prompt }], + temperature: 0.2, + }), + }); + if (!response.ok) { + throw new Error( + `${baseUrl}: ${response.status} ${await response.text()}`, + ); + } + return extractMessage(await response.json()); +} + +/** The repository's guidance, when it has some, as prompt context. */ +function readGuide() { + for (const name of ["AGENTS.md", "CLAUDE.md", "rules.md"]) { + try { + return readFileSync(name, "utf8"); + } catch { + // Try the next one; a repository without any is fine. + } + } + return ""; +} + +async function main() { + const env = process.env; + const missing = missingConfig(env); + if (missing.length > 0) { + console.error(`missing configuration: ${missing.join(", ")}`); + process.exit(78); + } + const repo = env.GITHUB_REPOSITORY; + const number = env.PR_NUMBER; + const token = env.GH_TOKEN; + + const pull = await github(`/repos/${repo}/pulls/${number}`, { token }); + const raw = await github(`/repos/${repo}/pulls/${number}`, { + token, + accept: "application/vnd.github.v3.diff", + }); + const { diff, truncated } = truncateDiff(raw); + + const review = await complete({ + baseUrl: env.LLM_BASE_URL, + key: env.LLM_API_KEY, + model: env.LLM_MODEL, + prompt: buildPrompt({ + number, + title: pull.title, + body: pull.body, + diff, + truncated, + guide: readGuide(), + }), + }); + + await github(`/repos/${repo}/issues/${number}/comments`, { + token, + method: "POST", + body: { + body: `${review}\n\nAutomated review — ${env.LLM_MODEL}. Not an ` + + `approval; the required checks decide whether this merges.`, + }, + }); + console.log(`posted a review on #${number}`); +} + +if (process.argv[1] && process.argv[1].endsWith("llm_review.mjs")) { + main().catch((error) => { + console.error(error.message); + process.exit(1); + }); +} diff --git a/scripts/llm_review.test.mjs b/scripts/llm_review.test.mjs new file mode 100644 index 0000000..404a355 --- /dev/null +++ b/scripts/llm_review.test.mjs @@ -0,0 +1,106 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { + buildPrompt, + extractMessage, + missingConfig, + truncateDiff, +} from "./llm_review.mjs"; + +test("names every missing setting at once", () => { + assert.deepEqual(missingConfig({}), [ + "LLM_API_KEY", + "LLM_BASE_URL", + "LLM_MODEL", + "GH_TOKEN", + ]); + assert.deepEqual( + missingConfig({ + LLM_API_KEY: "k", + LLM_BASE_URL: "u", + LLM_MODEL: "m", + GH_TOKEN: "t", + }), + [], + ); +}); + +/** + * An empty model slug must count as missing. Passing "" to the provider is a + * 400 on somebody else's pull request, reported as a review failure rather + * than as the configuration mistake it is. + */ +test("treats an empty setting as missing", () => { + const missing = missingConfig({ + LLM_API_KEY: "k", + LLM_BASE_URL: "u", + LLM_MODEL: "", + GH_TOKEN: "t", + }); + assert.deepEqual(missing, ["LLM_MODEL"]); +}); + +test("leaves a diff that fits alone", () => { + const diff = "diff --git a/a b/a\n+one\n"; + assert.deepEqual(truncateDiff(diff, 1000), { diff, truncated: false }); +}); + +/** + * The cut has to land on a file boundary. Handing the model half a hunk + * invites it to review a fragment as though it were the whole change. + */ +test("truncates a long diff at a file boundary", () => { + const first = `diff --git a/a b/a\n${"+x\n".repeat(50)}`; + const second = `diff --git a/b b/b\n${"+y\n".repeat(50)}`; + const { diff, truncated } = truncateDiff(first + second, first.length + 20); + assert.equal(truncated, true); + assert.equal(diff, first.replace(/\n$/, "")); + assert.ok(!diff.includes("b/b")); +}); + +test("still truncates when no boundary is reachable", () => { + const huge = `diff --git a/a b/a\n${"+x\n".repeat(200)}`; + const { diff, truncated } = truncateDiff(huge, 50); + assert.equal(truncated, true); + assert.equal(diff.length, 50); +}); + +test("tells the model when it is seeing a partial diff", () => { + const whole = buildPrompt({ number: 1, title: "t", diff: "d" }); + assert.doesNotMatch(whole, /truncated/); + const part = buildPrompt({ number: 1, title: "t", diff: "d", truncated: true }); + assert.match(part, /truncated/); +}); + +test("carries the repository's guidance into the prompt", () => { + const prompt = buildPrompt({ + number: 7, + title: "Add a thing", + diff: "d", + guide: "NEVER use any", + }); + assert.match(prompt, /#7: Add a thing/); + assert.match(prompt, /NEVER use any/); +}); + +test("reads the assistant message", () => { + const payload = { choices: [{ message: { content: " looks fine " } }] }; + assert.equal(extractMessage(payload), "looks fine"); +}); + +/** + * A provider that answers 200 with an error body, or with an empty message, + * must not post an empty review that reads like approval. + */ +test("refuses an empty or malformed completion", () => { + assert.throws(() => extractMessage({}), /no message/); + assert.throws( + () => extractMessage({ choices: [{ message: { content: " " } }] }), + /no message/, + ); + assert.throws( + () => extractMessage({ error: { message: "bad model" } }), + /no message/, + ); +}); diff --git a/tests/test_serving_bundle.py b/tests/test_serving_bundle.py index c50e0c5..3643411 100644 --- a/tests/test_serving_bundle.py +++ b/tests/test_serving_bundle.py @@ -148,7 +148,12 @@ def test_rebuild_is_bit_exact_in_a_subprocess(tmp_path: Any) -> None: env=dict(os.environ, PYTHONPATH=root), capture_output=True, text=True, - timeout=180, + # Generous because this is a cold Python process importing Torch, and + # the runners share one workstation with every other project's CI. At + # 180s it timed out under load and reported a bit-exactness failure + # that had not happened, which is the worst kind of red: it looks like + # the thing the test is named after. + timeout=600, ) assert result.returncode == 0, result.stderr assert result.stdout.strip() == "ok"