diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..d37d95f --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,34 @@ +version: 2 + +# Grouped so a week's updates arrive as one reviewable change per ecosystem +# rather than one per dependency. The packages under `packages/` share a lock +# discipline, so pip updates are grouped across the repository. +updates: + - package-ecosystem: pip + directory: / + schedule: + interval: weekly + open-pull-requests-limit: 3 + groups: + # Torch decides what the engine can run and how large every artifact + # is, so it lands on its own rather than inside a bulk update. + torch: + patterns: + - torch + - torchvision + python: + patterns: + - "*" + exclude-patterns: + - torch + - torchvision + + - package-ecosystem: github-actions + directory: / + schedule: + interval: weekly + open-pull-requests-limit: 3 + groups: + actions: + patterns: + - "*" diff --git a/.github/workflows/agent-issue.yml b/.github/workflows/agent-issue.yml new file mode 100644 index 0000000..40fd74a --- /dev/null +++ b/.github/workflows/agent-issue.yml @@ -0,0 +1,134 @@ +name: Work an issue + +# Turns a labelled issue into a pull request without a person opening a +# session. Label an issue `agent`, or comment `/agent`, and the work starts. +# +# The model is reached through aider, which speaks to any OpenAI-compatible +# endpoint, so the provider is configuration rather than code: +# +# secret LLM_API_KEY the provider key +# secret AUTOMATION_TOKEN a PAT or App token +# variable LLM_BASE_URL e.g. https://openrouter.ai/api/v1 +# variable LLM_MODEL the provider's exact model slug +# +# `AUTOMATION_TOKEN` is not optional. GitHub does not start workflow runs for +# events raised by the built-in `GITHUB_TOKEN`, so a pull request opened with +# it sits with no checks and can never satisfy auto-merge. +# +# The agent cannot merge, cannot approve, and cannot push to main: `main` +# requires a pull request and its checks. That is the whole reason it is +# reasonable to let a model write here unattended. + +on: + issues: + types: [labeled] + issue_comment: + types: [created] + +permissions: + contents: read + issues: write + +jobs: + work: + # `issue_comment` fires for pull requests too; `.issue.pull_request` is + # absent on a real issue, which is how the two are told apart. + if: > + (github.event_name == 'issues' && + github.event.label.name == 'agent') || + (github.event_name == 'issue_comment' && + !github.event.issue.pull_request && + startsWith(github.event.comment.body, '/agent')) + runs-on: [self-hosted, linux, x64, spikeforge-ci] + timeout-minutes: 60 + steps: + - name: Check the configuration exists + shell: bash + run: | + missing="" + [ -z "${{ secrets.LLM_API_KEY }}" ] && missing="$missing LLM_API_KEY" + [ -z "${{ secrets.AUTOMATION_TOKEN }}" ] && missing="$missing AUTOMATION_TOKEN" + [ -z "${{ vars.LLM_BASE_URL }}" ] && missing="$missing LLM_BASE_URL" + [ -z "${{ vars.LLM_MODEL }}" ] && missing="$missing LLM_MODEL" + if [ -n "$missing" ]; then + echo "::error::missing configuration:$missing" + exit 1 + fi + + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + token: ${{ secrets.AUTOMATION_TOKEN }} + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install the agent + run: python -m pip install --upgrade aider-chat + + - name: Write the instructions + shell: bash + env: + ISSUE_TITLE: ${{ github.event.issue.title }} + ISSUE_BODY: ${{ github.event.issue.body }} + run: | + # Through a file, not an argument: an issue body is arbitrary text + # and interpolating it into a shell command would let its contents + # decide what runs. + { + echo "Work this issue." + echo + echo "Title: $ISSUE_TITLE" + echo + echo "$ISSUE_BODY" + echo + echo "Read this repository's own guidance first -- rules.md, and" + echo "AGENTS.md or CLAUDE.md if present. It is binding." + echo + echo "Five published distributions and two applications build" + echo "against this repository. If you change a version, update" + echo "compatibility.json to match: the dashboard pins both its" + echo "end-to-end suite and the engine inside the desktop" + echo "application to that file, so leaving it behind ships a" + echo "mismatched build." + } > /tmp/agent-instructions.txt + cat /tmp/agent-instructions.txt + + - name: Make the change + env: + OPENAI_API_KEY: ${{ secrets.LLM_API_KEY }} + OPENAI_API_BASE: ${{ vars.LLM_BASE_URL }} + run: | + set -euo pipefail + branch="agent/issue-${{ github.event.issue.number }}" + git config user.name "spikeforge-agent" + git config user.email "contact@capsizegames.com" + git checkout -b "$branch" + aider --yes --no-analytics --no-gitignore \ + --model "openai/${{ vars.LLM_MODEL }}" \ + --message-file /tmp/agent-instructions.txt + if git diff --quiet HEAD; then + echo "::error::the agent produced no change" + exit 1 + fi + + - name: Check its work + run: | + python -m pip install -e ".[dev]" || python -m pip install -e . + python -m pytest -x -q + + - name: Open the pull request + env: + GH_TOKEN: ${{ secrets.AUTOMATION_TOKEN }} + run: | + set -euo pipefail + branch="agent/issue-${{ github.event.issue.number }}" + git push --set-upstream origin "$branch" + gh pr create --base main --head "$branch" \ + --title "${{ github.event.issue.title }}" \ + --body "Closes #${{ github.event.issue.number }}. + + Written by \`${{ vars.LLM_MODEL }}\` from the issue text. Nobody has + read this. The test suite passed on the runner before it was opened; + the required checks decide whether it merges." diff --git a/.github/workflows/agent-review.yml b/.github/workflows/agent-review.yml new file mode 100644 index 0000000..ae7a3f2 --- /dev/null +++ b/.github/workflows/agent-review.yml @@ -0,0 +1,59 @@ +name: Review a pull request + +# Reviews every pull request. With nobody reading the diff, this is the only +# thing that looks at a change as a change rather than as a test result. +# +# It talks to any OpenAI-compatible chat-completions endpoint -- OpenRouter, +# DeepInfra, or anything else speaking that protocol -- so the provider and +# model are configuration, not code: +# +# secret LLM_API_KEY the provider key +# variable LLM_BASE_URL e.g. https://openrouter.ai/api/v1 +# variable LLM_MODEL the provider's exact model slug +# +# It posts a comment. It cannot approve, block, or merge: the required checks +# decide that. + +on: + pull_request: + types: [opened, synchronize, reopened] + +permissions: + contents: read + pull-requests: write + +jobs: + review: + if: ${{ !github.event.pull_request.draft }} + runs-on: [self-hosted, linux, x64, spikeforge-ci] + timeout-minutes: 20 + steps: + - name: Skip without a provider + id: guard + shell: bash + run: | + if [ -z "${{ secrets.LLM_API_KEY }}" ] \ + || [ -z "${{ vars.LLM_BASE_URL }}" ] \ + || [ -z "${{ vars.LLM_MODEL }}" ]; then + echo "LLM_API_KEY, LLM_BASE_URL or LLM_MODEL is unset;" + echo "no review will be posted." + echo "skip=true" >> "$GITHUB_OUTPUT" + fi + + - uses: actions/checkout@v4 + if: steps.guard.outputs.skip != 'true' + + - uses: actions/setup-node@v4 + if: steps.guard.outputs.skip != 'true' + with: + node-version: "22" + + - name: Post the review + if: steps.guard.outputs.skip != 'true' + env: + LLM_API_KEY: ${{ secrets.LLM_API_KEY }} + LLM_BASE_URL: ${{ vars.LLM_BASE_URL }} + LLM_MODEL: ${{ vars.LLM_MODEL }} + GH_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ github.event.pull_request.number }} + run: node scripts/llm_review.mjs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e5c8e22..d85b9cd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,10 +21,16 @@ jobs: steps: - uses: actions/checkout@v4 + # No `cache: pip` anywhere in this file. This fleet's runners keep + # `_work` on another drive behind a symlink, and setup-python's + # dependency-file glob does not see through it: it reports that no + # pyproject.toml exists and fails the job before anything is built, + # while all seven of them sit in `packages/`. A self-hosted runner also + # keeps `~/.cache/pip` between jobs on its own, so the action's cache + # was buying nothing here even when it worked. - uses: actions/setup-python@v5 with: python-version: "3.12" - cache: pip - name: Install the core dev extra run: | @@ -56,7 +62,6 @@ jobs: - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - cache: pip - name: Install torch CPU wheels run: | @@ -89,7 +94,6 @@ jobs: - uses: actions/setup-python@v5 with: python-version: "3.12" - cache: pip - name: Install torch CPU wheels run: | @@ -141,7 +145,6 @@ jobs: - uses: actions/setup-python@v5 with: python-version: "3.12" - cache: pip - name: Install torch CPU wheels run: | @@ -182,7 +185,6 @@ jobs: - uses: actions/setup-python@v5 with: python-version: "3.12" - cache: pip - name: Install torch CPU wheels run: | @@ -243,7 +245,6 @@ jobs: - uses: actions/setup-python@v5 with: python-version: "3.12" - cache: pip - name: Install torch CPU wheels run: | @@ -310,7 +311,6 @@ jobs: - uses: actions/setup-python@v5 with: python-version: "3.12" - cache: pip - name: Install docs dependencies run: | @@ -337,7 +337,6 @@ jobs: - uses: actions/setup-python@v5 with: python-version: "3.12" - cache: pip - name: Install the renderer PyPI itself uses run: | @@ -405,7 +404,6 @@ jobs: - uses: actions/setup-python@v5 with: python-version: "3.12" - cache: pip - name: Build the core wheel with no extras run: | diff --git a/scripts/llm_review.mjs b/scripts/llm_review.mjs new file mode 100644 index 0000000..3256fdc --- /dev/null +++ b/scripts/llm_review.mjs @@ -0,0 +1,186 @@ +#!/usr/bin/env node +/** + * Review a pull request with an OpenAI-compatible model and post the result. + * + * Written against the chat-completions shape rather than any one vendor's SDK, + * so OpenRouter, DeepInfra, or anything else speaking that protocol works by + * changing two variables. Nothing here is specific to a model. + * + * Configuration, all from the environment: + * + * LLM_API_KEY the provider key (secret) + * LLM_BASE_URL e.g. https://openrouter.ai/api/v1 + * LLM_MODEL the provider's exact model slug + * GH_TOKEN token used to read the diff and post the comment + * GITHUB_REPOSITORY, PR_NUMBER + * + * The model slug is deliberately not defaulted. A wrong default fails at the + * provider with an opaque error on somebody else's pull request, and guessing + * one is worse than refusing to start. + */ + +import { readFileSync } from "node:fs"; + +/** Diffs beyond this are truncated; large ones exhaust the context window. */ +const MAX_DIFF_BYTES = 120_000; + +const REQUIRED = ["LLM_API_KEY", "LLM_BASE_URL", "LLM_MODEL", "GH_TOKEN"]; + +export function missingConfig(env) { + return REQUIRED.filter((name) => !env[name]); +} + +/** + * Cut a diff down to a size the model can read, at a file boundary. + * + * Truncating mid-hunk hands the model a fragment it may read as the whole + * change, so the cut lands on the last `diff --git` before the limit and the + * omission is stated in the text. + */ +export function truncateDiff(diff, limit = MAX_DIFF_BYTES) { + if (diff.length <= limit) return { diff, truncated: false }; + const head = diff.slice(0, limit); + const lastFile = head.lastIndexOf("\ndiff --git "); + const cut = lastFile > 0 ? head.slice(0, lastFile) : head; + return { diff: cut, truncated: true }; +} + +/** The instructions the model is judged against. */ +export function buildPrompt({ number, title, body, diff, truncated, guide }) { + const omitted = truncated + ? "\n\nThis diff was truncated. Review only what is shown, and say so." + : ""; + return [ + `Review pull request #${number}: ${title}`, + body ? `\nIts description:\n${body}` : "", + guide ? `\nThe repository's own guidance is binding:\n${guide}` : "", + "\nReport only defects you can point at in the diff. Say plainly when", + "you find none; a review that invents work to look useful is worse than", + "a short one. Rank what you do find, most serious first.", + "\nWeigh hardest the things this repository has shipped broken:", + "- a new file under desktop/ missing from the files: list in", + " electron-builder.yml, which silently will not be packaged;", + "- a check that passes while the product is broken, such as asserting a", + " server answered rather than that the application works;", + "- a startup path that can leave the application running with no window;", + "- public-facing copy stating something the code does not do;", + "- the hard limits in the guidance: file and function length, `any`,", + " non-null assertions, suppression comments, dead code.", + "\nWrite GitHub-flavoured markdown. Do not approve or request changes;", + "the required checks decide whether this merges.", + `\nThe diff:\n\n${diff}${omitted}`, + ].join("\n"); +} + +/** Pull the assistant's text out of a chat-completions response. */ +export function extractMessage(payload) { + const text = payload?.choices?.[0]?.message?.content; + if (typeof text !== "string" || text.trim() === "") { + throw new Error( + `the model returned no message: ${JSON.stringify(payload).slice(0, 400)}`, + ); + } + return text.trim(); +} + +async function github(path, { token, accept, method, body }) { + const response = await fetch(`https://api.github.com${path}`, { + method: method ?? "GET", + headers: { + Accept: accept ?? "application/vnd.github+json", + Authorization: `Bearer ${token}`, + "Content-Type": "application/json", + }, + body: body ? JSON.stringify(body) : undefined, + }); + if (!response.ok) { + throw new Error( + `GitHub ${method ?? "GET"} ${path}: ${response.status} ` + + `${await response.text()}`, + ); + } + return accept?.includes("diff") ? response.text() : response.json(); +} + +async function complete({ baseUrl, key, model, prompt }) { + const response = await fetch(`${baseUrl.replace(/\/$/, "")}/chat/completions`, { + method: "POST", + headers: { + Authorization: `Bearer ${key}`, + "Content-Type": "application/json", + }, + body: JSON.stringify({ + model, + messages: [{ role: "user", content: prompt }], + temperature: 0.2, + }), + }); + if (!response.ok) { + throw new Error( + `${baseUrl}: ${response.status} ${await response.text()}`, + ); + } + return extractMessage(await response.json()); +} + +/** The repository's guidance, when it has some, as prompt context. */ +function readGuide() { + for (const name of ["AGENTS.md", "CLAUDE.md", "rules.md"]) { + try { + return readFileSync(name, "utf8"); + } catch { + // Try the next one; a repository without any is fine. + } + } + return ""; +} + +async function main() { + const env = process.env; + const missing = missingConfig(env); + if (missing.length > 0) { + console.error(`missing configuration: ${missing.join(", ")}`); + process.exit(78); + } + const repo = env.GITHUB_REPOSITORY; + const number = env.PR_NUMBER; + const token = env.GH_TOKEN; + + const pull = await github(`/repos/${repo}/pulls/${number}`, { token }); + const raw = await github(`/repos/${repo}/pulls/${number}`, { + token, + accept: "application/vnd.github.v3.diff", + }); + const { diff, truncated } = truncateDiff(raw); + + const review = await complete({ + baseUrl: env.LLM_BASE_URL, + key: env.LLM_API_KEY, + model: env.LLM_MODEL, + prompt: buildPrompt({ + number, + title: pull.title, + body: pull.body, + diff, + truncated, + guide: readGuide(), + }), + }); + + await github(`/repos/${repo}/issues/${number}/comments`, { + token, + method: "POST", + body: { + body: `${review}\n\nAutomated review — ${env.LLM_MODEL}. Not an ` + + `approval; the required checks decide whether this merges.`, + }, + }); + console.log(`posted a review on #${number}`); +} + +if (process.argv[1] && process.argv[1].endsWith("llm_review.mjs")) { + main().catch((error) => { + console.error(error.message); + process.exit(1); + }); +} diff --git a/scripts/llm_review.test.mjs b/scripts/llm_review.test.mjs new file mode 100644 index 0000000..404a355 --- /dev/null +++ b/scripts/llm_review.test.mjs @@ -0,0 +1,106 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { + buildPrompt, + extractMessage, + missingConfig, + truncateDiff, +} from "./llm_review.mjs"; + +test("names every missing setting at once", () => { + assert.deepEqual(missingConfig({}), [ + "LLM_API_KEY", + "LLM_BASE_URL", + "LLM_MODEL", + "GH_TOKEN", + ]); + assert.deepEqual( + missingConfig({ + LLM_API_KEY: "k", + LLM_BASE_URL: "u", + LLM_MODEL: "m", + GH_TOKEN: "t", + }), + [], + ); +}); + +/** + * An empty model slug must count as missing. Passing "" to the provider is a + * 400 on somebody else's pull request, reported as a review failure rather + * than as the configuration mistake it is. + */ +test("treats an empty setting as missing", () => { + const missing = missingConfig({ + LLM_API_KEY: "k", + LLM_BASE_URL: "u", + LLM_MODEL: "", + GH_TOKEN: "t", + }); + assert.deepEqual(missing, ["LLM_MODEL"]); +}); + +test("leaves a diff that fits alone", () => { + const diff = "diff --git a/a b/a\n+one\n"; + assert.deepEqual(truncateDiff(diff, 1000), { diff, truncated: false }); +}); + +/** + * The cut has to land on a file boundary. Handing the model half a hunk + * invites it to review a fragment as though it were the whole change. + */ +test("truncates a long diff at a file boundary", () => { + const first = `diff --git a/a b/a\n${"+x\n".repeat(50)}`; + const second = `diff --git a/b b/b\n${"+y\n".repeat(50)}`; + const { diff, truncated } = truncateDiff(first + second, first.length + 20); + assert.equal(truncated, true); + assert.equal(diff, first.replace(/\n$/, "")); + assert.ok(!diff.includes("b/b")); +}); + +test("still truncates when no boundary is reachable", () => { + const huge = `diff --git a/a b/a\n${"+x\n".repeat(200)}`; + const { diff, truncated } = truncateDiff(huge, 50); + assert.equal(truncated, true); + assert.equal(diff.length, 50); +}); + +test("tells the model when it is seeing a partial diff", () => { + const whole = buildPrompt({ number: 1, title: "t", diff: "d" }); + assert.doesNotMatch(whole, /truncated/); + const part = buildPrompt({ number: 1, title: "t", diff: "d", truncated: true }); + assert.match(part, /truncated/); +}); + +test("carries the repository's guidance into the prompt", () => { + const prompt = buildPrompt({ + number: 7, + title: "Add a thing", + diff: "d", + guide: "NEVER use any", + }); + assert.match(prompt, /#7: Add a thing/); + assert.match(prompt, /NEVER use any/); +}); + +test("reads the assistant message", () => { + const payload = { choices: [{ message: { content: " looks fine " } }] }; + assert.equal(extractMessage(payload), "looks fine"); +}); + +/** + * A provider that answers 200 with an error body, or with an empty message, + * must not post an empty review that reads like approval. + */ +test("refuses an empty or malformed completion", () => { + assert.throws(() => extractMessage({}), /no message/); + assert.throws( + () => extractMessage({ choices: [{ message: { content: " " } }] }), + /no message/, + ); + assert.throws( + () => extractMessage({ error: { message: "bad model" } }), + /no message/, + ); +}); diff --git a/tests/test_serving_bundle.py b/tests/test_serving_bundle.py index c50e0c5..3643411 100644 --- a/tests/test_serving_bundle.py +++ b/tests/test_serving_bundle.py @@ -148,7 +148,12 @@ def test_rebuild_is_bit_exact_in_a_subprocess(tmp_path: Any) -> None: env=dict(os.environ, PYTHONPATH=root), capture_output=True, text=True, - timeout=180, + # Generous because this is a cold Python process importing Torch, and + # the runners share one workstation with every other project's CI. At + # 180s it timed out under load and reported a bit-exactness failure + # that had not happened, which is the worst kind of red: it looks like + # the thing the test is named after. + timeout=600, ) assert result.returncode == 0, result.stderr assert result.stdout.strip() == "ok"