From e12e486b1473c1f671c3931f753469ea8801d2bd Mon Sep 17 00:00:00 2001 From: KB Date: Thu, 30 Jul 2026 09:33:54 +0530 Subject: [PATCH 01/56] prototype: managed-compute guardrail state machine Throwaway. Answers whether agent-proposes/Atlas-decides holds together, and who enforces an approved deadline. POST /leases and /release are faked because the real ones provision GPUs and bill. GET /options and POST /estimate are free, so [o] probes them for real to verify response shapes. Models the Atlas behaviour read from source but NOT verified at runtime, so each assumption is a toggle rather than a baked-in fact: gate mode (first-hour vs total), server-side deadline present or absent, client alive or dead, release succeeding or failing. Run: cd backend/cli && bun run prototype:guardrail --- backend/cli/package.json | 3 +- .../src/compute/PROTOTYPE-guardrail-model.ts | 232 ++++++++++++++++ .../src/compute/PROTOTYPE-guardrail-repl.ts | 260 ++++++++++++++++++ 3 files changed, 494 insertions(+), 1 deletion(-) create mode 100644 backend/cli/src/compute/PROTOTYPE-guardrail-model.ts create mode 100644 backend/cli/src/compute/PROTOTYPE-guardrail-repl.ts diff --git a/backend/cli/package.json b/backend/cli/package.json index 49b6680e..0db00965 100644 --- a/backend/cli/package.json +++ b/backend/cli/package.json @@ -11,7 +11,8 @@ "typecheck": "tsgo --noEmit", "test": "bun test --timeout 15000", "build": "bun run script/build.ts", - "dev": "bun run --conditions=browser ./src/index.ts" + "dev": "bun run --conditions=browser ./src/index.ts", + "prototype:guardrail": "bun run ./src/compute/PROTOTYPE-guardrail-repl.ts" }, "bin": { "openscience": "./bin/openscience" diff --git a/backend/cli/src/compute/PROTOTYPE-guardrail-model.ts b/backend/cli/src/compute/PROTOTYPE-guardrail-model.ts new file mode 100644 index 00000000..720c975d --- /dev/null +++ b/backend/cli/src/compute/PROTOTYPE-guardrail-model.ts @@ -0,0 +1,232 @@ +/** + * PROTOTYPE — throwaway. Not wired into the product. Delete or lift, don't ship. + * + * ── The question ──────────────────────────────────────────────────────────── + * We want managed GPU compute where the AGENT PROPOSES a duration and ATLAS + * DECIDES whether it can be afforded — because OpenScience is open-source, so a + * decision made client-side is a decision a fork can delete. + * + * Does that state machine hold together? Four sub-questions: + * (a) what must an agent see to propose a sensible duration? + * (b) what does a rejection look like, and can the agent act on it? + * (c) what happens at the deadline, and who actually enforces it? + * (d) what does the client do when release fails or can't be confirmed? + * + * ── What is real vs modelled ──────────────────────────────────────────────── + * Atlas today (read from ~/codes/InkVell/atlas, origin/main): + * - POST /api/compute/estimate returns a RATE only. No duration, no total. + * - POST /api/compute/leases has NO duration field and NO expires_at column. + * It derives a flat 24h TTL from the user's plan, and verifies the wallet + * can fund ONE HOUR (402 otherwise). + * - The only server clocks are that 24h TTL and a 10-min heartbeat check. + * - All four leasable providers silently DISCARD the timeout argument, so + * provider-native auto-termination does not exist. Something must call + * release, or the VM runs on. + * + * That last group is why this prototype exists: it lets you toggle each + * assumption and watch the money. + * + * NOTE: the Atlas code above was read but NOT verified at runtime, so this + * models its apparent behaviour, not confirmed behaviour. + * ──────────────────────────────────────────────────────────────────────────── + */ + +/** What the agent asks for. Untrusted — it only ever proposes. */ +export interface Proposal { + provider: string + sku: string + /** The agent's own guess at how long the work needs. */ + hours: number +} + +/** What Atlas returns from /estimate today: a rate, and nothing about duration. */ +export interface Quote { + funding: "managed" | "byok" | "unavailable" + rateCentsPerHour: number + balanceCents: number +} + +/** + * Which affordability rule the server applies. + * - "first-hour" is what Atlas does TODAY: can you fund one hour? + * - "total" is the proposed change: can you fund rate x approved_hours? + * Toggling this is the point — see what "first-hour" approves. + */ +export type GateMode = "first-hour" | "total" + +export interface DecideInput { + proposal: Proposal + quote: Quote + /** Server-side ceiling. Atlas uses plan.gpu_sandbox_max_ttl_hours (24, all plans). */ + planTtlHours: number + mode: GateMode +} + +export type Verdict = + | { + ok: true + /** May be less than proposed — the plan TTL clamps it. */ + approvedHours: number + clampedFrom?: number + costCents: number + /** Free for BYOK; the wallet is never touched. */ + billed: boolean + } + | { + ok: false + /** Mirrors the shapes Atlas actually returns. */ + code: "insufficient_credit" | "unavailable" | "invalid" + message: string + /** The actionable part: what WOULD be approved. This is (b). */ + remedy?: { affordableHours: number; neededCents: number; availableCents: number } + } + +export function quoteTotal(rateCentsPerHour: number, hours: number): number { + return Math.round(rateCentsPerHour * hours) +} + +/** + * THE decision. Runs server-side in the real design, which is the whole point: + * a fork of the open-source client cannot reach it. + */ +export function decide(input: DecideInput): Verdict { + const { proposal, quote, planTtlHours, mode } = input + + if (quote.funding === "unavailable") + return { ok: false, code: "unavailable", message: `${proposal.sku} is not available on ${proposal.provider}.` } + + if (!Number.isFinite(proposal.hours) || proposal.hours <= 0) + return { ok: false, code: "invalid", message: "Proposed duration must be a positive number of hours." } + + const approvedHours = Math.min(proposal.hours, planTtlHours) + const clampedFrom = approvedHours < proposal.hours ? proposal.hours : undefined + + // BYOK runs on the user's own provider account. We never bill, so there is + // nothing to gate on affordability. + if (quote.funding === "byok") + return { ok: true, approvedHours, clampedFrom, costCents: 0, billed: false } + + const total = quoteTotal(quote.rateCentsPerHour, approvedHours) + const required = mode === "first-hour" ? quote.rateCentsPerHour : total + + if (required > quote.balanceCents) { + const affordableHours = quote.rateCentsPerHour > 0 ? quote.balanceCents / quote.rateCentsPerHour : 0 + return { + ok: false, + code: "insufficient_credit", + message: `Needs ${(required / 100).toFixed(2)} USD, wallet has ${(quote.balanceCents / 100).toFixed(2)} USD.`, + remedy: { + affordableHours: Math.floor(affordableHours * 10) / 10, + neededCents: required, + availableCents: quote.balanceCents, + }, + } + } + + return { ok: true, approvedHours, clampedFrom, costCents: total, billed: true } +} + +// ── The running lease ─────────────────────────────────────────────────────── + +export type Phase = + | "running" + | "past-deadline" + | "released" + /** Nobody is left to call release. This is the financial exposure. */ + | "orphaned" + +export interface Lease { + id: string + rateCentsPerHour: number + approvedHours: number + startedAtMs: number + /** Absent when the server has no deadline column — i.e. Atlas as it stands. */ + expiresAtMs?: number + phase: Phase + /** Wall-clock cost so far. Atlas meters per second, never rounding up to an hour. */ + spentCents: number + releaseAttempts: number + note?: string +} + +export interface World { + /** Does the SERVER hold the deadline and enforce it in its sweep? */ + serverEnforcesDeadline: boolean + /** Is our process still alive to run its own timer? */ + clientAlive: boolean + /** Simulate release calls failing (provider flake, network, auth). */ + releaseFails: boolean + /** Atlas's absolute backstop: plan.gpu_sandbox_max_ttl_hours. */ + planTtlHours: number +} + +export function spend(lease: Lease, nowMs: number): number { + const secs = Math.max(0, (nowMs - lease.startedAtMs) / 1000) + return Math.round((lease.rateCentsPerHour * secs) / 3600) +} + +export function overrunHours(lease: Lease, nowMs: number): number { + const elapsed = (nowMs - lease.startedAtMs) / 3_600_000 + return Math.max(0, elapsed - lease.approvedHours) +} + +/** What SHOULD happen next, given the world. Pure — the caller applies it. */ +export type Action = + | { kind: "none" } + | { kind: "server-releases"; why: string } + | { kind: "client-should-release"; why: string } + | { kind: "nobody-will-release"; why: string; exposureCents: number } + +export function evaluate(lease: Lease, nowMs: number, world: World): Action { + if (lease.phase === "released" || lease.phase === "orphaned") return { kind: "none" } + + const elapsedHours = (nowMs - lease.startedAtMs) / 3_600_000 + const pastApproved = elapsedHours >= lease.approvedHours + const pastPlanTtl = elapsedHours >= world.planTtlHours + + // The absolute backstop fires regardless of anything else. + if (pastPlanTtl) return { kind: "server-releases", why: `plan TTL of ${world.planTtlHours}h reached` } + + if (!pastApproved) return { kind: "none" } + + // Past the approved duration. Who notices? + if (world.serverEnforcesDeadline && lease.expiresAtMs !== undefined) + return { kind: "server-releases", why: "server-side expires_at reached" } + + if (world.clientAlive) return { kind: "client-should-release", why: "client deadline timer fired" } + + // Nobody is watching. This is the case the design has to prevent. + const untilBackstopH = Math.max(0, world.planTtlHours - elapsedHours) + return { + kind: "nobody-will-release", + why: `no server deadline and the client is gone; billing until the ${world.planTtlHours}h backstop`, + exposureCents: Math.round(lease.rateCentsPerHour * untilBackstopH), + } +} + +export type ReleaseResult = + | { ok: true; already: boolean } + | { ok: false; retryable: boolean; message: string } + +/** + * Release, modelling the outcomes Atlas actually returns. 409 (already + * released) is SUCCESS for our purposes — the VM is gone either way. + */ +export function attemptRelease(lease: Lease, world: World): ReleaseResult { + if (lease.phase === "released") return { ok: true, already: true } + if (world.releaseFails) + return { ok: false, retryable: true, message: "release failed (network/provider); VM may still be running" } + return { ok: true, already: false } +} + +/** Fail closed: never assume cleanup happened. Give up only after N tries. */ +export const MAX_RELEASE_ATTEMPTS = 3 + +export function formatCents(cents: number): string { + return `$${(cents / 100).toFixed(2)}` +} + +export function formatHours(h: number): string { + if (h < 1) return `${Math.round(h * 60)}m` + return `${h.toFixed(h < 10 ? 1 : 0)}h` +} diff --git a/backend/cli/src/compute/PROTOTYPE-guardrail-repl.ts b/backend/cli/src/compute/PROTOTYPE-guardrail-repl.ts new file mode 100644 index 00000000..9e893417 --- /dev/null +++ b/backend/cli/src/compute/PROTOTYPE-guardrail-repl.ts @@ -0,0 +1,260 @@ +#!/usr/bin/env bun +/** + * PROTOTYPE — throwaway TUI. Run: cd backend/cli && bun run prototype:guardrail + * + * Drives the managed-compute guardrail state machine in + * PROTOTYPE-guardrail-model.ts. See that file for the question this answers. + * + * POST /leases and /release are FAKED — the real ones provision GPUs and bill + * real money. GET /options and POST /estimate are free, so [o] hits them for + * real to verify response shapes (needs ATLAS_TOKEN + optional ATLAS_BASE). + * + * This shell is disposable. The model next door is the liftable part. + */ +import { + decide, + evaluate, + attemptRelease, + spend, + overrunHours, + quoteTotal, + formatCents, + formatHours, + MAX_RELEASE_ATTEMPTS, + type Proposal, + type Quote, + type GateMode, + type Lease, + type World, + type Verdict, +} from "./PROTOTYPE-guardrail-model" + +const B = "\x1b[1m" +const D = "\x1b[2m" +const R = "\x1b[0m" +const G = "\x1b[32m" +const Y = "\x1b[33m" +const RED = "\x1b[31m" +const C = "\x1b[36m" +const INV = "\x1b[7m" + +const HOUR = 3_600_000 + +// A plausible managed H100 rate, from the Modal table in the Atlas source. +let quote: Quote = { funding: "managed", rateCentsPerHour: 699, balanceCents: 1500 } +let proposal: Proposal = { provider: "lambda", sku: "gpu_1x_h100_pcie", hours: 4 } +let mode: GateMode = "first-hour" +let world: World = { + serverEnforcesDeadline: false, + clientAlive: true, + releaseFails: false, + planTtlHours: 24, +} +let verdict: Verdict | null = null +let lease: Lease | null = null +let now = Date.now() +let realProbe: string[] = [] +let log: string[] = [] + +function say(line: string) { + log.unshift(line) + log = log.slice(0, 6) +} + +function launch() { + verdict = decide({ proposal, quote, planTtlHours: world.planTtlHours, mode }) + if (!verdict.ok) { + say(`${RED}REJECTED${R} ${verdict.code} — ${verdict.message}`) + if (verdict.remedy) + say( + ` ${D}remedy the agent can act on:${R} propose ≤ ${formatHours(verdict.remedy.affordableHours)} ` + + `(needs ${formatCents(verdict.remedy.neededCents)}, has ${formatCents(verdict.remedy.availableCents)})`, + ) + lease = null + return + } + lease = { + id: `lease-${Math.abs(now % 100000)}`, + rateCentsPerHour: quote.funding === "byok" ? 0 : quote.rateCentsPerHour, + approvedHours: verdict.approvedHours, + startedAtMs: now, + // The server only records a deadline if we build that column. + expiresAtMs: world.serverEnforcesDeadline ? now + verdict.approvedHours * HOUR : undefined, + phase: "running", + spentCents: 0, + releaseAttempts: 0, + } + const clamp = verdict.clampedFrom ? ` ${Y}(clamped from ${formatHours(verdict.clampedFrom)})${R}` : "" + say( + `${G}APPROVED${R} ${formatHours(verdict.approvedHours)}${clamp} — ` + + `${verdict.billed ? formatCents(verdict.costCents) : "free (BYOK)"}`, + ) +} + +function tick(hours: number) { + now += hours * HOUR + if (!lease || lease.phase === "released" || lease.phase === "orphaned") return + lease.spentCents = spend(lease, now) + const action = evaluate(lease, now, world) + if (action.kind === "server-releases") { + lease.phase = "released" + lease.note = action.why + say(`${G}server released${R} — ${action.why} · billed ${formatCents(lease.spentCents)}`) + } else if (action.kind === "client-should-release") { + lease.phase = "past-deadline" + say(`${Y}deadline passed${R} — ${action.why}. Press [r] to release.`) + } else if (action.kind === "nobody-will-release") { + lease.phase = "orphaned" + lease.note = action.why + say(`${RED}${INV} ORPHANED ${R} ${action.why}`) + say(` ${RED}projected extra spend: ${formatCents(action.exposureCents)}${R}`) + } +} + +function release() { + if (!lease || lease.phase === "released") return say(`${D}nothing to release${R}`) + lease.releaseAttempts++ + const res = attemptRelease(lease, world) + if (res.ok) { + lease.phase = "released" + lease.spentCents = spend(lease, now) + say(`${G}released${R}${res.already ? " (409 already — still success)" : ""} · billed ${formatCents(lease.spentCents)}`) + return + } + say(`${RED}release failed${R} (attempt ${lease.releaseAttempts}/${MAX_RELEASE_ATTEMPTS}) — ${res.message}`) + if (lease.releaseAttempts >= MAX_RELEASE_ATTEMPTS) { + lease.phase = "orphaned" + lease.note = "release failed repeatedly — fail closed, surface loudly" + say(`${RED}${INV} FAIL CLOSED ${R} gave up after ${MAX_RELEASE_ATTEMPTS} — must alert, never assume cleanup`) + } +} + +async function probeReal() { + const base = process.env["ATLAS_BASE"] || "https://app.syntheticsciences.ai" + const token = process.env["ATLAS_TOKEN"] + realProbe = [`${D}base ${base}${R}`] + if (!token) { + realProbe.push(`${Y}set ATLAS_TOKEN=thk_… to probe the real (free) endpoints${R}`) + return + } + for (const [label, path, body] of [ + ["GET /api/compute/options", "/api/compute/options", null], + ["POST /api/compute/estimate", "/api/compute/estimate", { provider: proposal.provider, sku: proposal.sku }], + ] as const) { + try { + const res = await fetch(`${base}${path}`, { + method: body ? "POST" : "GET", + headers: { Authorization: `Bearer ${token}`, "Content-Type": "application/json" }, + body: body ? JSON.stringify(body) : undefined, + }) + const text = (await res.text()).slice(0, 160).replace(/\s+/g, " ") + realProbe.push(`${res.ok ? G : RED}${res.status}${R} ${label} ${D}${text}${R}`) + } catch (err) { + realProbe.push(`${RED}ERR${R} ${label} ${D}${err instanceof Error ? err.message : String(err)}${R}`) + } + } +} + +function render() { + console.clear() + console.log(`${B}managed-compute guardrail — state-machine prototype${R}`) + console.log(`${D}Q: does agent-proposes / Atlas-decides hold up? who enforces the deadline?${R}\n`) + + const total = quoteTotal(quote.rateCentsPerHour, Math.min(proposal.hours, world.planTtlHours)) + console.log( + `${B}PROPOSAL${R} ${proposal.sku} ${D}on${R} ${proposal.provider} ` + + `${C}${formatHours(proposal.hours)}${R} ${D}→ ${formatCents(total)} at ${formatCents(quote.rateCentsPerHour)}/h${R}`, + ) + console.log( + `${B}WALLET${R} ${formatCents(quote.balanceCents)} ${B}FUNDING${R} ${quote.funding} ` + + `${B}PLAN TTL${R} ${world.planTtlHours}h`, + ) + console.log( + `${B}GATE${R} ${mode === "first-hour" ? `${Y}first-hour (Atlas today)${R}` : `${G}total (proposed)${R}`}` + + ` ${B}SERVER DEADLINE${R} ${world.serverEnforcesDeadline ? `${G}yes${R}` : `${RED}no (Atlas today)${R}`}` + + ` ${B}CLIENT${R} ${world.clientAlive ? `${G}alive${R}` : `${RED}dead${R}`}` + + ` ${B}RELEASE${R} ${world.releaseFails ? `${RED}failing${R}` : `${G}ok${R}`}\n`, + ) + + if (lease) { + const el = (now - lease.startedAtMs) / HOUR + const over = overrunHours(lease, now) + const phase = + lease.phase === "running" + ? `${G}running${R}` + : lease.phase === "past-deadline" + ? `${Y}past-deadline${R}` + : lease.phase === "released" + ? `${D}released${R}` + : `${RED}${INV} ORPHANED ${R}` + console.log(`${B}LEASE${R} ${lease.id} ${phase}`) + console.log( + ` ${D}approved${R} ${formatHours(lease.approvedHours)} ${D}elapsed${R} ${formatHours(el)}` + + (over > 0 ? ` ${RED}overrun ${formatHours(over)}${R}` : "") + + ` ${D}billed${R} ${formatCents(spend(lease, now))}`, + ) + if (lease.note) console.log(` ${D}${lease.note}${R}`) + } else console.log(`${D}no lease — press [enter] to submit the proposal${R}`) + + if (realProbe.length) { + console.log(`\n${B}REAL API PROBE${R} ${D}(free endpoints only)${R}`) + for (const l of realProbe) console.log(` ${l}`) + } + + if (log.length) { + console.log(`\n${B}LOG${R}`) + for (const l of log) console.log(` ${l}`) + } + + console.log( + `\n${D}[h/H] hours -/+ [b/B] wallet -/+ [g] gate mode [s] server deadline [x] kill client` + + `\n[f] release failure [enter] submit [t] +1h [T] +6h [r] release [o] probe real [n] reset [q] quit${R}`, + ) +} + +function reset() { + now = Date.now() + lease = null + verdict = null + log = [] + say(`${D}reset${R}`) +} + +async function main() { + if (!process.stdin.isTTY || typeof process.stdin.setRawMode !== "function") { + console.error("This prototype is interactive — run it in a terminal:\n bun run prototype:guardrail\n") + process.exit(1) + } + process.stdin.setRawMode(true) + process.stdin.resume() + render() + for await (const chunk of process.stdin) { + const k = chunk.toString() + if (k === "q" || k === "") break + if (k === "h") proposal.hours = Math.max(0.5, proposal.hours - 0.5) + else if (k === "H") proposal.hours = Math.min(48, proposal.hours + 0.5) + else if (k === "b") quote.balanceCents = Math.max(0, quote.balanceCents - 500) + else if (k === "B") quote.balanceCents += 500 + else if (k === "g") mode = mode === "first-hour" ? "total" : "first-hour" + else if (k === "s") world.serverEnforcesDeadline = !world.serverEnforcesDeadline + else if (k === "x") world.clientAlive = !world.clientAlive + else if (k === "f") world.releaseFails = !world.releaseFails + else if (k === "\r" || k === "\n") launch() + else if (k === "t") tick(1) + else if (k === "T") tick(6) + else if (k === "r") release() + else if (k === "n") reset() + else if (k === "o") { + say(`${D}probing real endpoints…${R}`) + render() + await probeReal() + } + render() + } + process.stdin.setRawMode(false) + console.clear() + console.log("prototype exited\n") + process.exit(0) +} + +main() From 9314dccf309eefe278aa7dbdd6d46c08687e6c2c Mon Sep 17 00:00:00 2001 From: KB Date: Thu, 30 Jul 2026 15:24:42 +0530 Subject: [PATCH 02/56] spec: managed compute budget cap Rewritten budget-first. The earlier duration-first draft grew a warning event, an extension path, a per-lease expires_at, and client-side early release -- every one an attempt to make a run SUCCEED rather than to stop a bill running away, and each depending on something that does not exist: a completion signal for arbitrary SSH commands, a live agent outliving a long job, or checkpointing. Separating safety (don't overspend) from productivity (don't waste money on a truncated run) collapses the design to two small Atlas changes: 1. Make compute_grants.hard_cap_cents a real running cap. It is debited once for hour one today and never re-checked, so it reads like a spend ceiling and is not one. Re-debit per billing tick; exhaustion releases via the path that already fires when the wallet empties. 2. Accept an optional budget_cents on lease creation and size the grant to it. Optional matters -- the dashboard and atlas compute:up both call that endpoint without it. The agent proposes a dollar budget rather than a duration, because it can judge whether an experiment is worth $30 and cannot predict whether a novel training run converges in 4 hours. Budget exhaustion is then arithmetic the server observes, so no completion signal, extension path, or agent liveness is required. Also records two corrections: atlas compute:up DOES exist (cli commands.mjs:922), so the system prompt is not broken and that acceptance criterion is dropped; and compute:up already takes max_price and dry_run. Accepted cost, stated explicitly: a budget-exhausted job loses its work. Roadmap 56 (checkpointing) is the tracked follow-on. Item 61 splits out. --- docs/specs/compute-guardrails-design.md | 238 ++++++++++++++++++++++++ 1 file changed, 238 insertions(+) create mode 100644 docs/specs/compute-guardrails-design.md diff --git a/docs/specs/compute-guardrails-design.md b/docs/specs/compute-guardrails-design.md new file mode 100644 index 00000000..209c0a9c --- /dev/null +++ b/docs/specs/compute-guardrails-design.md @@ -0,0 +1,238 @@ +# Managed compute budget cap — design + +Status: approved, ready for implementation planning +Date: 2026-07-30 +Roadmap items: **55** (budget guardrails + kill switches), **103** (cost approval gates), and the gate half of +**51/2** (agent-facing compute tool) + +Spans two repos. This document is the **contract**; each side gets its own implementation plan because they +have separate test runs and deploys. + +- `atlas` (Python/FastAPI) — the decision and the enforcement +- `openscience` (Bun/TypeScript) — a thin client that relays a proposal and obeys the verdict + +> **The Atlas behaviour described below was read from source, not verified at runtime.** Treat every claim about +> current behaviour as "what the code appears to do" and confirm against the deployed service before relying on +> it. The prototype models each such assumption as a toggle for exactly this reason. + +## The problem, stated narrowly + +The agent can start GPU work and nothing bounds the cost. `POST /api/compute/leases` checks only that the +wallet can fund **one hour**, so a 4-hour job on a 1-hour balance is approved and then strands mid-run — the +prototype confirmed this: a 4-hour H100 at $6.99/h against a $15.00 wallet is **approved at a cost of $27.96**. + +Roadmap **51/2** — exposing compute to the agent as a tool — is gated on fixing that. Wiring an LLM to an +unmetered spend path would be materially worse than today's human-only exposure. + +## The scoping decision that shapes everything below + +Two different problems kept getting tangled during design. They are separated deliberately here. + +| | Problem | Status | +| ---------------- | ---------------------------------------------- | ----------------------------------------------- | +| **Safety** | Don't spend more than was authorised | **This spec. Completely solvable now.** | +| **Productivity** | Don't waste money on a run that gets truncated | Needs roadmap **56** (checkpointing). Not this. | + +An earlier draft of this design tried to solve both, and grew a warning event, an extension-request path, a +per-lease `expires_at`, and client-side early release. Every one of those was an attempt to make a run +_succeed_, not to stop a bill running away — and each depended on something that does not exist (a completion +signal for arbitrary SSH commands, a live agent session outliving a long job, or checkpointing). + +**This spec solves safety completely and does not pretend to solve productivity.** What that costs is stated +under "What you are accepting". + +## Trust boundary + +**The agent proposes; the server decides.** The proposed budget is untrusted input. + +This is structural, not stylistic. OpenScience is open-source, so a decision made client-side is one a fork can +delete — and the agent has `bash`, so it is running _inside_ the client. Only a decision made over HTTP, behind +auth, in a process the agent is not running, is one it cannot influence. The gate is real only if it is remote. + +Corollary: **OpenScience holds no pricing logic, no balance logic, and no approval logic.** It relays a proposal +and obeys a verdict. + +## Why budget rather than duration + +The agent proposes _"this is worth up to $30"_, not _"this needs 4 hours"_. + +**An agent can judge the first and cannot predict the second.** "Is this experiment worth $30?" is a value +judgement LLMs handle well. "Will this converge in 4 hours?" is a prediction about a novel training run that +nobody can make — and a wrong duration estimate is what created the need for an extension path, a warning +window, and a re-estimation loop in the earlier draft. + +Three consequences, all simplifying: + +- **No completion signal needed.** Budget exhaustion is a fact the server observes. There is nothing to detect, + which matters because a lease is a VM, not a job — `POST /compute/leases` has no notion of the work running on + it, and the only completion signal anywhere (`agent_telemetry` rows with `done`/`error_trace`) is written by + the Atlas agent runtime, not by an arbitrary command run over SSH. +- **No extension path needed.** Exhaustion is arithmetic, not a guess that might need revising. +- **No agent liveness needed.** The server enforces whether or not the session that started the job survived. + +And it uses a primitive that already exists rather than adding one — see below. + +## What to build + +### Atlas — change 1: make `hard_cap_cents` a real running cap + +`compute_grants.hard_cap_cents` already reads like a running spend ceiling. It is not one: `acquire_lease` +debits it **once, for one hour**, and the billing tick then calls `usage_service.charge` + `mark_billed` without +ever calling `debit_grant` again. So `spent_cents` freezes at hour one and the atomic ceiling in +`compute_repo.debit_grant` (`AND (spent_cents + ?) <= hard_cap_cents`) is never re-evaluated. + +The fix: **the billing tick re-debits the grant by the same delta it charges.** When the debit would exceed the +cap, release the lease — reusing the path that already fires when the wallet runs dry +(`compute_billing_service` catching `InsufficientCredits` → `_safe_release`). + +**This is not a double charge.** The grant and the wallet are different ledgers: the wallet is money, the grant +is an authorisation envelope drawn against it. The tick already debits the wallet via `usage_service.charge`; it +will now also decrement the envelope. Two records, one charge. An implementer who "de-duplicates" these has +removed the cap. + +This is the whole feature. It is a small change to money-handling code, so it lands as **its own commit with its +own tests**, separate from change 2, so a failing `fly deploy -a thesis-dev` can be attributed to one or the +other. + +### Atlas — change 2: accept a budget on lease creation + +``` +POST /api/compute/leases +Request: { provider, sku, region?, node_id?, budget_cents?: number } +``` + +`budget_cents` is **optional**, and this matters: the Atlas dashboard already calls this endpoint +(`frontend/src/api/account.ts:284`) without it, and so does `atlas compute:up`. Absent means today's behaviour — +grant sized to the plan TTL at the hourly rate. Present means the grant is sized to `budget_cents` instead, and +change 1 then enforces it. + +Rejection reuses the existing structured `402`, extended with what _would_ fit: + +``` +402 { error: "insufficient_cli_credit", needed_cents, available_cents, + affordable_budget_cents, actions: ["byok", "topup"], message } +``` + +**A budget larger than the wallet is clamped, not rejected.** The wallet is always the outer bound — if it +empties first, the existing exhaustion path releases the lease regardless of what the grant permits. So a +$1000 budget against a $15 balance is not an error, it simply buys $15 of compute. But the caller must not be +left believing otherwise: the response reports the **effective** cap (`min(budget_cents, effective_balance)`) +so the agent can tell the user what was actually authorised rather than what was asked for. + +**Managed leases only.** BYOK runs on the user's own provider account, which we neither meter nor bill, so a +budget cap there would be a number we cannot enforce. BYOK ignores `budget_cents`. + +### OpenScience — one tool + +The agent lists options, picks a SKU itself, and proposes a budget: + +1. `GET /api/compute/options` → the agent sees live per-SKU rates and picks. **Selection is the agent's job**, + which is why OpenScience needs no selection logic and no wrapper around `compute:up`. +2. Submit `{provider, sku, budget_cents}`. **Refuse to launch without a verdict that came back from Atlas.** +3. On `402`, surface `affordable_budget_cents` to the user and stop. **Never auto-retry at a smaller budget** — + a truncated training run is not a cheaper result, it is a discarded one, and an agent that quietly downsizes + scientific work produces invalid output while appearing to succeed. +4. On `429` (concurrency cap, currently 2 managed GPU leases), surface it rather than retrying. +5. Release on request. **No client-side deadline timer** — the server is the enforcer, and the client has no + completion signal to improve on it with. + +## The two clocks that remain + +| Bound | Owner | Fires when | +| ---------------- | ------------------- | ------------------------------ | +| `hard_cap_cents` | Atlas billing tick | the approved money is spent | +| Plan TTL (24h) | Atlas billing sweep | anything has run absurdly long | + +Both already exist as mechanisms; only the first is being made functional. **No `expires_at` column is added** — +time is not the thing being authorised, and a second time bound alongside the plan TTL would be redundant. + +Billing ticks every 60 seconds, so a budget can overrun by up to a minute of rate (~$0.12 on an H100). Approved +budgets are therefore ceilings-plus-a-minute and must never be described as exact. + +## What you are accepting + +**A budget-exhausted job loses its work.** You paid the budget and got a partial run. This is already true +today; the spec does not make it worse, but it does not fix it either. **Roadmap 56 (checkpointing) is the fix, +and should be tracked as the follow-on that makes this good rather than merely safe.** Until then, a warning +event before exhaustion would be advice nobody can act on. + +**Time is unbounded within a budget.** A cheap CPU lease could run for days inside a small budget. The 24-hour +plan TTL is the only backstop, which is deliberate. + +## Testing + +**Atlas** follows `backend/tests/test_compute_billing.py`: a `_FakeProvider` registered into the provider +registry, `aiosqlite` + `run_migrations` for an isolated DB, assertions on the verdict and on whether release +was called. Runs under plain `pytest` before the `fly deploy -a thesis-dev` check, so the deploy verifies +integration rather than being the code's first execution. + +Cases that must be covered: + +- A tick whose delta would exceed `hard_cap_cents` **releases the lease**; one that fits does not. +- `spent_cents` tracks cumulative charge across several ticks rather than freezing after the first. +- A lease created **without** `budget_cents` behaves exactly as before (dashboard and `compute:up` compatibility). +- A budget smaller than one hour's rate is rejected at creation with a `402` carrying + `affordable_budget_cents`. +- BYOK leases ignore `budget_cents` and are never debited. +- A budget larger than the wallet is clamped, and the response reports the effective cap rather than the asked-for one. +- The plan TTL still fires independently of any budget. + +**OpenScience** follows its house pattern — stub `globalThis.fetch`, exercise the real tool. Cover: refusing to +launch without a verdict, surfacing `402` without retrying, surfacing `429`, and release. + +Every new assertion must be shown failing against the specific mutation it guards before being committed. On the +preceding `science_fetch` branch, seven assertion defects were found and **all seven were in plan-authored test +code**; the ones that held up were proven against a _deletion_, not merely an inversion. + +## Migration + +`compute_grants.hard_cap_cents` already exists — no schema change for change 1. Change 2 adds no column either; +`budget_cents` only sizes the grant at creation. Existing in-flight grants keep whatever cap they were created +with, and change 1 begins enforcing it from the next tick, which is the safe direction. + +## Corrections to earlier analysis + +Recorded because both errors reached a draft of this document. + +- **`atlas compute:up` exists.** It is at `cli/src/atlas-runtime/commands.mjs:922` as a `LOCAL` command (aliases + `compute:launch`, `compute:lease`), alongside `compute:list` and `compute:ssh`. An earlier draft claimed it + existed in no version — that was wrong, caused by a comment two lines below it saying compute provisioning is + not part of the CLI. **Consequence: the system prompt at `session/prompt.ts:1554-1562` is not broken** and + needs no fix. It was previously an acceptance criterion; it is removed. +- **`compute:up` already takes `max_price` and `dry_run`.** A per-hour price ceiling and a no-spend preview + already exist in the CLI's parameter set, closer to this design than earlier drafts represented. Adding + `budget_cents` to `compute:up` as well would let CLI users have the same cap — worth doing, out of scope here. + +## Out of scope + +- **Roadmap 61** (per-job secrets, never into logs) — belongs to the _local_ runner `compute/jobs.ts`, which has + no billing involvement. Logs there go straight to a file descriptor unredacted while every job inherits the + user's Modal, RunPod, Lambda, Vast, W&B and HuggingFace keys. Real problem, separate workstream. +- **Roadmap 56** (checkpointing) — the follow-on that makes budget exhaustion survivable. +- **Roadmap 4** (real BYOK provider API clients) — five separable vendor integrations. +- **Roadmap 52** (`bun:sqlite` for the local runner's state). +- Adding `budget_cents` to `atlas compute:up`, a per-lease `expires_at`, warning events, extension requests, + client-side deadline timers, and any client-side price table. + +## Acceptance criteria + +1. The billing tick re-debits the grant, so `spent_cents` tracks cumulative spend instead of freezing at hour + one. +2. A tick whose delta would exceed `hard_cap_cents` releases the lease via the existing release path. +3. `POST /api/compute/leases` accepts optional `budget_cents` and sizes the grant to it. +4. Omitting `budget_cents` preserves today's behaviour exactly — the dashboard and `compute:up` keep working. +5. A budget that cannot fund the first hour is rejected with `402` carrying `affordable_budget_cents`. +6. A budget exceeding the wallet is clamped to the effective balance, and the response reports the effective cap. +7. BYOK leases ignore `budget_cents` and are never debited. +8. The 24-hour plan TTL still fires independently. +9. `pytest` passes with no network access; change 1 is a separate commit with its own tests. +10. The OpenScience tool refuses to launch without an Atlas verdict, surfaces `402` and `429` without retrying, + and holds no pricing or approval logic. + +## Prototype + +`backend/cli/src/compute/PROTOTYPE-guardrail-model.ts` and `PROTOTYPE-guardrail-repl.ts` (openscience, +`e12e486`), runnable via `bun run prototype:guardrail`. It was built duration-first, so its `decide()` reasons +about hours rather than a budget — the _gate-mode_ finding (first-hour versus total) is what carried over and +motivated this design. Each unverified Atlas behaviour is a toggle, so verifying the real backend means flipping +switches rather than rewriting the model. From d6c61d1d30bf79cabf0c90a8c41777914e6e3395 Mon Sep 17 00:00:00 2001 From: KB Date: Thu, 30 Jul 2026 16:56:48 +0530 Subject: [PATCH 03/56] spec: compute mode detection (byok / managed / none) OpenScience decides how GPU work is paid for from a config value that never checks whether the user has any provider keys. computeBillingMode() returns config.billing?.compute ?? "byok", so a user with zero keys resolves to BYOK -- claiming BYOK with nothing to BYOK with. Replaces it with runtime detection, mirroring what billing.llm already does (its schema says 'unset = auto-detect from the resolved credential', backed by resolveCredentialSource; billing.compute says 'unset = byok', a static default with no detection function). Adds a third state. BillingMode is managed|byok, so 'the user has no keys AND managed compute is unavailable' has nowhere to live and silently resolves to a mode that cannot work. Making 'none' expressible is the point -- it turns a broken instruction into an honest 'connect a provider key'. Records what the prompt currently claims and why each part is false: the published CLI has no compute:up (the Atlas repo does, at the same version number), atlas doctor reports no compute field, and managed compute is off by default behind COMPUTE_RESELL_ENABLED. An agent in managed mode therefore runs an unknown command, cannot check the sanctioned availability signal, and is pointed at the user's own uncapped provider keys as the remedy. Scope is openscience only, no Atlas changes. The budget cap stays parked. --- docs/specs/compute-mode-detection-design.md | 239 ++++++++++++++++++++ 1 file changed, 239 insertions(+) create mode 100644 docs/specs/compute-mode-detection-design.md diff --git a/docs/specs/compute-mode-detection-design.md b/docs/specs/compute-mode-detection-design.md new file mode 100644 index 00000000..03965a7c --- /dev/null +++ b/docs/specs/compute-mode-detection-design.md @@ -0,0 +1,239 @@ +# Compute mode detection — design + +Status: proposed, for review +Date: 2026-07-30 +Scope: `openscience` only. **No Atlas changes required.** +Roadmap: contributes to **5** (fix existing compute gaps); unblocks honest messaging for **55**/**103** + +## Summary + +OpenScience decides how GPU work gets paid for using a config value that never checks whether the user +actually has any provider keys. This replaces that with **runtime detection**: if provider credentials are +present, we're in BYOK mode; if not, managed; and if neither is usable, we say so instead of guessing. + +The change is small and self-contained. Its value is that it makes one currently-unrepresentable state — +_"there is no compute available"_ — expressible, which is the state we handle worst today. + +## Problem + +### 1. The mode is a config value that can contradict reality + +```ts +// src/session/billing-gate.ts:34 +export async function computeBillingMode(): Promise { + return (await Config.get()).billing?.compute ?? "byok" +} +``` + +It never inspects the environment. A brand-new user with zero provider keys resolves to `"byok"` — claiming +BYOK with nothing to BYOK with. + +Note the asymmetry with LLM billing, which already does this correctly. From the config schema +(`src/config/config.ts`): + +- `billing.llm` — _"Unset or null = **auto-detect** from the resolved credential."_ Backed by + `resolveCredentialSource(providerID, modelID)`, which inspects the actual credential and returns + `byok | managed | oauth-free`. +- `billing.compute` — _"**Unset = byok**."_ A static default. No detection function exists. + +**This design makes compute behave the way LLM already does.** + +### 2. The "no compute available" state cannot be expressed + +`BillingMode` is `"managed" | "byok"`. There is no third value, so the case _"the user has no keys **and** +managed compute isn't available"_ has nowhere to live. Today it silently resolves to one of the two working +modes and the agent is told to use a path that cannot work. + +### 3. The prompt is only injected when the mode is explicitly set — and is wrong when it is + +```ts +// src/session/prompt.ts:1553 +if (COMPUTE_AGENTS.has(input.agent.name) && (await Config.get()).billing?.compute) { +``` + +`COMPUTE_AGENTS` is `research`, `biology`, `physics`, `ml`. Two distinct problems: + +**When `billing.compute` is unset (the default), no guidance is injected at all.** The agent receives no +information about how compute is funded and picks an approach from the skill catalog with no idea whether the +user's keys exist. + +**When it is set to `managed`, the injected text is inaccurate:** + +> _"Run GPU/training work through the bundled `atlas compute` CLI (e.g. `atlas compute:up`), which bills +> Credits. Do not fall back to the user's own GPU providers unless `atlas doctor` reports managed compute +> unavailable."_ + +Three things in that sentence do not hold: + +- **`atlas compute:up` is not in the published CLI.** `@synsci/atlas@0.13.2` on npm contains no `compute:` + command. The Atlas repo's `cli/` does (`commands.mjs:922`), at the _same version number_ — so the source + and the published artifact disagree and the pin `^0.13.2` resolves to the one without it. +- **`atlas doctor` reports nothing about compute.** Verified against a live run: the keys are + `config_path`, `profile`, `base_url`, `auth`, `backend`, `package.skills`, `integrations`, `spool`, + `warnings`, `ok`. There is no compute field, so the stated condition is unobservable. +- **Managed compute is off by default server-side.** Atlas gates it on `COMPUTE_RESELL_ENABLED`, which + defaults to `false` (`backend/app/config.py:383`), plus a configured operator key. Without both, every + provider reports `funding: "unavailable"`. + +So an agent in managed mode runs an unknown command, cannot check the sanctioned availability signal, and is +pointed at _"the user's own GPU providers"_ as the remedy — which in managed mode is precisely the set of keys +that doesn't exist. Where keys _do_ exist, that fallback spends on the user's own uncapped provider account. + +## Design + +### Three states, detected at runtime + +```ts +export type ComputeSource = "byok" | "managed" | "none" +``` + +| Provider keys present? | Managed available? | Resolved | Agent is told | +| ---------------------- | ------------------ | --------- | ---------------------------------------------------------------- | +| yes | — | `byok` | use the user's connected providers via the cloud-compute skills | +| no | yes | `managed` | use managed compute, billed to the wallet | +| no | no | `none` | **no compute is available — connect a provider key in Settings** | + +BYOK wins when keys are present. It is free to the user, it works today, and it needs nothing from Atlas. + +### `billing.compute` becomes an override, not the source of truth + +Detection supplies the default; the existing setting still lets a user force a mode. This keeps the config +meaningful without letting it assert something false. + +- unset → use detection +- `"byok"` → force BYOK. If no keys are present, resolve to `none` rather than pretending. +- `"managed"` → force managed. If managed is unavailable, resolve to `none`. + +An override may narrow the outcome to `none`; it may never manufacture a capability that isn't there. + +### What counts as a provider key + +From `PROVIDER_ENV` in `src/server/routes/settings/compute.ts:170` plus Modal's pair: + +| Provider | Env vars | +| --------------- | ------------------------------------------------------------- | +| Modal | `MODAL_TOKEN_ID` **and** `MODAL_TOKEN_SECRET` (both required) | +| Lambda | `LAMBDA_API_KEY` or `LAMBDA_LABS_API_KEY` | +| RunPod | `RUNPOD_API_KEY` | +| Vast | `VAST_API_KEY` | +| Prime Intellect | `PRIME_API_KEY` or `PRIME_INTELLECT_API_KEY` | +| TensorPool | `TENSORPOOL_KEY` or `TENSORPOOL_API_KEY` | + +Any one provider fully configured is sufficient for `byok`. Modal is the only pair — a half-pasted Modal +credential maps to nothing and must not count, which mirrors the existing behaviour at `compute.ts:185`. + +### Ordering requirement — the main footgun + +Keys reach `process.env` from three places, all legitimate BYOK: + +1. The user's shell or `.env` +2. The **Credentials** settings panel — injected by `applyCredentialEnv()` at `src/index.ts:102` +3. The **Compute** settings panel — injected by `ComputeSettings.applyComputeEnv()` at `src/index.ts:106` + +**Detection must run after line 106.** Both injections are wrapped in `.catch(() => {})` and fail silently, so +detecting too early reports `none` for a user who has keys configured through the UI. Detection should be lazy +(resolved on first use, cached) rather than computed during boot, so ordering cannot regress silently. + +### Determining whether managed is available + +One authenticated call to `GET /api/compute/options`, which already annotates each provider with `funding` +(`managed` when reselling is on and an operator key exists, else `unavailable`). If no provider reports +`managed`, managed is unavailable. + +Cache the result for the process lifetime with a short TTL. Treat a failed or unauthenticated call as +**unavailable** — failing toward `none` produces an honest "connect a key" message, whereas failing toward +`managed` reproduces today's bug of promising a capability we haven't confirmed. + +**This is only reached when no provider keys are present**, so a BYOK user never pays the network call. + +### Prompt behaviour + +Inject for `COMPUTE_AGENTS` on **every** turn, not only when the config is explicitly set — the agent needs to +know how compute is funded regardless of whether the user has expressed a preference. + +- **byok** — run GPU work on the user's connected providers via the cloud-compute skills. Never launch managed + leases. +- **managed** — run GPU work through managed compute, billed to Credits. Do not fall back to the user's own + providers. +- **none** — no compute is available. Do not attempt GPU work. Tell the user to connect a provider key in + Settings → Compute, or to top up for managed compute. + +Remove the `atlas compute:up` instruction and the `atlas doctor` condition. Neither is currently true, and +mode resolution now answers the question the `doctor` check was trying to answer. + +## Testing + +House pattern: no mocks, exercise the real resolver, no network in tests. + +- Each provider in isolation resolves to `byok`. +- **Modal with only `MODAL_TOKEN_ID` and no other provider** does not resolve to `byok` — it falls through to + the managed/none branch exactly as if no key were set. +- No keys plus managed available → `managed`. +- No keys plus managed unavailable → `none`. +- No keys plus a failed availability call → `none`. +- Override `"byok"` with no keys → `none`. +- Override `"managed"` with managed unavailable → `none`. +- Keys present → the availability call is **not** made. +- Detection reflects a key injected by `applyComputeEnv()` after startup (the ordering guarantee). +- The prompt text differs across all three modes and mentions neither `compute:up` nor `atlas doctor`. + +Every new assertion must be demonstrated failing against the specific mutation it guards — ideally the +_deletion_ of the logic, not merely its inversion. On the preceding `science_fetch` branch seven assertion +defects were found and all seven were in plan-authored test code; the ones that held up were the ones proven +against deletion. + +## Out of scope + +- **The managed-compute budget cap** (roadmap 55/103). Parked in + `docs/specs/compute-guardrails-design.md`, which an Opus review found unsound; it also cannot be validated + until managed compute is actually switched on somewhere. +- **Turning managed compute on** — an Atlas deployment decision (`COMPUTE_RESELL_ENABLED` plus operator keys). +- **Publishing `compute:up`**, and the source/npm version divergence at `0.13.2`. Worth an independent fix: + a consumer pinning `^0.13.2` cannot tell which artifact they'll get. +- **Roadmap 61** (per-job secrets, never into logs) — the local runner `compute/jobs.ts`, unrelated to billing. +- Any change to how `billing.llm` resolves. + +## Acceptance criteria + +1. A resolver returns `byok | managed | none` from the runtime environment, not from a static default. +2. Any single fully-configured provider yields `byok`; a half-configured Modal credential does not. +3. With no keys and managed unavailable — including when the availability check fails — the result is `none`. +4. `billing.compute` can narrow the result to `none` but can never assert an unavailable capability. +5. A key injected by either settings panel at boot is detected (resolution happens after `src/index.ts:106`). +6. The availability call is skipped entirely when provider keys are present. +7. `COMPUTE_AGENTS` receive mode guidance on every turn, including when `billing.compute` is unset. +8. No prompt references `atlas compute:up` or `atlas doctor` for compute availability. +9. In `none`, the agent is instructed not to attempt GPU work and to tell the user how to enable it. +10. `bun test` passes with no network access. + +## Open questions for review + +1. **Should a BYOK user with keys for one provider but asking about another get `byok` or a + partial answer?** This design says `byok` if any provider is configured, and leaves provider choice to the + agent and the skills. A per-provider resolution would be more precise and more complex. +2. **Should `none` be a hard block or a warning?** Currently the agent is told not to attempt GPU work, but + nothing enforces it — it still has `bash`. Enforcement would mean gating the cloud-compute skills, which is + a larger change. +3. **TTL on the availability cache.** Process lifetime is simplest. A user who tops up or connects an operator + key mid-session would need a restart to see `managed` appear. +4. **Does the `billing.compute` description need updating** in the config schema? It currently says + _"Unset = byok"_, which this change makes false. + +## Appendix: what was verified, and how + +Every claim above was checked against code or a live run rather than inferred. Recorded because three earlier +conclusions in this investigation were wrong, and the corrections are the reason this design exists. + +| Claim | Verified by | +| ------------------------------------------------------ | ------------------------------------------------------------------------------------------------- | +| `computeBillingMode()` reads config only | `src/session/billing-gate.ts:34` | +| Prompt injects only when explicitly set | `src/session/prompt.ts:1553` | +| `COMPUTE_AGENTS` = research, biology, physics, ml | `src/session/prompt.ts:75` | +| Env injection order and silent failure | `src/index.ts:102,106` | +| Provider env var names | `src/server/routes/settings/compute.ts:170-176,185` | +| `compute:up` absent from published npm 0.13.2 | `npm pack @synsci/atlas`, grep of the tarball | +| `compute:up` present in Atlas repo at the same version | `atlas` `origin/main:cli/src/atlas-runtime/commands.mjs:922`; `cli/package.json` version `0.13.2` | +| `atlas doctor` reports no compute field | live `atlas doctor` output | +| Managed compute off by default | `atlas` `origin/main:backend/app/config.py:383` | +| `funding` derivation | `atlas` `origin/main:backend/app/routes/compute.py:117-122` | +| LLM billing already auto-detects | `src/config/config.ts` `billing.llm`; `billing-gate.ts:63` | From c95762d3ac81c230a757daca7a8b9ccb3a70d8ab Mon Sep 17 00:00:00 2001 From: KB Date: Thu, 30 Jul 2026 19:53:58 +0530 Subject: [PATCH 04/56] spec: agent learns compute mode from a tool, not a prompt injection The first draft injected mode guidance into every turn for COMPUTE_AGENTS. Wrong, for a reason that matters more than token cost: the mode changes mid-session. A user connects a Modal key in Settings while a session is running, and a reminder injected at turn 3 is false by turn 12. Replaces it with a compute_status tool returning {mode, providers, managed_available, guidance, balance_usd?}, resolved per call. The tool DESCRIPTION carries the constraint that an injection was doing well ("check before running GPU work") at no extra per-turn cost, since tool definitions are in every request anyway. So the description constrains and the result informs; nothing needs injecting. Resolving on demand also removes the boot-ordering footgun rather than documenting it: provider keys arrive from three places, two of them injected at src/index.ts:102 and :106 behind silent .catch handlers, so anything resolved at startup can be wrong and can regress if someone reorders boot. By tool-call time every injection has run. This is the read half of roadmap 51/2 -- compute_status now, compute_submit when managed compute is real and the budget cap is sound. Drops the availability-cache-TTL open question, which was an attempt to paper over staleness that the tool removes outright. Adds two: whether to keep a one-line prompt pointer in case the agent never calls the tool, and what timeout compute_status may block for. --- docs/specs/compute-mode-detection-design.md | 130 ++++++++++++++++---- 1 file changed, 104 insertions(+), 26 deletions(-) diff --git a/docs/specs/compute-mode-detection-design.md b/docs/specs/compute-mode-detection-design.md index 03965a7c..f27c50fe 100644 --- a/docs/specs/compute-mode-detection-design.md +++ b/docs/specs/compute-mode-detection-design.md @@ -131,8 +131,12 @@ Keys reach `process.env` from three places, all legitimate BYOK: 3. The **Compute** settings panel — injected by `ComputeSettings.applyComputeEnv()` at `src/index.ts:106` **Detection must run after line 106.** Both injections are wrapped in `.catch(() => {})` and fail silently, so -detecting too early reports `none` for a user who has keys configured through the UI. Detection should be lazy -(resolved on first use, cached) rather than computed during boot, so ordering cannot regress silently. +detecting too early reports `none` for a user who has keys configured through the UI. + +The robust way to guarantee that is not to order boot steps carefully — it is to **resolve on demand, when the +tool is called, and never at startup.** By then every injection has run, so the ordering constraint cannot be +violated and cannot silently regress if someone reorders `src/index.ts` later. This is the same property that +makes a tool the right shape in the first place. ### Determining whether managed is available @@ -140,26 +144,81 @@ One authenticated call to `GET /api/compute/options`, which already annotates ea (`managed` when reselling is on and an operator key exists, else `unavailable`). If no provider reports `managed`, managed is unavailable. -Cache the result for the process lifetime with a short TTL. Treat a failed or unauthenticated call as -**unavailable** — failing toward `none` produces an honest "connect a key" message, whereas failing toward -`managed` reproduces today's bug of promising a capability we haven't confirmed. +Treat a failed, unauthenticated, or timed-out call as **unavailable** — failing toward `none` produces an honest +"connect a key" message, whereas failing toward `managed` reproduces today's bug of promising a capability we +haven't confirmed. **This is only reached when no provider keys are present**, so a BYOK user never pays the network call. -### Prompt behaviour +**Caching:** a short in-process TTL (single-digit seconds) is fine to stop a chatty agent hammering the endpoint +within one turn, but it must not be a startup-time or process-lifetime cache. The whole reason this is a tool +rather than a prompt injection is that the answer changes mid-session — a long cache reintroduces exactly the +staleness the tool exists to avoid. Key detection itself reads `process.env` and needs no cache at all. + +### How the agent learns the mode: a tool, not a prompt injection + +**The agent pulls the mode from a tool. Nothing is injected per turn.** + +An earlier draft injected mode guidance into every turn for `COMPUTE_AGENTS`. That was wrong for a reason that +matters more than token cost: **the mode can change mid-session.** A user connects a Modal key in +Settings → Compute while a session is running, and a reminder injected at turn 3 is false by turn 12. A tool +returns the state at the moment it is asked. + +It also composes with action. The agent needs the mode only because it is about to run GPU work, so a call at +that moment can return the mode _and_ the specifics worth having — which providers are configured, whether +managed is available, and the balance and rates when it is. Injected prose is information divorced from the +decision, and it does not scale: adding rates or balance to an every-turn injection is expensive, while adding +them to a tool result is free. + +This also matches the pattern this codebase already treats as correct. `tool/science.ts` exposes three tools +over 42 connectors rather than 42 tool definitions — capability discovered on demand, tool count flat. -Inject for `COMPUTE_AGENTS` on **every** turn, not only when the config is explicitly set — the agent needs to -know how compute is funded regardless of whether the user has expressed a preference. +#### `compute_status` + +No parameters. Returns the resolved mode plus what the agent needs to act on it: + +```ts +{ + mode: "byok" | "managed" | "none", + providers: string[], // configured BYOK providers, e.g. ["modal", "runpod"] + managed_available: boolean, + guidance: string, // the mode-specific rule, see below + balance_usd?: number // managed only +} +``` -- **byok** — run GPU work on the user's connected providers via the cloud-compute skills. Never launch managed - leases. -- **managed** — run GPU work through managed compute, billed to Credits. Do not fall back to the user's own - providers. -- **none** — no compute is available. Do not attempt GPU work. Tell the user to connect a provider key in - Settings → Compute, or to top up for managed compute. +`guidance` carries the behavioural rule, delivered at the point of relevance: -Remove the `atlas compute:up` instruction and the `atlas doctor` condition. Neither is currently true, and -mode resolution now answers the question the `doctor` check was trying to answer. +| mode | guidance | +| --------- | -------------------------------------------------------------------------------------------------------------------------------------------------- | +| `byok` | Run GPU work on the user's connected providers via the cloud-compute skills. Do not launch managed leases. | +| `managed` | Run GPU work through managed compute, billed to Credits. Do not use the user's own provider keys — they are not funded here. | +| `none` | No compute is available. Do not attempt GPU work. Tell the user to connect a provider key in Settings → Compute, or to top up for managed compute. | + +#### The tool description is the prompt + +Constraints have to reach the agent _before_ it starts down a path, which is the one thing an injection did +well. The tool **description** does that job at no extra per-turn cost, because tool definitions are in every +request regardless: _"Check how GPU compute is funded before running any GPU work. Returns byok, managed, or +none, with the providers available and the rule that applies."_ + +So the description constrains and the result informs. Nothing needs injecting. + +#### Prompt changes + +Delete the `atlas compute:up` instruction and the `atlas doctor` condition from `prompt.ts:1554-1562`. Both are +false today, independent of this design, and mode resolution now answers the question the `doctor` check was +reaching for. + +Whether to keep a minimal pointer in the prompt is left open — see the open questions. The default position is +no injection at all. + +#### Relationship to roadmap 51/2 + +This is the read half of the agent-facing compute tool. `compute_status` now; a `compute_submit` alongside it +when managed compute is actually switched on and the budget cap from +`docs/specs/compute-guardrails-design.md` is sound. Same seam, so the later work adds a tool rather than +reshaping this one. ## Testing @@ -175,7 +234,16 @@ House pattern: no mocks, exercise the real resolver, no network in tests. - Override `"managed"` with managed unavailable → `none`. - Keys present → the availability call is **not** made. - Detection reflects a key injected by `applyComputeEnv()` after startup (the ordering guarantee). -- The prompt text differs across all three modes and mentions neither `compute:up` nor `atlas doctor`. +- **A key connected mid-session changes the answer on the next call** — the staleness property that motivated a + tool over an injection. Resolve, inject a key, resolve again, assert the mode changed. + +For the tool, following the house pattern of stubbing `globalThis.fetch` and exercising the real tool: + +- `compute_status` returns each of the three modes with matching `guidance` text. +- `byok` lists the configured providers in `providers`. +- `managed` includes `balance_usd`; `byok` and `none` do not. +- No prompt in `session/prompt/*.txt` or `prompt.ts` references `compute:up` or `atlas doctor` for compute + availability. Every new assertion must be demonstrated failing against the specific mutation it guards — ideally the _deletion_ of the logic, not merely its inversion. On the preceding `science_fetch` branch seven assertion @@ -201,23 +269,33 @@ against deletion. 4. `billing.compute` can narrow the result to `none` but can never assert an unavailable capability. 5. A key injected by either settings panel at boot is detected (resolution happens after `src/index.ts:106`). 6. The availability call is skipped entirely when provider keys are present. -7. `COMPUTE_AGENTS` receive mode guidance on every turn, including when `billing.compute` is unset. -8. No prompt references `atlas compute:up` or `atlas doctor` for compute availability. -9. In `none`, the agent is instructed not to attempt GPU work and to tell the user how to enable it. -10. `bun test` passes with no network access. +7. A `compute_status` tool returns the mode, the configured providers, and mode-specific `guidance`, resolving + on each call rather than from a value cached at startup. +8. A credential connected mid-session is reflected on the next `compute_status` call without a restart. +9. Nothing is injected into the prompt per turn for compute mode; the tool's description carries the + "check before running GPU work" instruction. +10. No prompt references `atlas compute:up` or `atlas doctor` for compute availability. +11. In `none`, the tool's `guidance` tells the agent not to attempt GPU work and how the user can enable it. +12. `bun test` passes with no network access. ## Open questions for review 1. **Should a BYOK user with keys for one provider but asking about another get `byok` or a partial answer?** This design says `byok` if any provider is configured, and leaves provider choice to the agent and the skills. A per-provider resolution would be more precise and more complex. -2. **Should `none` be a hard block or a warning?** Currently the agent is told not to attempt GPU work, but - nothing enforces it — it still has `bash`. Enforcement would mean gating the cloud-compute skills, which is - a larger change. -3. **TTL on the availability cache.** Process lifetime is simplest. A user who tops up or connects an operator - key mid-session would need a restart to see `managed` appear. +2. **Should `none` be a hard block or a warning?** The tool's `guidance` tells the agent not to attempt GPU + work, but nothing enforces it — it still has `bash` and the cloud-compute skills. Enforcement would mean + gating those skills, which is a larger change. Worth deciding explicitly rather than by omission. +3. **Should the prompt keep a one-line pointer to the tool?** The default position here is no injection at all, + on the grounds that the tool description already carries the instruction. The risk is an agent that never + calls the tool and reaches for `bash` directly. A single line — _"call `compute_status` before GPU work"_ — + would cost a handful of tokens per turn and close that gap. This is the one place where the tool-versus-prompt + trade-off is genuinely unresolved. 4. **Does the `billing.compute` description need updating** in the config schema? It currently says _"Unset = byok"_, which this change makes false. +5. **How long may `compute_status` block?** In `none`/`managed` it makes one authenticated call to + `/api/compute/options`. A slow or hanging Atlas would stall the agent mid-turn, so it needs a short timeout + with `none` as the timeout result — but "short" should be a stated number, not left to the implementer. ## Appendix: what was verified, and how From 70c758b4dfaec9e7eee937719ae224ff76a1f8dc Mon Sep 17 00:00:00 2001 From: KB Date: Thu, 30 Jul 2026 20:27:04 +0530 Subject: [PATCH 05/56] spec: resolve compute mode in SkillTool.init, filter catalog by usable provider Folds in the dynamic-loading flow. Three changes. Resolution point: SkillTool is defined with an async init that already builds and filters its catalog, and registry.ts:187 calls t.init({agent}) inside tools() -- so that init runs per request. Resolving there gives per-turn freshness for free and makes the boot-ordering hazard unreachable by construction, since every env injection at index.ts:102/:106 has long since run by the time a turn is served. Both call sites must share one resolver; two implementations of "which providers are usable" would drift silently. Filter, do not auto-load. Only usable providers' skills are listed, so the agent picks the right one because it is the only one offered. Auto-injecting provider markdown would fight tool/skill.ts, which exists so content is pulled on demand, and these files are large enough to be costly on turns unrelated to compute. A provider is usable only with a key AND a skill. RunPod and Vast have keys that inject and no skill to load, so a RunPod-only user would otherwise resolve to byok with an empty toolbox. They now resolve to managed/none with an honest message. This surfaces roadmap item 5 rather than causing it, and is added as an open question: write the skills, drop the providers, or label them as pending. --- docs/specs/compute-mode-detection-design.md | 146 ++++++++++++++------ 1 file changed, 103 insertions(+), 43 deletions(-) diff --git a/docs/specs/compute-mode-detection-design.md b/docs/specs/compute-mode-detection-design.md index f27c50fe..7213dfcd 100644 --- a/docs/specs/compute-mode-detection-design.md +++ b/docs/specs/compute-mode-detection-design.md @@ -87,13 +87,14 @@ that doesn't exist. Where keys _do_ exist, that fallback spends on the user's ow export type ComputeSource = "byok" | "managed" | "none" ``` -| Provider keys present? | Managed available? | Resolved | Agent is told | -| ---------------------- | ------------------ | --------- | ---------------------------------------------------------------- | -| yes | — | `byok` | use the user's connected providers via the cloud-compute skills | -| no | yes | `managed` | use managed compute, billed to the wallet | -| no | no | `none` | **no compute is available — connect a provider key in Settings** | +| Usable provider? | Managed available? | Resolved | Agent is told | +| ---------------- | ------------------ | --------- | ---------------------------------------------------------------- | +| yes | — | `byok` | use the user's connected providers via the cloud-compute skills | +| no | yes | `managed` | use managed compute, billed to the wallet | +| no | no | `none` | **no compute is available — connect a provider key in Settings** | -BYOK wins when keys are present. It is free to the user, it works today, and it needs nothing from Atlas. +BYOK wins when a usable provider is present. It is free to the user, it works today, and it needs nothing from +Atlas. "Usable" means a key **and** a skill — see below, because two providers have a key and no skill. ### `billing.compute` becomes an override, not the source of truth @@ -101,26 +102,58 @@ Detection supplies the default; the existing setting still lets a user force a m meaningful without letting it assert something false. - unset → use detection -- `"byok"` → force BYOK. If no keys are present, resolve to `none` rather than pretending. +- `"byok"` → force BYOK. If no _usable_ provider is present, resolve to `none` rather than pretending. - `"managed"` → force managed. If managed is unavailable, resolve to `none`. An override may narrow the outcome to `none`; it may never manufacture a capability that isn't there. -### What counts as a provider key +### What counts as a usable provider: a key **and** a skill -From `PROVIDER_ENV` in `src/server/routes/settings/compute.ts:170` plus Modal's pair: +A key alone is not enough. The agent runs GPU work by loading a provider's skill, so a provider with a +credential but no skill gives the agent nothing to act on. -| Provider | Env vars | -| --------------- | ------------------------------------------------------------- | -| Modal | `MODAL_TOKEN_ID` **and** `MODAL_TOKEN_SECRET` (both required) | -| Lambda | `LAMBDA_API_KEY` or `LAMBDA_LABS_API_KEY` | -| RunPod | `RUNPOD_API_KEY` | -| Vast | `VAST_API_KEY` | -| Prime Intellect | `PRIME_API_KEY` or `PRIME_INTELLECT_API_KEY` | -| TensorPool | `TENSORPOOL_KEY` or `TENSORPOOL_API_KEY` | +From `PROVIDER_ENV` (`src/server/routes/settings/compute.ts:170`) plus Modal's pair, cross-referenced against +the skill tree: -Any one provider fully configured is sufficient for `byok`. Modal is the only pair — a half-pasted Modal -credential maps to nothing and must not count, which mirrors the existing behaviour at `compute.ts:185`. +| Provider | Env vars | Skill | Usable | +| --------------- | ------------------------------------------------------------- | -------------------------------------------------------- | ------ | +| Modal | `MODAL_TOKEN_ID` **and** `MODAL_TOKEN_SECRET` (both required) | `cloud-compute/modal`, `cloud-compute/modal-ml-training` | yes | +| Lambda | `LAMBDA_API_KEY` or `LAMBDA_LABS_API_KEY` | `cloud-compute/lambda-labs` | yes | +| TensorPool | `TENSORPOOL_KEY` or `TENSORPOOL_API_KEY` | `cloud-compute/tensorpool` | yes | +| Prime Intellect | `PRIME_API_KEY` or `PRIME_INTELLECT_API_KEY` | `ml-training/prime-intellect-lab` | yes | +| **RunPod** | `RUNPOD_API_KEY` | **none** | **no** | +| **Vast** | `VAST_API_KEY` | **none** | **no** | + +**Rule: a provider is BYOK-usable only when it has both.** So `byok` requires at least one provider with a key +_and_ a skill. A user whose only credential is RunPod resolves to `managed`/`none` with an honest message, +rather than to `byok` with an empty toolbox. + +Modal is the only credential pair — a half-pasted Modal token maps to nothing and must not count, mirroring +`compute.ts:185`. + +This rule surfaces roadmap item **5** rather than causing it: RunPod and Vast keys inject with no consumer +today, and `RUNPOD_API_KEY` is even named to the model in all six session prompts. Detection makes that gap +visible instead of silent. Either write those two skills or stop offering the providers — see open questions. + +### Filtering the skill catalog + +`SkillTool` (`src/tool/skill.ts:32`) is defined with an async init that builds its catalog and already filters +it — today by `PermissionNext.evaluate("skill", skill.name, agent.permission)`. Crucially, +`registry.ts:187` calls `await t.init({ agent })` inside `tools()`, so **that init runs per request**. + +That makes it the right seam, for two reasons: + +- **Freshness is free.** The catalog is rebuilt every turn, so a credential connected mid-session appears on + the next turn with no cache to invalidate. +- **Ordering is guaranteed by construction.** By the time a turn is served, every env injection at + `src/index.ts:102` and `:106` has long since run, so detection cannot observe a half-initialised environment. + +**Filter the catalog; do not auto-load the markdown.** Only the skills of usable providers are listed, so the +agent picks the right one because it is the only one offered. Auto-injecting a provider's markdown would fight +the existing mechanism — `tool/skill.ts` exists precisely so content is pulled on demand — and these files are +large enough that unprompted injection is expensive on turns that have nothing to do with compute. + +In `managed` and `none`, no BYOK provider skill is listed at all. ### Ordering requirement — the main footgun @@ -133,10 +166,14 @@ Keys reach `process.env` from three places, all legitimate BYOK: **Detection must run after line 106.** Both injections are wrapped in `.catch(() => {})` and fail silently, so detecting too early reports `none` for a user who has keys configured through the UI. -The robust way to guarantee that is not to order boot steps carefully — it is to **resolve on demand, when the -tool is called, and never at startup.** By then every injection has run, so the ordering constraint cannot be -violated and cannot silently regress if someone reorders `src/index.ts` later. This is the same property that -makes a tool the right shape in the first place. +The robust way to guarantee that is not to order boot steps carefully — it is to **resolve on demand and never +at startup**, at either of the two points that already run per request: `SkillTool`'s init when the catalog is +built, and `compute_status` when the agent calls it. By then every injection has run, so the constraint cannot +be violated and cannot silently regress if someone reorders `src/index.ts` later. + +**Resolution must be a single shared function** used by both call sites. Two independent implementations of +"which providers are usable" would drift, and the failure would be quiet: a catalog listing a provider the +status tool says is unavailable, or the reverse. ### Determining whether managed is available @@ -148,7 +185,7 @@ Treat a failed, unauthenticated, or timed-out call as **unavailable** — failin "connect a key" message, whereas failing toward `managed` reproduces today's bug of promising a capability we haven't confirmed. -**This is only reached when no provider keys are present**, so a BYOK user never pays the network call. +**This is only reached when no usable provider is present**, so a BYOK user never pays the network call. **Caching:** a short in-process TTL (single-digit seconds) is fine to stop a chatty agent hammering the endpoint within one turn, but it must not be a startup-time or process-lifetime cache. The whole reason this is a tool @@ -224,7 +261,8 @@ reshaping this one. House pattern: no mocks, exercise the real resolver, no network in tests. -- Each provider in isolation resolves to `byok`. +- Each provider **that has a skill**, in isolation, resolves to `byok`. +- **A RunPod-only or Vast-only environment does NOT resolve to `byok`** — key without skill is not usable. - **Modal with only `MODAL_TOKEN_ID` and no other provider** does not resolve to `byok` — it falls through to the managed/none branch exactly as if no key were set. - No keys plus managed available → `managed`. @@ -240,11 +278,23 @@ House pattern: no mocks, exercise the real resolver, no network in tests. For the tool, following the house pattern of stubbing `globalThis.fetch` and exercising the real tool: - `compute_status` returns each of the three modes with matching `guidance` text. -- `byok` lists the configured providers in `providers`. +- `byok` lists the usable providers in `providers`. - `managed` includes `balance_usd`; `byok` and `none` do not. - No prompt in `session/prompt/*.txt` or `prompt.ts` references `compute:up` or `atlas doctor` for compute availability. +For the catalog filtering, exercising the real `SkillTool.init`: + +- With only a Modal credential, the catalog lists the Modal skills and **not** `lambda-labs`, `tensorpool`, or + `prime-intellect-lab`. +- With a RunPod-only credential, **no** provider skill is listed. +- In `managed` and in `none`, no BYOK provider skill is listed. +- A credential added between two `init()` calls changes the catalog on the second — the per-turn freshness + property, and the reason this lives in init rather than at startup. +- Non-compute skills are unaffected by mode in every case. +- `SkillTool.init` and `compute_status` never disagree about which providers are usable (they call the same + resolver). + Every new assertion must be demonstrated failing against the specific mutation it guards — ideally the _deletion_ of the logic, not merely its inversion. On the preceding `science_fetch` branch seven assertion defects were found and all seven were in plan-authored test code; the ones that held up were the ones proven @@ -263,37 +313,47 @@ against deletion. ## Acceptance criteria -1. A resolver returns `byok | managed | none` from the runtime environment, not from a static default. -2. Any single fully-configured provider yields `byok`; a half-configured Modal credential does not. -3. With no keys and managed unavailable — including when the availability check fails — the result is `none`. -4. `billing.compute` can narrow the result to `none` but can never assert an unavailable capability. -5. A key injected by either settings panel at boot is detected (resolution happens after `src/index.ts:106`). -6. The availability call is skipped entirely when provider keys are present. -7. A `compute_status` tool returns the mode, the configured providers, and mode-specific `guidance`, resolving +1. A resolver returns `byok | managed | none` from the runtime environment, not from a static default, and is + the single shared implementation used by both `SkillTool.init` and `compute_status`. +2. A provider counts toward `byok` only with both a key and a skill; a half-configured Modal credential does + not count, and a RunPod-only or Vast-only environment does not resolve to `byok`. +3. The skill catalog lists provider skills only for usable providers, and lists none in `managed` or `none`. + Non-compute skills are unaffected. +4. Provider markdown is never auto-injected — the agent still loads it through the `skill` tool. +5. With no keys and managed unavailable — including when the availability check fails — the result is `none`. +6. `billing.compute` can narrow the result to `none` but can never assert an unavailable capability. +7. A key injected by either settings panel at boot is detected (resolution happens after `src/index.ts:106`). +8. The availability call is skipped entirely when a usable provider is present. +9. A `compute_status` tool returns the mode, the usable providers, and mode-specific `guidance`, resolving on each call rather than from a value cached at startup. -8. A credential connected mid-session is reflected on the next `compute_status` call without a restart. -9. Nothing is injected into the prompt per turn for compute mode; the tool's description carries the - "check before running GPU work" instruction. -10. No prompt references `atlas compute:up` or `atlas doctor` for compute availability. -11. In `none`, the tool's `guidance` tells the agent not to attempt GPU work and how the user can enable it. -12. `bun test` passes with no network access. +10. A credential connected mid-session is reflected on the next `compute_status` call without a restart. +11. Nothing is injected into the prompt per turn for compute mode; the tool's description carries the + "check before running GPU work" instruction. +12. No prompt references `atlas compute:up` or `atlas doctor` for compute availability. +13. In `none`, the tool's `guidance` tells the agent not to attempt GPU work and how the user can enable it. +14. `bun test` passes with no network access. ## Open questions for review 1. **Should a BYOK user with keys for one provider but asking about another get `byok` or a partial answer?** This design says `byok` if any provider is configured, and leaves provider choice to the agent and the skills. A per-provider resolution would be more precise and more complex. -2. **Should `none` be a hard block or a warning?** The tool's `guidance` tells the agent not to attempt GPU +2. **What do we do about RunPod and Vast?** Both accept a key in Settings, inject env vars, and have no skill + for the agent to load — so under this design they never make a user BYOK-usable. Three options: write the + two skills, remove the providers from the Compute panel, or keep them and show "key stored — skill coming". + Doing nothing means a user can connect RunPod, see it accepted, and still be told no compute is available. + This is roadmap item **5**; it is listed here because this design is what makes it user-visible. +3. **Should `none` be a hard block or a warning?** The tool's `guidance` tells the agent not to attempt GPU work, but nothing enforces it — it still has `bash` and the cloud-compute skills. Enforcement would mean gating those skills, which is a larger change. Worth deciding explicitly rather than by omission. -3. **Should the prompt keep a one-line pointer to the tool?** The default position here is no injection at all, +4. **Should the prompt keep a one-line pointer to the tool?** The default position here is no injection at all, on the grounds that the tool description already carries the instruction. The risk is an agent that never calls the tool and reaches for `bash` directly. A single line — _"call `compute_status` before GPU work"_ — would cost a handful of tokens per turn and close that gap. This is the one place where the tool-versus-prompt trade-off is genuinely unresolved. -4. **Does the `billing.compute` description need updating** in the config schema? It currently says +5. **Does the `billing.compute` description need updating** in the config schema? It currently says _"Unset = byok"_, which this change makes false. -5. **How long may `compute_status` block?** In `none`/`managed` it makes one authenticated call to +6. **How long may `compute_status` block?** In `none`/`managed` it makes one authenticated call to `/api/compute/options`. A slow or hanging Atlas would stall the agent mid-turn, so it needs a short timeout with `none` as the timeout result — but "short" should be a stated number, not left to the implementer. From 414bd5c903bece9b044d0c829a34b2fe11cfe8a4 Mon Sep 17 00:00:00 2001 From: KB Date: Thu, 30 Jul 2026 20:34:21 +0530 Subject: [PATCH 06/56] spec: park the budget-cap design, it was marked approved and is not buildable The doc said 'approved, ready for implementation planning'. It is neither, and leaving that header on a shared branch invites someone to build it. Two independent reasons it cannot proceed. Managed compute is switched off -- COMPUTE_RESELL_ENABLED defaults to false, so every provider reports funding 'unavailable' and there is no live overspend to guard. And an independent review returned 'Unsound as written' with five verified findings, the worst being that lease_reaper terminates a silent lease at 600s, so a budget lease running SSH work would die having spent ~1.17 dollars and the cap would never bind. Also retracts a retraction. The doc's own 'corrections' section claimed atlas compute:up exists and therefore the system prompt needs no fix. It exists in the Atlas repo but NOT in the published 0.13.2 that the ^0.13.2 pin resolves to -- source and npm disagree at an identical version number. The original finding was right; the correction was the error. The prompt fix moves to compute-mode-detection-design.md. The analysis is kept, not deleted -- it remains the best record of how Atlas compute billing works, and the five findings become the implementation checklist if managed compute is ever switched on. --- docs/specs/compute-guardrails-design.md | 81 ++++++++++++++++++++++--- 1 file changed, 71 insertions(+), 10 deletions(-) diff --git a/docs/specs/compute-guardrails-design.md b/docs/specs/compute-guardrails-design.md index 209c0a9c..a41c626a 100644 --- a/docs/specs/compute-guardrails-design.md +++ b/docs/specs/compute-guardrails-design.md @@ -1,7 +1,58 @@ # Managed compute budget cap — design -Status: approved, ready for implementation planning -Date: 2026-07-30 +Status: **PARKED — do not implement.** Superseded for now by +[`compute-mode-detection-design.md`](./compute-mode-detection-design.md). +Date: 2026-07-30 · parked 2026-07-30 + +> ## ⚠️ Read this before anything below +> +> **This design was reviewed and found unsound, and the problem it solves is not currently reachable.** +> The analysis below is kept because it is still the best record of how Atlas compute billing works. The +> _proposal_ should not be built as written. +> +> ### Why it is parked +> +> **1. Managed compute is switched off.** Atlas gates it on `COMPUTE_RESELL_ENABLED`, which defaults to +> `false` (`backend/app/config.py:383`), plus a configured operator key. Without both, every provider reports +> `funding: "unavailable"` and `POST /leases` refuses. **There is no live overspend to guard**, so this is a +> guardrail for a road that is not open. +> +> **2. An independent review returned "Unsound as written."** Five findings, each verified in source: +> +> - **The lease reaper kills these leases at ten minutes.** `lease_reaper` sweeps every 60s over all +> unfinished leases and reaps anything silent for `HEARTBEAT_STALE_SECONDS` = 600. A workload run over SSH +> emits no `agent_telemetry`, so a $30 budget lease would be terminated having spent about $1.17. The budget +> would never bind. This also falsifies the claim below that no agent liveness is needed — Atlas requires +> liveness in the form of telemetry. +> - **`budget_cents` already exists** on the agent-spawn path with a default of `500` +> (`agent_tools.py:1386`). Making the cap real converts a display-only $5 into a hard kill on a shipped +> feature. The migration section below calls this a no-op; it is not. +> - **The acquire-time debit is never rolled back**, so re-debiting per tick double-counts hour one. A $10 +> budget at $6.99/h would die at 25.8 minutes rather than ~1.4 hours; a one-hour budget would buy 60 seconds. +> The test list below does not catch this — the property that matters, _a budget of $B at $R/h lasts ≈B/R +> hours_, appears in neither the tests nor the acceptance criteria. +> - **Sequential leases are unbounded.** The cap is per-grant and a grant is per-lease, so an agent can release +> and re-acquire without limit. The concurrency cap of 2 does not bound cumulative spend. +> - **No idempotency on the money path.** The billing tick performs independent committing writes; a crash +> between them double-charges on the next tick, and adding a grant debit widens the window. +> +> ### Two factual errors in the text below +> +> - **"`atlas compute:up` exists"** — it exists in the Atlas repo (`cli/…/commands.mjs:922`) but **not in the +> published `@synsci/atlas@0.13.2`**, which is what `^0.13.2` resolves to. Source and npm disagree at the +> same version number. The "Corrections to earlier analysis" section below asserts the opposite and is wrong. +> - **The system prompt is therefore still broken**, contrary to what that section says. It instructs the agent +> to run a command absent from the installed CLI, then to check `atlas doctor` for managed availability — +> which reports no compute field at all — and offers "the user's own GPU providers" as the fallback. +> +> ### What to do instead +> +> Fixing that prompt is the live work, and it is handled by +> [`compute-mode-detection-design.md`](./compute-mode-detection-design.md), which needs nothing from Atlas. +> +> **Revive this document only when managed compute is actually switched on somewhere**, and then treat the five +> findings above as the implementation checklist rather than as blockers. + Roadmap items: **55** (budget guardrails + kill switches), **103** (cost approval gates), and the gate half of **51/2** (agent-facing compute tool) @@ -194,14 +245,24 @@ with, and change 1 begins enforcing it from the next tick, which is the safe dir Recorded because both errors reached a draft of this document. -- **`atlas compute:up` exists.** It is at `cli/src/atlas-runtime/commands.mjs:922` as a `LOCAL` command (aliases - `compute:launch`, `compute:lease`), alongside `compute:list` and `compute:ssh`. An earlier draft claimed it - existed in no version — that was wrong, caused by a comment two lines below it saying compute provisioning is - not part of the CLI. **Consequence: the system prompt at `session/prompt.ts:1554-1562` is not broken** and - needs no fix. It was previously an acceptance criterion; it is removed. -- **`compute:up` already takes `max_price` and `dry_run`.** A per-hour price ceiling and a no-spend preview - already exist in the CLI's parameter set, closer to this design than earlier drafts represented. Adding - `budget_cents` to `compute:up` as well would let CLI users have the same cap — worth doing, out of scope here. +- ~~**`atlas compute:up` exists.**~~ **This correction was itself wrong — see the banner at the top.** It exists + in the Atlas _repo_ but not in the published `@synsci/atlas@0.13.2`, which is what the `^0.13.2` pin + resolves to. Source and npm disagree at an identical version number. The original finding — that the prompt + points at a command the installed CLI does not have — was right, and the retraction below is the error. + It is at `cli/src/atlas-runtime/commands.mjs:922` as a `LOCAL` command (aliases + `compute:launch`, `compute:lease`), alongside `compute:list` and `compute:ssh` — **in the repo.** An earlier + draft claimed it existed in no version, which was wrong about the source and right about the artifact. + ~~**Consequence: the system prompt at `session/prompt.ts:1554-1562` is not broken** and needs no fix. It was + previously an acceptance criterion; it is removed.~~ **That consequence does not follow.** The published + package has no `compute:` command, so the prompt does point at something the installed CLI cannot run. The + prompt fix belongs back in scope — it now lives in `compute-mode-detection-design.md`. +- **`compute:up` already takes `max_price` and `dry_run`** — again, **in the repo only**. A per-hour price + ceiling and a no-spend preview exist in the source parameter set but ship to nobody until the CLI is + published. Adding `budget_cents` there too would let CLI users have the same cap, once any of it ships. + +**The lesson worth carrying:** three separate conclusions in this investigation came from reading source and +were wrong about the deployed reality — the CLI's contents, whether the prompt was broken, and whether managed +compute was reachable at all. Verify against the running system before designing against it. ## Out of scope From ac88f1ff3c221069d4807f679f7c6b5508995133 Mon Sep 17 00:00:00 2001 From: KB Date: Thu, 30 Jul 2026 21:01:50 +0530 Subject: [PATCH 07/56] feat(compute): resolve usable GPU providers from key and skill MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Establishes ComputeMode, the single shared rule for what makes a GPU provider "usable": a credential in process.env AND a catalogued skill. Tasks 2-4 consume PROVIDERS, SKILLS, and usable() from here. Mutation-proof (Step 5), each applied then reverted: 1. Delete `if (!keyed(spec.env)) continue` in usable() -> fails "a catalogued skill with NO key is not usable" (also collaterally fails 5 other tests, since every catalogued skill now counts as usable regardless of key) 2. Replace `spec.skills.some(...)` branch with unconditional `providers.push(id)` -> fails "a key with NO catalogued skill is not usable" 3. In keyed(), change `group.every(...)` to `group.some(...)` -> fails "modal needs BOTH token vars — id alone is not a key" 4. In keyed(), change `!!process.env[name]` to `name in process.env` -> fails "an empty-string key does not count as set" 5. Hoist the catalog Set to a module-scope memo, computed once -> "a key injected after the first call is seen on the next call" still passes, exactly as expected: env is read fresh every call, so that test can't see a stale skill catalog. It fails two other tests instead, via cross-test cache pollution (an earlier test's catalog leaks into a later one within the same run) — collateral damage, not a defect in this test. Freshness of the skill list itself is Instance.state's job; Task 4 covers per-turn catalog freshness. usable() intentionally does NOT memoize the catalog. All five mutations reproduced the outcomes named in the task brief; no test defects found. --- backend/cli/src/compute/mode.ts | 94 ++++++++++++++++++ backend/cli/test/compute/mode.test.ts | 131 ++++++++++++++++++++++++++ 2 files changed, 225 insertions(+) create mode 100644 backend/cli/src/compute/mode.ts create mode 100644 backend/cli/test/compute/mode.test.ts diff --git a/backend/cli/src/compute/mode.ts b/backend/cli/src/compute/mode.ts new file mode 100644 index 00000000..dac20b31 --- /dev/null +++ b/backend/cli/src/compute/mode.ts @@ -0,0 +1,94 @@ +import { Skill } from "@/skill" + +/** + * Runtime resolution of how GPU compute is funded. + * + * `billing.compute` used to answer this from config alone, which meant a + * brand-new user with zero provider keys resolved to "byok" — claiming BYOK + * with nothing to BYOK with. This module answers it from the environment + * instead, and can say "none", which is the state we previously handled worst. + * + * Resolution deliberately happens ON DEMAND and never at startup. Provider keys + * reach process.env from three places — the user's shell, the Credentials panel + * (`applyCredentialEnv`, src/index.ts:102) and the Compute panel + * (`applyComputeEnv`, src/index.ts:106) — and the latter two are wrapped in + * `.catch(() => {})`. Detecting at boot would report "none" for a user whose + * keys are configured through the UI. Both call sites (SkillTool.init and the + * compute_status tool) run per request, long after those injections, so the + * ordering constraint cannot be violated and cannot silently regress if someone + * reorders src/index.ts later. + */ +export namespace ComputeMode { + export type Source = "byok" | "managed" | "none" + + /** + * A provider is BYOK-usable only with BOTH a credential and a skill: the agent + * runs GPU work by loading a provider's skill, so a key with no skill gives it + * nothing to act on. + * + * `env` is a list of ALTERNATIVE groups; a group is satisfied when every var in + * it is set and non-empty. Modal is the only pair — its single pasted key + * splits into a token id + secret, and a half-pasted one maps to nothing + * (mirroring `mapProviderEnv`, server/routes/settings/compute.ts:181). + * + * `skills` are frontmatter `name` values, NOT directory names and NOT + * category-prefixed. Only these names are subject to mode filtering; the other + * cloud-compute skills (tinker, skypilot, fireworks, together) are inference + * APIs and orchestrators keyed by their own credentials, not GPU leases this + * mode governs, and are never hidden. + */ + export const PROVIDERS: Record = { + modal: { + env: [["MODAL_TOKEN_ID", "MODAL_TOKEN_SECRET"]], + skills: ["modal-serverless-gpu", "modal-ml-training", "modal-research-gpu"], + }, + lambda: { + env: [["LAMBDA_API_KEY"], ["LAMBDA_LABS_API_KEY"]], + skills: ["lambda-labs-gpu-cloud"], + }, + tensorpool: { + env: [["TENSORPOOL_KEY"], ["TENSORPOOL_API_KEY"]], + skills: ["tensorpool-gpu-cloud"], + }, + prime: { + env: [["PRIME_API_KEY"], ["PRIME_INTELLECT_API_KEY"]], + skills: ["prime-intellect-lab"], + }, + runpod: { + env: [["RUNPOD_API_KEY"]], + skills: ["runpod-gpu-cloud"], + }, + vast: { + env: [["VAST_API_KEY"]], + skills: ["vast-ai-gpu-cloud"], + }, + } + + /** Every provider skill name — the exact set the catalog filter operates on. */ + export const SKILLS = new Set(Object.values(PROVIDERS).flatMap((p) => p.skills)) + + /** Read process.env directly rather than Env.get: applyComputeEnv writes to + * process.env first and mirrors to Env only when instance state exists, so + * process.env is the one source that is always populated. */ + function keyed(groups: string[][]): boolean { + return groups.some((group) => group.every((name) => !!process.env[name])) + } + + /** + * Split configured providers into those the agent can actually act on and + * those with a stored key but no catalogued skill. The second list exists so + * `none` can say *why* — a user who connected a key and is then told no + * compute is available deserves better than silence. + */ + export async function usable() { + const catalog = new Set(await Skill.all().then((all) => all.map((skill) => skill.name))) + const providers: string[] = [] + const unusable: string[] = [] + for (const [id, spec] of Object.entries(PROVIDERS)) { + if (!keyed(spec.env)) continue + if (spec.skills.some((name) => catalog.has(name))) providers.push(id) + else unusable.push(id) + } + return { providers, unusable } + } +} diff --git a/backend/cli/test/compute/mode.test.ts b/backend/cli/test/compute/mode.test.ts new file mode 100644 index 00000000..249ea211 --- /dev/null +++ b/backend/cli/test/compute/mode.test.ts @@ -0,0 +1,131 @@ +import { test, expect, afterEach, describe } from "bun:test" +import path from "path" +import { ComputeMode } from "../../src/compute/mode" +import { Instance } from "../../src/project/instance" +import { tmpdir } from "../fixture/fixture" + +const ENV = [ + "MODAL_TOKEN_ID", + "MODAL_TOKEN_SECRET", + "LAMBDA_API_KEY", + "LAMBDA_LABS_API_KEY", + "TENSORPOOL_KEY", + "TENSORPOOL_API_KEY", + "PRIME_API_KEY", + "PRIME_INTELLECT_API_KEY", + "RUNPOD_API_KEY", + "VAST_API_KEY", +] + +function clearEnv() { + for (const name of ENV) delete process.env[name] +} + +afterEach(clearEnv) + +/** A tmpdir project seeded with real SKILL.md files, so Skill.all() finds them + * without a network catalog. `OPENSCIENCE_DISABLE_BUNDLED_SKILLS` in preload.ts + * keeps the dev skills/ dir and the server index out, so the test controls the + * catalog exactly. */ +async function withSkills(names: string[], fn: () => Promise): Promise { + await using tmp = await tmpdir({ + git: true, + init: async (dir) => { + for (const name of names) { + await Bun.write( + path.join(dir, ".openscience", "skill", name, "SKILL.md"), + `---\nname: ${name}\ndescription: Test fixture for ${name}.\ncategory: cloud-compute\n---\n\n# ${name}\n`, + ) + } + }, + }) + return Instance.provide({ directory: tmp.path, fn }) +} + +describe("ComputeMode.usable", () => { + test("a provider with a key and a skill is usable", async () => { + clearEnv() + process.env["LAMBDA_API_KEY"] = "secret_abc" + const result = await withSkills(["lambda-labs-gpu-cloud"], () => ComputeMode.usable()) + expect(result.providers).toEqual(["lambda"]) + expect(result.unusable).toEqual([]) + }) + + test("the alternate env spelling also counts", async () => { + clearEnv() + process.env["LAMBDA_LABS_API_KEY"] = "secret_abc" + const result = await withSkills(["lambda-labs-gpu-cloud"], () => ComputeMode.usable()) + expect(result.providers).toEqual(["lambda"]) + }) + + test("a key with NO catalogued skill is not usable", async () => { + clearEnv() + process.env["RUNPOD_API_KEY"] = "rpa_abc" + const result = await withSkills([], () => ComputeMode.usable()) + expect(result.providers).toEqual([]) + expect(result.unusable).toEqual(["runpod"]) + }) + + test("a catalogued skill with NO key is not usable", async () => { + clearEnv() + const result = await withSkills(["lambda-labs-gpu-cloud"], () => ComputeMode.usable()) + expect(result.providers).toEqual([]) + expect(result.unusable).toEqual([]) + }) + + test("modal needs BOTH token vars — id alone is not a key", async () => { + clearEnv() + process.env["MODAL_TOKEN_ID"] = "ak-abc" + const result = await withSkills(["modal-serverless-gpu"], () => ComputeMode.usable()) + expect(result.providers).toEqual([]) + expect(result.unusable).toEqual([]) + }) + + test("modal with both token vars is usable", async () => { + clearEnv() + process.env["MODAL_TOKEN_ID"] = "ak-abc" + process.env["MODAL_TOKEN_SECRET"] = "as-def" + const result = await withSkills(["modal-serverless-gpu"], () => ComputeMode.usable()) + expect(result.providers).toEqual(["modal"]) + }) + + test("an empty-string key does not count as set", async () => { + clearEnv() + process.env["TENSORPOOL_KEY"] = "" + const result = await withSkills(["tensorpool-gpu-cloud"], () => ComputeMode.usable()) + expect(result.providers).toEqual([]) + expect(result.unusable).toEqual([]) + }) + + test("every provider resolves in isolation, given its own skill", async () => { + const cases: Array<[string, Record, string]> = [ + ["modal", { MODAL_TOKEN_ID: "ak-a", MODAL_TOKEN_SECRET: "as-b" }, "modal-serverless-gpu"], + ["lambda", { LAMBDA_API_KEY: "k" }, "lambda-labs-gpu-cloud"], + ["tensorpool", { TENSORPOOL_KEY: "k" }, "tensorpool-gpu-cloud"], + ["prime", { PRIME_API_KEY: "k" }, "prime-intellect-lab"], + ["runpod", { RUNPOD_API_KEY: "k" }, "runpod-gpu-cloud"], + ["vast", { VAST_API_KEY: "k" }, "vast-ai-gpu-cloud"], + ] + for (const [id, env, skill] of cases) { + clearEnv() + Object.assign(process.env, env) + const result = await withSkills([skill], () => ComputeMode.usable()) + expect(result.providers).toEqual([id]) + } + }) + + test("SKILLS covers every name in PROVIDERS and nothing else", async () => { + const declared = Object.values(ComputeMode.PROVIDERS).flatMap((p) => p.skills) + expect([...ComputeMode.SKILLS].sort()).toEqual([...new Set(declared)].sort()) + expect(ComputeMode.SKILLS.size).toBeGreaterThan(0) + }) + + test("a key injected after the first call is seen on the next call", async () => { + clearEnv() + await withSkills(["lambda-labs-gpu-cloud"], async () => { + expect((await ComputeMode.usable()).providers).toEqual([]) + process.env["LAMBDA_API_KEY"] = "secret_late" + expect((await ComputeMode.usable()).providers).toEqual(["lambda"]) + }) + }) +}) From dc125b99065746f01af47fbd4eb0cb1db2f9c6e7 Mon Sep 17 00:00:00 2001 From: KB Date: Thu, 30 Jul 2026 21:12:33 +0530 Subject: [PATCH 08/56] test(compute): close three coverage gaps in mode.test.ts (review round 1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reviewer found the implementation spec-compliant with no logic defects, but three test-design gaps in the Task 1 suite. All fixes are test-side; src/compute/mode.ts is unchanged. 1. Two of modal's three skill names (modal-ml-training, modal-research-gpu) had zero coverage — every modal test seeded only modal-serverless-gpu, so the .some() disjunction was never proven to resolve modal off its 2nd/3rd name. Added a test that seeds each name alone (plus both Modal env vars) and asserts ["modal"]. 2. The SKILLS test compared ComputeMode.SKILLS against Object.values(PROVIDERS).flatMap(p => p.skills) — the exact expression SKILLS is implemented as, so it could never catch a misspelled skill string inside PROVIDERS. Replaced the derived right-hand side with a hardcoded literal array of the eight required names. 3. No test ever had two or more providers keyed at once, so the PROVIDERS-declaration-order contract on `providers`/`unusable` (which Tasks 2-4 rely on) was unguarded. Added one test keying vast, modal, lambda together (set out of declaration order) asserting ["modal", "lambda", "vast"], and one keying prime + tensorpool (unusable, no skills seeded) plus lambda (usable) asserting unusable == ["tensorpool", "prime"] — declaration order, not env-set order. Mutation-proof (Step 5 discipline), each applied then reverted: - spec.skills.some(...) -> catalog.has(spec.skills[0]) (checks only the first skill name) -> fails "each of modal's three skill names resolves modal on its own" (2nd/3rd iterations) - Corrupt PROVIDERS.modal.skills[1] ("modal-ml-training" -> "modal-ml-training-x") -> fails both "each of modal's three skill names..." and "SKILLS covers every name in PROVIDERS and nothing else" (single mutation proves both Finding 1 and Finding 2's guards) - Iterate Object.entries(PROVIDERS).sort(([a], [b]) => a.localeCompare(b)) instead of declaration order -> fails both "providers keyed together return in PROVIDERS declaration order, not set order" and "unusable providers also return in PROVIDERS declaration order, not set order" All three findings' guards held on first mutation; no test needed a second-order fix. --- backend/cli/test/compute/mode.test.ts | 50 ++++++++++++++++++++++++--- 1 file changed, 46 insertions(+), 4 deletions(-) diff --git a/backend/cli/test/compute/mode.test.ts b/backend/cli/test/compute/mode.test.ts index 249ea211..f61e836c 100644 --- a/backend/cli/test/compute/mode.test.ts +++ b/backend/cli/test/compute/mode.test.ts @@ -89,6 +89,17 @@ describe("ComputeMode.usable", () => { expect(result.providers).toEqual(["modal"]) }) + test("each of modal's three skill names resolves modal on its own", async () => { + const names = ["modal-serverless-gpu", "modal-ml-training", "modal-research-gpu"] + for (const name of names) { + clearEnv() + process.env["MODAL_TOKEN_ID"] = "ak-abc" + process.env["MODAL_TOKEN_SECRET"] = "as-def" + const result = await withSkills([name], () => ComputeMode.usable()) + expect(result.providers).toEqual(["modal"]) + } + }) + test("an empty-string key does not count as set", async () => { clearEnv() process.env["TENSORPOOL_KEY"] = "" @@ -114,10 +125,19 @@ describe("ComputeMode.usable", () => { } }) - test("SKILLS covers every name in PROVIDERS and nothing else", async () => { - const declared = Object.values(ComputeMode.PROVIDERS).flatMap((p) => p.skills) - expect([...ComputeMode.SKILLS].sort()).toEqual([...new Set(declared)].sort()) - expect(ComputeMode.SKILLS.size).toBeGreaterThan(0) + test("SKILLS covers every name in PROVIDERS and nothing else", () => { + const required = [ + "modal-serverless-gpu", + "modal-ml-training", + "modal-research-gpu", + "lambda-labs-gpu-cloud", + "tensorpool-gpu-cloud", + "prime-intellect-lab", + "runpod-gpu-cloud", + "vast-ai-gpu-cloud", + ] + expect([...ComputeMode.SKILLS].sort()).toEqual([...required].sort()) + expect(ComputeMode.SKILLS.size).toBe(required.length) }) test("a key injected after the first call is seen on the next call", async () => { @@ -128,4 +148,26 @@ describe("ComputeMode.usable", () => { expect((await ComputeMode.usable()).providers).toEqual(["lambda"]) }) }) + + test("providers keyed together return in PROVIDERS declaration order, not set order", async () => { + clearEnv() + process.env["VAST_API_KEY"] = "v" + process.env["MODAL_TOKEN_ID"] = "ak-a" + process.env["MODAL_TOKEN_SECRET"] = "as-b" + process.env["LAMBDA_API_KEY"] = "l" + const result = await withSkills(["vast-ai-gpu-cloud", "modal-serverless-gpu", "lambda-labs-gpu-cloud"], () => + ComputeMode.usable(), + ) + expect(result.providers).toEqual(["modal", "lambda", "vast"]) + }) + + test("unusable providers also return in PROVIDERS declaration order, not set order", async () => { + clearEnv() + process.env["PRIME_API_KEY"] = "p" + process.env["TENSORPOOL_KEY"] = "t" + process.env["LAMBDA_API_KEY"] = "l" + const result = await withSkills(["lambda-labs-gpu-cloud"], () => ComputeMode.usable()) + expect(result.providers).toEqual(["lambda"]) + expect(result.unusable).toEqual(["tensorpool", "prime"]) + }) }) From c90f2cc3ac9e04de5bda982575827fe98a7e3153 Mon Sep 17 00:00:00 2001 From: KB Date: Thu, 30 Jul 2026 21:24:03 +0530 Subject: [PATCH 09/56] refactor(compute): a credential alone makes a GPU provider usable --- backend/cli/src/compute/mode.ts | 35 ++++++-------- backend/cli/test/compute/mode.test.ts | 66 +++++++++++---------------- 2 files changed, 39 insertions(+), 62 deletions(-) diff --git a/backend/cli/src/compute/mode.ts b/backend/cli/src/compute/mode.ts index dac20b31..4c579f49 100644 --- a/backend/cli/src/compute/mode.ts +++ b/backend/cli/src/compute/mode.ts @@ -1,5 +1,3 @@ -import { Skill } from "@/skill" - /** * Runtime resolution of how GPU compute is funded. * @@ -22,10 +20,6 @@ export namespace ComputeMode { export type Source = "byok" | "managed" | "none" /** - * A provider is BYOK-usable only with BOTH a credential and a skill: the agent - * runs GPU work by loading a provider's skill, so a key with no skill gives it - * nothing to act on. - * * `env` is a list of ALTERNATIVE groups; a group is satisfied when every var in * it is set and non-empty. Modal is the only pair — its single pasted key * splits into a token id + secret, and a half-pasted one maps to nothing @@ -56,11 +50,11 @@ export namespace ComputeMode { }, runpod: { env: [["RUNPOD_API_KEY"]], - skills: ["runpod-gpu-cloud"], + skills: [], }, vast: { env: [["VAST_API_KEY"]], - skills: ["vast-ai-gpu-cloud"], + skills: [], }, } @@ -75,20 +69,17 @@ export namespace ComputeMode { } /** - * Split configured providers into those the agent can actually act on and - * those with a stored key but no catalogued skill. The second list exists so - * `none` can say *why* — a user who connected a key and is then told no - * compute is available deserves better than silence. + * The credentialed GPU providers, in declaration order. + * + * A credential is the whole test. An earlier revision also required a + * matching skill, on the theory that a provider with no skill gives the agent + * nothing to act on — but a capable agent drives a documented cloud API from a + * key, so that conjunction only produced a false "no compute available" for + * users holding a perfectly workable key. A skill, where one exists, is a + * quality boost; the catalog filter still offers a provider's skills only when + * that provider is credentialed. */ - export async function usable() { - const catalog = new Set(await Skill.all().then((all) => all.map((skill) => skill.name))) - const providers: string[] = [] - const unusable: string[] = [] - for (const [id, spec] of Object.entries(PROVIDERS)) { - if (!keyed(spec.env)) continue - if (spec.skills.some((name) => catalog.has(name))) providers.push(id) - else unusable.push(id) - } - return { providers, unusable } + export function usable(): string[] { + return Object.keys(PROVIDERS).filter((id) => keyed(PROVIDERS[id].env)) } } diff --git a/backend/cli/test/compute/mode.test.ts b/backend/cli/test/compute/mode.test.ts index f61e836c..b89ddb08 100644 --- a/backend/cli/test/compute/mode.test.ts +++ b/backend/cli/test/compute/mode.test.ts @@ -27,7 +27,7 @@ afterEach(clearEnv) * without a network catalog. `OPENSCIENCE_DISABLE_BUNDLED_SKILLS` in preload.ts * keeps the dev skills/ dir and the server index out, so the test controls the * catalog exactly. */ -async function withSkills(names: string[], fn: () => Promise): Promise { +async function withSkills(names: string[], fn: () => T): Promise { await using tmp = await tmpdir({ git: true, init: async (dir) => { @@ -47,38 +47,33 @@ describe("ComputeMode.usable", () => { clearEnv() process.env["LAMBDA_API_KEY"] = "secret_abc" const result = await withSkills(["lambda-labs-gpu-cloud"], () => ComputeMode.usable()) - expect(result.providers).toEqual(["lambda"]) - expect(result.unusable).toEqual([]) + expect(result).toEqual(["lambda"]) }) test("the alternate env spelling also counts", async () => { clearEnv() process.env["LAMBDA_LABS_API_KEY"] = "secret_abc" const result = await withSkills(["lambda-labs-gpu-cloud"], () => ComputeMode.usable()) - expect(result.providers).toEqual(["lambda"]) + expect(result).toEqual(["lambda"]) }) - test("a key with NO catalogued skill is not usable", async () => { + test("a key with NO catalogued skill IS usable — the agent drives the provider API directly", async () => { clearEnv() process.env["RUNPOD_API_KEY"] = "rpa_abc" - const result = await withSkills([], () => ComputeMode.usable()) - expect(result.providers).toEqual([]) - expect(result.unusable).toEqual(["runpod"]) + expect(await withSkills([], () => ComputeMode.usable())).toEqual(["runpod"]) }) test("a catalogued skill with NO key is not usable", async () => { clearEnv() const result = await withSkills(["lambda-labs-gpu-cloud"], () => ComputeMode.usable()) - expect(result.providers).toEqual([]) - expect(result.unusable).toEqual([]) + expect(result).toEqual([]) }) test("modal needs BOTH token vars — id alone is not a key", async () => { clearEnv() process.env["MODAL_TOKEN_ID"] = "ak-abc" const result = await withSkills(["modal-serverless-gpu"], () => ComputeMode.usable()) - expect(result.providers).toEqual([]) - expect(result.unusable).toEqual([]) + expect(result).toEqual([]) }) test("modal with both token vars is usable", async () => { @@ -86,26 +81,29 @@ describe("ComputeMode.usable", () => { process.env["MODAL_TOKEN_ID"] = "ak-abc" process.env["MODAL_TOKEN_SECRET"] = "as-def" const result = await withSkills(["modal-serverless-gpu"], () => ComputeMode.usable()) - expect(result.providers).toEqual(["modal"]) + expect(result).toEqual(["modal"]) }) - test("each of modal's three skill names resolves modal on its own", async () => { - const names = ["modal-serverless-gpu", "modal-ml-training", "modal-research-gpu"] - for (const name of names) { - clearEnv() - process.env["MODAL_TOKEN_ID"] = "ak-abc" - process.env["MODAL_TOKEN_SECRET"] = "as-def" - const result = await withSkills([name], () => ComputeMode.usable()) - expect(result.providers).toEqual(["modal"]) - } + test("PROVIDERS pins the exact skill names the catalog filter matches on", async () => { + expect([...ComputeMode.SKILLS].sort()).toEqual( + [ + "lambda-labs-gpu-cloud", + "modal-ml-training", + "modal-research-gpu", + "modal-serverless-gpu", + "prime-intellect-lab", + "tensorpool-gpu-cloud", + ].sort(), + ) + expect(ComputeMode.PROVIDERS["runpod"].skills).toEqual([]) + expect(ComputeMode.PROVIDERS["vast"].skills).toEqual([]) }) test("an empty-string key does not count as set", async () => { clearEnv() process.env["TENSORPOOL_KEY"] = "" const result = await withSkills(["tensorpool-gpu-cloud"], () => ComputeMode.usable()) - expect(result.providers).toEqual([]) - expect(result.unusable).toEqual([]) + expect(result).toEqual([]) }) test("every provider resolves in isolation, given its own skill", async () => { @@ -121,7 +119,7 @@ describe("ComputeMode.usable", () => { clearEnv() Object.assign(process.env, env) const result = await withSkills([skill], () => ComputeMode.usable()) - expect(result.providers).toEqual([id]) + expect(result).toEqual([id]) } }) @@ -133,8 +131,6 @@ describe("ComputeMode.usable", () => { "lambda-labs-gpu-cloud", "tensorpool-gpu-cloud", "prime-intellect-lab", - "runpod-gpu-cloud", - "vast-ai-gpu-cloud", ] expect([...ComputeMode.SKILLS].sort()).toEqual([...required].sort()) expect(ComputeMode.SKILLS.size).toBe(required.length) @@ -143,9 +139,9 @@ describe("ComputeMode.usable", () => { test("a key injected after the first call is seen on the next call", async () => { clearEnv() await withSkills(["lambda-labs-gpu-cloud"], async () => { - expect((await ComputeMode.usable()).providers).toEqual([]) + expect(await ComputeMode.usable()).toEqual([]) process.env["LAMBDA_API_KEY"] = "secret_late" - expect((await ComputeMode.usable()).providers).toEqual(["lambda"]) + expect(await ComputeMode.usable()).toEqual(["lambda"]) }) }) @@ -158,16 +154,6 @@ describe("ComputeMode.usable", () => { const result = await withSkills(["vast-ai-gpu-cloud", "modal-serverless-gpu", "lambda-labs-gpu-cloud"], () => ComputeMode.usable(), ) - expect(result.providers).toEqual(["modal", "lambda", "vast"]) - }) - - test("unusable providers also return in PROVIDERS declaration order, not set order", async () => { - clearEnv() - process.env["PRIME_API_KEY"] = "p" - process.env["TENSORPOOL_KEY"] = "t" - process.env["LAMBDA_API_KEY"] = "l" - const result = await withSkills(["lambda-labs-gpu-cloud"], () => ComputeMode.usable()) - expect(result.providers).toEqual(["lambda"]) - expect(result.unusable).toEqual(["tensorpool", "prime"]) + expect(result).toEqual(["modal", "lambda", "vast"]) }) }) From f78a9d44d8b293bbf6ec7c1cac68f57257b705fa Mon Sep 17 00:00:00 2001 From: KB Date: Thu, 30 Jul 2026 21:40:53 +0530 Subject: [PATCH 10/56] feat(compute): resolve byok/managed/none at runtime, config becomes an override Adds the managed-availability probe against Atlas's /api/compute/options and the full ComputeMode.resolve() that turns credentialed providers plus that probe into one of byok/managed/none. A failed, unauthenticated, or timed-out (3s) probe always resolves toward none, never managed, so a degraded backend can't promise a capability nobody confirmed. A usable BYOK provider skips the network call entirely. billing.compute now acts as an override that can only narrow the outcome to none, never manufacture a capability that isn't there. --- backend/cli/src/compute/mode.ts | 102 +++++++++++++ backend/cli/test/compute/mode.test.ts | 202 +++++++++++++++++++++++++- 2 files changed, 303 insertions(+), 1 deletion(-) diff --git a/backend/cli/src/compute/mode.ts b/backend/cli/src/compute/mode.ts index 4c579f49..a5e22494 100644 --- a/backend/cli/src/compute/mode.ts +++ b/backend/cli/src/compute/mode.ts @@ -16,6 +16,9 @@ * ordering constraint cannot be violated and cannot silently regress if someone * reorders src/index.ts later. */ +import { Config } from "@/config/config" +import { API_BASE, OpenScience } from "@/openscience" + export namespace ComputeMode { export type Source = "byok" | "managed" | "none" @@ -82,4 +85,103 @@ export namespace ComputeMode { export function usable(): string[] { return Object.keys(PROVIDERS).filter((id) => keyed(PROVIDERS[id].env)) } + + export interface Resolution { + mode: Source + /** Credentialed BYOK providers, in PROVIDERS declaration order. */ + providers: string[] + managed: boolean + /** Wallet balance in USD. Present only when mode === "managed". */ + balance?: number + } + + /** Hard ceiling on how long resolution may block an agent turn. Atlas's own + * 60s default is far too long to sit in front of a tool call; a slow or + * hanging backend must degrade to "none", not stall the turn. */ + const TIMEOUT = 3_000 + + /** Short in-process TTL, enough to stop a chatty agent hammering the endpoint + * inside one turn and no longer. The whole reason this is a tool rather than + * a prompt injection is that the answer changes mid-session, so a long cache + * would reintroduce exactly the staleness the tool exists to avoid. */ + const TTL = 5_000 + + let cache: { at: number; value: { managed: boolean; balance?: number } } | undefined + + /** Drop the availability cache. Called by tests; also safe after a connect. */ + export function invalidate() { + cache = undefined + } + + /** + * One authenticated call to /api/compute/options, which already annotates each + * provider with `funding` — "managed" when reselling is on and an operator key + * exists, else "unavailable". A failed, unauthenticated or timed-out call is + * treated as UNAVAILABLE: failing toward "none" produces an honest "connect a + * key" message, whereas failing toward "managed" would reproduce the bug this + * design exists to fix, promising a capability we never confirmed. + */ + async function available() { + if (cache && Date.now() - cache.at < TTL) return cache.value + const value = await probe() + cache = { at: Date.now(), value } + return value + } + + async function probe(): Promise<{ managed: boolean; balance?: number }> { + const session = await OpenScience.getSession().catch(() => null) + if (!session) return { managed: false } + try { + const res = await fetch(`${API_BASE}/api/compute/options`, { + headers: { Authorization: `Bearer ${session.api_key}` }, + signal: AbortSignal.timeout(TIMEOUT), + }) + if (!res.ok) return { managed: false } + const data = await res.json() + const providers = Array.isArray(data?.providers) ? data.providers : [] + const managed = providers.some((entry: { funding?: string }) => entry?.funding === "managed") + if (!managed) return { managed: false } + const cents = data?.cli_effective_balance_cents + return { managed: true, balance: typeof cents === "number" ? cents / 100 : undefined } + } catch { + return { managed: false } + } + } + + /** + * The single shared entry point. `billing.compute` is an OVERRIDE, not the + * source of truth: it may narrow the outcome to "none", but it may never + * manufacture a capability that isn't there. + */ + export async function resolve(): Promise { + const providers = usable() + const override = (await Config.get()).billing?.compute + + if (override === "byok") { + return { mode: providers.length ? "byok" : "none", providers, managed: false } + } + + if (override === "managed") { + const managed = await available() + return { + mode: managed.managed ? "managed" : "none", + providers, + managed: managed.managed, + balance: managed.managed ? managed.balance : undefined, + } + } + + // BYOK wins when a credentialed provider is present: it is free to the user, + // it works today, and it needs nothing from Atlas. This is also why a BYOK + // user never pays for the availability call. + if (providers.length) return { mode: "byok", providers, managed: false } + + const managed = await available() + return { + mode: managed.managed ? "managed" : "none", + providers, + managed: managed.managed, + balance: managed.managed ? managed.balance : undefined, + } + } } diff --git a/backend/cli/test/compute/mode.test.ts b/backend/cli/test/compute/mode.test.ts index b89ddb08..94a453ad 100644 --- a/backend/cli/test/compute/mode.test.ts +++ b/backend/cli/test/compute/mode.test.ts @@ -1,7 +1,9 @@ -import { test, expect, afterEach, describe } from "bun:test" +import { test, expect, afterEach, beforeEach, describe } from "bun:test" import path from "path" +import fs from "fs/promises" import { ComputeMode } from "../../src/compute/mode" import { Instance } from "../../src/project/instance" +import { Global } from "../../src/global" import { tmpdir } from "../fixture/fixture" const ENV = [ @@ -157,3 +159,201 @@ describe("ComputeMode.usable", () => { expect(result).toEqual(["modal", "lambda", "vast"]) }) }) + +const OPTIONS_URL = "/api/compute/options" +const SESSION = path.join(Global.Path.data, "openscience-session.json") +const realFetch = globalThis.fetch + +/** Record of every URL the resolver fetched, so "the call is skipped" is a + * positive assertion rather than an absence of failure. */ +let calls: string[] = [] + +function stubOptions(body: unknown, status = 200) { + globalThis.fetch = (async (input: string | URL | Request) => { + const url = String(input instanceof Request ? input.url : input) + calls.push(url) + if (!url.includes(OPTIONS_URL)) return realFetch(input as never) + return new Response(JSON.stringify(body), { status, headers: { "content-type": "application/json" } }) + }) as typeof fetch +} + +async function signIn() { + await fs.mkdir(Global.Path.data, { recursive: true }) + await Bun.write(SESSION, JSON.stringify({ api_key: "thk_test.secret", user_id: "u1" })) +} + +const MANAGED_ON = { + options: [], + providers: [ + { provider: "lambda", has_byok: false, has_operator: true, funding: "managed", count: 3 }, + { provider: "vast", has_byok: false, has_operator: false, funding: "unavailable", count: 0 }, + ], + resell_enabled: true, + cli_effective_balance_cents: 1234, +} + +const MANAGED_OFF = { + options: [], + providers: [{ provider: "lambda", has_byok: false, has_operator: false, funding: "unavailable", count: 0 }], + resell_enabled: false, + cli_effective_balance_cents: 1234, +} + +describe("ComputeMode.resolve", () => { + beforeEach(() => { + clearEnv() + calls = [] + ComputeMode.invalidate() + }) + + afterEach(async () => { + globalThis.fetch = realFetch + await fs.rm(SESSION, { force: true }).catch(() => {}) + }) + + test("a usable provider resolves to byok WITHOUT calling the availability endpoint", async () => { + await signIn() + stubOptions(MANAGED_ON) + process.env["LAMBDA_API_KEY"] = "k" + const result = await withSkills(["lambda-labs-gpu-cloud"], () => ComputeMode.resolve()) + expect(result.mode).toBe("byok") + expect(result.providers).toEqual(["lambda"]) + expect(result.balance).toBeUndefined() + expect(calls.filter((url) => url.includes(OPTIONS_URL))).toEqual([]) + }) + + test("no keys plus managed available resolves to managed, with the balance", async () => { + await signIn() + stubOptions(MANAGED_ON) + const result = await withSkills(["lambda-labs-gpu-cloud"], () => ComputeMode.resolve()) + expect(result.mode).toBe("managed") + expect(result.managed).toBe(true) + expect(result.balance).toBe(12.34) + }) + + test("no keys plus managed unavailable resolves to none", async () => { + await signIn() + stubOptions(MANAGED_OFF) + const result = await withSkills(["lambda-labs-gpu-cloud"], () => ComputeMode.resolve()) + expect(result.mode).toBe("none") + expect(result.managed).toBe(false) + expect(result.balance).toBeUndefined() + }) + + test("a failing availability call resolves to none, not managed", async () => { + await signIn() + globalThis.fetch = (async (input: string | URL | Request): Promise => { + throw new Error("network down") + }) as typeof fetch + const result = await withSkills([], () => ComputeMode.resolve()) + expect(result.mode).toBe("none") + expect(result.managed).toBe(false) + }) + + test("a non-ok availability response resolves to none", async () => { + await signIn() + stubOptions({ detail: "unauthorized" }, 401) + const result = await withSkills([], () => ComputeMode.resolve()) + expect(result.mode).toBe("none") + }) + + test("no session means managed is unavailable and no call is made", async () => { + await fs.rm(SESSION, { force: true }).catch(() => {}) + stubOptions(MANAGED_ON) + const result = await withSkills([], () => ComputeMode.resolve()) + expect(result.mode).toBe("none") + expect(calls.filter((url) => url.includes(OPTIONS_URL))).toEqual([]) + }) + + test("a key with no skill still resolves to byok and skips the availability call", async () => { + await signIn() + stubOptions(MANAGED_ON) + process.env["RUNPOD_API_KEY"] = "rpa_x" + const result = await withSkills([], () => ComputeMode.resolve()) + expect(result.mode).toBe("byok") + expect(result.providers).toEqual(["runpod"]) + expect(calls.filter((url) => url.includes(OPTIONS_URL))).toEqual([]) + }) + + test("the availability answer is cached within the TTL", async () => { + await signIn() + stubOptions(MANAGED_ON) + await withSkills([], async () => { + await ComputeMode.resolve() + await ComputeMode.resolve() + }) + expect(calls.filter((url) => url.includes(OPTIONS_URL)).length).toBe(1) + }) + + test("invalidate() drops the cache", async () => { + await signIn() + stubOptions(MANAGED_ON) + await withSkills([], async () => { + await ComputeMode.resolve() + ComputeMode.invalidate() + await ComputeMode.resolve() + }) + expect(calls.filter((url) => url.includes(OPTIONS_URL)).length).toBe(2) + }) +}) + +describe("ComputeMode.resolve override", () => { + beforeEach(() => { + clearEnv() + calls = [] + ComputeMode.invalidate() + }) + afterEach(async () => { + globalThis.fetch = realFetch + await fs.rm(SESSION, { force: true }).catch(() => {}) + }) + + /** Same tmpdir fixture as withSkills, plus an openscience.json setting + * billing.compute. */ + async function withOverride(mode: "byok" | "managed", skills: string[], fn: () => Promise): Promise { + await using tmp = await tmpdir({ + git: true, + init: async (dir) => { + for (const name of skills) { + await Bun.write( + path.join(dir, ".openscience", "skill", name, "SKILL.md"), + `---\nname: ${name}\ndescription: Test fixture for ${name}.\ncategory: cloud-compute\n---\n\n# ${name}\n`, + ) + } + await Bun.write(path.join(dir, "openscience.json"), JSON.stringify({ billing: { compute: mode } })) + }, + }) + return Instance.provide({ directory: tmp.path, fn }) + } + + test("override byok with a usable provider stays byok", async () => { + await signIn() + stubOptions(MANAGED_ON) + process.env["LAMBDA_API_KEY"] = "k" + const result = await withOverride("byok", ["lambda-labs-gpu-cloud"], () => ComputeMode.resolve()) + expect(result.mode).toBe("byok") + }) + + test("override byok with NO usable provider narrows to none, never managed", async () => { + await signIn() + stubOptions(MANAGED_ON) + const result = await withOverride("byok", ["lambda-labs-gpu-cloud"], () => ComputeMode.resolve()) + expect(result.mode).toBe("none") + expect(calls.filter((url) => url.includes(OPTIONS_URL))).toEqual([]) + }) + + test("override managed with managed unavailable narrows to none", async () => { + await signIn() + stubOptions(MANAGED_OFF) + const result = await withOverride("managed", [], () => ComputeMode.resolve()) + expect(result.mode).toBe("none") + }) + + test("override managed beats a usable provider when managed IS available", async () => { + await signIn() + stubOptions(MANAGED_ON) + process.env["LAMBDA_API_KEY"] = "k" + const result = await withOverride("managed", ["lambda-labs-gpu-cloud"], () => ComputeMode.resolve()) + expect(result.mode).toBe("managed") + }) +}) From 879c3c199aa96ed096a3da16ab960c1e7ade182b Mon Sep 17 00:00:00 2001 From: KB Date: Thu, 30 Jul 2026 21:54:13 +0530 Subject: [PATCH 11/56] fix(compute): close three gaps in resolve()'s test coverage and dedupe MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round-1 review findings on the compute-mode-detection resolver: - No test proved a forced managed override narrows to none when the user also has a usable BYOK provider and the managed probe is unavailable. Added a test that sets both, asserts mode is none and providers still reports the credential. - The cache suite proved same-turn reuse and invalidate(), but never that the cache actually expires — a process-lifetime cache would have passed every existing test. Added a Date.now-stubbed test that crosses the 5s TTL boundary and reads a changed probe result on the other side; tightened the existing within-TTL test to use the same stub so the two bracket the boundary rather than overlap. - Folded the two byte-identical Resolution-construction blocks (managed-override arm, no-override/no-provider fallback) into a single funded() helper. --- backend/cli/src/compute/mode.ts | 31 ++++++++++++++------------ backend/cli/test/compute/mode.test.ts | 32 +++++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 14 deletions(-) diff --git a/backend/cli/src/compute/mode.ts b/backend/cli/src/compute/mode.ts index a5e22494..3787d3b5 100644 --- a/backend/cli/src/compute/mode.ts +++ b/backend/cli/src/compute/mode.ts @@ -148,6 +148,21 @@ export namespace ComputeMode { } } + /** Build a Resolution purely from an availability probe's verdict — never + * from `providers`, which is passed through only for display. Shared by the + * managed-override arm and the no-override/no-provider fallback: both trust + * `available()` completely and must never fall back to "byok" just because + * a credential happens to be present (that would silently defeat the + * managed override — see the "managed with a usable provider" test). */ + function funded(providers: string[], state: { managed: boolean; balance?: number }): Resolution { + return { + mode: state.managed ? "managed" : "none", + providers, + managed: state.managed, + balance: state.managed ? state.balance : undefined, + } + } + /** * The single shared entry point. `billing.compute` is an OVERRIDE, not the * source of truth: it may narrow the outcome to "none", but it may never @@ -162,13 +177,7 @@ export namespace ComputeMode { } if (override === "managed") { - const managed = await available() - return { - mode: managed.managed ? "managed" : "none", - providers, - managed: managed.managed, - balance: managed.managed ? managed.balance : undefined, - } + return funded(providers, await available()) } // BYOK wins when a credentialed provider is present: it is free to the user, @@ -176,12 +185,6 @@ export namespace ComputeMode { // user never pays for the availability call. if (providers.length) return { mode: "byok", providers, managed: false } - const managed = await available() - return { - mode: managed.managed ? "managed" : "none", - providers, - managed: managed.managed, - balance: managed.managed ? managed.balance : undefined, - } + return funded(providers, await available()) } } diff --git a/backend/cli/test/compute/mode.test.ts b/backend/cli/test/compute/mode.test.ts index 94a453ad..fbb604d9 100644 --- a/backend/cli/test/compute/mode.test.ts +++ b/backend/cli/test/compute/mode.test.ts @@ -163,6 +163,7 @@ describe("ComputeMode.usable", () => { const OPTIONS_URL = "/api/compute/options" const SESSION = path.join(Global.Path.data, "openscience-session.json") const realFetch = globalThis.fetch +const realNow = Date.now /** Record of every URL the resolver fetched, so "the call is skipped" is a * positive assertion rather than an absence of failure. */ @@ -208,6 +209,7 @@ describe("ComputeMode.resolve", () => { afterEach(async () => { globalThis.fetch = realFetch + Date.now = realNow await fs.rm(SESSION, { force: true }).catch(() => {}) }) @@ -278,13 +280,32 @@ describe("ComputeMode.resolve", () => { test("the availability answer is cached within the TTL", async () => { await signIn() stubOptions(MANAGED_ON) + let now = realNow() + Date.now = () => now await withSkills([], async () => { await ComputeMode.resolve() + now += 4_999 // still inside the 5s TTL await ComputeMode.resolve() }) expect(calls.filter((url) => url.includes(OPTIONS_URL)).length).toBe(1) }) + test("the cache expires once the TTL elapses, forcing a re-probe", async () => { + await signIn() + stubOptions(MANAGED_ON) + let now = realNow() + Date.now = () => now + await withSkills([], async () => { + const first = await ComputeMode.resolve() + expect(first.mode).toBe("managed") + now += 5_001 // past the 5s TTL — the cached verdict must be treated as stale + stubOptions(MANAGED_OFF) + const second = await ComputeMode.resolve() + expect(second.mode).toBe("none") + }) + expect(calls.filter((url) => url.includes(OPTIONS_URL)).length).toBe(2) + }) + test("invalidate() drops the cache", async () => { await signIn() stubOptions(MANAGED_ON) @@ -349,6 +370,17 @@ describe("ComputeMode.resolve override", () => { expect(result.mode).toBe("none") }) + test("override managed with a usable provider still narrows to none when managed is unavailable", async () => { + await signIn() + stubOptions(MANAGED_OFF) + process.env["LAMBDA_API_KEY"] = "k" + const result = await withOverride("managed", ["lambda-labs-gpu-cloud"], () => ComputeMode.resolve()) + expect(result.mode).toBe("none") + // The credential is real — it's just not the funded path under a forced + // managed override, so it must still be reported, not hidden. + expect(result.providers).toEqual(["lambda"]) + }) + test("override managed beats a usable provider when managed IS available", async () => { await signIn() stubOptions(MANAGED_ON) From 18a210b17a27e0ec09a3e96c01a71751a22d6cd0 Mon Sep 17 00:00:00 2001 From: KB Date: Thu, 30 Jul 2026 22:06:47 +0530 Subject: [PATCH 12/56] feat(tool): add compute_status so the agent pulls its compute mode Nothing is injected into the prompt per turn because compute mode can change mid-session (a user connects a provider key in Settings while a session is running). The agent instead calls this tool before doing any GPU/training/cluster work; its description carries the constraint, its result carries the mode, usable providers, and mode-specific guidance. --- backend/cli/src/tool/compute.ts | 58 +++++++ backend/cli/src/tool/registry.ts | 2 + backend/cli/test/tool/compute-status.test.ts | 161 +++++++++++++++++++ 3 files changed, 221 insertions(+) create mode 100644 backend/cli/src/tool/compute.ts create mode 100644 backend/cli/test/tool/compute-status.test.ts diff --git a/backend/cli/src/tool/compute.ts b/backend/cli/src/tool/compute.ts new file mode 100644 index 00000000..33ec9384 --- /dev/null +++ b/backend/cli/src/tool/compute.ts @@ -0,0 +1,58 @@ +import z from "zod" +import { Tool } from "./tool" +import { ComputeMode } from "@/compute/mode" + +/** + * The agent PULLS its compute mode from here; nothing is injected per turn. + * + * An earlier design injected mode guidance into every turn. That was wrong for a + * reason that matters more than token cost: the mode changes mid-session. A user + * connects a Modal key in Settings ▸ Compute while a session is running, and a + * reminder injected at turn 3 is false by turn 12. A tool returns the state at + * the moment it is asked. + * + * The DESCRIPTION carries the constraint — it reaches the agent before it starts + * down a path, which is the one thing an injection did well, and tool definitions + * are in every request regardless, so it costs nothing extra. The RESULT carries + * the specifics. Adding rates or a balance to an every-turn injection would be + * expensive; adding them here is free. + */ + +const GUIDANCE: Record = { + byok: "Run GPU work on the user's connected providers via the cloud-compute skills. Do not launch managed leases — they bill Credits and are not the funded path here.", + managed: + "Run GPU work through managed compute, billed to Credits. Do not use the user's own provider keys — they are not funded here.", + none: "No compute is available. Do not attempt GPU work. Tell the user to connect a provider key in Settings ▸ Compute, or to top up for managed compute.", +} + +export const ComputeStatusTool = Tool.define("compute_status", { + description: [ + "Check how GPU compute is funded before running any GPU, training, or cluster work.", + "Returns one of byok, managed, or none, the providers available, and the rule that applies.", + "Call this first — the answer can change mid-session as the user connects or removes keys.", + ].join(" "), + parameters: z.object({}), + async execute(_params, _ctx) { + const state = await ComputeMode.resolve() + const lines = [ + `**mode**: ${state.mode}`, + `**providers**: ${state.providers.length ? state.providers.join(", ") : "none configured"}`, + `**managed available**: ${state.managed ? "yes" : "no"}`, + ] + if (state.balance !== undefined) lines.push(`**balance**: $${state.balance.toFixed(2)}`) + lines.push("", GUIDANCE[state.mode]) + + return { + title: `Compute: ${state.mode}`, + output: lines.join("\n"), + metadata: { + mode: state.mode, + providers: state.providers, + managed_available: state.managed, + balance_usd: state.balance, + }, + } + }, +}) + +export const ComputeTools = [ComputeStatusTool] diff --git a/backend/cli/src/tool/registry.ts b/backend/cli/src/tool/registry.ts index 3f5096af..34e794cd 100644 --- a/backend/cli/src/tool/registry.ts +++ b/backend/cli/src/tool/registry.ts @@ -33,6 +33,7 @@ import { ArtifactTool } from "./artifact" import { LearnTool } from "./learn" import { ScienceTools } from "./science" import { ProvenanceTools } from "./provenance" +import { ComputeTools } from "./compute" import { NotebookTool } from "./notebook" import { RKernelTool } from "./rkernel" @@ -131,6 +132,7 @@ export namespace ToolRegistry { ...BiologyTools, ...ScienceTools, ...ProvenanceTools, + ...ComputeTools, NotebookTool, RKernelTool, ArtifactTool, diff --git a/backend/cli/test/tool/compute-status.test.ts b/backend/cli/test/tool/compute-status.test.ts new file mode 100644 index 00000000..6417b7dc --- /dev/null +++ b/backend/cli/test/tool/compute-status.test.ts @@ -0,0 +1,161 @@ +import { test, expect, describe, beforeEach, afterEach } from "bun:test" +import path from "path" +import fs from "fs/promises" +import { ComputeStatusTool } from "../../src/tool/compute" +import { ComputeMode } from "../../src/compute/mode" +import { ToolRegistry } from "../../src/tool/registry" +import { Instance } from "../../src/project/instance" +import { Global } from "../../src/global" +import { tmpdir } from "../fixture/fixture" + +const ENV = ["LAMBDA_API_KEY", "RUNPOD_API_KEY", "MODAL_TOKEN_ID", "MODAL_TOKEN_SECRET"] +const SESSION = path.join(Global.Path.data, "openscience-session.json") +const realFetch = globalThis.fetch + +const CTX = { + sessionID: "ses_test", + messageID: "msg_test", + agent: "research", + abort: new AbortController().signal, + messages: [], + metadata: () => {}, + ask: async () => {}, +} + +function stub(body: unknown, status = 200) { + globalThis.fetch = (async (input: string | URL | Request) => { + const url = String(input instanceof Request ? input.url : input) + if (!url.includes("/api/compute/options")) return realFetch(input as never) + return new Response(JSON.stringify(body), { status, headers: { "content-type": "application/json" } }) + }) as typeof fetch +} + +const MANAGED_ON = { + providers: [{ provider: "lambda", funding: "managed", has_byok: false, has_operator: true, count: 2 }], + resell_enabled: true, + cli_effective_balance_cents: 4200, +} +const MANAGED_OFF = { providers: [], resell_enabled: false, cli_effective_balance_cents: 0 } + +async function run(skills: string[], fn?: () => Promise) { + await using tmp = await tmpdir({ + git: true, + init: async (dir) => { + for (const name of skills) { + await Bun.write( + path.join(dir, ".openscience", "skill", name, "SKILL.md"), + `---\nname: ${name}\ndescription: Fixture ${name}.\ncategory: cloud-compute\n---\n\n# ${name}\n`, + ) + } + }, + }) + return Instance.provide({ + directory: tmp.path, + fn: async () => { + await fn?.() + const tool = await ComputeStatusTool.init({}) + return tool.execute({}, CTX as never) + }, + }) +} + +describe("compute_status", () => { + beforeEach(async () => { + for (const name of ENV) delete process.env[name] + ComputeMode.invalidate() + await fs.mkdir(Global.Path.data, { recursive: true }) + await Bun.write(SESSION, JSON.stringify({ api_key: "thk_t.s", user_id: "u1" })) + }) + afterEach(async () => { + globalThis.fetch = realFetch + for (const name of ENV) delete process.env[name] + await fs.rm(SESSION, { force: true }).catch(() => {}) + }) + + test("byok reports the mode, the usable providers, and byok guidance", async () => { + stub(MANAGED_ON) + process.env["LAMBDA_API_KEY"] = "k" + const result = await run(["lambda-labs-gpu-cloud"]) + expect(result.metadata.mode).toBe("byok") + expect(result.metadata.providers).toEqual(["lambda"]) + expect(result.output).toContain("lambda") + expect(result.output.toLowerCase()).toContain("do not launch managed") + expect(result.metadata.balance_usd).toBeUndefined() + }) + + test("managed reports the balance and managed guidance", async () => { + stub(MANAGED_ON) + const result = await run([]) + expect(result.metadata.mode).toBe("managed") + expect(result.metadata.balance_usd).toBe(42) + expect(result.output).toContain("42") + expect(result.output.toLowerCase()).toContain("credits") + }) + + test("none tells the agent not to attempt GPU work and how to enable it", async () => { + stub(MANAGED_OFF) + const result = await run([]) + expect(result.metadata.mode).toBe("none") + expect(result.output.toLowerCase()).toContain("do not attempt gpu work") + expect(result.output).toContain("Settings") + expect(result.metadata.balance_usd).toBeUndefined() + }) + + test("a provider with a key but no skill is still reported as usable byok", async () => { + stub(MANAGED_OFF) + process.env["RUNPOD_API_KEY"] = "rpa_x" + const result = await run([]) + expect(result.metadata.mode).toBe("byok") + expect(result.metadata.providers).toEqual(["runpod"]) + expect(result.output).toContain("runpod") + }) + + test("the three modes produce three DIFFERENT guidance strings", async () => { + stub(MANAGED_ON) + process.env["LAMBDA_API_KEY"] = "k" + const byok = await run(["lambda-labs-gpu-cloud"]) + delete process.env["LAMBDA_API_KEY"] + ComputeMode.invalidate() + const managed = await run([]) + stub(MANAGED_OFF) + ComputeMode.invalidate() + const none = await run([]) + // Comparing whole `output` strings is a false positive: the leading + // `**mode**: byok|managed|none` line always differs by itself, so the + // assertion would pass even if GUIDANCE collapsed to one shared string. + // Isolate the guidance sentence — the text after the blank-line + // separator the tool always inserts before it — so this actually + // exercises the property under test. + const guidance = (output: string) => output.split("\n\n").at(-1) + const texts = [byok.output, managed.output, none.output].map(guidance) + expect(new Set(texts).size).toBe(3) + }) + + test("a credential connected between two calls changes the answer, no restart", async () => { + stub(MANAGED_OFF) + const before = await run([]) + expect(before.metadata.mode).toBe("none") + process.env["LAMBDA_API_KEY"] = "connected-mid-session" + const after = await run(["lambda-labs-gpu-cloud"]) + expect(after.metadata.mode).toBe("byok") + }) + + test("the description instructs the agent to check before running GPU work", async () => { + const tool = await ComputeStatusTool.init({}) + expect(tool.description.toLowerCase()).toContain("before") + expect(tool.description.toLowerCase()).toContain("gpu") + expect(tool.description).toContain("byok") + expect(tool.description).toContain("managed") + expect(tool.description).toContain("none") + }) + + test("the tool is registered", async () => { + await using tmp = await tmpdir({ git: true }) + await Instance.provide({ + directory: tmp.path, + fn: async () => { + expect(await ToolRegistry.ids()).toContain("compute_status") + }, + }) + }) +}) From 2940bd3fdc26a535385d899bd4eb916bb95c674a Mon Sep 17 00:00:00 2001 From: KB Date: Fri, 31 Jul 2026 09:19:16 +0530 Subject: [PATCH 13/56] fix(tool): harden compute_status test assertions per review round 1 Three fixes to test/tool/compute-status.test.ts: - The guidance-distinctness test compared whole `output` strings via a formatting artifact (text after a blank-line separator). A harmless refactor that drops the blank line while keeping three genuinely distinct GUIDANCE strings would have failed it for no reason, and it never actually asserted the guidance differed. Replaced with a contract check: each mode's short, load-bearing guidance phrase must appear in that mode's output and only that mode's output. - The no-skill byok test reused test 1's output-line mutation as its proof of non-vacuity. Retargeted to a distinct mutation this layer owns: deleting `providers` from the returned metadata object. The underlying skill-independence rule is owned and tested directly by ComputeMode in test/compute/mode.ts; this test is a passthrough check by design. - Nothing asserted that the managed balance comes from the same /api/compute/options call that decided availability, not a second round trip. Added call-count tracking (mirrors mode.test.ts) and an explicit assertion of exactly one call. --- backend/cli/test/tool/compute-status.test.ts | 48 ++++++++++++++++---- 1 file changed, 39 insertions(+), 9 deletions(-) diff --git a/backend/cli/test/tool/compute-status.test.ts b/backend/cli/test/tool/compute-status.test.ts index 6417b7dc..c944754b 100644 --- a/backend/cli/test/tool/compute-status.test.ts +++ b/backend/cli/test/tool/compute-status.test.ts @@ -22,9 +22,15 @@ const CTX = { ask: async () => {}, } +/** Every URL fetched since the last reset, so "only one network call" is a + * positive assertion rather than an absence of failure (mirrors + * test/compute/mode.test.ts's `calls`). */ +let calls: string[] = [] + function stub(body: unknown, status = 200) { globalThis.fetch = (async (input: string | URL | Request) => { const url = String(input instanceof Request ? input.url : input) + calls.push(url) if (!url.includes("/api/compute/options")) return realFetch(input as never) return new Response(JSON.stringify(body), { status, headers: { "content-type": "application/json" } }) }) as typeof fetch @@ -62,6 +68,7 @@ async function run(skills: string[], fn?: () => Promise) { describe("compute_status", () => { beforeEach(async () => { for (const name of ENV) delete process.env[name] + calls = [] ComputeMode.invalidate() await fs.mkdir(Global.Path.data, { recursive: true }) await Bun.write(SESSION, JSON.stringify({ api_key: "thk_t.s", user_id: "u1" })) @@ -83,13 +90,16 @@ describe("compute_status", () => { expect(result.metadata.balance_usd).toBeUndefined() }) - test("managed reports the balance and managed guidance", async () => { + test("managed reports the balance and managed guidance, from a single network call", async () => { stub(MANAGED_ON) const result = await run([]) expect(result.metadata.mode).toBe("managed") expect(result.metadata.balance_usd).toBe(42) expect(result.output).toContain("42") expect(result.output.toLowerCase()).toContain("credits") + // balance_usd must come from the SAME /api/compute/options response that + // decided managed availability, never a second round trip. + expect(calls.filter((url) => url.includes("/api/compute/options")).length).toBe(1) }) test("none tells the agent not to attempt GPU work and how to enable it", async () => { @@ -121,14 +131,34 @@ describe("compute_status", () => { ComputeMode.invalidate() const none = await run([]) // Comparing whole `output` strings is a false positive: the leading - // `**mode**: byok|managed|none` line always differs by itself, so the - // assertion would pass even if GUIDANCE collapsed to one shared string. - // Isolate the guidance sentence — the text after the blank-line - // separator the tool always inserts before it — so this actually - // exercises the property under test. - const guidance = (output: string) => output.split("\n\n").at(-1) - const texts = [byok.output, managed.output, none.output].map(guidance) - expect(new Set(texts).size).toBe(3) + // `**mode**: byok|managed|none` line always differs by itself, so a + // whole-string comparison would pass even if GUIDANCE collapsed to one + // shared string. A formatting-position trick (e.g. "text after the last + // blank line") is equally fragile — it breaks the moment the separator + // between the report and the guidance changes shape, which is a pure + // formatting edit that should never fail this test. + // + // Assert against the actual contract instead: each mode's GUIDANCE entry + // carries a short, semantically load-bearing phrase that could not + // survive a collapse to one shared string, and that phrase must appear + // in that mode's output and ONLY that mode's output. + const PHRASE = { + byok: "do not launch managed", + managed: "do not use the user's own provider keys", + none: "do not attempt gpu work", + } + const output = { + byok: byok.output.toLowerCase(), + managed: managed.output.toLowerCase(), + none: none.output.toLowerCase(), + } + for (const mode of Object.keys(PHRASE) as (keyof typeof PHRASE)[]) { + expect(output[mode]).toContain(PHRASE[mode]) + for (const other of Object.keys(PHRASE) as (keyof typeof PHRASE)[]) { + if (other === mode) continue + expect(output[mode]).not.toContain(PHRASE[other]) + } + } }) test("a credential connected between two calls changes the answer, no restart", async () => { From 3a941fb2ea2ad475da8102c7ea119d384b55383d Mon Sep 17 00:00:00 2001 From: KB Date: Fri, 31 Jul 2026 09:32:25 +0530 Subject: [PATCH 14/56] feat(skill): offer GPU provider skills only for usable providers --- backend/cli/src/tool/skill.ts | 18 +- .../test/tool/skill-compute-filter.test.ts | 199 ++++++++++++++++++ 2 files changed, 216 insertions(+), 1 deletion(-) create mode 100644 backend/cli/test/tool/skill-compute-filter.test.ts diff --git a/backend/cli/src/tool/skill.ts b/backend/cli/src/tool/skill.ts index 48d1a5e9..cc668d32 100644 --- a/backend/cli/src/tool/skill.ts +++ b/backend/cli/src/tool/skill.ts @@ -8,6 +8,7 @@ import { PermissionNext } from "../permission/next" import { OpenScience } from "@/openscience" import { RSILifecycle } from "@/session/rsi/lifecycle" import { Global } from "@/global" +import { ComputeMode } from "@/compute/mode" // Lightweight fuzzy score: rewards substring containment + shared bigrams. // Returns 0..1. No external deps needed for a "did you mean?" hint. @@ -34,13 +35,28 @@ export const SkillTool = Tool.define("skill", async (ctx) => { // Filter skills by agent permissions if agent provided const agent = ctx?.agent - const accessibleSkills = agent + const permitted = agent ? skills.filter((skill) => { const rule = PermissionNext.evaluate("skill", skill.name, agent.permission) return rule.action !== "deny" }) : skills + // Filter the GPU provider skills by the resolved compute mode, so the agent + // picks the right provider because it is the only one offered. This init runs + // per request (registry.ts calls it inside tools()), which buys two things for + // free: a credential connected mid-session shows up on the next turn with no + // cache to invalidate, and resolution always happens after src/index.ts's env + // injections rather than racing them. + // + // This is a LISTING filter, not a gate. `none` is guidance, not enforcement — + // a hidden skill can still be loaded by exact name, and the agent still has + // bash. Gating the load path is a larger change and is deliberately out of + // scope; see docs/specs/compute-mode-detection-design.md open question 3. + const compute = await ComputeMode.resolve() + const offered = new Set(compute.providers.flatMap((id) => ComputeMode.PROVIDERS[id].skills)) + const accessibleSkills = permitted.filter((skill) => !ComputeMode.SKILLS.has(skill.name) || offered.has(skill.name)) + // Group skills by category for the description const categories: Record = {} const uncategorized: Skill.Info[] = [] diff --git a/backend/cli/test/tool/skill-compute-filter.test.ts b/backend/cli/test/tool/skill-compute-filter.test.ts new file mode 100644 index 00000000..14d9c9ed --- /dev/null +++ b/backend/cli/test/tool/skill-compute-filter.test.ts @@ -0,0 +1,199 @@ +import { test, expect, describe, beforeEach, afterEach } from "bun:test" +import path from "path" +import fs from "fs/promises" +import { SkillTool } from "../../src/tool/skill" +import { ComputeMode } from "../../src/compute/mode" +import { Instance } from "../../src/project/instance" +import { Global } from "../../src/global" +import { tmpdir } from "../fixture/fixture" + +const ENV = ["LAMBDA_API_KEY", "RUNPOD_API_KEY", "MODAL_TOKEN_ID", "MODAL_TOKEN_SECRET", "TENSORPOOL_KEY"] +const SESSION = path.join(Global.Path.data, "openscience-session.json") +const realFetch = globalThis.fetch + +// Every provider skill, plus two skills that must never be filtered: a +// non-compute one and a cloud-compute skill that maps to no panel provider. +const ALL = [ + ["modal-serverless-gpu", "cloud-compute"], + ["lambda-labs-gpu-cloud", "cloud-compute"], + ["tensorpool-gpu-cloud", "cloud-compute"], + ["prime-intellect-lab", "ml-training"], + ["tinker-fine-tuning", "cloud-compute"], + ["rdkit", "chemistry"], +] as const + +function stub(managed: boolean) { + globalThis.fetch = (async (input: string | URL | Request) => { + const url = String(input instanceof Request ? input.url : input) + if (!url.includes("/api/compute/options")) return realFetch(input as never) + return new Response( + JSON.stringify({ + providers: managed ? [{ provider: "lambda", funding: "managed" }] : [], + resell_enabled: managed, + cli_effective_balance_cents: 500, + }), + { status: 200, headers: { "content-type": "application/json" } }, + ) + }) as typeof fetch +} + +async function project(fn: (dir: string) => Promise) { + return tmpdir({ + git: true, + init: async (dir) => { + for (const [name, category] of ALL) { + await Bun.write( + path.join(dir, ".openscience", "skill", name, "SKILL.md"), + `---\nname: ${name}\ndescription: Fixture ${name}.\ncategory: ${category}\n---\n\n# ${name}\n`, + ) + } + await fn(dir) + }, + }) +} + +/** Which of the six provider skills does the tool offer? Read from the tool's + * own category listing, which is what the model sees. */ +async function offered(): Promise { + const tool = await SkillTool.init({}) + const found: string[] = [] + for (const category of ["cloud-compute", "ml-training"]) { + const result = await tool + .execute({ category }, { + sessionID: "s", + messageID: "m", + agent: "research", + abort: new AbortController().signal, + messages: [], + metadata: () => {}, + ask: async () => {}, + } as never) + .catch(() => undefined) + if (result) found.push(result.output) + } + const text = found.join("\n") + return [...ComputeMode.SKILLS].filter((name) => text.includes(`**${name}**`)).sort() +} + +async function nonComputeVisible(): Promise { + const tool = await SkillTool.init({}) + const result = await tool.execute({ category: "chemistry" }, { + sessionID: "s", + messageID: "m", + agent: "research", + abort: new AbortController().signal, + messages: [], + metadata: () => {}, + ask: async () => {}, + } as never) + return result.output.includes("**rdkit**") +} + +async function tinkerVisible(): Promise { + const tool = await SkillTool.init({}) + const result = await tool.execute({ category: "cloud-compute" }, { + sessionID: "s", + messageID: "m", + agent: "research", + abort: new AbortController().signal, + messages: [], + metadata: () => {}, + ask: async () => {}, + } as never) + return result.output.includes("**tinker-fine-tuning**") +} + +describe("skill catalog filtering by compute mode", () => { + beforeEach(async () => { + for (const name of ENV) delete process.env[name] + ComputeMode.invalidate() + await fs.mkdir(Global.Path.data, { recursive: true }) + await Bun.write(SESSION, JSON.stringify({ api_key: "thk_t.s", user_id: "u1" })) + }) + afterEach(async () => { + globalThis.fetch = realFetch + for (const name of ENV) delete process.env[name] + await fs.rm(SESSION, { force: true }).catch(() => {}) + }) + + test("with only a Modal credential, only Modal's skills are offered", async () => { + stub(false) + process.env["MODAL_TOKEN_ID"] = "ak-a" + process.env["MODAL_TOKEN_SECRET"] = "as-b" + await using tmp = await project(async () => {}) + const names = await Instance.provide({ directory: tmp.path, fn: offered }) + expect(names).toEqual(["modal-serverless-gpu"]) + }) + + test("a RunPod credential is byok but contributes no skills — nobody else's are offered either", async () => { + stub(false) + process.env["RUNPOD_API_KEY"] = "rpa_x" + await using tmp = await project(async () => {}) + await Instance.provide({ + directory: tmp.path, + fn: async () => { + // RunPod carries skills: [] (Decision 2), so being credentialed makes the + // user byok without unlocking any other provider's skills. + expect((await ComputeMode.resolve()).mode).toBe("byok") + expect(await offered()).toEqual([]) + }, + }) + }) + + test("in managed, no BYOK provider skill is offered", async () => { + stub(true) + await using tmp = await project(async () => {}) + const names = await Instance.provide({ directory: tmp.path, fn: offered }) + expect(names).toEqual([]) + }) + + test("in none, no BYOK provider skill is offered", async () => { + stub(false) + await using tmp = await project(async () => {}) + const names = await Instance.provide({ directory: tmp.path, fn: offered }) + expect(names).toEqual([]) + }) + + test("non-compute skills are unaffected in every mode", async () => { + for (const managed of [true, false]) { + stub(managed) + ComputeMode.invalidate() + await using tmp = await project(async () => {}) + expect(await Instance.provide({ directory: tmp.path, fn: nonComputeVisible })).toBe(true) + } + }) + + test("cloud-compute skills that map to no panel provider are never hidden", async () => { + stub(false) + await using tmp = await project(async () => {}) + expect(await Instance.provide({ directory: tmp.path, fn: tinkerVisible })).toBe(true) + }) + + test("a credential added between two init() calls changes the catalog on the second", async () => { + stub(false) + await using tmp = await project(async () => {}) + await Instance.provide({ + directory: tmp.path, + fn: async () => { + expect(await offered()).toEqual([]) + process.env["TENSORPOOL_KEY"] = "tp-late" + expect(await offered()).toEqual(["tensorpool-gpu-cloud"]) + }, + }) + }) + + test("SkillTool.init and compute_status never disagree about usable providers", async () => { + stub(false) + process.env["LAMBDA_API_KEY"] = "k" + await using tmp = await project(async () => {}) + await Instance.provide({ + directory: tmp.path, + fn: async () => { + const state = await ComputeMode.resolve() + const names = await offered() + const expected = state.providers.flatMap((id) => ComputeMode.PROVIDERS[id].skills) + expect(names.sort()).toEqual([...new Set(expected)].sort()) + }, + }) + }) +}) From ed28cdca9e77b7e2105b53f60c66837f1dbfbea0 Mon Sep 17 00:00:00 2001 From: KB Date: Fri, 31 Jul 2026 09:49:17 +0530 Subject: [PATCH 15/56] test(skill): stop duplicating compute filter logic, complete ENV scrub list Test 8 re-derived skill.ts's own offered-skills formula instead of comparing against real tools, and never actually called compute_status despite its name. Split it into two hardcoded-literal scenarios that invoke SkillTool and ComputeStatusTool directly and compare their verdicts, one credentialed and one bare. The ENV scrub list only cleared 5 of the 10 credential vars in ComputeMode.PROVIDERS, so a real PRIME_API_KEY or VAST_API_KEY in a developer's shell could leak into tests asserting an empty catalog. Brought it to the same 10-var list already used in test/compute/mode.test.ts. Also closed a coverage gap: the two "unaffected in every mode" tests never exercised byok, so extended both to a credentialed scenario. --- .../test/tool/skill-compute-filter.test.ts | 101 ++++++++++++------ 1 file changed, 66 insertions(+), 35 deletions(-) diff --git a/backend/cli/test/tool/skill-compute-filter.test.ts b/backend/cli/test/tool/skill-compute-filter.test.ts index 14d9c9ed..446e5325 100644 --- a/backend/cli/test/tool/skill-compute-filter.test.ts +++ b/backend/cli/test/tool/skill-compute-filter.test.ts @@ -2,15 +2,41 @@ import { test, expect, describe, beforeEach, afterEach } from "bun:test" import path from "path" import fs from "fs/promises" import { SkillTool } from "../../src/tool/skill" +import { ComputeStatusTool } from "../../src/tool/compute" import { ComputeMode } from "../../src/compute/mode" import { Instance } from "../../src/project/instance" import { Global } from "../../src/global" import { tmpdir } from "../fixture/fixture" -const ENV = ["LAMBDA_API_KEY", "RUNPOD_API_KEY", "MODAL_TOKEN_ID", "MODAL_TOKEN_SECRET", "TENSORPOOL_KEY"] +// All ten credential variables across ComputeMode.PROVIDERS (modal x2, lambda x2, +// tensorpool x2, prime x2, runpod x1, vast x1) — matches test/compute/mode.test.ts's +// ENV list. A partial list lets a developer's own ambient shell keys (e.g. a real +// PRIME_API_KEY) leak into tests asserting an empty catalog. +const ENV = [ + "MODAL_TOKEN_ID", + "MODAL_TOKEN_SECRET", + "LAMBDA_API_KEY", + "LAMBDA_LABS_API_KEY", + "TENSORPOOL_KEY", + "TENSORPOOL_API_KEY", + "PRIME_API_KEY", + "PRIME_INTELLECT_API_KEY", + "RUNPOD_API_KEY", + "VAST_API_KEY", +] const SESSION = path.join(Global.Path.data, "openscience-session.json") const realFetch = globalThis.fetch +const CTX = { + sessionID: "s", + messageID: "m", + agent: "research", + abort: new AbortController().signal, + messages: [], + metadata: () => {}, + ask: async () => {}, +} + // Every provider skill, plus two skills that must never be filtered: a // non-compute one and a cloud-compute skill that maps to no panel provider. const ALL = [ @@ -58,17 +84,7 @@ async function offered(): Promise { const tool = await SkillTool.init({}) const found: string[] = [] for (const category of ["cloud-compute", "ml-training"]) { - const result = await tool - .execute({ category }, { - sessionID: "s", - messageID: "m", - agent: "research", - abort: new AbortController().signal, - messages: [], - metadata: () => {}, - ask: async () => {}, - } as never) - .catch(() => undefined) + const result = await tool.execute({ category }, CTX as never).catch(() => undefined) if (result) found.push(result.output) } const text = found.join("\n") @@ -77,32 +93,24 @@ async function offered(): Promise { async function nonComputeVisible(): Promise { const tool = await SkillTool.init({}) - const result = await tool.execute({ category: "chemistry" }, { - sessionID: "s", - messageID: "m", - agent: "research", - abort: new AbortController().signal, - messages: [], - metadata: () => {}, - ask: async () => {}, - } as never) + const result = await tool.execute({ category: "chemistry" }, CTX as never) return result.output.includes("**rdkit**") } async function tinkerVisible(): Promise { const tool = await SkillTool.init({}) - const result = await tool.execute({ category: "cloud-compute" }, { - sessionID: "s", - messageID: "m", - agent: "research", - abort: new AbortController().signal, - messages: [], - metadata: () => {}, - ask: async () => {}, - } as never) + const result = await tool.execute({ category: "cloud-compute" }, CTX as never) return result.output.includes("**tinker-fine-tuning**") } +/** The compute_status tool's own verdict — the second surface that must agree + * with SkillTool's catalog filter about which providers are usable. */ +async function computeStatus(): Promise<{ mode: string; providers: string[] }> { + const tool = await ComputeStatusTool.init({}) + const result = await tool.execute({}, CTX as never) + return result.metadata as { mode: string; providers: string[] } +} + describe("skill catalog filtering by compute mode", () => { beforeEach(async () => { for (const name of ENV) delete process.env[name] @@ -161,12 +169,25 @@ describe("skill catalog filtering by compute mode", () => { await using tmp = await project(async () => {}) expect(await Instance.provide({ directory: tmp.path, fn: nonComputeVisible })).toBe(true) } + + // byok: a credentialed provider must not affect a skill outside its scope either. + stub(false) + process.env["LAMBDA_API_KEY"] = "k" + ComputeMode.invalidate() + await using byok = await project(async () => {}) + expect(await Instance.provide({ directory: byok.path, fn: nonComputeVisible })).toBe(true) }) test("cloud-compute skills that map to no panel provider are never hidden", async () => { stub(false) await using tmp = await project(async () => {}) expect(await Instance.provide({ directory: tmp.path, fn: tinkerVisible })).toBe(true) + + // byok: a credentialed provider must not hide a skill outside ComputeMode.SKILLS either. + process.env["LAMBDA_API_KEY"] = "k" + ComputeMode.invalidate() + await using byok = await project(async () => {}) + expect(await Instance.provide({ directory: byok.path, fn: tinkerVisible })).toBe(true) }) test("a credential added between two init() calls changes the catalog on the second", async () => { @@ -182,17 +203,27 @@ describe("skill catalog filtering by compute mode", () => { }) }) - test("SkillTool.init and compute_status never disagree about usable providers", async () => { + test("credentialed: SkillTool and compute_status agree lambda is the usable provider", async () => { stub(false) process.env["LAMBDA_API_KEY"] = "k" await using tmp = await project(async () => {}) await Instance.provide({ directory: tmp.path, fn: async () => { - const state = await ComputeMode.resolve() - const names = await offered() - const expected = state.providers.flatMap((id) => ComputeMode.PROVIDERS[id].skills) - expect(names.sort()).toEqual([...new Set(expected)].sort()) + expect(await offered()).toEqual(["lambda-labs-gpu-cloud"]) + expect((await computeStatus()).providers).toEqual(["lambda"]) + }, + }) + }) + + test("no credentials: SkillTool and compute_status agree nothing is usable", async () => { + stub(false) + await using tmp = await project(async () => {}) + await Instance.provide({ + directory: tmp.path, + fn: async () => { + expect(await offered()).toEqual([]) + expect((await computeStatus()).mode).toBe("none") }, }) }) From 12a4369528df1b5ccc18e09f97508675179f89ac Mon Sep 17 00:00:00 2001 From: KB Date: Fri, 31 Jul 2026 10:05:36 +0530 Subject: [PATCH 16/56] fix(prompt): drop the false atlas compute:up guidance for a compute_status pointer --- backend/cli/src/config/config.ts | 2 +- backend/cli/src/session/billing-gate.ts | 5 -- backend/cli/src/session/prompt.ts | 16 +++--- .../cli/test/session/compute-prompt.test.ts | 49 +++++++++++++++++++ 4 files changed, 57 insertions(+), 15 deletions(-) create mode 100644 backend/cli/test/session/compute-prompt.test.ts diff --git a/backend/cli/src/config/config.ts b/backend/cli/src/config/config.ts index 5660a1f8..e76666f2 100644 --- a/backend/cli/src/config/config.ts +++ b/backend/cli/src/config/config.ts @@ -1059,7 +1059,7 @@ export namespace Config { .enum(["managed", "byok"]) .optional() .describe( - "How GPU/compute is paid for. 'managed' runs on Atlas-provisioned compute billed to your wallet (via the bundled atlas CLI); 'byok' uses your own connected GPU providers (Modal, Tinker, TensorPool, …). Unset = byok.", + "How GPU/compute is paid for. 'managed' runs on Atlas-provisioned compute billed to your wallet; 'byok' uses your own connected GPU providers (Modal, Lambda Labs, TensorPool, Prime Intellect, RunPod, Vast.ai). Unset = auto-detect from your connected providers. Setting this can only narrow the result — if the mode you pick isn't actually available, compute resolves to none rather than pretending.", ), }) .optional() diff --git a/backend/cli/src/session/billing-gate.ts b/backend/cli/src/session/billing-gate.ts index cc77ec2b..a77650d9 100644 --- a/backend/cli/src/session/billing-gate.ts +++ b/backend/cli/src/session/billing-gate.ts @@ -30,11 +30,6 @@ export async function llmBillingMode(): Promise { return (await Config.get()).billing?.llm ?? undefined } -/** The user-facing compute spend toggle. Defaults to "byok" (own GPU providers). */ -export async function computeBillingMode(): Promise { - return (await Config.get()).billing?.compute ?? "byok" -} - /** First-party providers whose OAuth path runs on the user's own subscription * and never debits Credits. */ const OAUTH_FREE_PROVIDERS = new Set([ diff --git a/backend/cli/src/session/prompt.ts b/backend/cli/src/session/prompt.ts index 7c5ae284..74b6cff6 100644 --- a/backend/cli/src/session/prompt.ts +++ b/backend/cli/src/session/prompt.ts @@ -43,7 +43,6 @@ import { Command } from "../command" import { $, fileURLToPath } from "bun" import { ConfigMarkdown } from "../config/markdown" import { Config } from "../config/config" -import { computeBillingMode } from "./billing-gate" import { SessionSummary } from "./summary" import { NamedError } from "@synsci/util/error" import { fn } from "@/util/fn" @@ -1547,19 +1546,18 @@ export namespace SessionPrompt { const userMessage = input.messages.findLast((msg) => msg.info.role === "user") if (!userMessage) return input.messages - // Compute spend preference — make the user's explicit managed/BYOK choice - // authoritative for GPU work. Only injected when the toggle is explicitly set - // (unset = the agent's own atlas-doctor-driven default, unchanged). - if (COMPUTE_AGENTS.has(input.agent.name) && (await Config.get()).billing?.compute) { - const managed = (await computeBillingMode()) === "managed" + // Compute funding is PULLED from the `compute_status` tool, not injected — + // the mode changes mid-session (a key connected in Settings ▸ Compute at + // turn 3 makes a reminder injected then false by turn 12). This line is a + // stateless pointer: it carries no mode, so it can never go stale, and it + // closes the gap where an agent reaches for bash without ever looking. + if (COMPUTE_AGENTS.has(input.agent.name)) { userMessage.parts.push({ id: Identifier.ascending("part"), messageID: userMessage.info.id, sessionID: userMessage.info.sessionID, type: "text", - text: managed - ? "Compute spend is set to MANAGED. Run GPU/training work through the bundled `atlas compute` CLI (e.g. `atlas compute:up`), which bills Credits. Do not fall back to the user's own GPU providers unless `atlas doctor` reports managed compute unavailable." - : "Compute spend is set to BYOK. Run GPU/training work on the user's own connected providers (Modal, Tinker, TensorPool, …) via the cloud-compute skills — do not launch managed `atlas compute` leases that bill Credits.", + text: "Call `compute_status` before running GPU, training, or cluster work. It reports how compute is funded and which providers are usable right now.", synthetic: true, }) } diff --git a/backend/cli/test/session/compute-prompt.test.ts b/backend/cli/test/session/compute-prompt.test.ts new file mode 100644 index 00000000..33f3302c --- /dev/null +++ b/backend/cli/test/session/compute-prompt.test.ts @@ -0,0 +1,49 @@ +import { test, expect, describe } from "bun:test" +import path from "path" + +const root = path.join(import.meta.dir, "..", "..", "src") + +async function sources() { + const files = await Array.fromAsync( + new Bun.Glob("session/**/*.{ts,txt}").scan({ cwd: root, absolute: true, onlyFiles: true }), + ) + return Promise.all(files.map(async (file) => [file, await Bun.file(file).text()] as const)) +} + +describe("compute prompt text", () => { + test("no prompt or session source references atlas compute:up", async () => { + const hits = (await sources()).filter(([, text]) => text.includes("compute:up")) + expect(hits.map(([file]) => path.relative(root, file))).toEqual([]) + }) + + test("no prompt or session source uses atlas doctor as the compute availability signal", async () => { + const hits = (await sources()).filter(([, text]) => /atlas doctor/i.test(text)) + expect(hits.map(([file]) => path.relative(root, file))).toEqual([]) + }) + + test("the compute reminder points at compute_status and carries no mode", async () => { + const text = await Bun.file(path.join(root, "session", "prompt.ts")).text() + expect(text).toContain("compute_status") + // The reminder must be stateless — a mode baked into an injected string is + // false the moment the user connects a key mid-session. + expect(text).not.toContain("Compute spend is set to") + }) + + test("computeBillingMode is gone and nothing imports it", async () => { + const gate = await Bun.file(path.join(root, "session", "billing-gate.ts")).text() + expect(gate).not.toContain("computeBillingMode") + const files = await Array.fromAsync(new Bun.Glob("**/*.ts").scan({ cwd: root, absolute: true, onlyFiles: true })) + const importers = ( + await Promise.all( + files.map(async (file) => ((await Bun.file(file).text()).includes("computeBillingMode") ? file : undefined)), + ) + ).filter(Boolean) + expect(importers).toEqual([]) + }) + + test("the billing.compute config description no longer claims 'Unset = byok'", async () => { + const text = await Bun.file(path.join(root, "config", "config.ts")).text() + expect(text).not.toContain("Unset = byok") + expect(text).toContain("auto-detect") + }) +}) From a723ac1b536ce504e4ddffccd0df0c8784c9117b Mon Sep 17 00:00:00 2001 From: KB Date: Fri, 31 Jul 2026 10:21:32 +0530 Subject: [PATCH 17/56] fix(prompt): research agent checks compute_status, not atlas doctor Task 5 removed the atlas-doctor-as-compute-signal fallback from src/session/, but the primary research agent's own Stage 5 prompt still gated managed compute on `atlas doctor --format=json`, which reports CLI auth, nothing about compute. Managed compute is gated server-side on COMPUTE_RESELL_ENABLED (default false), so an authenticated CLI told the agent managed compute worked when it didn't. Stage 5 now calls compute_status directly and routes on its byok/managed/ none verdict. Also fixes a stale `modal` skill reference that no longer resolves under the Task 4 catalog filter (real name: modal-serverless-gpu). Widened compute-prompt.test.ts to scan agent/prompt/*.txt alongside session/**, and replaced the blanket atlas-doctor string check with a paragraph-scoped one so the legitimate atlas-doctor CLI-availability check at research.txt:81-84 stays permitted. --- backend/cli/src/agent/prompt/research.txt | 12 +++--- .../cli/test/session/compute-prompt.test.ts | 41 ++++++++++++++++--- 2 files changed, 43 insertions(+), 10 deletions(-) diff --git a/backend/cli/src/agent/prompt/research.txt b/backend/cli/src/agent/prompt/research.txt index 2a9269c5..f5852792 100644 --- a/backend/cli/src/agent/prompt/research.txt +++ b/backend/cli/src/agent/prompt/research.txt @@ -251,12 +251,14 @@ implementation in up-to-date usage rather than potentially stale training data. ### Stage 5: COMPUTE Execute computational work. If `methodology.md` exists, follow the pipeline defined there. -- Managed compute (Daytona-backed) runs through the bundled `atlas` CLI when your Atlas - session is active. Run `atlas doctor --format=json` first; if it reports the CLI is - unavailable/unauthenticated, print a one-line note and fall back to the BYOK cloud-compute - skills below (Modal, Tinker, TensorPool, Prime Intellect, HF Jobs) — never block on it. +- Call the `compute_status` tool before launching any GPU work. It reports how compute is + funded right now — `byok`, `managed`, or `none` — which providers are usable, and the rule + that applies. This is the only compute-availability signal — CLI auth status is not one. +- If it returns `byok`, load the cloud-compute skill for one of the providers it lists. + If `managed`, run the work through managed compute. If `none`, do not launch GPU work — + tell the user to connect a provider key in Settings ▸ Compute. - Load: `modal-research-gpu` for GPU-accelerated scientific computing -- Load: `modal` for general serverless GPU (inference, serving) +- Load: `modal-serverless-gpu` for general serverless GPU (inference, serving) - Load: domain libraries as needed (see Scientific Computing skills) - Present cost estimate and get approval before launching jobs - Run computations, monitor progress, collect outputs diff --git a/backend/cli/test/session/compute-prompt.test.ts b/backend/cli/test/session/compute-prompt.test.ts index 33f3302c..d57efa1e 100644 --- a/backend/cli/test/session/compute-prompt.test.ts +++ b/backend/cli/test/session/compute-prompt.test.ts @@ -4,9 +4,14 @@ import path from "path" const root = path.join(import.meta.dir, "..", "..", "src") async function sources() { - const files = await Array.fromAsync( - new Bun.Glob("session/**/*.{ts,txt}").scan({ cwd: root, absolute: true, onlyFiles: true }), - ) + const globs = ["session/**/*.{ts,txt}", "agent/prompt/*.txt"] + const files = ( + await Promise.all( + globs.map((pattern) => + Array.fromAsync(new Bun.Glob(pattern).scan({ cwd: root, absolute: true, onlyFiles: true })), + ), + ) + ).flat() return Promise.all(files.map(async (file) => [file, await Bun.file(file).text()] as const)) } @@ -16,11 +21,37 @@ describe("compute prompt text", () => { expect(hits.map(([file]) => path.relative(root, file))).toEqual([]) }) - test("no prompt or session source uses atlas doctor as the compute availability signal", async () => { - const hits = (await sources()).filter(([, text]) => /atlas doctor/i.test(text)) + test("the file set is non-empty and covers both prompt trees", async () => { + const files = (await sources()).map(([file]) => path.relative(root, file)) + expect(files.length).toBeGreaterThan(20) + expect(files).toContain("session/prompt.ts") + expect(files).toContain("agent/prompt/research.txt") + }) + + test("no prompt uses atlas doctor as the compute availability signal", async () => { + // `atlas doctor` legitimately reports whether the atlas CLI is present and + // authenticated (research.txt uses it that way before loading graph state). + // What it does NOT report is anything about compute — so any paragraph that + // mentions both compute and `atlas doctor` is reading a signal that isn't there. + const hits = (await sources()).filter(([, text]) => + text.split(/\n\s*\n/).some((para) => /atlas doctor/i.test(para) && /\bcompute\b/i.test(para)), + ) expect(hits.map(([file]) => path.relative(root, file))).toEqual([]) }) + test("agent prompts point at compute_status for GPU funding", async () => { + const text = await Bun.file(path.join(root, "agent", "prompt", "research.txt")).text() + expect(text).toContain("compute_status") + }) + + test("prompts name skills that exist in the provider map", async () => { + // `modal` is not a skill name — the real ones are modal-serverless-gpu, + // modal-ml-training, modal-research-gpu. A prompt naming a skill the catalog + // does not have sends the agent to load something that cannot resolve. + const text = await Bun.file(path.join(root, "agent", "prompt", "research.txt")).text() + expect(text).not.toMatch(/`modal`/) + }) + test("the compute reminder points at compute_status and carries no mode", async () => { const text = await Bun.file(path.join(root, "session", "prompt.ts")).text() expect(text).toContain("compute_status") From 028a267a2f13c368e62ca465b3bc2dbf30fe94de Mon Sep 17 00:00:00 2001 From: KB Date: Fri, 31 Jul 2026 10:35:12 +0530 Subject: [PATCH 18/56] fix(prompt): correct remaining bare `modal` skill refs, sharpen guard test Round-1 review found the modal fix incomplete: the skill-index appendix in research.txt still listed bare `modal` at two more spots (the Inference & Deployment and Cloud Compute sections), which is a directory name, not the frontmatter `name` the skill tool resolves on. Corrected both to modal-serverless-gpu. Left the two prose mentions of "Modal" (capitalized, naming the company/platform, not invoking a skill) untouched. The guard test only matched the backtick-wrapped literal, so it never saw the appendix and would have missed a bare-`modal` regression in Stage 5 too. Replaced it with a check that extracts every modal*-shaped token from research.txt and verifies each against ComputeMode.PROVIDERS.modal.skills (the real source of truth), comparing the prompt's tokens against the map rather than the reverse, and renamed it to describe exactly what it checks. --- backend/cli/src/agent/prompt/research.txt | 4 ++-- .../cli/test/session/compute-prompt.test.ts | 19 ++++++++++++++----- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/backend/cli/src/agent/prompt/research.txt b/backend/cli/src/agent/prompt/research.txt index f5852792..a27feee4 100644 --- a/backend/cli/src/agent/prompt/research.txt +++ b/backend/cli/src/agent/prompt/research.txt @@ -352,7 +352,7 @@ Tracking: weights-and-biases, mlflow, tensorboard, langsmith, phoenix ### Inference & Deployment — serving, quantization, benchmarking Serving: vllm, sglang, tensorrt-llm, llama-cpp, outlines Quantization: bitsandbytes, awq, gptq, hqq, gguf -Platforms: modal, lambda-labs, tensorpool, skypilot, fireworks-ai, groq, together-ai +Platforms: modal-serverless-gpu, lambda-labs, tensorpool, skypilot, fireworks-ai, groq, together-ai Eval: lm-evaluation-harness, bigcode-evaluation-harness, llm-as-judge-evaluation, hugging-face-evaluation ### Data & Embeddings — DataFrames, datasets, tokenizers, vector stores @@ -386,7 +386,7 @@ Writing: scientific-writing, ml-paper-writing, research-grants, venue-templates, Docs: scientific-slides, paper-2-web, latex-posters, pptx-posters, markitdown, market-research-reports ### Cloud Compute — GPU provisioning, serverless, distributed -GPU: modal-research-gpu, modal, lambda-labs, tensorpool, prime-intellect-lab, skypilot +GPU: modal-research-gpu, modal-serverless-gpu, lambda-labs, tensorpool, prime-intellect-lab, skypilot Distributed: ray-train, ray-data Managed: tinker, hugging-face-jobs Utility: get-available-resources diff --git a/backend/cli/test/session/compute-prompt.test.ts b/backend/cli/test/session/compute-prompt.test.ts index d57efa1e..72814978 100644 --- a/backend/cli/test/session/compute-prompt.test.ts +++ b/backend/cli/test/session/compute-prompt.test.ts @@ -1,5 +1,6 @@ import { test, expect, describe } from "bun:test" import path from "path" +import { ComputeMode } from "../../src/compute/mode" const root = path.join(import.meta.dir, "..", "..", "src") @@ -44,12 +45,20 @@ describe("compute prompt text", () => { expect(text).toContain("compute_status") }) - test("prompts name skills that exist in the provider map", async () => { - // `modal` is not a skill name — the real ones are modal-serverless-gpu, - // modal-ml-training, modal-research-gpu. A prompt naming a skill the catalog - // does not have sends the agent to load something that cannot resolve. + test("modal skill mentions in research.txt resolve against ComputeMode.PROVIDERS", async () => { + // `modal` (bare, backticked or not) is a directory name, not a skill name — + // the frontmatter `name` values the skill tool actually resolves on are + // modal-serverless-gpu, modal-ml-training, modal-research-gpu, which is + // exactly ComputeMode.PROVIDERS.modal.skills. Pull every lowercase + // modal*-shaped token out of the prompt (skill tokens are always + // lowercase-hyphenated; "Modal" the company name in prose is capitalized + // and so never matches) and check it against that list — not the other way + // around, since the map trivially agrees with itself. const text = await Bun.file(path.join(root, "agent", "prompt", "research.txt")).text() - expect(text).not.toMatch(/`modal`/) + const tokens = [...new Set(text.match(/\bmodal[a-z-]*\b/g) ?? [])] + expect(tokens.length).toBeGreaterThan(0) + const valid = new Set(ComputeMode.PROVIDERS.modal.skills) + expect(tokens.filter((token) => !valid.has(token))).toEqual([]) }) test("the compute reminder points at compute_status and carries no mode", async () => { From 25ea4ede40f5f68508f333fc99ee647b1e1f5c6d Mon Sep 17 00:00:00 2001 From: KB Date: Fri, 31 Jul 2026 10:58:11 +0530 Subject: [PATCH 19/56] fix(compute): key the skill catalog filter off mode, not providers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SkillTool.init offered a credentialed provider's GPU skills whenever the provider was usable, even under a managed override where resolve() reports that same providers list for display only. A user with a connected key who switches Settings > Spend > Compute to Managed still saw that provider's skill offered, contradicting compute_status's "not funded here" guidance. It also called the full resolve(), whose no-provider path reaches the authenticated /api/compute/options probe (up to 3s) for a mode value the filter never read — a per-LLM-step network round trip for signed-in users with no GPU keys. Add ComputeMode.offered(): synchronous except for Config, exact in every state without the availability probe, because offered is non-empty iff resolve()'s mode is "byok" and that never depends on the probe. skill.ts now calls offered() instead of resolve(). Prove the equivalence with a matrix test across credential x override x managed-availability, asserting offered() matches resolve()'s byok arm exactly and never touches the network. Also replace the toothless "in managed, no BYOK skill offered" test (zero credentials, so providers was already empty regardless of the filter) with one that credentials a provider under a forced managed override, so it actually exercises the mode/providers disagreement the old filter got wrong. --- backend/cli/src/compute/mode.ts | 30 ++++++++ backend/cli/src/tool/skill.ts | 10 ++- backend/cli/test/compute/mode.test.ts | 68 +++++++++++++++++++ .../test/tool/skill-compute-filter.test.ts | 24 +++++-- 4 files changed, 126 insertions(+), 6 deletions(-) diff --git a/backend/cli/src/compute/mode.ts b/backend/cli/src/compute/mode.ts index 3787d3b5..cf45e738 100644 --- a/backend/cli/src/compute/mode.ts +++ b/backend/cli/src/compute/mode.ts @@ -187,4 +187,34 @@ export namespace ComputeMode { return funded(providers, await available()) } + + /** + * The skill names the catalog filter should offer: exactly the skills of + * the credentialed providers when — and only when — those providers are the + * FUNDED path, i.e. `resolve()`'s mode is "byok". Empty in every other + * state, including "managed" with a real credential sitting unused (see + * `Resolution.providers`'s doc comment) — offering it there would dangle + * the user's own uncapped provider account in front of an agent that has + * just been told not to touch it. + * + * This mirrors `resolve()`'s byok arms exactly but never reaches the + * availability probe, because it doesn't need to: whether the mode is + * "byok" is fully decided by `usable()` (env-only) and the override alone. + * + * - override "byok" -> byok iff a credential exists; no network either way. + * - override "managed" -> mode is "managed" or "none"; either way offered is empty. + * - unset -> byok iff a credential exists; the network arm only + * runs when there are no credentials, and then offered + * is empty regardless of what it returns. + * + * So this is synchronous except for `Config.get()` — zero I/O, safe to call + * every LLM step. + */ + export async function offered(): Promise> { + const providers = usable() + const override = (await Config.get()).billing?.compute + if (override === "managed") return new Set() + if (!providers.length) return new Set() + return new Set(providers.flatMap((id) => PROVIDERS[id].skills)) + } } diff --git a/backend/cli/src/tool/skill.ts b/backend/cli/src/tool/skill.ts index cc668d32..b081b1c4 100644 --- a/backend/cli/src/tool/skill.ts +++ b/backend/cli/src/tool/skill.ts @@ -49,12 +49,18 @@ export const SkillTool = Tool.define("skill", async (ctx) => { // cache to invalidate, and resolution always happens after src/index.ts's env // injections rather than racing them. // + // ComputeMode.offered(), not resolve(): it answers the one question this + // filter needs (which skills are the FUNDED path) without the availability + // probe resolve() sometimes needs for the *mode label* — see its doc + // comment. That keeps this init synchronous-except-for-Config, so a + // signed-in user with no GPU keys never pays a per-step network round trip + // for a value this filter doesn't read. + // // This is a LISTING filter, not a gate. `none` is guidance, not enforcement — // a hidden skill can still be loaded by exact name, and the agent still has // bash. Gating the load path is a larger change and is deliberately out of // scope; see docs/specs/compute-mode-detection-design.md open question 3. - const compute = await ComputeMode.resolve() - const offered = new Set(compute.providers.flatMap((id) => ComputeMode.PROVIDERS[id].skills)) + const offered = await ComputeMode.offered() const accessibleSkills = permitted.filter((skill) => !ComputeMode.SKILLS.has(skill.name) || offered.has(skill.name)) // Group skills by category for the description diff --git a/backend/cli/test/compute/mode.test.ts b/backend/cli/test/compute/mode.test.ts index fbb604d9..914d8f8f 100644 --- a/backend/cli/test/compute/mode.test.ts +++ b/backend/cli/test/compute/mode.test.ts @@ -389,3 +389,71 @@ describe("ComputeMode.resolve override", () => { expect(result.mode).toBe("managed") }) }) + +describe("ComputeMode.offered", () => { + beforeEach(() => { + clearEnv() + calls = [] + ComputeMode.invalidate() + }) + afterEach(async () => { + globalThis.fetch = realFetch + await fs.rm(SESSION, { force: true }).catch(() => {}) + }) + + /** Prove the equivalence by construction rather than by example: across + * every combination of credential presence, override, and managed + * availability, `offered()` must be non-empty exactly when `resolve()` + * reports mode "byok", must equal that mode's providers' skills, and must + * never touch the network — a positive assertion (an empty recorded call + * list), not merely the absence of a thrown error. */ + test("offered() is non-empty iff resolve() is byok, matches its providers' skills, and never calls the availability endpoint", async () => { + const overrides = [undefined, "byok", "managed"] as const + for (const credential of [true, false]) { + for (const override of overrides) { + for (const managedAvailable of [true, false]) { + clearEnv() + calls = [] + ComputeMode.invalidate() + await signIn() + stubOptions(managedAvailable ? MANAGED_ON : MANAGED_OFF) + if (credential) process.env["LAMBDA_API_KEY"] = "k" + + await using tmp = await tmpdir({ + git: true, + init: async (dir) => { + await Bun.write( + path.join(dir, ".openscience", "skill", "lambda-labs-gpu-cloud", "SKILL.md"), + `---\nname: lambda-labs-gpu-cloud\ndescription: Test fixture.\ncategory: cloud-compute\n---\n\n# lambda-labs-gpu-cloud\n`, + ) + if (override) { + await Bun.write(path.join(dir, "openscience.json"), JSON.stringify({ billing: { compute: override } })) + } + }, + }) + + await Instance.provide({ + directory: tmp.path, + fn: async () => { + const state = `credential=${credential} override=${override ?? "unset"} managedAvailable=${managedAvailable}` + const resolved = await ComputeMode.resolve() + calls = [] // isolate the assertion below to offered()'s own network usage + const result = await ComputeMode.offered() + + expect(calls.filter((url) => url.includes(OPTIONS_URL))).toEqual([]) + + if (resolved.mode === "byok") { + expect(result.size, state).toBeGreaterThan(0) + expect([...result].sort(), state).toEqual( + resolved.providers.flatMap((id) => ComputeMode.PROVIDERS[id].skills).sort(), + ) + } else { + expect(result.size, state).toBe(0) + } + }, + }) + } + } + } + }) +}) diff --git a/backend/cli/test/tool/skill-compute-filter.test.ts b/backend/cli/test/tool/skill-compute-filter.test.ts index 446e5325..cde2692d 100644 --- a/backend/cli/test/tool/skill-compute-filter.test.ts +++ b/backend/cli/test/tool/skill-compute-filter.test.ts @@ -148,11 +148,27 @@ describe("skill catalog filtering by compute mode", () => { }) }) - test("in managed, no BYOK provider skill is offered", async () => { + test("in managed, a credentialed provider's skill is still not offered — mode governs, not providers", async () => { + // Regression guard for the filter keying off `providers` (the credentialed + // set, reported verbatim in every mode) instead of `mode`. Lambda IS + // credentialed here — ComputeMode.resolve().providers is non-empty — but + // billing.compute forces the managed override and the probe confirms + // managed is funded, so mode is "managed", not "byok". A filter that + // trusted `providers` would offer lambda's skill anyway; this must not. stub(true) - await using tmp = await project(async () => {}) - const names = await Instance.provide({ directory: tmp.path, fn: offered }) - expect(names).toEqual([]) + process.env["LAMBDA_API_KEY"] = "k" + await using tmp = await project(async (dir) => { + await Bun.write(path.join(dir, "openscience.json"), JSON.stringify({ billing: { compute: "managed" } })) + }) + await Instance.provide({ + directory: tmp.path, + fn: async () => { + const resolved = await ComputeMode.resolve() + expect(resolved.mode).toBe("managed") + expect(resolved.providers).toEqual(["lambda"]) + expect(await offered()).toEqual([]) + }, + }) }) test("in none, no BYOK provider skill is offered", async () => { From 0fe8f5c13adfcafdf1cb10c1a427525cecd70c0c Mon Sep 17 00:00:00 2001 From: KB Date: Fri, 31 Jul 2026 11:05:36 +0530 Subject: [PATCH 20/56] fix(billing): make billing.compute nullable, matching llm's auto semantics MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit billing.compute changed meaning from a static config default to a runtime- resolved override, but the settings API and UI never followed: the GET route coerced unset to "byok" (so the UI showed BYOK as active when nothing had been chosen), the PUT schema had no null case (so there was no way to set it back to auto), and the doc comment still said "compute defaults to byok". A user with no GPU keys and managed available would correctly resolve to managed, see BYOK apparently active in Settings, click Managed, reconsider, and click BYOK to "undo" it — only to persist billing.compute: "byok" and silently lose managed compute with no explanation and no way back except editing the config file by hand. Mirror what llm already does: BillingState/BillingPatch.compute is now nullable, readState() returns null instead of coercing to "byok", and the config.ts schema for billing.compute is nullable too (a persisted null must round-trip through Config's validation, the same way llm's already does). Add an "Auto" card to Billing.tsx's COMPUTE_MODES, matching LLM_MODES' shape and copy conventions, and widen the compute grid to fit the third card. Add settings-billing.test.ts coverage for the unset-round-trips-as-unset and PUT-null-sets-auto cases, plus a beforeEach reset of Config's in-process config cache — a read-only GET test would otherwise observe a previous test's in-memory state after that test's own afterEach had already deleted the file underneath it. --- backend/cli/src/config/config.ts | 3 +- .../cli/src/server/routes/settings/billing.ts | 16 ++++---- .../cli/test/server/settings-billing.test.ts | 41 ++++++++++++++++++- .../src/components/settings/Billing.tsx | 9 +++- 4 files changed, 58 insertions(+), 11 deletions(-) diff --git a/backend/cli/src/config/config.ts b/backend/cli/src/config/config.ts index e76666f2..e99a1730 100644 --- a/backend/cli/src/config/config.ts +++ b/backend/cli/src/config/config.ts @@ -1057,9 +1057,10 @@ export namespace Config { ), compute: z .enum(["managed", "byok"]) + .nullable() .optional() .describe( - "How GPU/compute is paid for. 'managed' runs on Atlas-provisioned compute billed to your wallet; 'byok' uses your own connected GPU providers (Modal, Lambda Labs, TensorPool, Prime Intellect, RunPod, Vast.ai). Unset = auto-detect from your connected providers. Setting this can only narrow the result — if the mode you pick isn't actually available, compute resolves to none rather than pretending.", + "How GPU/compute is paid for. 'managed' runs on Atlas-provisioned compute billed to your wallet; 'byok' uses your own connected GPU providers (Modal, Lambda Labs, TensorPool, Prime Intellect, RunPod, Vast.ai). Unset or null = auto-detect from your connected providers. Setting this can only narrow the result — if the mode you pick isn't actually available, compute resolves to none rather than pretending.", ), }) .optional() diff --git a/backend/cli/src/server/routes/settings/billing.ts b/backend/cli/src/server/routes/settings/billing.ts index b551c5b2..b8fecf96 100644 --- a/backend/cli/src/server/routes/settings/billing.ts +++ b/backend/cli/src/server/routes/settings/billing.ts @@ -10,11 +10,13 @@ const log = Log.create({ service: "settings-billing" }) // The two independent spend toggles (Settings → Spend), backed by the strict // config (`billing.llm` / `billing.compute`). "managed" runs on Credits; -// "byok" runs on the user's own keys/OAuth and is never billed. LLM is nullable -// (unset = auto-detect from the resolved credential); compute defaults to byok. +// "byok" runs on the user's own keys/OAuth and is never billed. Both are +// nullable: unset/null means auto-detect — llm from the resolved credential, +// compute from ComputeMode.resolve() (connected providers, then managed +// availability). export const BillingState = z.object({ llm: z.enum(["managed", "byok"]).nullable(), - compute: z.enum(["managed", "byok"]), + compute: z.enum(["managed", "byok"]).nullable(), wallet: z.object({ signedIn: z.boolean().describe("Whether an Atlas session (thk_ key) is available"), balanceUsd: z.number().describe("Credit balance in USD; -1 when signed out or unavailable"), @@ -22,11 +24,11 @@ export const BillingState = z.object({ }) export type BillingState = z.infer -// `llm: null` sets the toggle back to auto (auto-detect from the resolved -// credential); omitting a field leaves it untouched. +// `llm: null` / `compute: null` sets the toggle back to auto (auto-detect); +// omitting a field leaves it untouched. const BillingPatch = z.object({ llm: z.enum(["managed", "byok"]).nullable().optional(), - compute: z.enum(["managed", "byok"]).optional(), + compute: z.enum(["managed", "byok"]).nullable().optional(), }) async function readState(): Promise { @@ -35,7 +37,7 @@ async function readState(): Promise { const balanceUsd = (session ? await OpenScience.getBalance().catch(() => null) : null) ?? -1 return { llm: cfg.billing?.llm ?? null, - compute: cfg.billing?.compute ?? "byok", + compute: cfg.billing?.compute ?? null, wallet: { signedIn: !!session, balanceUsd }, } } diff --git a/backend/cli/test/server/settings-billing.test.ts b/backend/cli/test/server/settings-billing.test.ts index 6a2c5916..a4b764fd 100644 --- a/backend/cli/test/server/settings-billing.test.ts +++ b/backend/cli/test/server/settings-billing.test.ts @@ -1,11 +1,21 @@ -import { test, expect, afterEach } from "bun:test" +import { test, expect, beforeEach, afterEach } from "bun:test" import path from "path" import fs from "fs/promises" import { Global } from "../../src/global" +import { Config } from "../../src/config/config" import { BillingSettingsRoutes } from "../../src/server/routes/settings/billing" const file = path.join(Global.Path.config, "openscience.json") +// Config.global is a lazy, in-process cache invalidated only by +// Config.updateGlobal()/replaceGlobal() calling .reset() — a bare GET never +// resets it. Force a fresh disk read before every test so a read-only test +// can never observe a previous test's in-memory state after that test's +// afterEach has already deleted the file out from under it. +beforeEach(() => { + Config.global.reset() +}) + afterEach(async () => { await fs.rm(file, { force: true }).catch(() => {}) }) @@ -51,3 +61,32 @@ test("PUT llm null sets the toggle back to auto", async () => { const written = JSON.parse(await Bun.file(file).text()) expect(written.billing.llm).toBeNull() }) + +test("GET compute with no config file round-trips as unset (null), not byok", async () => { + // No config file at all — the state readState() must report for a brand + // new user, who has never touched Settings > Spend > Compute. Coercing + // this to "byok" makes the UI show BYOK as active when nothing was chosen, + // and is exactly the bug that let a user narrow themselves into "none" by + // clicking BYOK to "undo" a state they never set. + const res = await BillingSettingsRoutes().request("/") + expect(res.status).toBe(200) + const state = await res.json() + expect(state.compute).toBeNull() +}) + +test("PUT compute null sets the toggle back to auto", async () => { + await fs.mkdir(Global.Path.config, { recursive: true }) + await Bun.write(file, JSON.stringify({ billing: { compute: "byok" } }, null, 2)) + + const res = await BillingSettingsRoutes().request("/", { + method: "PUT", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ compute: null }), + }) + expect(res.status).toBe(200) + const state = await res.json() + expect(state.compute).toBeNull() + + const written = JSON.parse(await Bun.file(file).text()) + expect(written.billing.compute).toBeNull() +}) diff --git a/frontend/workspace/src/components/settings/Billing.tsx b/frontend/workspace/src/components/settings/Billing.tsx index 1b76a2b1..088ef9f2 100644 --- a/frontend/workspace/src/components/settings/Billing.tsx +++ b/frontend/workspace/src/components/settings/Billing.tsx @@ -83,6 +83,11 @@ const COMPUTE_MODES = [ title: "BYOK", body: "Your own connected GPU providers (Settings → Compute). Your provider bills you directly.", }, + { + value: null, + title: "Auto", + body: "Detect from your setup — a connected provider key runs BYOK, otherwise managed compute when available.", + }, ] export default function Billing(): JSX.Element { @@ -117,7 +122,7 @@ export default function Billing(): JSX.Element { if (res.data) return setBilling(res.data) setBillingError("Couldn't load spend settings.") } - const updateBilling = async (patch: { llm?: "managed" | "byok" | null; compute?: "managed" | "byok" }) => { + const updateBilling = async (patch: { llm?: "managed" | "byok" | null; compute?: "managed" | "byok" | null }) => { setBillingBusy(true) setBillingError(undefined) const res = await sdk.client.settings.billing.update(patch) @@ -275,7 +280,7 @@ export default function Billing(): JSX.Element {
Compute -
+
{(m) => ( Date: Fri, 31 Jul 2026 11:05:51 +0530 Subject: [PATCH 21/56] test(compute): scope the auto-detect assertion, complete the ENV scrub list Two assertions that couldn't fail: compute-prompt.test.ts checked config.ts as a whole for "auto-detect", but billing.llm's description (untouched by this branch) already contains that word, so the assertion passed regardless of what billing.compute's own description said. Scope it to the compute field's description specifically. compute-status.test.ts scrubbed 4 of the 10 credential vars ComputeMode. PROVIDERS actually checks, unlike the sibling skill-compute-filter.test.ts which scrubs all ten after a prior review caught the same gap there. An ambient PRIME_API_KEY, TENSORPOOL_KEY, VAST_API_KEY or LAMBDA_LABS_API_KEY in a developer's shell produced false failures in three tests. Derive the scrub list from ComputeMode.PROVIDERS instead of hand-typing it, so it can't drift from the sibling file's list again; verified programmatically that the derived list matches PROVIDERS exactly and that an ambient PRIME_API_KEY no longer leaks through. --- backend/cli/test/session/compute-prompt.test.ts | 13 +++++++++++-- backend/cli/test/tool/compute-status.test.ts | 7 ++++++- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/backend/cli/test/session/compute-prompt.test.ts b/backend/cli/test/session/compute-prompt.test.ts index 72814978..a0746139 100644 --- a/backend/cli/test/session/compute-prompt.test.ts +++ b/backend/cli/test/session/compute-prompt.test.ts @@ -83,7 +83,16 @@ describe("compute prompt text", () => { test("the billing.compute config description no longer claims 'Unset = byok'", async () => { const text = await Bun.file(path.join(root, "config", "config.ts")).text() - expect(text).not.toContain("Unset = byok") - expect(text).toContain("auto-detect") + // Scoped to the compute field's own description, not the whole file — + // billing.llm's untouched description already contains "auto-detect", so + // an unscoped scan would pass even if compute's description regressed to + // something like "Defaults to byok when unset". + const start = text.indexOf("compute: z") + expect(start).toBeGreaterThan(-1) + const end = text.indexOf("username: z", start) + expect(end).toBeGreaterThan(start) + const description = text.slice(start, end) + expect(description).not.toContain("Unset = byok") + expect(description).toContain("auto-detect") }) }) diff --git a/backend/cli/test/tool/compute-status.test.ts b/backend/cli/test/tool/compute-status.test.ts index c944754b..e28ba3a9 100644 --- a/backend/cli/test/tool/compute-status.test.ts +++ b/backend/cli/test/tool/compute-status.test.ts @@ -8,7 +8,12 @@ import { Instance } from "../../src/project/instance" import { Global } from "../../src/global" import { tmpdir } from "../fixture/fixture" -const ENV = ["LAMBDA_API_KEY", "RUNPOD_API_KEY", "MODAL_TOKEN_ID", "MODAL_TOKEN_SECRET"] +// Every credential variable across ComputeMode.PROVIDERS, derived rather than +// hand-typed so it can never drift out of sync with the sibling scrub list in +// test/tool/skill-compute-filter.test.ts (a hand-typed subset previously let +// an ambient PRIME_API_KEY, TENSORPOOL_KEY, VAST_API_KEY, or +// LAMBDA_LABS_API_KEY leak into these tests and produce false failures). +const ENV = Object.values(ComputeMode.PROVIDERS).flatMap((provider) => provider.env.flat()) const SESSION = path.join(Global.Path.data, "openscience-session.json") const realFetch = globalThis.fetch From d0ad9948edd4e2f11e476748eb6d47443df1bcba Mon Sep 17 00:00:00 2001 From: KB Date: Fri, 31 Jul 2026 11:11:04 +0530 Subject: [PATCH 22/56] chore(sdk): regenerate SDK for billing.compute's nullable schema MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ran tooling/repo/generate.ts (openapi.json -> hey-api client codegen). The generated openapi.json, sdk.gen.ts, and types.gen.ts still described compute as 'managed'|'byok' non-nullable and claimed spend was billed "via the bundled atlas CLI" with "Unset = byok" — both false since this branch deleted the atlas CLI dependency and made compute an override with an auto (null) state. Regeneration picks up config.ts's now-nullable schema and corrected description everywhere it's duplicated in the spec (Config, SettingsBillingGetResponse x2, SettingsBillingUpdateData). The generator's repo-wide `bunx prettier --write .` pass (tooling/repo/ format.ts) also reformatted two unrelated files under pre-existing formatting drift (backend/cli/src/compute/PROTOTYPE-guardrail-{model,repl}.ts, whitespace/line-wrap only) — reverted before this commit, not part of it. --- tooling/sdk/js/src/v2/gen/sdk.gen.ts | 2 +- tooling/sdk/js/src/v2/gen/types.gen.ts | 10 ++--- tooling/sdk/openapi.json | 62 +++++++++++++++++++------- 3 files changed, 51 insertions(+), 23 deletions(-) diff --git a/tooling/sdk/js/src/v2/gen/sdk.gen.ts b/tooling/sdk/js/src/v2/gen/sdk.gen.ts index 55b22e47..a6893717 100644 --- a/tooling/sdk/js/src/v2/gen/sdk.gen.ts +++ b/tooling/sdk/js/src/v2/gen/sdk.gen.ts @@ -1210,7 +1210,7 @@ export class Billing extends HeyApiClient { public update( parameters?: { llm?: "managed" | "byok" | null - compute?: "managed" | "byok" + compute?: "managed" | "byok" | null }, options?: Options, ) { diff --git a/tooling/sdk/js/src/v2/gen/types.gen.ts b/tooling/sdk/js/src/v2/gen/types.gen.ts index 710cb283..b5f576af 100644 --- a/tooling/sdk/js/src/v2/gen/types.gen.ts +++ b/tooling/sdk/js/src/v2/gen/types.gen.ts @@ -1710,9 +1710,9 @@ export type Config = { */ llm?: "managed" | "byok" | null /** - * How GPU/compute is paid for. 'managed' runs on Atlas-provisioned compute billed to your wallet (via the bundled atlas CLI); 'byok' uses your own connected GPU providers (Modal, Tinker, TensorPool, …). Unset = byok. + * How GPU/compute is paid for. 'managed' runs on Atlas-provisioned compute billed to your wallet; 'byok' uses your own connected GPU providers (Modal, Lambda Labs, TensorPool, Prime Intellect, RunPod, Vast.ai). Unset or null = auto-detect from your connected providers. Setting this can only narrow the result — if the mode you pick isn't actually available, compute resolves to none rather than pretending. */ - compute?: "managed" | "byok" + compute?: "managed" | "byok" | null } /** * Custom username to display in conversations instead of system username @@ -3761,7 +3761,7 @@ export type SettingsBillingGetResponses = { */ 200: { llm: "managed" | "byok" | null - compute: "managed" | "byok" + compute: "managed" | "byok" | null wallet: { /** * Whether an Atlas session (thk_ key) is available @@ -3780,7 +3780,7 @@ export type SettingsBillingGetResponse = SettingsBillingGetResponses[keyof Setti export type SettingsBillingUpdateData = { body?: { llm?: "managed" | "byok" | null - compute?: "managed" | "byok" + compute?: "managed" | "byok" | null } path?: never query?: never @@ -3793,7 +3793,7 @@ export type SettingsBillingUpdateResponses = { */ 200: { llm: "managed" | "byok" | null - compute: "managed" | "byok" + compute: "managed" | "byok" | null wallet: { /** * Whether an Atlas session (thk_ key) is available diff --git a/tooling/sdk/openapi.json b/tooling/sdk/openapi.json index c9f16bd3..7053439d 100644 --- a/tooling/sdk/openapi.json +++ b/tooling/sdk/openapi.json @@ -4505,10 +4505,17 @@ ] }, "compute": { - "type": "string", - "enum": [ - "managed", - "byok" + "anyOf": [ + { + "type": "string", + "enum": [ + "managed", + "byok" + ] + }, + { + "type": "null" + } ] }, "wallet": { @@ -4572,10 +4579,17 @@ ] }, "compute": { - "type": "string", - "enum": [ - "managed", - "byok" + "anyOf": [ + { + "type": "string", + "enum": [ + "managed", + "byok" + ] + }, + { + "type": "null" + } ] }, "wallet": { @@ -4627,10 +4641,17 @@ ] }, "compute": { - "type": "string", - "enum": [ - "managed", - "byok" + "anyOf": [ + { + "type": "string", + "enum": [ + "managed", + "byok" + ] + }, + { + "type": "null" + } ] } } @@ -19770,11 +19791,18 @@ ] }, "compute": { - "description": "How GPU/compute is paid for. 'managed' runs on Atlas-provisioned compute billed to your wallet (via the bundled atlas CLI); 'byok' uses your own connected GPU providers (Modal, Tinker, TensorPool, …). Unset = byok.", - "type": "string", - "enum": [ - "managed", - "byok" + "description": "How GPU/compute is paid for. 'managed' runs on Atlas-provisioned compute billed to your wallet; 'byok' uses your own connected GPU providers (Modal, Lambda Labs, TensorPool, Prime Intellect, RunPod, Vast.ai). Unset or null = auto-detect from your connected providers. Setting this can only narrow the result — if the mode you pick isn't actually available, compute resolves to none rather than pretending.", + "anyOf": [ + { + "type": "string", + "enum": [ + "managed", + "byok" + ] + }, + { + "type": "null" + } ] } } From 6b4bdee62d7efb7ec8961ea721bd060f7135a153 Mon Sep 17 00:00:00 2001 From: KB Date: Fri, 31 Jul 2026 11:31:17 +0530 Subject: [PATCH 23/56] fix(compute): close the cache-warming gap in offered()'s zero-network proof Re-review round 2, three closing items: 1. (Minor, the only real defect) test/compute/mode.test.ts's offered() matrix test called resolve() first, which can warm the 5s availability cache; a subsequent offered() implementation that itself called available() would be served from that warm cache and record no fetch, passing the "no network call" assertion regardless. Proved by swapping offered()'s body for the equivalent-but-probe-hitting `resolve()`-then-filter form and confirming it passed 28/28 before this fix. Add ComputeMode.invalidate() right after resetting `calls`, so a probe-hitting implementation has nowhere to hide; reran the same swap and confirmed it now fails with the recorded /api/compute/options URL, then reverted the swap. 2. mode.ts's doc comment and the matrix test's title both stated offered() is non-empty "iff" mode is byok. False in one direction: a runpod- or vast-only credential resolves to byok with offered() empty, since those providers carry skills: []. Reworded to the two one-way implications the code actually guarantees. 3. "zero I/O, safe to call every LLM step" overclaimed. Config.get() is Instance-memoized but its first read per instance can fetch a well-known config URL (config.ts:82-105) when that auth type is configured. Reworded to say precisely what changed: the per-step Atlas compute-availability round trip is eliminated; Config.get()'s own (at most once per instance) cost is unrelated and unchanged. --- backend/cli/src/compute/mode.ts | 30 +++++++++++++++++++-------- backend/cli/test/compute/mode.test.ts | 16 +++++++++----- 2 files changed, 32 insertions(+), 14 deletions(-) diff --git a/backend/cli/src/compute/mode.ts b/backend/cli/src/compute/mode.ts index cf45e738..fcd8ab5e 100644 --- a/backend/cli/src/compute/mode.ts +++ b/backend/cli/src/compute/mode.ts @@ -189,13 +189,19 @@ export namespace ComputeMode { } /** - * The skill names the catalog filter should offer: exactly the skills of - * the credentialed providers when — and only when — those providers are the - * FUNDED path, i.e. `resolve()`'s mode is "byok". Empty in every other - * state, including "managed" with a real credential sitting unused (see - * `Resolution.providers`'s doc comment) — offering it there would dangle - * the user's own uncapped provider account in front of an agent that has - * just been told not to touch it. + * The skill names the catalog filter should offer. The invariant is two + * one-way implications, not a single "iff" on non-emptiness: + * - `offered()` non-empty IMPLIES `resolve()`'s mode is "byok" — only the + * funded path ever offers anything. + * - mode "byok" IMPLIES `offered()` equals the credentialed providers' + * skills exactly, which is the EMPTY set when those providers carry no + * catalogued skill (`runpod`, `vast`: `skills: []`). A byok user with + * only a RunPod key correctly sees nothing offered here — RunPod has no + * skill to offer, though the agent can still drive its API directly. + * Empty in every other mode too, including "managed" with a real credential + * sitting unused (see `Resolution.providers`'s doc comment) — offering it + * there would dangle the user's own uncapped provider account in front of + * an agent that has just been told not to touch it. * * This mirrors `resolve()`'s byok arms exactly but never reaches the * availability probe, because it doesn't need to: whether the mode is @@ -207,8 +213,14 @@ export namespace ComputeMode { * runs when there are no credentials, and then offered * is empty regardless of what it returns. * - * So this is synchronous except for `Config.get()` — zero I/O, safe to call - * every LLM step. + * `available()`/`probe()` — the per-LLM-step Atlas round trip this function + * exists to eliminate — is never reached here. The only I/O is + * `Config.get()`, which `Instance.state` memoizes per project instance + * after its first read within that instance; that first read can itself + * issue a fetch when a `wellknown` auth entry is configured + * (config.ts:82-105), but that cost belongs to `Config.get()` and is paid + * at most once per instance, not once per step — it is not a cost this + * function adds. */ export async function offered(): Promise> { const providers = usable() diff --git a/backend/cli/test/compute/mode.test.ts b/backend/cli/test/compute/mode.test.ts index 914d8f8f..25996029 100644 --- a/backend/cli/test/compute/mode.test.ts +++ b/backend/cli/test/compute/mode.test.ts @@ -403,11 +403,12 @@ describe("ComputeMode.offered", () => { /** Prove the equivalence by construction rather than by example: across * every combination of credential presence, override, and managed - * availability, `offered()` must be non-empty exactly when `resolve()` - * reports mode "byok", must equal that mode's providers' skills, and must - * never touch the network — a positive assertion (an empty recorded call - * list), not merely the absence of a thrown error. */ - test("offered() is non-empty iff resolve() is byok, matches its providers' skills, and never calls the availability endpoint", async () => { + * availability, `offered()` must equal `resolve()`'s providers' skills + * exactly when the resolved mode is "byok" (empty otherwise, including + * when byok's providers carry no catalogued skill), and must never touch + * the network — a positive assertion (an empty recorded call list), not + * merely the absence of a thrown error. */ + test("offered() equals resolve()'s byok providers' skills when mode is byok, empty otherwise, and never calls the availability endpoint", async () => { const overrides = [undefined, "byok", "managed"] as const for (const credential of [true, false]) { for (const override of overrides) { @@ -438,6 +439,11 @@ describe("ComputeMode.offered", () => { const state = `credential=${credential} override=${override ?? "unset"} managedAvailable=${managedAvailable}` const resolved = await ComputeMode.resolve() calls = [] // isolate the assertion below to offered()'s own network usage + // resolve() may have just warmed the 5s availability cache — without + // dropping it, a probe-hitting offered() would be served from cache + // and never reach fetch, so the "no network call" assertion below + // would pass even for an implementation that calls available(). + ComputeMode.invalidate() const result = await ComputeMode.offered() expect(calls.filter((url) => url.includes(OPTIONS_URL))).toEqual([]) From 5e54c82d94fb468d2df6531df2e833521f80872e Mon Sep 17 00:00:00 2001 From: KB Date: Fri, 31 Jul 2026 15:34:57 +0530 Subject: [PATCH 24/56] spec(compute): revive the budget-cap design, five findings become the checklist Managed compute is switched ON in production (resell_enabled: true, four operator providers, 292 options), so the banner's load-bearing reason for parking - no live overspend to guard - is false. That was read from a default in config.py and contradicted by the deployed service: the fourth time this investigation drew a wrong conclusion from source, and the first made by the correction to the third. Folds in the design proposed this session and the edge case it raised: - change 0, the prerequisite: the lease reaper terminates any lease with no telemetry ~10 min after creation, and create_lease mints no runner token, so a user lease cannot prove liveness. A $30 budget dies having spent $1.17 and no budget can bind. Scope heartbeat staleness to leases that have a runner token. - change 3: a rolling window cap, because a per-lease cap does not bound sequential leases. - change 4: attach a persistent volume so exhaustion costs the compute rather than the work. Atlas already has a volumes API that leases do not use; RunPod gets volumeInGb, which dies with the pod. - change 5: budget extension, admitted only because its absence changes nothing about enforcement. - RunPod as the managed default: the only provider leaving no account-level key artifact. Records the Vast and Prime key leaks found alongside. Headline acceptance criterion is now the property the last attempt's tests and criteria both omitted: a budget of $B at $R/h lasts about B/R hours. The prompt half of the original problem is resolved and marked so. --- docs/specs/compute-guardrails-design.md | 320 +++++++++++++++++++----- 1 file changed, 263 insertions(+), 57 deletions(-) diff --git a/docs/specs/compute-guardrails-design.md b/docs/specs/compute-guardrails-design.md index a41c626a..67d29379 100644 --- a/docs/specs/compute-guardrails-design.md +++ b/docs/specs/compute-guardrails-design.md @@ -1,23 +1,55 @@ # Managed compute budget cap — design -Status: **PARKED — do not implement.** Superseded for now by -[`compute-mode-detection-design.md`](./compute-mode-detection-design.md). -Date: 2026-07-30 · parked 2026-07-30 +Status: **REVIVED — ready to plan.** Parked 2026-07-30, revived 2026-07-31. +Date: 2026-07-30 · revived 2026-07-31 > ## ⚠️ Read this before anything below > -> **This design was reviewed and found unsound, and the problem it solves is not currently reachable.** -> The analysis below is kept because it is still the best record of how Atlas compute billing works. The -> _proposal_ should not be built as written. +> **The parking condition has expired, and the five findings below are now the implementation checklist — +> exactly as the parked banner instructed.** The design's core (budget not duration, server decides, +> `hard_cap_cents` made real) survives review intact. What changed is that the problem became reachable and +> the gaps became work items. > -> ### Why it is parked +> ### What changed on 2026-07-31 > -> **1. Managed compute is switched off.** Atlas gates it on `COMPUTE_RESELL_ENABLED`, which defaults to -> `false` (`backend/app/config.py:383`), plus a configured operator key. Without both, every provider reports -> `funding: "unavailable"` and `POST /leases` refuses. **There is no live overspend to guard**, so this is a -> guardrail for a road that is not open. +> **Managed compute is switched ON in production.** The parked banner's first and load-bearing reason — +> "there is no live overspend to guard" — is false as of a live check against `thesis-synsc`: > -> **2. An independent review returned "Unsound as written."** Five findings, each verified in source: +> ``` +> GET /api/compute/options → resell_enabled: true +> lambda / runpod / vast / prime_intellect → funding: "managed" +> 292 launchable options +> ``` +> +> `COMPUTE_RESELL_ENABLED` still _defaults_ to `false` (`backend/app/config.py:383`), which is what the +> original analysis read — but production sets it. **This is the third time in this investigation that reading +> source gave the wrong answer about deployed reality.** Verify against the running system. +> +> So there is a live, unmetered spend path today: `POST /compute/leases` checks only the first hour, and +> `atlas compute:up` is the human-facing door to it. +> +> ### The prerequisite that blocks everything +> +> **Finding 1 below is not a checklist item — it is a hard prerequisite, and it is a live defect independent +> of this spec.** The lease reaper terminates _any_ lease that emits no telemetry roughly ten minutes after +> creation: +> +> - `compute_repo.list_unfinished_leases` is explicitly category-agnostic — +> `WHERE status NOT IN ('released','failed')`, no exemption for user leases. +> - `lease_reaper.sweep_once` branch 3 falls back to `_lease_started` (= `started_at or created_at`) when +> there is no telemetry, and reaps past `HEARTBEAT_STALE_SECONDS` = 600. +> - `create_lease` mints **no runner token**, and `POST /api/agent/runner/telemetry` requires one +> (`x-thesis-runner-token`, scoped to a lease). +> +> **A user-launched lease therefore has no way to prove liveness, and is destroyed before it is useful.** +> Provisioning eats several minutes of the ten. Until this is fixed, no budget can bind — a $30 budget lease +> dies having spent about $1.17 — and `atlas compute:up` cannot run a research task no matter what else ships. +> +> The fix is to scope heartbeat staleness to leases that are _supposed_ to report: those with a runner token. +> User leases are already bounded by three independent mechanisms — plan TTL, wallet exhaustion, and explicit +> release — and adding a budget cap makes four. They do not need a liveness probe they cannot answer. +> +> ### The five findings, now the checklist > > - **The lease reaper kills these leases at ten minutes.** `lease_reaper` sweeps every 60s over all > unfinished leases and reaps anything silent for `HEARTBEAT_STALE_SECONDS` = 600. A workload run over SSH @@ -36,22 +68,22 @@ Date: 2026-07-30 · parked 2026-07-30 > - **No idempotency on the money path.** The billing tick performs independent committing writes; a crash > between them double-charges on the next tick, and adding a grant debit widens the window. > -> ### Two factual errors in the text below -> -> - **"`atlas compute:up` exists"** — it exists in the Atlas repo (`cli/…/commands.mjs:922`) but **not in the -> published `@synsci/atlas@0.13.2`**, which is what `^0.13.2` resolves to. Source and npm disagree at the -> same version number. The "Corrections to earlier analysis" section below asserts the opposite and is wrong. -> - **The system prompt is therefore still broken**, contrary to what that section says. It instructs the agent -> to run a command absent from the installed CLI, then to check `atlas doctor` for managed availability — -> which reports no compute field at all — and offers "the user's own GPU providers" as the fallback. +> ### Status of the two factual errors > -> ### What to do instead +> - **"`atlas compute:up` exists"** — still true only of the repo. Verified again 2026-07-31 by unpacking the +> registry artifact: published `@synsci/atlas@0.13.2` contains **155 command specs and zero `compute:`**. +> The chronology explains it — `3e1d1ca` removed the compute commands, `0.13.1` and `0.13.2` shipped without +> them, then `205bbc0` re-added them and four commits developed them further **with no version bump**. `main` +> still declares `0.13.2`, identical to the artifact that lacks them. The fix is a release, not code. +> - **The system prompt was broken and is now FIXED.** Shipped on `feat/compute-guardrails` — the false +> `atlas compute:up` / `atlas doctor` guidance is deleted from both `session/prompt.ts` and the `research` +> agent prompt, replaced by a `compute_status` tool that resolves `byok | managed | none` at runtime. > -> Fixing that prompt is the live work, and it is handled by -> [`compute-mode-detection-design.md`](./compute-mode-detection-design.md), which needs nothing from Atlas. +> ### Scope note inherited from that work > -> **Revive this document only when managed compute is actually switched on somewhere**, and then treat the five -> findings above as the implementation checklist rather than as blockers. +> `compute_status` now tells an agent in `managed` mode to "run GPU work through managed compute" — and no +> mechanism exists for it to do so. That guidance is honest about funding but not about capability, and it is +> the user-visible reason this document is being revived rather than left parked. Roadmap items: **55** (budget guardrails + kill switches), **103** (cost approval gates), and the gate half of **51/2** (agent-facing compute tool) @@ -89,8 +121,22 @@ per-lease `expires_at`, and client-side early release. Every one of those was an _succeed_, not to stop a bill running away — and each depended on something that does not exist (a completion signal for arbitrary SSH commands, a live agent session outliving a long job, or checkpointing). -**This spec solves safety completely and does not pretend to solve productivity.** What that costs is stated -under "What you are accepting". +**Safety is still what this spec guarantees.** Changes 0–3 are the whole of it, and they depend on nothing +outside Atlas. + +The 2026-07-31 revision adds two productivity changes — and the test that admitted them is deliberately narrow: +**does it work when nobody is watching?** + +- **Change 4 (volumes) passes.** It moves files off the box before anything fails. It needs no completion + signal, no live session, and no client cooperation, because the volume simply outlives the pod. +- **Change 5 (extension) passes only because it is optional.** If no extension arrives, exhaustion proceeds + unchanged. Nothing blocks on a decision, so a dead agent costs nothing. + +The rejected features failed that test: each needed something alive to act on a signal. That is the line — +**a productivity feature is admissible here only if its absence changes nothing about enforcement.** A per-lease +`expires_at`, client-side timers, and auto-extension remain out for exactly that reason. + +What still isn't solved is stated under "What you are accepting". ## Trust boundary @@ -118,13 +164,39 @@ Three consequences, all simplifying: which matters because a lease is a VM, not a job — `POST /compute/leases` has no notion of the work running on it, and the only completion signal anywhere (`agent_telemetry` rows with `done`/`error_trace`) is written by the Atlas agent runtime, not by an arbitrary command run over SSH. -- **No extension path needed.** Exhaustion is arithmetic, not a guess that might need revising. +- **No extension path needed _for safety_.** Exhaustion is arithmetic, not a guess that might need revising. + An extension is therefore a convenience, never a correctness requirement — see change 5, which adds one + deliberately and keeps it outside the enforcement path. - **No agent liveness needed.** The server enforces whether or not the session that started the job survived. + This is the property change 0 restores: today the reaper demands a liveness signal that a user lease cannot + produce, which inverts exactly this design goal. And it uses a primitive that already exists rather than adding one — see below. ## What to build +Six changes, in dependency order. **Change 0 is a prerequisite** — without it none of the rest can be observed +to work, because the box dies first. + +### Atlas — change 0: stop the reaper killing user leases + +`lease_reaper.sweep_once` applies heartbeat staleness to every unfinished lease. Only agent-spawned leases can +answer it, because only they are issued a runner token. Scope the check to leases that have one: + +```python +# branch 3 — heartbeat staleness +if reason is None and lease.get("status") != "provisioning" and _has_runner_token(lease): + ... +``` + +Leases without a runner token stay bounded by plan TTL, wallet exhaustion, explicit release, and (after change + +1. the budget cap. The reaper's other branches — provider-terminal and provisioning-timeout — continue to apply + to every lease and should not be narrowed. + +**This is a live user-facing bug, not scaffolding for this spec.** It ships first, on its own, with its own +test: a lease with no runner token survives past `HEARTBEAT_STALE_SECONDS`; one with a token still gets reaped. + ### Atlas — change 1: make `hard_cap_cents` a real running cap `compute_grants.hard_cap_cents` already reads like a running spend ceiling. It is not one: `acquire_lease` @@ -173,6 +245,79 @@ so the agent can tell the user what was actually authorised rather than what was **Managed leases only.** BYOK runs on the user's own provider account, which we neither meter nor bill, so a budget cap there would be a number we cannot enforce. BYOK ignores `budget_cents`. +### Atlas — change 3: bound cumulative spend, not just per-lease spend + +The cap is per-grant and a grant is per-lease, so an agent can release and re-acquire without limit. The +concurrency cap of 2 bounds how many boxes run at once, not what they cost in total. A $30 budget honoured +twenty times is $600. + +Add a **rolling window cap** checked at lease creation: the sum of grants opened by this user in the trailing +window must not exceed the plan's ceiling. Window and ceiling are plan config, alongside +`gpu_sandbox_max_ttl_hours`. Rejection reuses the `402` shape with a distinct `error` code so the client can +tell "this box is too expensive" from "you have spent enough today". + +Design this in now. Retrofitting a cumulative cap after users depend on a per-lease one changes the meaning of +a number they already trust. + +### Atlas — change 4: attach a persistent volume so exhaustion costs compute, not work + +Atlas already has a volumes API — `POST /api/compute/volumes` (10 GB–10 TB), `list_volumes`, `delete_volume` +with a detach requirement. **Leases do not use it.** `LeaseRequest` has no volume field, and the RunPod +provider passes `volumeInGb: 20`, which is a _pod-scoped_ volume RunPod destroys with the pod. + +- Add `volume_id?` to `LeaseRequest`. +- Pass it to RunPod as `networkVolumeId`, mounted at `/workspace`. + +Budget exhaustion then destroys the pod and leaves the work. The user relaunches against the same volume and +continues. The economics strongly favour it: a network volume costs cents per GB-month against dollars per +GPU-hour, so preserving the work costs approximately nothing next to the compute that produced it. + +**Honest limit:** a volume preserves _files_, not _process state_. A run killed mid-epoch still dies; it +resumes only if it was checkpointing to `/workspace`. The volume is the substrate roadmap **56** needs, not a +replacement for it — but it converts "you lost the job" into "you lost the GPU", which is the difference +between a wasted budget and a wasted hour. + +### Atlas — change 5: extend a live budget + +``` +POST /api/compute/leases/{lease_id}/budget +Request: { additional_cents } +Response: { hard_cap_cents, spent_cents, effective_cap_cents } +``` + +Raises `hard_cap_cents` on the existing grant, clamped by the wallet and by the rolling cap from change 3. +Same 402 shapes on refusal. Requires no new state — it edits a number change 1 already reads every tick. + +Three constraints keep this from re-opening the door the original draft closed: + +- **Extension is pull, never push.** Atlas never auto-extends from a remaining balance. Spending without being + asked is precisely what a budget exists to prevent, and a budget that quietly refills is not a budget. +- **It is not part of enforcement.** If no extension arrives, exhaustion proceeds exactly as change 1 defines. + Nothing waits for a decision, so a dead agent changes nothing. +- **No warning event is required for it to work.** A notification at ~80% is worth adding for humans, but it is + advice, not a mechanism, and the cap must not depend on anyone reading it. + +### Why RunPod is the managed default + +Verified across all four reseller providers: each generates a fresh Ed25519 keypair per lease and the provider +only ever sees the public half, so **the key handed back opens exactly one box** everywhere. They differ in +what they leave behind: + +| Provider | How the public key attaches | Account artifact | Cleaned up on release | +| --------------- | --------------------------------------------------------------------- | ---------------- | ------------------------------------ | +| **RunPod** | injected via the `PUBLIC_KEY` env var the base images consume on boot | **none** | nothing to clean | +| Lambda | registered in the account key registry | yes | yes — on release _and_ failed launch | +| Vast | posted to account `/ssh/` **and** `/instances/{id}/ssh/` | yes | **no** | +| Prime Intellect | `POST /ssh_keys/`, referenced as `sshKeyId` | yes | **no** | + +RunPod is the only one with no account-level trace at all, which is the right property when Atlas owns the box +lifecycle. Its pod-creation body also takes an arbitrary `env` dict — the same lever the Modal spawn path uses +to pass a token and callback URL — so anything Atlas later wants running on boot needs no SSH bootstrap. + +**Two Atlas bugs fall out of this table, in the operator account rather than users':** Vast and Prime Intellect +leak one public key per lease, unbounded and forever. Lambda's pattern is the fix. Out of scope here; worth +their own ticket. + ### OpenScience — one tool The agent lists options, picks a SKU itself, and proposes a budget: @@ -187,25 +332,34 @@ The agent lists options, picks a SKU itself, and proposes a budget: 5. Release on request. **No client-side deadline timer** — the server is the enforcer, and the client has no completion signal to improve on it with. -## The two clocks that remain +## The bounds that remain -| Bound | Owner | Fires when | -| ---------------- | ------------------- | ------------------------------ | -| `hard_cap_cents` | Atlas billing tick | the approved money is spent | -| Plan TTL (24h) | Atlas billing sweep | anything has run absurdly long | +| Bound | Owner | Fires when | Status after this spec | +| ------------------- | -------------------- | ---------------------------------------- | ----------------------- | +| `hard_cap_cents` | Atlas billing tick | the approved money is spent | made functional (ch. 1) | +| Rolling window cap | Atlas lease creation | cumulative spend hits the period ceiling | new (ch. 3) | +| Wallet exhaustion | Atlas billing tick | the money actually runs out | already works | +| Plan TTL (24h) | Atlas billing sweep | anything has run absurdly long | already works | +| Heartbeat staleness | Atlas lease reaper | an _agent-spawned_ lease stops reporting | narrowed (ch. 0) | -Both already exist as mechanisms; only the first is being made functional. **No `expires_at` column is added** — -time is not the thing being authorised, and a second time bound alongside the plan TTL would be redundant. +Every one is server-side; none can be influenced by the client. **No `expires_at` column is added** — time is +not the thing being authorised, and a second time bound alongside the plan TTL would be redundant. + +Note the shape of change 0: it _removes_ a bound from user leases. That is safe precisely because the other +four still apply, and it is required because the bound it removes is one those leases cannot satisfy. Billing ticks every 60 seconds, so a budget can overrun by up to a minute of rate (~$0.12 on an H100). Approved budgets are therefore ceilings-plus-a-minute and must never be described as exact. ## What you are accepting -**A budget-exhausted job loses its work.** You paid the budget and got a partial run. This is already true -today; the spec does not make it worse, but it does not fix it either. **Roadmap 56 (checkpointing) is the fix, -and should be tracked as the follow-on that makes this good rather than merely safe.** Until then, a warning -event before exhaustion would be advice nobody can act on. +**A budget-exhausted job loses its GPU. With change 4 it need not lose its work.** Attaching a persistent +volume moves the files off the box, so exhaustion costs the compute rather than the run — provided the job +checkpointed to `/workspace`. Roadmap **56** (checkpointing) remains the thing that makes this genuinely good; +change 4 is the substrate it needs, and without it roadmap 56 has nowhere durable to write. + +The residual loss is real and accepted: **a run killed mid-epoch that was not checkpointing is gone.** No +server-side mechanism can fix that, because the server cannot know what the process was holding in memory. **Time is unbounded within a budget.** A cheap CPU lease could run for days inside a small budget. The 24-hour plan TTL is the only backstop, which is deliberate. @@ -219,6 +373,18 @@ integration rather than being the code's first execution. Cases that must be covered: +- **The property the last attempt missed: a budget of $B at $R/h lasts ≈ B/R hours.** Assert the elapsed + billable duration, not just that a release eventually happened. This is what catches the un-rolled-back + acquire-time debit — a $10 budget at $6.99/h dying at 25.8 minutes instead of ~1.4 hours passes every + release-happened assertion while being off by 3×. +- A lease **with no runner token survives past `HEARTBEAT_STALE_SECONDS`**; one with a token is still reaped + (change 0). Without this, every budget test silently measures a ten-minute reap instead of the cap. +- Money-path writes are **idempotent**: replaying a tick that already committed does not double-charge. +- The **rolling window cap** rejects an N+1th lease whose grant would exceed the period ceiling, even when each + individual budget is affordable. +- A lease created with `volume_id` mounts it, and **releasing the lease does not delete the volume**. +- Extension raises the cap, is clamped by wallet and rolling cap, and **exhaustion proceeds normally when no + extension arrives**. - A tick whose delta would exceed `hard_cap_cents` **releases the lease**; one that fits does not. - `spent_cents` tracks cumulative charge across several ticks rather than freezing after the first. - A lease created **without** `budget_cents` behaves exactly as before (dashboard and `compute:up` compatibility). @@ -241,6 +407,23 @@ code**; the ones that held up were proven against a _deletion_, not merely an in `budget_cents` only sizes the grant at creation. Existing in-flight grants keep whatever cap they were created with, and change 1 begins enforcing it from the next tick, which is the safe direction. +Per change: + +- **Change 0** is behavioural only, and its direction is _fewer_ terminations. Leases currently being reaped at + ten minutes will start surviving — which is the intent, but it means live GPU boxes that used to die on their + own now run until a real bound fires. Ship it with change 1 close behind, or ship it while the only bounds are + plan TTL and wallet, and accept that a forgotten box costs up to the TTL. +- **Change 3** needs plan config for the window and ceiling, and a query over recent grants. If grants are not + already indexed by `(user_id, created_at)`, that index is the migration. +- **Change 4** adds a nullable `volume_id` to the lease row. Volume lifecycle is already modelled by + `compute_volume_repo`; releasing a lease must **not** cascade a delete. +- **Change 5** adds no column — it edits `hard_cap_cents` in place. + +**One migration hazard, from finding 2.** `budget_cents` already exists on the agent-spawn path defaulting to +`500`, where it is display-only. Change 1 makes caps real, so every already-shipped spawn silently acquires a +hard $5 kill. Either raise that default deliberately or exempt spawn-path grants until their budgets are chosen +with enforcement in mind. **This is not a no-op, and the previous draft called it one.** + ## Corrections to earlier analysis Recorded because both errors reached a draft of this document. @@ -253,16 +436,22 @@ Recorded because both errors reached a draft of this document. `compute:launch`, `compute:lease`), alongside `compute:list` and `compute:ssh` — **in the repo.** An earlier draft claimed it existed in no version, which was wrong about the source and right about the artifact. ~~**Consequence: the system prompt at `session/prompt.ts:1554-1562` is not broken** and needs no fix. It was - previously an acceptance criterion; it is removed.~~ **That consequence does not follow.** The published - package has no `compute:` command, so the prompt does point at something the installed CLI cannot run. The - prompt fix belongs back in scope — it now lives in `compute-mode-detection-design.md`. + previously an acceptance criterion; it is removed.~~ **That consequence did not follow.** The published + package has no `compute:` command, so the prompt did point at something the installed CLI cannot run. + **Resolved 2026-07-31** on `feat/compute-guardrails`: the guidance is deleted from `session/prompt.ts` and + from the `research` agent prompt, and replaced by runtime detection via a `compute_status` tool. - **`compute:up` already takes `max_price` and `dry_run`** — again, **in the repo only**. A per-hour price ceiling and a no-spend preview exist in the source parameter set but ship to nobody until the CLI is published. Adding `budget_cents` there too would let CLI users have the same cap, once any of it ships. -**The lesson worth carrying:** three separate conclusions in this investigation came from reading source and -were wrong about the deployed reality — the CLI's contents, whether the prompt was broken, and whether managed -compute was reachable at all. Verify against the running system before designing against it. +**The lesson worth carrying:** **four** separate conclusions in this investigation came from reading source and +were wrong about the deployed reality — the CLI's contents, whether the prompt was broken, whether managed +compute was reachable at all, and then (2026-07-31) the parked banner's own claim that reselling was off, which +was read from a default and contradicted by production. Verify against the running system before designing +against it. + +The fourth is the sharpest, because it was made _by the correction to the third_. A document written to warn +about this exact failure repeated it one section later. ## Out of scope @@ -272,22 +461,39 @@ compute was reachable at all. Verify against the running system before designing - **Roadmap 56** (checkpointing) — the follow-on that makes budget exhaustion survivable. - **Roadmap 4** (real BYOK provider API clients) — five separable vendor integrations. - **Roadmap 52** (`bun:sqlite` for the local runner's state). -- Adding `budget_cents` to `atlas compute:up`, a per-lease `expires_at`, warning events, extension requests, - client-side deadline timers, and any client-side price table. +- A per-lease `expires_at`, client-side deadline timers, and any client-side price table. (Extension requests + are now **in** scope — change 5 — but remain outside the enforcement path.) +- **Publishing the Atlas CLI.** `compute:*` has sat unpublished in `main` since `205bbc0` with no version bump; + `npm` still serves the pre-removal `0.13.2`. Worth its own release, and worth adding `budget_cents` to + `compute:up` when it happens — but a release, not this design. +- **Fixing the CLI's usability gaps.** Reviewed 2026-07-31: it prints the one-time SSH private key and never + saves it (so the `ssh_command` it prints cannot work), has no file-transfer command, no exec, and no compute + tests. Its resolver is genuinely good; everything around it is unfinished. Separate workstream. +- **Vast and Prime Intellect SSH key leaks** — one public key per lease left in the operator account forever. + Lambda's delete-on-release pattern is the fix. ## Acceptance criteria -1. The billing tick re-debits the grant, so `spent_cents` tracks cumulative spend instead of freezing at hour +0. **A lease with no runner token is not reaped for heartbeat staleness**, and one with a token still is. Until + this holds, no other criterion can be observed — the box dies at ten minutes regardless. +1. **A budget of $B at rate $R/h lasts ≈ B/R hours**, asserted on elapsed billable duration. This is the + headline property and the one the previous attempt's tests and criteria both omitted. +2. The money path is idempotent: a replayed tick does not double-charge. +3. A cumulative rolling cap bounds spend across sequential leases, not merely within one. +4. `volume_id` attaches a persistent volume that **survives lease release**. +5. Extension raises the cap when affordable, is refused with a structured `402` when not, and never fires + automatically. +6. The billing tick re-debits the grant, so `spent_cents` tracks cumulative spend instead of freezing at hour one. -2. A tick whose delta would exceed `hard_cap_cents` releases the lease via the existing release path. -3. `POST /api/compute/leases` accepts optional `budget_cents` and sizes the grant to it. -4. Omitting `budget_cents` preserves today's behaviour exactly — the dashboard and `compute:up` keep working. -5. A budget that cannot fund the first hour is rejected with `402` carrying `affordable_budget_cents`. -6. A budget exceeding the wallet is clamped to the effective balance, and the response reports the effective cap. -7. BYOK leases ignore `budget_cents` and are never debited. -8. The 24-hour plan TTL still fires independently. -9. `pytest` passes with no network access; change 1 is a separate commit with its own tests. -10. The OpenScience tool refuses to launch without an Atlas verdict, surfaces `402` and `429` without retrying, +7. A tick whose delta would exceed `hard_cap_cents` releases the lease via the existing release path. +8. `POST /api/compute/leases` accepts optional `budget_cents` and sizes the grant to it. +9. Omitting `budget_cents` preserves today's behaviour exactly — the dashboard and `compute:up` keep working. +10. A budget that cannot fund the first hour is rejected with `402` carrying `affordable_budget_cents`. +11. A budget exceeding the wallet is clamped to the effective balance, and the response reports the effective cap. +12. BYOK leases ignore `budget_cents` and are never debited. +13. The 24-hour plan TTL still fires independently. +14. `pytest` passes with no network access; changes 0 and 1 are each a separate commit with their own tests. +15. The OpenScience tool refuses to launch without an Atlas verdict, surfaces `402` and `429` without retrying, and holds no pricing or approval logic. ## Prototype From 8486b104c7f3afd245569acc7519d54f567d8473 Mon Sep 17 00:00:00 2001 From: KB Date: Fri, 31 Jul 2026 15:35:22 +0530 Subject: [PATCH 25/56] spec(compute): fix the count in the revival banner The banner said 'third time' while the lesson section says four. Four is right: CLI contents, whether the prompt was broken, whether managed compute was reachable, and the parked banner's own claim that reselling was off. --- docs/specs/compute-guardrails-design.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/specs/compute-guardrails-design.md b/docs/specs/compute-guardrails-design.md index 67d29379..6dc4ec24 100644 --- a/docs/specs/compute-guardrails-design.md +++ b/docs/specs/compute-guardrails-design.md @@ -22,8 +22,9 @@ Date: 2026-07-30 · revived 2026-07-31 > ``` > > `COMPUTE_RESELL_ENABLED` still _defaults_ to `false` (`backend/app/config.py:383`), which is what the -> original analysis read — but production sets it. **This is the third time in this investigation that reading -> source gave the wrong answer about deployed reality.** Verify against the running system. +> original analysis read — but production sets it. **This is the fourth time in this investigation that reading +> source gave the wrong answer about deployed reality** (see "Corrections to earlier analysis"), and the first +> one made _by_ a correction to an earlier mistake. Verify against the running system. > > So there is a live, unmetered spend path today: `POST /compute/leases` checks only the first hour, and > `atlas compute:up` is the human-facing door to it. From 39df1bf039c5bc7149ef43219c3e558c587f813d Mon Sep 17 00:00:00 2001 From: KB Date: Fri, 31 Jul 2026 16:01:57 +0530 Subject: [PATCH 26/56] spec(compute): record the compute:up SKU race, verified against prod Ran the CLI from source: compute:up --dry-run --gpu h100 fails with HTTP 400 Unknown SKU because Vast's offer ids churn between the options fetch and the estimate call. Vast supplies 204 of 292 live options so it is nearly always the cheapest pick, making the default path fail while --provider lambda and --provider runpod succeed. No retry exists. Server-side selection fixes this and the duplicate-resolver problem at the same time. --- docs/specs/compute-guardrails-design.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/specs/compute-guardrails-design.md b/docs/specs/compute-guardrails-design.md index 6dc4ec24..f5747aaf 100644 --- a/docs/specs/compute-guardrails-design.md +++ b/docs/specs/compute-guardrails-design.md @@ -470,6 +470,16 @@ about this exact failure repeated it one section later. - **Fixing the CLI's usability gaps.** Reviewed 2026-07-31: it prints the one-time SSH private key and never saves it (so the `ssh_command` it prints cannot work), has no file-transfer command, no exec, and no compute tests. Its resolver is genuinely good; everything around it is unfinished. Separate workstream. +- **The `compute:up` SKU race.** Verified by running the CLI from source against production: `compute:up` + fetches `/compute/options`, picks, then calls `/compute/estimate` — and Vast's SKUs are ephemeral marketplace + offer IDs that churn in between. Since Vast supplies 204 of the 292 live options it is almost always the + cheapest pick, so **the default path and `--gpu h100` both fail** with a raw `HTTP 400: Unknown SKU`, while + `--provider lambda` and `--provider runpod` (stable instance types) succeed. There is no retry. + + Two fixes, either sufficient: re-resolve once on a 400 and pick the next-best offer, or make selection + server-side so fetch-and-lease is atomic inside Atlas. The second also removes the duplicate resolver + described under "OpenScience — one tool". + - **Vast and Prime Intellect SSH key leaks** — one public key per lease left in the operator account forever. Lambda's delete-on-release pattern is the fix. From 353dbee5974481ec9158430f804e6fc8005d5d7b Mon Sep 17 00:00:00 2001 From: KB Date: Fri, 31 Jul 2026 16:02:20 +0530 Subject: [PATCH 27/56] spec(compute): split verified-at-runtime claims from source-only ones The blanket 'read from source, not verified at runtime' caveat is now misleading - it would have readers distrust the live checks against production. Separates what was confirmed against thesis-synsc from what still needs confirming, since the doc's own history is four wrong conclusions drawn from source. --- docs/specs/compute-guardrails-design.md | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/docs/specs/compute-guardrails-design.md b/docs/specs/compute-guardrails-design.md index f5747aaf..4d79e296 100644 --- a/docs/specs/compute-guardrails-design.md +++ b/docs/specs/compute-guardrails-design.md @@ -95,9 +95,19 @@ have separate test runs and deploys. - `atlas` (Python/FastAPI) — the decision and the enforcement - `openscience` (Bun/TypeScript) — a thin client that relays a proposal and obeys the verdict -> **The Atlas behaviour described below was read from source, not verified at runtime.** Treat every claim about -> current behaviour as "what the code appears to do" and confirm against the deployed service before relying on -> it. The prototype models each such assumption as a toggle for exactly this reason. +> **Verification status (updated 2026-07-31).** The original draft was written entirely from source, which is +> how it acquired four wrong conclusions about deployed reality. Much of it has since been checked against the +> running system, so distinguish: +> +> - **Verified against production `thesis-synsc`:** reselling is on with four operator providers and 292 live +> options; `/compute/estimate` returns `funding: "managed"` with a real rate and runway; the published npm +> artifact contains no `compute:` command; the CLI runs from source and its `compute:up` default path fails +> on the Vast SKU race while `--provider lambda|runpod` succeeds. +> - **Still read from source only:** the billing tick's debit behaviour, grant accounting, the reaper's exact +> reap path, and every claim about what changing them would do. Confirm these against the deployed service — +> or better, against a test — before relying on them. +> +> The prototype models each unverified assumption as a toggle for exactly this reason. ## The problem, stated narrowly From 0c76899955abff32cd813a96dda347da3cd5afd7 Mon Sep 17 00:00:00 2001 From: KB Date: Fri, 31 Jul 2026 16:46:28 +0530 Subject: [PATCH 28/56] spec(compute): merge the two compute specs into one current document The mode-detection spec was shipped but still asserted a 'key AND skill' rule that was overruled during implementation, a skill-name table that matched nothing, and that managed compute was unavailable. The guardrails spec was revived but overlapped it heavily. Two documents disagreed with each other and with production. compute-management-design.md is now the single spec: Part 1 - mode detection, shipped, with the corrections folded in and a table of what was verified against the running binary and backend Part 2 - guardrails, to build, changes 0-5 with change 0 (the reaper killing user leases at ten minutes) called out as a live-defect prerequisite rather than a checklist item Opens with production reality verified against thesis-synsc, because the predecessors' shared failure was reasoning from source about a deployed system - four times, the last made by a correction to the third. Both predecessors keep their content as historical record behind a superseded banner; the mode-detection banner names its two wrong claims explicitly so nobody builds from them. --- docs/specs/compute-guardrails-design.md | 9 +- docs/specs/compute-management-design.md | 435 ++++++++++++++++++++ docs/specs/compute-mode-detection-design.md | 14 + 3 files changed, 457 insertions(+), 1 deletion(-) create mode 100644 docs/specs/compute-management-design.md diff --git a/docs/specs/compute-guardrails-design.md b/docs/specs/compute-guardrails-design.md index 4d79e296..3fba8009 100644 --- a/docs/specs/compute-guardrails-design.md +++ b/docs/specs/compute-guardrails-design.md @@ -1,6 +1,13 @@ # Managed compute budget cap — design -Status: **REVIVED — ready to plan.** Parked 2026-07-30, revived 2026-07-31. +> **SUPERSEDED 2026-07-31 by [`compute-management-design.md`](./compute-management-design.md), which is the +> single current spec for compute and carries this design forward as its Part 2.** +> +> Kept for the material that did not survive the merge: the full five-finding review, the prototype's +> gate-mode evidence, the corrections log, and the record of two false starts. **Build from the merged +> document, not this one.** + +Status: **REVIVED — superseded by the merged spec.** Parked 2026-07-30, revived 2026-07-31, merged 2026-07-31. Date: 2026-07-30 · revived 2026-07-31 > ## ⚠️ Read this before anything below diff --git a/docs/specs/compute-management-design.md b/docs/specs/compute-management-design.md new file mode 100644 index 00000000..b6bb35a5 --- /dev/null +++ b/docs/specs/compute-management-design.md @@ -0,0 +1,435 @@ +# Compute management — design + +Status: **Part 1 shipped. Part 2 ready to plan.** +Date: 2026-07-31 · supersedes [`compute-mode-detection-design.md`](./compute-mode-detection-design.md) +(shipped) and [`compute-guardrails-design.md`](./compute-guardrails-design.md) (revived). +Roadmap: **5**, **51/2**, **55**, **103**; unblocks **56**. + +One document for how OpenScience and Atlas decide **who pays for GPU work, whether it is possible at all, and +what bounds the bill**. It replaces two documents that disagreed with each other and with production. + +| Part | What | Status | +| ----- | ------------------------------------------------------------------------ | ---------------------------------------------------- | +| **1** | Resolve `byok \| managed \| none` at runtime and tell the agent honestly | **Shipped** on `feat/compute-guardrails` | +| **2** | Bound what managed compute can spend, and make it usable at all | **To build.** Change 0 is a live-defect prerequisite | + +--- + +## Production reality, verified 2026-07-31 + +Both predecessor documents were written from source and drew **four** wrong conclusions about the deployed +system. Everything in this section was checked against running services, and claims elsewhere are labelled. + +**Managed compute is ON.** `GET /api/compute/options` against `thesis-synsc`: + +``` +resell_enabled: true cli_effective_balance_cents: 19883 +lambda managed operator:true 24 options +runpod managed operator:true 37 options +vast managed operator:true 204 options +prime_intellect managed operator:true 27 options +hyperbolic / coreweave / together / fluidstack / paperspace / nebius + unavailable operator:false 0 options (ScaffoldProvider, not wired) +``` + +292 launchable options. `POST /api/compute/estimate` for `lambda/cpu_4x_general` returns +`funding: "managed"`, 20¢/hr, `sufficient: true`, `runway_hours: 994.1`. **The spend path is live today.** + +`COMPUTE_RESELL_ENABLED` still _defaults_ to `false` (`backend/app/config.py:383`) — which is what the earlier +analysis read, and why it concluded the opposite. Production sets it. + +**The Atlas CLI runs from source with no build step**, authenticated, with all five `compute:*` commands +present — but `@synsci/atlas@0.13.2` on npm has 155 command specs and **zero** `compute:`. `3e1d1ca` removed +them, `0.13.1` and `0.13.2` shipped without them, `205bbc0` re-added them, and no version bump followed. +Source and artifact disagree at an identical version. + +--- + +# Part 1 — Mode detection (shipped) + +## What it fixed + +`computeBillingMode()` returned `config.billing?.compute ?? "byok"` and never inspected the environment. A user +with zero GPU credentials resolved to `byok` — claiming BYOK with nothing to BYOK with — and "no compute is +available" had no representation at all. Meanwhile the prompt told the agent to run `atlas compute:up` (absent +from the published CLI) and to check `atlas doctor` for compute availability (it reports no compute field). + +## The three states + +```ts +export type ComputeSource = "byok" | "managed" | "none" +``` + +| Credentialed provider? | Managed available? | Resolved | Agent is told | +| ---------------------- | ------------------ | --------- | ---------------------------------------------------------- | +| yes | — | `byok` | use the connected providers via the cloud-compute skills | +| no | yes | `managed` | use managed compute, billed to Credits | +| no | no | `none` | no compute available — connect a key in Settings ▸ Compute | + +BYOK wins when a credential is present: it is free to the user, works today, and needs nothing from Atlas. +That is also why a BYOK user never pays for the availability network call. + +### A credential is the whole test + +An earlier draft required a credential **and** a matching skill, reasoning that a provider with no skill gave +the agent nothing to act on. **Overruled during implementation:** a capable agent drives a documented cloud API +from a bare key, so the conjunction only produced a false `none` for users holding a perfectly workable key. + +| Provider | Env (any group satisfies; all vars within a group required) | Catalogued skills | +| --------------- | ----------------------------------------------------------- | ----------------------------------------------------------------- | +| Modal | `MODAL_TOKEN_ID` **+** `MODAL_TOKEN_SECRET` | `modal-serverless-gpu`, `modal-ml-training`, `modal-research-gpu` | +| Lambda | `LAMBDA_API_KEY` \| `LAMBDA_LABS_API_KEY` | `lambda-labs-gpu-cloud` | +| TensorPool | `TENSORPOOL_KEY` \| `TENSORPOOL_API_KEY` | `tensorpool-gpu-cloud` | +| Prime Intellect | `PRIME_API_KEY` \| `PRIME_INTELLECT_API_KEY` | `prime-intellect-lab` | +| RunPod | `RUNPOD_API_KEY` | none | +| Vast | `VAST_API_KEY` | none | + +Modal is the only credential pair; a half-pasted token counts for nothing. Skill names are SKILL.md frontmatter +`name` values — **not** directory names and not category-prefixed. The predecessor document's table +(`cloud-compute/modal`, `cloud-compute/lambda-labs`) matched nothing; hardcoding it would have filtered nothing. + +Dropping the skill requirement also removed a failure mode: skills reach a shipped binary from the server +catalog, so under the old rule a catalog that renamed `lambda-labs-gpu-cloud` would have silently marked Lambda +unusable for a user whose key was fine. + +## `billing.compute` is an override, never the source of truth + +- unset / `null` → detection +- `"byok"` → BYOK if a credential exists, else `none` +- `"managed"` → managed if available, else `none` + +**An override may narrow to `none`; it may never manufacture a capability.** Forcing `managed` while holding a +Lambda key still yields `none` when managed is unavailable — it does not silently fall back to BYOK. + +The setting is nullable end-to-end (config schema, settings PUT, and an "Auto" card in the UI) because without +that a user who clicked BYOK with no keys was trapped in `none` with no way back. + +## Two seams, one resolver, resolved per request + +Credentials reach `process.env` from three places: the user's shell, the Credentials panel +(`applyCredentialEnv`, `src/index.ts:102`), and the Compute panel (`applyComputeEnv`, `:106`). Both injections +are wrapped in `.catch(() => {})`. + +**Resolution therefore happens on demand and never at startup**, at two points that already run per request: + +- `SkillTool.init` — `registry.ts` calls `t.init({ agent })` inside `tools()`, so the catalog is rebuilt every + turn and a mid-session credential appears on the next turn with no cache to invalidate. +- the `compute_status` tool, whenever the agent asks. + +This makes the ordering constraint unbreakable by construction rather than by careful boot sequencing — and it +proved itself in the wild: the developer's Modal credential lives in none of the three expected places, arriving +instead via dashboard sync → `~/.config/openscience/synced-env.json` → `preload-env` replay. Startup detection +would have reported `none`. + +Availability is one authenticated `GET /api/compute/options` with a **3s timeout** and a **5s TTL** cache. +A failed, unauthenticated or timed-out call resolves to **unavailable** — failing toward `managed` would +reproduce the original bug of promising an unconfirmed capability. The cache holds the availability verdict +only; credentials are never cached. + +## The catalog filter + +`ComputeMode.offered()` returns the skills the agent may see, and is non-empty **only in `byok`**. It needs no +network in any state: with an override of `"managed"` the answer is empty regardless, and with no credential it +is empty regardless, so the availability probe is never required to decide it. + +Only the six mapped providers' skills are filtered. `fireworks-ai-inference`, `together-ai-inference`, +`tinker-fine-tuning`, `tinker-training-cost` and `skypilot-multi-cloud-orchestration` share the `cloud-compute` +category but are inference APIs and orchestrators keyed by their own credentials — never hidden. RSI-learned +skills are never hidden. + +**This is a listing filter, not a gate.** A hidden skill remains loadable by exact name and the agent still has +`bash`. `none` is guidance, not enforcement; gating the load path was considered and deliberately declined. + +## `compute_status` + +No parameters. Returns `{mode, providers, managed_available, balance_usd?}` plus mode-specific guidance, and +resolves on every call. + +The mode changes mid-session — a key connected at turn 3 makes a reminder injected then false by turn 12 — so +nothing is injected per turn. The tool **description** carries the constraint ("check before running GPU work"), +which costs nothing because tool definitions are in every request regardless; the **result** carries the +specifics. `session/prompt.ts` keeps only a stateless one-line pointer for `COMPUTE_AGENTS`. + +`balance_usd` comes from `cli_effective_balance_cents` in the same availability response — no second call. + +## Verified live, not just tested + +| Case | Result | +| --------------------------------------- | ----------------------------------------------------------------------------------- | +| Real machine (Modal via dashboard sync) | `Compute: byok` | +| `billing.compute=managed` forced | `Compute: managed` | +| `billing.compute=byok`, no credentials | `Compute: none`, probe skipped | +| Catalog in `byok` | Modal's 3 shown; `lambda-labs-gpu-cloud`, `tensorpool-gpu-cloud` hidden | +| Catalog in `none` | all 5 provider skills hidden; non-provider and learned skills intact | +| Settings API | `GET` returns `null` (not coerced); `PUT "byok"` persists; `PUT null` restores Auto | + +Suite: 1488 pass / 1 skip / 1 pre-existing unrelated failure. + +--- + +# Part 2 — Guardrails and usability (to build) + +## The gap Part 1 exposed + +`compute_status` now tells an agent in `managed` mode to _"run GPU work through managed compute, billed to +Credits"_ — and **no mechanism exists**. There is no `compute_submit`, and `compute:up` is unpublished. Since +managed is live, every keyless user lands there. + +Simultaneously the spend path has no ceiling: `POST /api/compute/leases` checks only that the wallet can fund +**one hour**, so a 4-hour job on a 1-hour balance is approved and strands mid-run. + +## Trust boundary + +**The agent proposes; the server decides.** OpenScience is open source and the agent has `bash`, so any +client-side check is one a fork can delete. Only a decision made over HTTP, behind auth, in a process the agent +does not run, is one it cannot influence. + +Corollary: **OpenScience holds no pricing, balance, or approval logic.** It relays a proposal and obeys a +verdict. + +## Why budget, not duration + +The agent proposes _"this is worth up to $30"_, not _"this needs 4 hours"_. An LLM can judge the first and +cannot predict the second — and a wrong duration estimate is what forced earlier drafts to grow extension +paths, warning windows and re-estimation loops. + +Consequences, all simplifying: exhaustion is a fact the server observes, so no completion signal is needed; it +is arithmetic, so no extension is needed for correctness; and it is server-side, so no agent liveness is needed. + +## The changes + +### Change 0 — stop the reaper killing user leases _(prerequisite, live defect)_ + +`lease_reaper.sweep_once` applies heartbeat staleness to every unfinished lease: + +- `compute_repo.list_unfinished_leases` is explicitly category-agnostic — + `WHERE status NOT IN ('released','failed')`. +- Branch 3 falls back to `_lease_started` (`started_at or created_at`) when there is no telemetry and reaps past + `HEARTBEAT_STALE_SECONDS` = 600. +- `create_lease` mints **no runner token**, and `POST /api/agent/runner/telemetry` requires one. + +**A user lease cannot prove liveness and is destroyed ~10 minutes after creation**, with provisioning eating +several of those minutes. A $30 budget lease dies having spent about $1.17. No budget can bind and +`atlas compute:up` cannot run a research task until this is fixed. + +Scope the check to leases that have a runner token. User leases stay bounded by plan TTL, wallet exhaustion, +explicit release, and (after change 1) the budget cap. The provider-terminal and provisioning-timeout branches +continue to apply to everything. + +Ships first, alone, with its own test. + +### Change 1 — make `hard_cap_cents` a real running cap + +`compute_grants.hard_cap_cents` reads like a running ceiling but is not one: `acquire_lease` debits it once for +one hour, and the billing tick then charges without calling `debit_grant` again, so `spent_cents` freezes. + +**The billing tick re-debits the grant by the same delta it charges**, releasing the lease when the debit would +exceed the cap — reusing the path that already fires on wallet exhaustion. + +This is **not** a double charge. The wallet is money; the grant is an authorisation envelope drawn against it. +An implementer who "de-duplicates" these has removed the cap. + +_Known trap (finding 3):_ the acquire-time debit is never rolled back, so a naive re-debit double-counts hour +one — a $10 budget at $6.99/h would die at 25.8 minutes instead of ~1.4 hours. + +### Change 2 — accept a budget on lease creation + +``` +POST /api/compute/leases +{ provider, sku, region?, node_id?, budget_cents?, volume_id? } +``` + +`budget_cents` is optional; absent preserves today's behaviour exactly (the dashboard and `compute:up` both call +this endpoint without it). Rejection reuses the structured 402, extended with `affordable_budget_cents`. + +**A budget larger than the wallet is clamped, not rejected** — the wallet is always the outer bound. The +response reports the **effective** cap so the agent can tell the user what was actually authorised. + +**Managed only.** BYOK runs on the user's own account, which Atlas neither meters nor bills. + +### Change 3 — bound cumulative spend + +The cap is per-grant and a grant is per-lease, so release-and-reacquire is unbounded; the concurrency cap of 2 +bounds concurrent boxes, not total cost. A $30 budget honoured twenty times is $600. + +Add a **rolling window cap** at lease creation, with window and ceiling as plan config alongside +`gpu_sandbox_max_ttl_hours`, and a distinct `error` code so clients can distinguish "this box is too expensive" +from "you have spent enough today". + +Design it in now — retrofitting changes the meaning of a number users already trust. + +### Change 4 — attach a persistent volume + +Atlas already has `POST /api/compute/volumes` (10 GB–10 TB), `list_volumes`, `delete_volume`. **Leases do not +use it**, and the RunPod provider passes `volumeInGb: 20`, a pod-scoped volume destroyed with the pod. + +Add `volume_id?` to the lease request and pass it to RunPod as `networkVolumeId`, mounted at `/workspace`. +Releasing a lease must **not** cascade a volume delete. + +Budget exhaustion then costs the compute, not the work. A network volume is cents per GB-month against dollars +per GPU-hour, so preserving the work costs approximately nothing. + +**Honest limit:** a volume preserves _files_, not _process state_. A run killed mid-epoch still dies unless it +checkpointed to `/workspace`. The volume is the substrate roadmap **56** needs, not a substitute for it. + +### Change 5 — extend a live budget + +``` +POST /api/compute/leases/{lease_id}/budget { additional_cents } +→ { hard_cap_cents, spent_cents, effective_cap_cents } +``` + +Raises the cap on the existing grant, clamped by wallet and rolling cap. No new state — it edits a number change +1 already reads every tick. + +Three constraints keep it from reopening the door earlier drafts closed: + +- **Pull, never push.** Atlas never auto-extends. A budget that quietly refills is not a budget. +- **Not part of enforcement.** If no extension arrives, exhaustion proceeds unchanged, so a dead agent costs + nothing. +- **No warning event required.** An 80% notification is worth adding for humans, but it is advice; the cap must + never depend on anyone reading it. + +### Why RunPod is the managed default + +Every provider generates a fresh Ed25519 keypair per lease and only ever shows the provider the public half, so +the returned key opens exactly one box everywhere. They differ in what they leave behind: + +| Provider | Key attachment | Account artifact | Cleaned up | +| --------------- | ---------------------------------------------- | ---------------- | ---------------------------------- | +| **RunPod** | `PUBLIC_KEY` env var consumed on boot | **none** | nothing to clean | +| Lambda | account key registry | yes | yes — on release and failed launch | +| Vast | account `/ssh/` **and** `/instances/{id}/ssh/` | yes | **no** | +| Prime Intellect | `POST /ssh_keys/` → `sshKeyId` | yes | **no** | + +RunPod is the only one with no account-level trace, which is the right property when Atlas owns the box +lifecycle. Its creation body also takes an arbitrary `env` dict — the same lever the Modal spawn path uses to +pass a runner token and callback URL — so anything Atlas later wants running on boot needs no SSH bootstrap. + +Vast is cheapest and deepest (204 of 292 options) but is interruptible spot, leaks a key per lease, and its SKUs +are ephemeral offer IDs (see Known defects). + +### OpenScience — one thin tool + +1. `compute_status` (shipped) to learn the mode. +2. `GET /api/compute/options` → the agent picks a SKU. **Selection is the agent's job**, so no client-side + resolver and no wrapper around `compute:up`. +3. Submit `{provider, sku, budget_cents, volume_id?}`. **Refuse to launch without a verdict from Atlas.** +4. On `402`, surface `affordable_budget_cents` and stop. **Never auto-retry at a smaller budget** — a truncated + training run is not a cheaper result, it is a discarded one. +5. On `429` (concurrency cap), surface rather than retry. +6. Release on request. **No client-side deadline timer.** + +## The bounds that remain + +| Bound | Owner | Fires when | Status | +| ------------------- | -------------- | ---------------------------------------- | ----------------------- | +| `hard_cap_cents` | billing tick | approved money is spent | made functional (ch. 1) | +| Rolling window cap | lease creation | cumulative spend hits the ceiling | new (ch. 3) | +| Wallet exhaustion | billing tick | money actually runs out | already works | +| Plan TTL (24h) | billing sweep | anything has run absurdly long | already works | +| Heartbeat staleness | lease reaper | an _agent-spawned_ lease stops reporting | narrowed (ch. 0) | + +All server-side; none client-influenceable. Change 0 _removes_ a bound from user leases, which is safe precisely +because the other four apply and necessary because it is one those leases cannot satisfy. + +Billing ticks every 60s, so a budget can overrun by up to a minute of rate (~$0.12 on an H100). Approved budgets +are ceilings-plus-a-minute and must never be described as exact. + +--- + +## What we deliberately did not build + +Design exploration reached a persistent runner daemon on the box, a command queue, a result stream, scoped +storage credentials and a possible PTY relay — a remote-execution platform, designed before a single box had +been leased through the agent, to fix what was at that point a false string. + +The test that settled it: **a productivity feature is admissible only if its absence changes nothing about +enforcement.** Volumes pass (files move off the box before anything fails). Optional extension passes. +A command relay, per-lease `expires_at`, client-side timers and auto-extension all fail — each needs something +alive to act on a signal. + +Two things from that exploration survive and are folded in above: enforcement belongs in Atlas, and RunPod is +the right managed default. + +--- + +## Known defects, owned elsewhere + +- **The `compute:up` SKU race.** Verified by running the CLI from source against production: it fetches + options, picks, then estimates — and Vast's offer IDs churn in between. Since Vast supplies 204 of 292 + options it is nearly always cheapest, so **the default path and `--gpu h100` fail** with a raw + `HTTP 400: Unknown SKU`, while `--provider lambda` and `--provider runpod` succeed. No retry exists. Fix by + re-resolving once on a 400, or by moving selection server-side (which also makes fetch-and-lease atomic). +- **CLI usability.** It prints the one-time SSH private key and never saves it, so the `ssh_command` it prints + cannot work. No file transfer, no exec, no compute tests. The resolver is genuinely good; everything around + it is unfinished. +- **CLI unpublished** — a release, not code. +- **Vast and Prime Intellect leak** one public key per lease into the operator account, forever. Lambda's + delete-on-release is the fix. +- **Stale skill names in agent prompts.** `research.txt`'s appendix and `ml.txt` name ~35 skills by directory + rather than frontmatter `name` (`vllm` → `serving-llms-vllm`, `peft` → `peft-fine-tuning`, …), so the agent is + told it has skills it cannot load. `skills/scholar-evaluation/SKILL.md` has no `name:` at all. +- **`budget_cents` already exists** on the spawn path defaulting to `500`, display-only. Change 1 makes caps + real, silently giving every shipped spawn a hard $5 kill. **Not a no-op.** + +## Out of scope + +Roadmap **61** (per-job secrets), **56** (checkpointing — the follow-on change 4 enables), **4** (BYOK provider +API clients), **52** (`bun:sqlite`). Per-lease `expires_at`, client-side deadline timers, client-side price +tables, auto-extension. Any change to how `billing.llm` resolves. + +## Testing + +Atlas follows `backend/tests/test_compute_billing.py` — `_FakeProvider`, `aiosqlite` + `run_migrations`, plain +`pytest` before any deploy. OpenScience stubs `globalThis.fetch` and exercises the real tool; no mocks, no +network. + +Cases that must be covered: + +- **A budget of $B at $R/h lasts ≈ B/R hours**, asserted on elapsed billable duration. This is the headline + property and the one the previous attempt omitted from both its tests and its criteria — a $10 budget dying at + 25.8 minutes passes every "release happened" assertion while being off by 3×. +- A lease **without** a runner token survives past `HEARTBEAT_STALE_SECONDS`; one with a token is still reaped. + Without this, every budget test silently measures a 10-minute reap. +- Money-path writes are idempotent — a replayed tick does not double-charge. +- The rolling cap rejects an N+1th lease even when each individual budget is affordable. +- `volume_id` mounts, and **release does not delete the volume**. +- Extension raises the cap, is clamped, and **exhaustion proceeds normally when none arrives**. +- A tick exceeding `hard_cap_cents` releases; one that fits does not. `spent_cents` accumulates across ticks. +- No `budget_cents` → today's behaviour exactly. BYOK ignores it. Plan TTL still fires independently. + +Every new assertion must be shown failing against the specific mutation it guards — ideally the _deletion_ of +the logic, not its inversion. On this branch alone, five plan-authored test defects were caught in review, +including an assertion that compared whole tool outputs and so passed even with all guidance collapsed to one +string. + +## Acceptance criteria (Part 2) + +0. A lease with no runner token is not reaped for heartbeat staleness; one with a token still is. +1. A budget of $B at rate $R/h lasts ≈ B/R hours, asserted on elapsed billable duration. +2. The money path is idempotent under a replayed tick. +3. A rolling cap bounds spend across sequential leases. +4. `volume_id` attaches a volume that survives lease release. +5. Extension raises the cap when affordable, refuses with a structured 402 when not, never fires automatically. +6. The billing tick re-debits the grant; `spent_cents` accumulates. +7. A tick exceeding the cap releases via the existing path. +8. `POST /leases` accepts `budget_cents`; omitting it preserves today's behaviour exactly. +9. A budget exceeding the wallet is clamped, and the response reports the effective cap. +10. BYOK ignores `budget_cents`. Plan TTL fires independently. +11. The OpenScience tool refuses to launch without a verdict, surfaces 402/429 without retrying, and holds no + pricing or approval logic. +12. `pytest` and `bun test` pass with no network; changes 0 and 1 are each their own commit. + +--- + +## Verification discipline + +Four conclusions in this investigation came from reading source and were wrong about deployed reality: the +CLI's contents, whether the prompt was broken, whether managed compute was reachable, and then the parked +banner's own claim that reselling was off — **the fourth made by a correction to the third.** A document +written to warn about this failure repeated it one section later. + +Claims in this document are labelled: the production-reality section and Part 1's verification table were +checked against running services. The billing tick's debit behaviour, grant accounting, and the reaper's exact +reap path are **read from source only** — confirm against a test before relying on them. diff --git a/docs/specs/compute-mode-detection-design.md b/docs/specs/compute-mode-detection-design.md index 7213dfcd..ec9f1a38 100644 --- a/docs/specs/compute-mode-detection-design.md +++ b/docs/specs/compute-mode-detection-design.md @@ -1,5 +1,19 @@ # Compute mode detection — design +> **SUPERSEDED 2026-07-31 by [`compute-management-design.md`](./compute-management-design.md), which is the +> single current spec for compute. This document is kept as the historical record of the shipped work and is +> no longer maintained.** +> +> Two things below are **wrong** and were corrected during implementation — do not build from them: +> +> - **The "key AND skill" rule was overruled.** A credential alone makes a provider usable; a capable agent +> drives a documented cloud API from a bare key. The conjunction only produced a false `none`. +> - **The skill-name table matches nothing.** Real frontmatter names are `modal-serverless-gpu`, +> `modal-ml-training`, `modal-research-gpu`, `lambda-labs-gpu-cloud`, `tensorpool-gpu-cloud`, +> `prime-intellect-lab` — not `cloud-compute/modal` etc. +> +> Also stale: this document assumes managed compute is unavailable. **It is live in production.** + Status: proposed, for review Date: 2026-07-30 Scope: `openscience` only. **No Atlas changes required.** From 12e32f43249d1d69c53c5b9343c5bf05345d18a1 Mon Sep 17 00:00:00 2001 From: KB Date: Fri, 31 Jul 2026 21:11:43 +0530 Subject: [PATCH 29/56] spec(compute): replace three specs with one, add the managed-lease design The three compute specs cross-referenced each other, two of them as superseded, and the surviving one carried claims that source disagrees with. Replace all three with a single document. New in Part B, decided this round: - Atlas resolves the SKU from {gpu, count, max_hourly_cents} and leases atomically, instead of the agent picking from 292 options. Fixes the Vast offer-ID race by construction rather than by client retry. - Three tools (launch/list/release), not one with an action parameter, so the permission rule can ask on launch and allow on list. - The agent never holds key material: the one-time private key goes to ~/.config/openscience/compute/.pem at 0600 and the tool returns key_path. Keeps it out of the transcript and compaction. - Atlas is the truth for what is running; the .pem is the only local state, because it is the only value that cannot be re-fetched. - compute_launch prompts by default, silenced only by explicit config. UX, not enforcement. Every Part B claim now carries a file:line citation, verified against the Atlas checkout at 7b0e9b6. That check corrected a predecessor finding: the acquire-time grant debit IS rolled back (lease_manager.py :77 and :209), just not before the first tick. The double-count trap is real; the reasoning printed for it was not. --- backend/cli/src/tool/skill.ts | 2 +- docs/specs/compute-design.md | 469 +++++++++++++++++ docs/specs/compute-guardrails-design.md | 533 -------------------- docs/specs/compute-management-design.md | 435 ---------------- docs/specs/compute-mode-detection-design.md | 391 -------------- 5 files changed, 470 insertions(+), 1360 deletions(-) create mode 100644 docs/specs/compute-design.md delete mode 100644 docs/specs/compute-guardrails-design.md delete mode 100644 docs/specs/compute-management-design.md delete mode 100644 docs/specs/compute-mode-detection-design.md diff --git a/backend/cli/src/tool/skill.ts b/backend/cli/src/tool/skill.ts index b081b1c4..bd2a3b34 100644 --- a/backend/cli/src/tool/skill.ts +++ b/backend/cli/src/tool/skill.ts @@ -59,7 +59,7 @@ export const SkillTool = Tool.define("skill", async (ctx) => { // This is a LISTING filter, not a gate. `none` is guidance, not enforcement — // a hidden skill can still be loaded by exact name, and the agent still has // bash. Gating the load path is a larger change and is deliberately out of - // scope; see docs/specs/compute-mode-detection-design.md open question 3. + // scope; see docs/specs/compute-design.md, Part A. const offered = await ComputeMode.offered() const accessibleSkills = permitted.filter((skill) => !ComputeMode.SKILLS.has(skill.name) || offered.has(skill.name)) diff --git a/docs/specs/compute-design.md b/docs/specs/compute-design.md new file mode 100644 index 00000000..051fee1b --- /dev/null +++ b/docs/specs/compute-design.md @@ -0,0 +1,469 @@ +# Compute — design + +Status: **Mode detection shipped. Managed leases to build.** +Date: 2026-07-31 · single current compute spec +Roadmap: **5**, **51/2**, **55**, **103**; unblocks **56** + +How OpenScience gets a GPU: who provisions it, who pays, and what stops it. Spans two repos — +`atlas` (Python/FastAPI) decides and enforces, `openscience` (Bun/TypeScript) relays and obeys. Each +side gets its own implementation plan; this document is the contract between them. + +Replaces `compute-management-design.md`, `compute-guardrails-design.md` and +`compute-mode-detection-design.md`, which disagreed with each other and, in places, with the code. + +--- + +## Two paths + +| Mode | Who provisions | Atlas role | What bounds it | +| --------- | --------------------------------- | ----------------------------------- | --------------------------- | +| `byok` | the agent, direct to the provider | none — never sees the key | the user's own account | +| `managed` | Atlas | provisions, meters, enforces, reaps | budget → wallet → cap → TTL | +| `none` | — | — | — | + +These are genuinely different mechanisms, not one flow with a funding flag. BYOK runs on the user's +provider account, which Atlas neither meters nor bills, so there is nothing for Atlas to decide and no +reason for it to custody a provider credential. **Everything below Part A is `managed` only.** + +--- + +# Part A — Mode detection (shipped) + +Shipped on `feat/compute-guardrails`. Summarised here because Part B depends on it; the detail lives in +the code and its tests. + +`computeBillingMode()` used to return `config.billing?.compute ?? "byok"` without inspecting the +environment, so a user with zero GPU credentials resolved to `byok` — claiming BYOK with nothing to BYOK +with — and "no compute available" had no representation at all. + +```ts +export type ComputeSource = "byok" | "managed" | "none" +``` + +**A credential is the whole test.** An earlier draft required a credential _and_ a matching skill; that +was overruled during implementation, because a capable agent drives a documented cloud API from a bare +key, so the conjunction only produced a false `none` for users holding a workable key. + +| Provider | Env (any group satisfies; all vars within a group required) | +| --------------- | ----------------------------------------------------------- | +| Modal | `MODAL_TOKEN_ID` **+** `MODAL_TOKEN_SECRET` | +| Lambda | `LAMBDA_API_KEY` \| `LAMBDA_LABS_API_KEY` | +| TensorPool | `TENSORPOOL_KEY` \| `TENSORPOOL_API_KEY` | +| Prime Intellect | `PRIME_API_KEY` \| `PRIME_INTELLECT_API_KEY` | +| RunPod | `RUNPOD_API_KEY` | +| Vast | `VAST_API_KEY` | + +BYOK wins whenever a credential is present: it is free to the user and needs nothing from Atlas — which +is also why a BYOK user never pays for the availability network call. + +`billing.compute` is an **override, never the source of truth**: unset → detection, `"byok"` → BYOK if a +credential exists else `none`, `"managed"` → managed if available else `none`. **An override may narrow +to `none`; it may never manufacture a capability.** + +Resolution happens per request, never at startup, at two points that already run per turn: `SkillTool.init` +and the `compute_status` tool. Credentials reach `process.env` from the shell, the Credentials panel and +the Compute panel — and, as the developer's own machine proved, sometimes from dashboard sync → +`synced-env.json` → `preload-env` replay. Startup detection would have reported `none`. + +Availability is one authenticated `GET /api/compute/options`, 3s timeout, 5s TTL cache. A failed, +unauthenticated or timed-out call resolves to **unavailable** — failing toward `managed` would reproduce +the original bug of promising an unconfirmed capability. + +`ComputeMode.offered()` filters the skill catalog and is non-empty **only in `byok`**. **This is a listing +filter, not a gate:** a hidden skill remains loadable by exact name and the agent still has `bash`. Gating +the load path was considered and declined. + +--- + +# Part B — Managed leases (to build) + +## The gap Part A exposed + +`compute_status` tells an agent in `managed` mode to run GPU work billed to Credits — and **no mechanism +exists**. There is no launch tool, and the Atlas CLI's `compute:*` commands are unpublished. Since managed +is live in production, every keyless user lands there. + +## Trust boundary + +**The agent proposes; the server decides.** OpenScience is open source and the agent has `bash`, so any +client-side check is one a fork can delete. Only a decision made over HTTP, behind auth, in a process the +agent does not run, is one it cannot influence. + +Corollary: **OpenScience holds no pricing, balance, selection or approval logic.** It relays a proposal +and obeys a verdict. + +## The flow + +``` +agent compute_status → mode=managed, balance_usd +agent compute_launch { gpu: "h100", count: 1, budget_cents: 3000, max_hourly_cents?, volume_id? } + └─ permission gate (default ask): + "RunPod H100 · $2.79/hr · cap $30.00 · balance $198.83" +Atlas resolve cheapest live offer matching gpu + count + max_hourly_cents + check wallet funds 1h AND rolling window has headroom + clamp budget to effective balance + mint Ed25519 pair · size grant to effective cap · launch pod + ← { lease_id, ip, ssh_user, private_key, effective_cap_cents, hourly_cents, provider, sku } +OS write private_key → ~/.config/openscience/compute/.pem (0600) + return { lease_id, ip, ssh_user, key_path, effective_cap_cents, hourly_cents } — no key material +agent bash: ssh -i @ … scp results back +agent compute_release { lease_id } → Atlas terminates · OS deletes the .pem +``` + +## Why budget, not balance + +The obvious design is to check the wallet each hour and stop the box when it can no longer fund another +one. That bound is **the user's entire balance** — a forgotten H100 against a $500 wallet costs $500, and +sequential re-leasing is unbounded on top of that. + +A per-run budget bounds the _run_. The agent proposes _"this is worth up to $30"_, not _"this needs four +hours"_ — an LLM can judge the first and cannot predict the second, and it is a wrong duration estimate +that forces extension paths, warning windows and re-estimation loops into a design. + +Three consequences, all simplifying: exhaustion is a fact the server observes, so no completion signal is +needed; it is arithmetic, so no extension is needed for correctness; and it is server-side, so no agent +liveness is needed. + +The wallet remains the outer bound. A $1000 budget against a $15 balance is not an error — it buys $15 of +compute — but **the response reports the effective cap** so the agent can tell the user what was actually +authorised rather than what was asked for. + +## Why Atlas resolves the SKU + +The agent states requirements (`gpu`, `count`, optional `max_hourly_cents`); Atlas picks the cheapest +matching live offer and leases it in one call. Three reasons, in order of weight: + +1. **It fixes the offer-ID race by construction.** `compute:up` fetches options, picks, then estimates — + and Vast's SKUs are ephemeral marketplace offer IDs that churn in between. Vast supplies 204 of 292 + live options and is therefore almost always the cheapest pick, so the default path fails with a raw + `HTTP 400: Unknown SKU`. Making fetch-and-lease atomic inside Atlas removes the window. +2. **292 options never enter the context window.** +3. **It keeps every price decision server-side**, which is what the trust boundary already required. + +The cost is an Atlas resolver that does not exist yet. The alternative — agent picks, client retries on +400 — puts ranking logic in OpenScience, which is exactly what the boundary forbids. + +## Why RunPod is the managed default + +Every provider generates a fresh Ed25519 keypair per lease and shows the provider only the public half, +so the returned key opens exactly one box everywhere. They differ in what they leave behind: + +| Provider | Key attachment | Account artifact | Cleaned up | +| --------------- | ---------------------------------------------- | ---------------- | ---------------------------------- | +| **RunPod** | `PUBLIC_KEY` env var consumed on boot | **none** | nothing to clean | +| Lambda | account key registry | yes | yes — on release and failed launch | +| Vast | account `/ssh/` **and** `/instances/{id}/ssh/` | yes | **no** | +| Prime Intellect | `POST /ssh_keys/` → `sshKeyId` | yes | **no** | + +RunPod is the only one leaving no account-level trace, which is the right property when Atlas owns the +box lifecycle. Its creation body also takes an arbitrary `env` dict, so anything Atlas later wants running +on boot needs no SSH bootstrap. + +Vast is cheapest and deepest but is interruptible spot and leaks a key per lease. **Vast and Prime +Intellect leak one public key per lease into the operator account, forever** — Lambda's delete-on-release +is the fix. Separate ticket. + +--- + +## Atlas changes + +### Change 0 — scope the lease reaper _(prerequisite, live defect)_ + +`lease_reaper.sweep_once` branch 3 (`backend/app/jobs/lease_reaper.py:141-144`) applies +`HEARTBEAT_STALE_SECONDS` = 600 (`config.py:69`) to everything returned by +`compute_repo.list_unfinished_leases`, whose own docstring reads _"category-agnostic"_ +(`compute_repo.py:456-462`). `create_lease` mints no runner token, and the telemetry endpoint requires +one. + +**A user lease cannot prove liveness and is destroyed ~10 minutes after creation**, with provisioning +eating several of those minutes. A $30 budget lease dies having spent about $1.17. No budget can bind +until this is fixed. + +Scope the heartbeat check to leases holding a runner token. User leases stay bounded by plan TTL, wallet +exhaustion, explicit release, and the budget cap. The provider-terminal (`:123`) and provisioning-timeout +(`:133`) branches continue to apply to everything. + +**Ships first, alone, with its own test.** + +### Change 1 — make `hard_cap_cents` a real running cap + +The column exists (`migrations.py:522`, `pg_migrations.py:775`) and the atomic ceiling exists +(`compute_repo.py:196` — `AND (spent_cents + ?) <= hard_cap_cents`). But `debit_grant` is called in +exactly four places, all in `lease_manager.py`: acquire for one hour (`:529`), wallet-insufficient +rollback (`:545`), reservation undo (`:77`), and settle true-up estimate→actual (`:209`). +**`compute_billing_service.tick_once` never calls it** — only `usage_service.charge` and `mark_billed`. +So `spent_cents` freezes at hour one and the ceiling is never re-evaluated. + +**The billing tick re-debits the grant by the same delta it charges**, releasing the lease when the debit +would exceed the cap — reusing the path that already fires on wallet exhaustion. + +**This is not a double charge.** The wallet is money; the grant is an authorisation envelope drawn against +it. An implementer who "de-duplicates" these has removed the cap. + +_Known trap:_ the acquire-time debit is reversed on failure and trued up at settle, but **not before the +first tick**. A naive re-debit therefore double-counts hour one — a $10 budget at $6.99/h would die at +25.8 minutes instead of ~1.4 hours. (A predecessor spec stated the debit "is never rolled back", which is +wrong; the rollback paths are `:77` and `:209`. The trap is real, the reasoning for it was not.) + +### Change 2 — accept a budget on lease creation + +``` +POST /api/compute/leases +{ provider?, sku?, gpu?, count?, max_hourly_cents?, region?, node_id?, budget_cents?, volume_id? } +``` + +`budget_cents` is optional; absent preserves today's behaviour exactly, which matters because the Atlas +dashboard and `compute:up` both call this endpoint without it. Rejection reuses the structured `402`, +extended with `affordable_budget_cents`. + +**A budget larger than the wallet is clamped, not rejected.** The response reports the **effective** cap. + +**Managed only.** BYOK ignores `budget_cents` and is never debited. + +### Change 3 — resolve a SKU from requirements + +Accept `{gpu, count, max_hourly_cents?}` in place of an explicit `sku`, resolve to the cheapest live +matching offer, and lease it in the same transaction. Explicit `provider`/`sku` continues to work. + +`GET /api/compute/options` (`routes/compute.py:214`) and `POST /api/compute/estimate` (`:244`) already do +the reads; what is new is doing them atomically with the lease. + +### Change 4 — bound cumulative spend + +The cap is per-grant and a grant is per-lease, so release-and-reacquire is unbounded. What exists today is +`MANAGED_GPU_CONCURRENT` (`lease_manager.py:563`), which bounds concurrent boxes, not total cost. A $30 +budget honoured twenty times is $600. + +Add a **rolling window cap** at lease creation, with window and ceiling as plan config alongside +`gpu_sandbox_max_ttl_hours`, and a distinct `error` code so clients can tell "this box is too expensive" +from "you have spent enough today". + +Design it in now — retrofitting changes the meaning of a number users already trust. If `compute_grants` +is not indexed by `(user_id, created_at)`, that index is the migration. + +### Change 5 — attach a persistent volume + +Atlas already has `POST /api/compute/volumes` (`routes/compute.py:565`), `list_volumes` and +`delete_volume`. **Leases do not use them**, and the RunPod provider passes `volumeInGb: 20` — a +pod-scoped volume destroyed with the pod. + +Add `volume_id?` to the lease request, pass it to RunPod as `networkVolumeId` mounted at `/workspace`. +**Releasing a lease must not cascade a volume delete.** + +Budget exhaustion then costs the compute, not the work — a network volume is cents per GB-month against +dollars per GPU-hour. + +**Honest limit:** a volume preserves _files_, not _process state_. A run killed mid-epoch still dies +unless it checkpointed to `/workspace`. The volume is the substrate roadmap **56** needs, not a substitute +for it. + +### Change 6 — extend a live budget + +``` +POST /api/compute/leases/{lease_id}/budget { additional_cents } +→ { hard_cap_cents, spent_cents, effective_cap_cents } +``` + +Raises the cap on the existing grant, clamped by wallet and rolling cap. No new state — it edits a number +change 1 already reads every tick. Three constraints keep it from reopening the door earlier drafts +closed: + +- **Pull, never push.** Atlas never auto-extends. A budget that quietly refills is not a budget. +- **Not part of enforcement.** If no extension arrives, exhaustion proceeds unchanged, so a dead agent + costs nothing. +- **No warning event required.** An 80% notification is worth adding for humans, but it is advice; the cap + must never depend on anyone reading it. + +--- + +## OpenScience changes + +### Three verbs + +| Tool | Input | Output | +| ----------------- | ----------------------------------------------------------------- | ------------------------------------------------------------------------------- | +| `compute_launch` | `gpu`, `count`, `budget_cents`, `max_hourly_cents?`, `volume_id?` | `lease_id`, `ip`, `ssh_user`, `key_path`, `effective_cap_cents`, `hourly_cents` | +| `compute_list` | — | running leases: id, provider, sku, ip, spent, cap | +| `compute_release` | `lease_id` | released | + +Plus `compute_status` (shipped, unchanged). Each verb maps 1:1 to an Atlas endpoint. Separate tools rather +than one `action` parameter, so the permission rule can ask on launch and allow on list. + +The agent does the actual work with plain `ssh`/`scp` from `bash`. **No relay, no exec wrapper, no +file-transfer helper** — that path is a remote-execution platform, and it was already designed once and +rejected. + +Behaviour: + +- **Refuse to launch without a verdict from Atlas.** +- On `402`, surface `affordable_budget_cents` and stop. **Never auto-retry at a smaller budget** — a + truncated training run is not a cheaper result, it is a discarded one. +- On `429` (concurrency cap), surface rather than retry. +- **No client-side deadline timer**, price table, or SKU ranking. + +### The agent never holds key material + +`compute_launch` writes the one-time private key to `~/.config/openscience/compute/.pem` at +`0600` and returns only `key_path`. The key stays out of the transcript, out of compaction, and out of +session storage. `compute_release` deletes it. + +This is also the fix for the existing Atlas CLI defect: it prints the private key and never saves it, so +the `ssh_command` it prints cannot work. + +### Atlas is the truth for what is running + +`compute_list` calls `GET /api/compute/leases` rather than reading a local ledger, so there is nothing to +drift and nothing to orphan on crash. The `.pem` is the only local state, because it is the only value +that cannot be re-fetched. + +### The approval gate + +`compute_launch` goes through the standard permission path. `PermissionNext.evaluate` already defaults to +`ask` when no rule matches (`src/permission/next.ts:237`) and `Permission` has a `.catchall` +(`src/config/config.ts:642`), so the gate is on by default and needs no schema change. The prompt shows +provider, SKU, hourly rate, proposed cap and current balance. + +Settings ▸ Compute exposes it as `permission.compute_launch`, so a user who wants unattended overnight +runs sets `allow` deliberately. **This is UX, not enforcement** — a fork can delete it. The server caps +are what actually bind. + +--- + +## The bounds that remain + +Verified against the Atlas checkout at HEAD `7b0e9b6`, source-read (not a running deploy). + +| Bound | Owner | Fires when | Today | +| ------------------- | -------------- | ------------------------------------- | ------------------------------------------- | +| `hard_cap_cents` | billing tick | approved money is spent | **column only, does not enforce** — ch. 1 | +| Rolling window cap | lease creation | cumulative spend hits the ceiling | **does not exist** — ch. 4 | +| Wallet exhaustion | billing tick | money actually runs out | **works** (`tick_once:152/225 → :172/239`) | +| Plan TTL (24h) | billing sweep | anything has run absurdly long | **works** (`_release_stale_gpu_leases:303`) | +| Explicit release | agent/user | asked | **works** (`routes/compute.py:513`) | +| Heartbeat staleness | lease reaper | an _agent-spawned_ lease stops report | **fires on user leases too** — ch. 0 | + +All server-side; none client-influenceable. Change 0 _removes_ a bound from user leases, which is safe +precisely because the others apply and necessary because it is one those leases cannot satisfy. + +`COMPUTE_BILLING_TICK_SECONDS` defaults to 60 (`compute_billing_service.py:44`), so a budget can overrun +by up to a minute of rate (~$0.12 on an H100). **Approved budgets are ceilings-plus-a-minute and must +never be described as exact.** + +--- + +## What we deliberately did not build + +Design exploration reached a persistent runner daemon on the box, a command queue, a result stream, +scoped storage credentials and a PTY relay — a remote-execution platform, designed before a single box had +been leased through the agent. + +The test that settled it: **a productivity feature is admissible only if its absence changes nothing about +enforcement.** Volumes pass (files move off the box before anything fails). Optional extension passes. A +command relay, per-lease `expires_at`, client-side timers and auto-extension all fail — each needs +something alive to act on a signal. + +Also excluded: an ephemeral sandbox/exec path alongside SSH leases. It has no user-facing endpoint today +and cannot run multi-hour training, which is the use case that motivated GPU compute at all. Volume +creation stays in the workspace UI rather than becoming a fourth tool. + +## What you are accepting + +- **A run killed mid-epoch that was not checkpointing is gone.** No server-side mechanism can fix that. + Change 5 converts "you lost the job" into "you lost the GPU"; roadmap **56** is what makes it good. +- **Time is unbounded within a budget.** A cheap CPU lease could run for days inside a small budget. The + 24-hour plan TTL is the only backstop, deliberately. +- **`none` is guidance, not enforcement.** The agent still has `bash`. + +## Known defects, owned elsewhere + +- **Atlas CLI unpublished.** `@synsci/atlas@0.13.2` on npm carries 155 command specs and zero `compute:`; + `3e1d1ca` removed them, `205bbc0` re-added them, no version bump followed. Source and artifact disagree + at an identical version. A release, not code. +- **CLI usability.** Prints the one-time private key and never saves it; no file transfer, no exec, no + compute tests. +- **Vast / Prime Intellect SSH key leaks** into the operator account, unbounded. +- **`budget_cents` already exists** on the agent-spawn path defaulting to `500`, display-only. Change 1 + makes caps real, silently giving every shipped spawn a hard $5 kill. **Not a no-op** — raise the default + deliberately or exempt spawn-path grants. +- **Stale skill names in agent prompts.** `research.txt`'s appendix and `ml.txt` name ~35 skills by + directory rather than frontmatter `name` (`vllm` → `serving-llms-vllm`), so the agent is told it has + skills it cannot load. `skills/scholar-evaluation/SKILL.md` has no `name:` at all. + +## Out of scope + +Roadmap **61** (per-job secrets), **56** (checkpointing), **4** (BYOK provider API clients), **52** +(`bun:sqlite`). Per-lease `expires_at`, client-side deadline timers, client-side price tables, +auto-extension, sandbox/exec paths. Any change to how `billing.llm` resolves. + +--- + +## Testing + +Atlas follows `backend/tests/test_compute_billing.py` — `_FakeProvider`, `aiosqlite` + `run_migrations`, +plain `pytest` before any deploy. OpenScience stubs `globalThis.fetch` and exercises the real tool; no +mocks, no network. + +- **A budget of $B at $R/h lasts ≈ B/R hours**, asserted on elapsed billable duration. This is the + headline property and the one the previous attempt omitted from both its tests and its criteria — a $10 + budget dying at 25.8 minutes passes every "release happened" assertion while being off by 3×. +- A lease **without** a runner token survives past `HEARTBEAT_STALE_SECONDS`; one with a token is still + reaped. Without this, every budget test silently measures a 10-minute reap. +- Money-path writes are idempotent — a replayed tick does not double-charge. +- The rolling cap rejects an N+1th lease even when each individual budget is affordable. +- SKU resolution picks the cheapest offer honouring `max_hourly_cents`, and leases atomically — a stale + offer ID cannot appear between resolve and lease. +- `volume_id` mounts, and **release does not delete the volume**. +- Extension raises the cap, is clamped, and **exhaustion proceeds normally when none arrives**. +- A tick exceeding `hard_cap_cents` releases; one that fits does not. `spent_cents` accumulates. +- No `budget_cents` → today's behaviour exactly. BYOK ignores it. Plan TTL still fires independently. +- OpenScience: the key is written `0600` and **never appears in the tool result**; release deletes it; + `402`/`429` surface without retry; launch without a verdict is refused. + +Every new assertion must be shown failing against the specific mutation it guards — ideally the _deletion_ +of the logic, not its inversion. On this branch alone, five plan-authored test defects were caught in +review, including an assertion that compared whole tool outputs and so passed even with all guidance +collapsed to one string. + +## Acceptance criteria + +0. A lease with no runner token is not reaped for heartbeat staleness; one with a token still is. +1. A budget of $B at rate $R/h lasts ≈ B/R hours, asserted on elapsed billable duration. +2. The money path is idempotent under a replayed tick. +3. The billing tick re-debits the grant; `spent_cents` accumulates; a tick exceeding the cap releases via + the existing path. +4. `POST /leases` accepts `budget_cents`; omitting it preserves today's behaviour exactly. +5. A budget exceeding the wallet is clamped, and the response reports the effective cap. +6. `{gpu, count, max_hourly_cents}` resolves to the cheapest matching live offer and leases atomically. +7. A rolling cap bounds spend across sequential leases. +8. `volume_id` attaches a volume that survives lease release. +9. Extension raises the cap when affordable, refuses with a structured `402` when not, never fires + automatically. +10. BYOK ignores `budget_cents`. Plan TTL fires independently. +11. `compute_launch` refuses to launch without a verdict, surfaces `402`/`429` without retrying, holds no + pricing or selection logic, writes the key `0600`, and never returns key material. +12. `compute_launch` prompts by default and is silenced only by explicit config. +13. `compute_list` reflects Atlas, not local state — a lease released out-of-band disappears from it. +14. `pytest` and `bun test` pass with no network; changes 0 and 1 are each their own commit. + +--- + +## Verification discipline + +Four conclusions in the predecessor investigation came from reading source and were wrong about deployed +reality: the CLI's contents, whether the prompt was broken, whether managed compute was reachable, and the +claim that reselling was off — **the fourth made by a correction to the third.** + +Labels in this document: + +- **Verified against production `thesis-synsc` (2026-07-31):** `resell_enabled: true`; lambda / runpod / + vast / prime_intellect operator-funded with 292 launchable options; `/compute/estimate` returns + `funding: "managed"` with a real rate and runway; the published npm artifact contains no `compute:` + command; `compute:up`'s default path fails on the Vast SKU race while `--provider lambda|runpod` + succeeds. +- **Verified against the Atlas checkout at HEAD `7b0e9b6` (source-read, 2026-07-31):** every `file:line` + citation in Part B — the reaper's branch structure, `debit_grant`'s four call sites and the tick's + absence from them, the 60s tick default, the 24h GPU TTL sweep, the absence of any rolling cap, and the + existing endpoint set. + +A source-read claim is not a deployed-behaviour claim. Confirm the money path against `pytest` before +relying on it. diff --git a/docs/specs/compute-guardrails-design.md b/docs/specs/compute-guardrails-design.md deleted file mode 100644 index 3fba8009..00000000 --- a/docs/specs/compute-guardrails-design.md +++ /dev/null @@ -1,533 +0,0 @@ -# Managed compute budget cap — design - -> **SUPERSEDED 2026-07-31 by [`compute-management-design.md`](./compute-management-design.md), which is the -> single current spec for compute and carries this design forward as its Part 2.** -> -> Kept for the material that did not survive the merge: the full five-finding review, the prototype's -> gate-mode evidence, the corrections log, and the record of two false starts. **Build from the merged -> document, not this one.** - -Status: **REVIVED — superseded by the merged spec.** Parked 2026-07-30, revived 2026-07-31, merged 2026-07-31. -Date: 2026-07-30 · revived 2026-07-31 - -> ## ⚠️ Read this before anything below -> -> **The parking condition has expired, and the five findings below are now the implementation checklist — -> exactly as the parked banner instructed.** The design's core (budget not duration, server decides, -> `hard_cap_cents` made real) survives review intact. What changed is that the problem became reachable and -> the gaps became work items. -> -> ### What changed on 2026-07-31 -> -> **Managed compute is switched ON in production.** The parked banner's first and load-bearing reason — -> "there is no live overspend to guard" — is false as of a live check against `thesis-synsc`: -> -> ``` -> GET /api/compute/options → resell_enabled: true -> lambda / runpod / vast / prime_intellect → funding: "managed" -> 292 launchable options -> ``` -> -> `COMPUTE_RESELL_ENABLED` still _defaults_ to `false` (`backend/app/config.py:383`), which is what the -> original analysis read — but production sets it. **This is the fourth time in this investigation that reading -> source gave the wrong answer about deployed reality** (see "Corrections to earlier analysis"), and the first -> one made _by_ a correction to an earlier mistake. Verify against the running system. -> -> So there is a live, unmetered spend path today: `POST /compute/leases` checks only the first hour, and -> `atlas compute:up` is the human-facing door to it. -> -> ### The prerequisite that blocks everything -> -> **Finding 1 below is not a checklist item — it is a hard prerequisite, and it is a live defect independent -> of this spec.** The lease reaper terminates _any_ lease that emits no telemetry roughly ten minutes after -> creation: -> -> - `compute_repo.list_unfinished_leases` is explicitly category-agnostic — -> `WHERE status NOT IN ('released','failed')`, no exemption for user leases. -> - `lease_reaper.sweep_once` branch 3 falls back to `_lease_started` (= `started_at or created_at`) when -> there is no telemetry, and reaps past `HEARTBEAT_STALE_SECONDS` = 600. -> - `create_lease` mints **no runner token**, and `POST /api/agent/runner/telemetry` requires one -> (`x-thesis-runner-token`, scoped to a lease). -> -> **A user-launched lease therefore has no way to prove liveness, and is destroyed before it is useful.** -> Provisioning eats several minutes of the ten. Until this is fixed, no budget can bind — a $30 budget lease -> dies having spent about $1.17 — and `atlas compute:up` cannot run a research task no matter what else ships. -> -> The fix is to scope heartbeat staleness to leases that are _supposed_ to report: those with a runner token. -> User leases are already bounded by three independent mechanisms — plan TTL, wallet exhaustion, and explicit -> release — and adding a budget cap makes four. They do not need a liveness probe they cannot answer. -> -> ### The five findings, now the checklist -> -> - **The lease reaper kills these leases at ten minutes.** `lease_reaper` sweeps every 60s over all -> unfinished leases and reaps anything silent for `HEARTBEAT_STALE_SECONDS` = 600. A workload run over SSH -> emits no `agent_telemetry`, so a $30 budget lease would be terminated having spent about $1.17. The budget -> would never bind. This also falsifies the claim below that no agent liveness is needed — Atlas requires -> liveness in the form of telemetry. -> - **`budget_cents` already exists** on the agent-spawn path with a default of `500` -> (`agent_tools.py:1386`). Making the cap real converts a display-only $5 into a hard kill on a shipped -> feature. The migration section below calls this a no-op; it is not. -> - **The acquire-time debit is never rolled back**, so re-debiting per tick double-counts hour one. A $10 -> budget at $6.99/h would die at 25.8 minutes rather than ~1.4 hours; a one-hour budget would buy 60 seconds. -> The test list below does not catch this — the property that matters, _a budget of $B at $R/h lasts ≈B/R -> hours_, appears in neither the tests nor the acceptance criteria. -> - **Sequential leases are unbounded.** The cap is per-grant and a grant is per-lease, so an agent can release -> and re-acquire without limit. The concurrency cap of 2 does not bound cumulative spend. -> - **No idempotency on the money path.** The billing tick performs independent committing writes; a crash -> between them double-charges on the next tick, and adding a grant debit widens the window. -> -> ### Status of the two factual errors -> -> - **"`atlas compute:up` exists"** — still true only of the repo. Verified again 2026-07-31 by unpacking the -> registry artifact: published `@synsci/atlas@0.13.2` contains **155 command specs and zero `compute:`**. -> The chronology explains it — `3e1d1ca` removed the compute commands, `0.13.1` and `0.13.2` shipped without -> them, then `205bbc0` re-added them and four commits developed them further **with no version bump**. `main` -> still declares `0.13.2`, identical to the artifact that lacks them. The fix is a release, not code. -> - **The system prompt was broken and is now FIXED.** Shipped on `feat/compute-guardrails` — the false -> `atlas compute:up` / `atlas doctor` guidance is deleted from both `session/prompt.ts` and the `research` -> agent prompt, replaced by a `compute_status` tool that resolves `byok | managed | none` at runtime. -> -> ### Scope note inherited from that work -> -> `compute_status` now tells an agent in `managed` mode to "run GPU work through managed compute" — and no -> mechanism exists for it to do so. That guidance is honest about funding but not about capability, and it is -> the user-visible reason this document is being revived rather than left parked. - -Roadmap items: **55** (budget guardrails + kill switches), **103** (cost approval gates), and the gate half of -**51/2** (agent-facing compute tool) - -Spans two repos. This document is the **contract**; each side gets its own implementation plan because they -have separate test runs and deploys. - -- `atlas` (Python/FastAPI) — the decision and the enforcement -- `openscience` (Bun/TypeScript) — a thin client that relays a proposal and obeys the verdict - -> **Verification status (updated 2026-07-31).** The original draft was written entirely from source, which is -> how it acquired four wrong conclusions about deployed reality. Much of it has since been checked against the -> running system, so distinguish: -> -> - **Verified against production `thesis-synsc`:** reselling is on with four operator providers and 292 live -> options; `/compute/estimate` returns `funding: "managed"` with a real rate and runway; the published npm -> artifact contains no `compute:` command; the CLI runs from source and its `compute:up` default path fails -> on the Vast SKU race while `--provider lambda|runpod` succeeds. -> - **Still read from source only:** the billing tick's debit behaviour, grant accounting, the reaper's exact -> reap path, and every claim about what changing them would do. Confirm these against the deployed service — -> or better, against a test — before relying on them. -> -> The prototype models each unverified assumption as a toggle for exactly this reason. - -## The problem, stated narrowly - -The agent can start GPU work and nothing bounds the cost. `POST /api/compute/leases` checks only that the -wallet can fund **one hour**, so a 4-hour job on a 1-hour balance is approved and then strands mid-run — the -prototype confirmed this: a 4-hour H100 at $6.99/h against a $15.00 wallet is **approved at a cost of $27.96**. - -Roadmap **51/2** — exposing compute to the agent as a tool — is gated on fixing that. Wiring an LLM to an -unmetered spend path would be materially worse than today's human-only exposure. - -## The scoping decision that shapes everything below - -Two different problems kept getting tangled during design. They are separated deliberately here. - -| | Problem | Status | -| ---------------- | ---------------------------------------------- | ----------------------------------------------- | -| **Safety** | Don't spend more than was authorised | **This spec. Completely solvable now.** | -| **Productivity** | Don't waste money on a run that gets truncated | Needs roadmap **56** (checkpointing). Not this. | - -An earlier draft of this design tried to solve both, and grew a warning event, an extension-request path, a -per-lease `expires_at`, and client-side early release. Every one of those was an attempt to make a run -_succeed_, not to stop a bill running away — and each depended on something that does not exist (a completion -signal for arbitrary SSH commands, a live agent session outliving a long job, or checkpointing). - -**Safety is still what this spec guarantees.** Changes 0–3 are the whole of it, and they depend on nothing -outside Atlas. - -The 2026-07-31 revision adds two productivity changes — and the test that admitted them is deliberately narrow: -**does it work when nobody is watching?** - -- **Change 4 (volumes) passes.** It moves files off the box before anything fails. It needs no completion - signal, no live session, and no client cooperation, because the volume simply outlives the pod. -- **Change 5 (extension) passes only because it is optional.** If no extension arrives, exhaustion proceeds - unchanged. Nothing blocks on a decision, so a dead agent costs nothing. - -The rejected features failed that test: each needed something alive to act on a signal. That is the line — -**a productivity feature is admissible here only if its absence changes nothing about enforcement.** A per-lease -`expires_at`, client-side timers, and auto-extension remain out for exactly that reason. - -What still isn't solved is stated under "What you are accepting". - -## Trust boundary - -**The agent proposes; the server decides.** The proposed budget is untrusted input. - -This is structural, not stylistic. OpenScience is open-source, so a decision made client-side is one a fork can -delete — and the agent has `bash`, so it is running _inside_ the client. Only a decision made over HTTP, behind -auth, in a process the agent is not running, is one it cannot influence. The gate is real only if it is remote. - -Corollary: **OpenScience holds no pricing logic, no balance logic, and no approval logic.** It relays a proposal -and obeys a verdict. - -## Why budget rather than duration - -The agent proposes _"this is worth up to $30"_, not _"this needs 4 hours"_. - -**An agent can judge the first and cannot predict the second.** "Is this experiment worth $30?" is a value -judgement LLMs handle well. "Will this converge in 4 hours?" is a prediction about a novel training run that -nobody can make — and a wrong duration estimate is what created the need for an extension path, a warning -window, and a re-estimation loop in the earlier draft. - -Three consequences, all simplifying: - -- **No completion signal needed.** Budget exhaustion is a fact the server observes. There is nothing to detect, - which matters because a lease is a VM, not a job — `POST /compute/leases` has no notion of the work running on - it, and the only completion signal anywhere (`agent_telemetry` rows with `done`/`error_trace`) is written by - the Atlas agent runtime, not by an arbitrary command run over SSH. -- **No extension path needed _for safety_.** Exhaustion is arithmetic, not a guess that might need revising. - An extension is therefore a convenience, never a correctness requirement — see change 5, which adds one - deliberately and keeps it outside the enforcement path. -- **No agent liveness needed.** The server enforces whether or not the session that started the job survived. - This is the property change 0 restores: today the reaper demands a liveness signal that a user lease cannot - produce, which inverts exactly this design goal. - -And it uses a primitive that already exists rather than adding one — see below. - -## What to build - -Six changes, in dependency order. **Change 0 is a prerequisite** — without it none of the rest can be observed -to work, because the box dies first. - -### Atlas — change 0: stop the reaper killing user leases - -`lease_reaper.sweep_once` applies heartbeat staleness to every unfinished lease. Only agent-spawned leases can -answer it, because only they are issued a runner token. Scope the check to leases that have one: - -```python -# branch 3 — heartbeat staleness -if reason is None and lease.get("status") != "provisioning" and _has_runner_token(lease): - ... -``` - -Leases without a runner token stay bounded by plan TTL, wallet exhaustion, explicit release, and (after change - -1. the budget cap. The reaper's other branches — provider-terminal and provisioning-timeout — continue to apply - to every lease and should not be narrowed. - -**This is a live user-facing bug, not scaffolding for this spec.** It ships first, on its own, with its own -test: a lease with no runner token survives past `HEARTBEAT_STALE_SECONDS`; one with a token still gets reaped. - -### Atlas — change 1: make `hard_cap_cents` a real running cap - -`compute_grants.hard_cap_cents` already reads like a running spend ceiling. It is not one: `acquire_lease` -debits it **once, for one hour**, and the billing tick then calls `usage_service.charge` + `mark_billed` without -ever calling `debit_grant` again. So `spent_cents` freezes at hour one and the atomic ceiling in -`compute_repo.debit_grant` (`AND (spent_cents + ?) <= hard_cap_cents`) is never re-evaluated. - -The fix: **the billing tick re-debits the grant by the same delta it charges.** When the debit would exceed the -cap, release the lease — reusing the path that already fires when the wallet runs dry -(`compute_billing_service` catching `InsufficientCredits` → `_safe_release`). - -**This is not a double charge.** The grant and the wallet are different ledgers: the wallet is money, the grant -is an authorisation envelope drawn against it. The tick already debits the wallet via `usage_service.charge`; it -will now also decrement the envelope. Two records, one charge. An implementer who "de-duplicates" these has -removed the cap. - -This is the whole feature. It is a small change to money-handling code, so it lands as **its own commit with its -own tests**, separate from change 2, so a failing `fly deploy -a thesis-dev` can be attributed to one or the -other. - -### Atlas — change 2: accept a budget on lease creation - -``` -POST /api/compute/leases -Request: { provider, sku, region?, node_id?, budget_cents?: number } -``` - -`budget_cents` is **optional**, and this matters: the Atlas dashboard already calls this endpoint -(`frontend/src/api/account.ts:284`) without it, and so does `atlas compute:up`. Absent means today's behaviour — -grant sized to the plan TTL at the hourly rate. Present means the grant is sized to `budget_cents` instead, and -change 1 then enforces it. - -Rejection reuses the existing structured `402`, extended with what _would_ fit: - -``` -402 { error: "insufficient_cli_credit", needed_cents, available_cents, - affordable_budget_cents, actions: ["byok", "topup"], message } -``` - -**A budget larger than the wallet is clamped, not rejected.** The wallet is always the outer bound — if it -empties first, the existing exhaustion path releases the lease regardless of what the grant permits. So a -$1000 budget against a $15 balance is not an error, it simply buys $15 of compute. But the caller must not be -left believing otherwise: the response reports the **effective** cap (`min(budget_cents, effective_balance)`) -so the agent can tell the user what was actually authorised rather than what was asked for. - -**Managed leases only.** BYOK runs on the user's own provider account, which we neither meter nor bill, so a -budget cap there would be a number we cannot enforce. BYOK ignores `budget_cents`. - -### Atlas — change 3: bound cumulative spend, not just per-lease spend - -The cap is per-grant and a grant is per-lease, so an agent can release and re-acquire without limit. The -concurrency cap of 2 bounds how many boxes run at once, not what they cost in total. A $30 budget honoured -twenty times is $600. - -Add a **rolling window cap** checked at lease creation: the sum of grants opened by this user in the trailing -window must not exceed the plan's ceiling. Window and ceiling are plan config, alongside -`gpu_sandbox_max_ttl_hours`. Rejection reuses the `402` shape with a distinct `error` code so the client can -tell "this box is too expensive" from "you have spent enough today". - -Design this in now. Retrofitting a cumulative cap after users depend on a per-lease one changes the meaning of -a number they already trust. - -### Atlas — change 4: attach a persistent volume so exhaustion costs compute, not work - -Atlas already has a volumes API — `POST /api/compute/volumes` (10 GB–10 TB), `list_volumes`, `delete_volume` -with a detach requirement. **Leases do not use it.** `LeaseRequest` has no volume field, and the RunPod -provider passes `volumeInGb: 20`, which is a _pod-scoped_ volume RunPod destroys with the pod. - -- Add `volume_id?` to `LeaseRequest`. -- Pass it to RunPod as `networkVolumeId`, mounted at `/workspace`. - -Budget exhaustion then destroys the pod and leaves the work. The user relaunches against the same volume and -continues. The economics strongly favour it: a network volume costs cents per GB-month against dollars per -GPU-hour, so preserving the work costs approximately nothing next to the compute that produced it. - -**Honest limit:** a volume preserves _files_, not _process state_. A run killed mid-epoch still dies; it -resumes only if it was checkpointing to `/workspace`. The volume is the substrate roadmap **56** needs, not a -replacement for it — but it converts "you lost the job" into "you lost the GPU", which is the difference -between a wasted budget and a wasted hour. - -### Atlas — change 5: extend a live budget - -``` -POST /api/compute/leases/{lease_id}/budget -Request: { additional_cents } -Response: { hard_cap_cents, spent_cents, effective_cap_cents } -``` - -Raises `hard_cap_cents` on the existing grant, clamped by the wallet and by the rolling cap from change 3. -Same 402 shapes on refusal. Requires no new state — it edits a number change 1 already reads every tick. - -Three constraints keep this from re-opening the door the original draft closed: - -- **Extension is pull, never push.** Atlas never auto-extends from a remaining balance. Spending without being - asked is precisely what a budget exists to prevent, and a budget that quietly refills is not a budget. -- **It is not part of enforcement.** If no extension arrives, exhaustion proceeds exactly as change 1 defines. - Nothing waits for a decision, so a dead agent changes nothing. -- **No warning event is required for it to work.** A notification at ~80% is worth adding for humans, but it is - advice, not a mechanism, and the cap must not depend on anyone reading it. - -### Why RunPod is the managed default - -Verified across all four reseller providers: each generates a fresh Ed25519 keypair per lease and the provider -only ever sees the public half, so **the key handed back opens exactly one box** everywhere. They differ in -what they leave behind: - -| Provider | How the public key attaches | Account artifact | Cleaned up on release | -| --------------- | --------------------------------------------------------------------- | ---------------- | ------------------------------------ | -| **RunPod** | injected via the `PUBLIC_KEY` env var the base images consume on boot | **none** | nothing to clean | -| Lambda | registered in the account key registry | yes | yes — on release _and_ failed launch | -| Vast | posted to account `/ssh/` **and** `/instances/{id}/ssh/` | yes | **no** | -| Prime Intellect | `POST /ssh_keys/`, referenced as `sshKeyId` | yes | **no** | - -RunPod is the only one with no account-level trace at all, which is the right property when Atlas owns the box -lifecycle. Its pod-creation body also takes an arbitrary `env` dict — the same lever the Modal spawn path uses -to pass a token and callback URL — so anything Atlas later wants running on boot needs no SSH bootstrap. - -**Two Atlas bugs fall out of this table, in the operator account rather than users':** Vast and Prime Intellect -leak one public key per lease, unbounded and forever. Lambda's pattern is the fix. Out of scope here; worth -their own ticket. - -### OpenScience — one tool - -The agent lists options, picks a SKU itself, and proposes a budget: - -1. `GET /api/compute/options` → the agent sees live per-SKU rates and picks. **Selection is the agent's job**, - which is why OpenScience needs no selection logic and no wrapper around `compute:up`. -2. Submit `{provider, sku, budget_cents}`. **Refuse to launch without a verdict that came back from Atlas.** -3. On `402`, surface `affordable_budget_cents` to the user and stop. **Never auto-retry at a smaller budget** — - a truncated training run is not a cheaper result, it is a discarded one, and an agent that quietly downsizes - scientific work produces invalid output while appearing to succeed. -4. On `429` (concurrency cap, currently 2 managed GPU leases), surface it rather than retrying. -5. Release on request. **No client-side deadline timer** — the server is the enforcer, and the client has no - completion signal to improve on it with. - -## The bounds that remain - -| Bound | Owner | Fires when | Status after this spec | -| ------------------- | -------------------- | ---------------------------------------- | ----------------------- | -| `hard_cap_cents` | Atlas billing tick | the approved money is spent | made functional (ch. 1) | -| Rolling window cap | Atlas lease creation | cumulative spend hits the period ceiling | new (ch. 3) | -| Wallet exhaustion | Atlas billing tick | the money actually runs out | already works | -| Plan TTL (24h) | Atlas billing sweep | anything has run absurdly long | already works | -| Heartbeat staleness | Atlas lease reaper | an _agent-spawned_ lease stops reporting | narrowed (ch. 0) | - -Every one is server-side; none can be influenced by the client. **No `expires_at` column is added** — time is -not the thing being authorised, and a second time bound alongside the plan TTL would be redundant. - -Note the shape of change 0: it _removes_ a bound from user leases. That is safe precisely because the other -four still apply, and it is required because the bound it removes is one those leases cannot satisfy. - -Billing ticks every 60 seconds, so a budget can overrun by up to a minute of rate (~$0.12 on an H100). Approved -budgets are therefore ceilings-plus-a-minute and must never be described as exact. - -## What you are accepting - -**A budget-exhausted job loses its GPU. With change 4 it need not lose its work.** Attaching a persistent -volume moves the files off the box, so exhaustion costs the compute rather than the run — provided the job -checkpointed to `/workspace`. Roadmap **56** (checkpointing) remains the thing that makes this genuinely good; -change 4 is the substrate it needs, and without it roadmap 56 has nowhere durable to write. - -The residual loss is real and accepted: **a run killed mid-epoch that was not checkpointing is gone.** No -server-side mechanism can fix that, because the server cannot know what the process was holding in memory. - -**Time is unbounded within a budget.** A cheap CPU lease could run for days inside a small budget. The 24-hour -plan TTL is the only backstop, which is deliberate. - -## Testing - -**Atlas** follows `backend/tests/test_compute_billing.py`: a `_FakeProvider` registered into the provider -registry, `aiosqlite` + `run_migrations` for an isolated DB, assertions on the verdict and on whether release -was called. Runs under plain `pytest` before the `fly deploy -a thesis-dev` check, so the deploy verifies -integration rather than being the code's first execution. - -Cases that must be covered: - -- **The property the last attempt missed: a budget of $B at $R/h lasts ≈ B/R hours.** Assert the elapsed - billable duration, not just that a release eventually happened. This is what catches the un-rolled-back - acquire-time debit — a $10 budget at $6.99/h dying at 25.8 minutes instead of ~1.4 hours passes every - release-happened assertion while being off by 3×. -- A lease **with no runner token survives past `HEARTBEAT_STALE_SECONDS`**; one with a token is still reaped - (change 0). Without this, every budget test silently measures a ten-minute reap instead of the cap. -- Money-path writes are **idempotent**: replaying a tick that already committed does not double-charge. -- The **rolling window cap** rejects an N+1th lease whose grant would exceed the period ceiling, even when each - individual budget is affordable. -- A lease created with `volume_id` mounts it, and **releasing the lease does not delete the volume**. -- Extension raises the cap, is clamped by wallet and rolling cap, and **exhaustion proceeds normally when no - extension arrives**. -- A tick whose delta would exceed `hard_cap_cents` **releases the lease**; one that fits does not. -- `spent_cents` tracks cumulative charge across several ticks rather than freezing after the first. -- A lease created **without** `budget_cents` behaves exactly as before (dashboard and `compute:up` compatibility). -- A budget smaller than one hour's rate is rejected at creation with a `402` carrying - `affordable_budget_cents`. -- BYOK leases ignore `budget_cents` and are never debited. -- A budget larger than the wallet is clamped, and the response reports the effective cap rather than the asked-for one. -- The plan TTL still fires independently of any budget. - -**OpenScience** follows its house pattern — stub `globalThis.fetch`, exercise the real tool. Cover: refusing to -launch without a verdict, surfacing `402` without retrying, surfacing `429`, and release. - -Every new assertion must be shown failing against the specific mutation it guards before being committed. On the -preceding `science_fetch` branch, seven assertion defects were found and **all seven were in plan-authored test -code**; the ones that held up were proven against a _deletion_, not merely an inversion. - -## Migration - -`compute_grants.hard_cap_cents` already exists — no schema change for change 1. Change 2 adds no column either; -`budget_cents` only sizes the grant at creation. Existing in-flight grants keep whatever cap they were created -with, and change 1 begins enforcing it from the next tick, which is the safe direction. - -Per change: - -- **Change 0** is behavioural only, and its direction is _fewer_ terminations. Leases currently being reaped at - ten minutes will start surviving — which is the intent, but it means live GPU boxes that used to die on their - own now run until a real bound fires. Ship it with change 1 close behind, or ship it while the only bounds are - plan TTL and wallet, and accept that a forgotten box costs up to the TTL. -- **Change 3** needs plan config for the window and ceiling, and a query over recent grants. If grants are not - already indexed by `(user_id, created_at)`, that index is the migration. -- **Change 4** adds a nullable `volume_id` to the lease row. Volume lifecycle is already modelled by - `compute_volume_repo`; releasing a lease must **not** cascade a delete. -- **Change 5** adds no column — it edits `hard_cap_cents` in place. - -**One migration hazard, from finding 2.** `budget_cents` already exists on the agent-spawn path defaulting to -`500`, where it is display-only. Change 1 makes caps real, so every already-shipped spawn silently acquires a -hard $5 kill. Either raise that default deliberately or exempt spawn-path grants until their budgets are chosen -with enforcement in mind. **This is not a no-op, and the previous draft called it one.** - -## Corrections to earlier analysis - -Recorded because both errors reached a draft of this document. - -- ~~**`atlas compute:up` exists.**~~ **This correction was itself wrong — see the banner at the top.** It exists - in the Atlas _repo_ but not in the published `@synsci/atlas@0.13.2`, which is what the `^0.13.2` pin - resolves to. Source and npm disagree at an identical version number. The original finding — that the prompt - points at a command the installed CLI does not have — was right, and the retraction below is the error. - It is at `cli/src/atlas-runtime/commands.mjs:922` as a `LOCAL` command (aliases - `compute:launch`, `compute:lease`), alongside `compute:list` and `compute:ssh` — **in the repo.** An earlier - draft claimed it existed in no version, which was wrong about the source and right about the artifact. - ~~**Consequence: the system prompt at `session/prompt.ts:1554-1562` is not broken** and needs no fix. It was - previously an acceptance criterion; it is removed.~~ **That consequence did not follow.** The published - package has no `compute:` command, so the prompt did point at something the installed CLI cannot run. - **Resolved 2026-07-31** on `feat/compute-guardrails`: the guidance is deleted from `session/prompt.ts` and - from the `research` agent prompt, and replaced by runtime detection via a `compute_status` tool. -- **`compute:up` already takes `max_price` and `dry_run`** — again, **in the repo only**. A per-hour price - ceiling and a no-spend preview exist in the source parameter set but ship to nobody until the CLI is - published. Adding `budget_cents` there too would let CLI users have the same cap, once any of it ships. - -**The lesson worth carrying:** **four** separate conclusions in this investigation came from reading source and -were wrong about the deployed reality — the CLI's contents, whether the prompt was broken, whether managed -compute was reachable at all, and then (2026-07-31) the parked banner's own claim that reselling was off, which -was read from a default and contradicted by production. Verify against the running system before designing -against it. - -The fourth is the sharpest, because it was made _by the correction to the third_. A document written to warn -about this exact failure repeated it one section later. - -## Out of scope - -- **Roadmap 61** (per-job secrets, never into logs) — belongs to the _local_ runner `compute/jobs.ts`, which has - no billing involvement. Logs there go straight to a file descriptor unredacted while every job inherits the - user's Modal, RunPod, Lambda, Vast, W&B and HuggingFace keys. Real problem, separate workstream. -- **Roadmap 56** (checkpointing) — the follow-on that makes budget exhaustion survivable. -- **Roadmap 4** (real BYOK provider API clients) — five separable vendor integrations. -- **Roadmap 52** (`bun:sqlite` for the local runner's state). -- A per-lease `expires_at`, client-side deadline timers, and any client-side price table. (Extension requests - are now **in** scope — change 5 — but remain outside the enforcement path.) -- **Publishing the Atlas CLI.** `compute:*` has sat unpublished in `main` since `205bbc0` with no version bump; - `npm` still serves the pre-removal `0.13.2`. Worth its own release, and worth adding `budget_cents` to - `compute:up` when it happens — but a release, not this design. -- **Fixing the CLI's usability gaps.** Reviewed 2026-07-31: it prints the one-time SSH private key and never - saves it (so the `ssh_command` it prints cannot work), has no file-transfer command, no exec, and no compute - tests. Its resolver is genuinely good; everything around it is unfinished. Separate workstream. -- **The `compute:up` SKU race.** Verified by running the CLI from source against production: `compute:up` - fetches `/compute/options`, picks, then calls `/compute/estimate` — and Vast's SKUs are ephemeral marketplace - offer IDs that churn in between. Since Vast supplies 204 of the 292 live options it is almost always the - cheapest pick, so **the default path and `--gpu h100` both fail** with a raw `HTTP 400: Unknown SKU`, while - `--provider lambda` and `--provider runpod` (stable instance types) succeed. There is no retry. - - Two fixes, either sufficient: re-resolve once on a 400 and pick the next-best offer, or make selection - server-side so fetch-and-lease is atomic inside Atlas. The second also removes the duplicate resolver - described under "OpenScience — one tool". - -- **Vast and Prime Intellect SSH key leaks** — one public key per lease left in the operator account forever. - Lambda's delete-on-release pattern is the fix. - -## Acceptance criteria - -0. **A lease with no runner token is not reaped for heartbeat staleness**, and one with a token still is. Until - this holds, no other criterion can be observed — the box dies at ten minutes regardless. -1. **A budget of $B at rate $R/h lasts ≈ B/R hours**, asserted on elapsed billable duration. This is the - headline property and the one the previous attempt's tests and criteria both omitted. -2. The money path is idempotent: a replayed tick does not double-charge. -3. A cumulative rolling cap bounds spend across sequential leases, not merely within one. -4. `volume_id` attaches a persistent volume that **survives lease release**. -5. Extension raises the cap when affordable, is refused with a structured `402` when not, and never fires - automatically. -6. The billing tick re-debits the grant, so `spent_cents` tracks cumulative spend instead of freezing at hour - one. -7. A tick whose delta would exceed `hard_cap_cents` releases the lease via the existing release path. -8. `POST /api/compute/leases` accepts optional `budget_cents` and sizes the grant to it. -9. Omitting `budget_cents` preserves today's behaviour exactly — the dashboard and `compute:up` keep working. -10. A budget that cannot fund the first hour is rejected with `402` carrying `affordable_budget_cents`. -11. A budget exceeding the wallet is clamped to the effective balance, and the response reports the effective cap. -12. BYOK leases ignore `budget_cents` and are never debited. -13. The 24-hour plan TTL still fires independently. -14. `pytest` passes with no network access; changes 0 and 1 are each a separate commit with their own tests. -15. The OpenScience tool refuses to launch without an Atlas verdict, surfaces `402` and `429` without retrying, - and holds no pricing or approval logic. - -## Prototype - -`backend/cli/src/compute/PROTOTYPE-guardrail-model.ts` and `PROTOTYPE-guardrail-repl.ts` (openscience, -`e12e486`), runnable via `bun run prototype:guardrail`. It was built duration-first, so its `decide()` reasons -about hours rather than a budget — the _gate-mode_ finding (first-hour versus total) is what carried over and -motivated this design. Each unverified Atlas behaviour is a toggle, so verifying the real backend means flipping -switches rather than rewriting the model. diff --git a/docs/specs/compute-management-design.md b/docs/specs/compute-management-design.md deleted file mode 100644 index b6bb35a5..00000000 --- a/docs/specs/compute-management-design.md +++ /dev/null @@ -1,435 +0,0 @@ -# Compute management — design - -Status: **Part 1 shipped. Part 2 ready to plan.** -Date: 2026-07-31 · supersedes [`compute-mode-detection-design.md`](./compute-mode-detection-design.md) -(shipped) and [`compute-guardrails-design.md`](./compute-guardrails-design.md) (revived). -Roadmap: **5**, **51/2**, **55**, **103**; unblocks **56**. - -One document for how OpenScience and Atlas decide **who pays for GPU work, whether it is possible at all, and -what bounds the bill**. It replaces two documents that disagreed with each other and with production. - -| Part | What | Status | -| ----- | ------------------------------------------------------------------------ | ---------------------------------------------------- | -| **1** | Resolve `byok \| managed \| none` at runtime and tell the agent honestly | **Shipped** on `feat/compute-guardrails` | -| **2** | Bound what managed compute can spend, and make it usable at all | **To build.** Change 0 is a live-defect prerequisite | - ---- - -## Production reality, verified 2026-07-31 - -Both predecessor documents were written from source and drew **four** wrong conclusions about the deployed -system. Everything in this section was checked against running services, and claims elsewhere are labelled. - -**Managed compute is ON.** `GET /api/compute/options` against `thesis-synsc`: - -``` -resell_enabled: true cli_effective_balance_cents: 19883 -lambda managed operator:true 24 options -runpod managed operator:true 37 options -vast managed operator:true 204 options -prime_intellect managed operator:true 27 options -hyperbolic / coreweave / together / fluidstack / paperspace / nebius - unavailable operator:false 0 options (ScaffoldProvider, not wired) -``` - -292 launchable options. `POST /api/compute/estimate` for `lambda/cpu_4x_general` returns -`funding: "managed"`, 20¢/hr, `sufficient: true`, `runway_hours: 994.1`. **The spend path is live today.** - -`COMPUTE_RESELL_ENABLED` still _defaults_ to `false` (`backend/app/config.py:383`) — which is what the earlier -analysis read, and why it concluded the opposite. Production sets it. - -**The Atlas CLI runs from source with no build step**, authenticated, with all five `compute:*` commands -present — but `@synsci/atlas@0.13.2` on npm has 155 command specs and **zero** `compute:`. `3e1d1ca` removed -them, `0.13.1` and `0.13.2` shipped without them, `205bbc0` re-added them, and no version bump followed. -Source and artifact disagree at an identical version. - ---- - -# Part 1 — Mode detection (shipped) - -## What it fixed - -`computeBillingMode()` returned `config.billing?.compute ?? "byok"` and never inspected the environment. A user -with zero GPU credentials resolved to `byok` — claiming BYOK with nothing to BYOK with — and "no compute is -available" had no representation at all. Meanwhile the prompt told the agent to run `atlas compute:up` (absent -from the published CLI) and to check `atlas doctor` for compute availability (it reports no compute field). - -## The three states - -```ts -export type ComputeSource = "byok" | "managed" | "none" -``` - -| Credentialed provider? | Managed available? | Resolved | Agent is told | -| ---------------------- | ------------------ | --------- | ---------------------------------------------------------- | -| yes | — | `byok` | use the connected providers via the cloud-compute skills | -| no | yes | `managed` | use managed compute, billed to Credits | -| no | no | `none` | no compute available — connect a key in Settings ▸ Compute | - -BYOK wins when a credential is present: it is free to the user, works today, and needs nothing from Atlas. -That is also why a BYOK user never pays for the availability network call. - -### A credential is the whole test - -An earlier draft required a credential **and** a matching skill, reasoning that a provider with no skill gave -the agent nothing to act on. **Overruled during implementation:** a capable agent drives a documented cloud API -from a bare key, so the conjunction only produced a false `none` for users holding a perfectly workable key. - -| Provider | Env (any group satisfies; all vars within a group required) | Catalogued skills | -| --------------- | ----------------------------------------------------------- | ----------------------------------------------------------------- | -| Modal | `MODAL_TOKEN_ID` **+** `MODAL_TOKEN_SECRET` | `modal-serverless-gpu`, `modal-ml-training`, `modal-research-gpu` | -| Lambda | `LAMBDA_API_KEY` \| `LAMBDA_LABS_API_KEY` | `lambda-labs-gpu-cloud` | -| TensorPool | `TENSORPOOL_KEY` \| `TENSORPOOL_API_KEY` | `tensorpool-gpu-cloud` | -| Prime Intellect | `PRIME_API_KEY` \| `PRIME_INTELLECT_API_KEY` | `prime-intellect-lab` | -| RunPod | `RUNPOD_API_KEY` | none | -| Vast | `VAST_API_KEY` | none | - -Modal is the only credential pair; a half-pasted token counts for nothing. Skill names are SKILL.md frontmatter -`name` values — **not** directory names and not category-prefixed. The predecessor document's table -(`cloud-compute/modal`, `cloud-compute/lambda-labs`) matched nothing; hardcoding it would have filtered nothing. - -Dropping the skill requirement also removed a failure mode: skills reach a shipped binary from the server -catalog, so under the old rule a catalog that renamed `lambda-labs-gpu-cloud` would have silently marked Lambda -unusable for a user whose key was fine. - -## `billing.compute` is an override, never the source of truth - -- unset / `null` → detection -- `"byok"` → BYOK if a credential exists, else `none` -- `"managed"` → managed if available, else `none` - -**An override may narrow to `none`; it may never manufacture a capability.** Forcing `managed` while holding a -Lambda key still yields `none` when managed is unavailable — it does not silently fall back to BYOK. - -The setting is nullable end-to-end (config schema, settings PUT, and an "Auto" card in the UI) because without -that a user who clicked BYOK with no keys was trapped in `none` with no way back. - -## Two seams, one resolver, resolved per request - -Credentials reach `process.env` from three places: the user's shell, the Credentials panel -(`applyCredentialEnv`, `src/index.ts:102`), and the Compute panel (`applyComputeEnv`, `:106`). Both injections -are wrapped in `.catch(() => {})`. - -**Resolution therefore happens on demand and never at startup**, at two points that already run per request: - -- `SkillTool.init` — `registry.ts` calls `t.init({ agent })` inside `tools()`, so the catalog is rebuilt every - turn and a mid-session credential appears on the next turn with no cache to invalidate. -- the `compute_status` tool, whenever the agent asks. - -This makes the ordering constraint unbreakable by construction rather than by careful boot sequencing — and it -proved itself in the wild: the developer's Modal credential lives in none of the three expected places, arriving -instead via dashboard sync → `~/.config/openscience/synced-env.json` → `preload-env` replay. Startup detection -would have reported `none`. - -Availability is one authenticated `GET /api/compute/options` with a **3s timeout** and a **5s TTL** cache. -A failed, unauthenticated or timed-out call resolves to **unavailable** — failing toward `managed` would -reproduce the original bug of promising an unconfirmed capability. The cache holds the availability verdict -only; credentials are never cached. - -## The catalog filter - -`ComputeMode.offered()` returns the skills the agent may see, and is non-empty **only in `byok`**. It needs no -network in any state: with an override of `"managed"` the answer is empty regardless, and with no credential it -is empty regardless, so the availability probe is never required to decide it. - -Only the six mapped providers' skills are filtered. `fireworks-ai-inference`, `together-ai-inference`, -`tinker-fine-tuning`, `tinker-training-cost` and `skypilot-multi-cloud-orchestration` share the `cloud-compute` -category but are inference APIs and orchestrators keyed by their own credentials — never hidden. RSI-learned -skills are never hidden. - -**This is a listing filter, not a gate.** A hidden skill remains loadable by exact name and the agent still has -`bash`. `none` is guidance, not enforcement; gating the load path was considered and deliberately declined. - -## `compute_status` - -No parameters. Returns `{mode, providers, managed_available, balance_usd?}` plus mode-specific guidance, and -resolves on every call. - -The mode changes mid-session — a key connected at turn 3 makes a reminder injected then false by turn 12 — so -nothing is injected per turn. The tool **description** carries the constraint ("check before running GPU work"), -which costs nothing because tool definitions are in every request regardless; the **result** carries the -specifics. `session/prompt.ts` keeps only a stateless one-line pointer for `COMPUTE_AGENTS`. - -`balance_usd` comes from `cli_effective_balance_cents` in the same availability response — no second call. - -## Verified live, not just tested - -| Case | Result | -| --------------------------------------- | ----------------------------------------------------------------------------------- | -| Real machine (Modal via dashboard sync) | `Compute: byok` | -| `billing.compute=managed` forced | `Compute: managed` | -| `billing.compute=byok`, no credentials | `Compute: none`, probe skipped | -| Catalog in `byok` | Modal's 3 shown; `lambda-labs-gpu-cloud`, `tensorpool-gpu-cloud` hidden | -| Catalog in `none` | all 5 provider skills hidden; non-provider and learned skills intact | -| Settings API | `GET` returns `null` (not coerced); `PUT "byok"` persists; `PUT null` restores Auto | - -Suite: 1488 pass / 1 skip / 1 pre-existing unrelated failure. - ---- - -# Part 2 — Guardrails and usability (to build) - -## The gap Part 1 exposed - -`compute_status` now tells an agent in `managed` mode to _"run GPU work through managed compute, billed to -Credits"_ — and **no mechanism exists**. There is no `compute_submit`, and `compute:up` is unpublished. Since -managed is live, every keyless user lands there. - -Simultaneously the spend path has no ceiling: `POST /api/compute/leases` checks only that the wallet can fund -**one hour**, so a 4-hour job on a 1-hour balance is approved and strands mid-run. - -## Trust boundary - -**The agent proposes; the server decides.** OpenScience is open source and the agent has `bash`, so any -client-side check is one a fork can delete. Only a decision made over HTTP, behind auth, in a process the agent -does not run, is one it cannot influence. - -Corollary: **OpenScience holds no pricing, balance, or approval logic.** It relays a proposal and obeys a -verdict. - -## Why budget, not duration - -The agent proposes _"this is worth up to $30"_, not _"this needs 4 hours"_. An LLM can judge the first and -cannot predict the second — and a wrong duration estimate is what forced earlier drafts to grow extension -paths, warning windows and re-estimation loops. - -Consequences, all simplifying: exhaustion is a fact the server observes, so no completion signal is needed; it -is arithmetic, so no extension is needed for correctness; and it is server-side, so no agent liveness is needed. - -## The changes - -### Change 0 — stop the reaper killing user leases _(prerequisite, live defect)_ - -`lease_reaper.sweep_once` applies heartbeat staleness to every unfinished lease: - -- `compute_repo.list_unfinished_leases` is explicitly category-agnostic — - `WHERE status NOT IN ('released','failed')`. -- Branch 3 falls back to `_lease_started` (`started_at or created_at`) when there is no telemetry and reaps past - `HEARTBEAT_STALE_SECONDS` = 600. -- `create_lease` mints **no runner token**, and `POST /api/agent/runner/telemetry` requires one. - -**A user lease cannot prove liveness and is destroyed ~10 minutes after creation**, with provisioning eating -several of those minutes. A $30 budget lease dies having spent about $1.17. No budget can bind and -`atlas compute:up` cannot run a research task until this is fixed. - -Scope the check to leases that have a runner token. User leases stay bounded by plan TTL, wallet exhaustion, -explicit release, and (after change 1) the budget cap. The provider-terminal and provisioning-timeout branches -continue to apply to everything. - -Ships first, alone, with its own test. - -### Change 1 — make `hard_cap_cents` a real running cap - -`compute_grants.hard_cap_cents` reads like a running ceiling but is not one: `acquire_lease` debits it once for -one hour, and the billing tick then charges without calling `debit_grant` again, so `spent_cents` freezes. - -**The billing tick re-debits the grant by the same delta it charges**, releasing the lease when the debit would -exceed the cap — reusing the path that already fires on wallet exhaustion. - -This is **not** a double charge. The wallet is money; the grant is an authorisation envelope drawn against it. -An implementer who "de-duplicates" these has removed the cap. - -_Known trap (finding 3):_ the acquire-time debit is never rolled back, so a naive re-debit double-counts hour -one — a $10 budget at $6.99/h would die at 25.8 minutes instead of ~1.4 hours. - -### Change 2 — accept a budget on lease creation - -``` -POST /api/compute/leases -{ provider, sku, region?, node_id?, budget_cents?, volume_id? } -``` - -`budget_cents` is optional; absent preserves today's behaviour exactly (the dashboard and `compute:up` both call -this endpoint without it). Rejection reuses the structured 402, extended with `affordable_budget_cents`. - -**A budget larger than the wallet is clamped, not rejected** — the wallet is always the outer bound. The -response reports the **effective** cap so the agent can tell the user what was actually authorised. - -**Managed only.** BYOK runs on the user's own account, which Atlas neither meters nor bills. - -### Change 3 — bound cumulative spend - -The cap is per-grant and a grant is per-lease, so release-and-reacquire is unbounded; the concurrency cap of 2 -bounds concurrent boxes, not total cost. A $30 budget honoured twenty times is $600. - -Add a **rolling window cap** at lease creation, with window and ceiling as plan config alongside -`gpu_sandbox_max_ttl_hours`, and a distinct `error` code so clients can distinguish "this box is too expensive" -from "you have spent enough today". - -Design it in now — retrofitting changes the meaning of a number users already trust. - -### Change 4 — attach a persistent volume - -Atlas already has `POST /api/compute/volumes` (10 GB–10 TB), `list_volumes`, `delete_volume`. **Leases do not -use it**, and the RunPod provider passes `volumeInGb: 20`, a pod-scoped volume destroyed with the pod. - -Add `volume_id?` to the lease request and pass it to RunPod as `networkVolumeId`, mounted at `/workspace`. -Releasing a lease must **not** cascade a volume delete. - -Budget exhaustion then costs the compute, not the work. A network volume is cents per GB-month against dollars -per GPU-hour, so preserving the work costs approximately nothing. - -**Honest limit:** a volume preserves _files_, not _process state_. A run killed mid-epoch still dies unless it -checkpointed to `/workspace`. The volume is the substrate roadmap **56** needs, not a substitute for it. - -### Change 5 — extend a live budget - -``` -POST /api/compute/leases/{lease_id}/budget { additional_cents } -→ { hard_cap_cents, spent_cents, effective_cap_cents } -``` - -Raises the cap on the existing grant, clamped by wallet and rolling cap. No new state — it edits a number change -1 already reads every tick. - -Three constraints keep it from reopening the door earlier drafts closed: - -- **Pull, never push.** Atlas never auto-extends. A budget that quietly refills is not a budget. -- **Not part of enforcement.** If no extension arrives, exhaustion proceeds unchanged, so a dead agent costs - nothing. -- **No warning event required.** An 80% notification is worth adding for humans, but it is advice; the cap must - never depend on anyone reading it. - -### Why RunPod is the managed default - -Every provider generates a fresh Ed25519 keypair per lease and only ever shows the provider the public half, so -the returned key opens exactly one box everywhere. They differ in what they leave behind: - -| Provider | Key attachment | Account artifact | Cleaned up | -| --------------- | ---------------------------------------------- | ---------------- | ---------------------------------- | -| **RunPod** | `PUBLIC_KEY` env var consumed on boot | **none** | nothing to clean | -| Lambda | account key registry | yes | yes — on release and failed launch | -| Vast | account `/ssh/` **and** `/instances/{id}/ssh/` | yes | **no** | -| Prime Intellect | `POST /ssh_keys/` → `sshKeyId` | yes | **no** | - -RunPod is the only one with no account-level trace, which is the right property when Atlas owns the box -lifecycle. Its creation body also takes an arbitrary `env` dict — the same lever the Modal spawn path uses to -pass a runner token and callback URL — so anything Atlas later wants running on boot needs no SSH bootstrap. - -Vast is cheapest and deepest (204 of 292 options) but is interruptible spot, leaks a key per lease, and its SKUs -are ephemeral offer IDs (see Known defects). - -### OpenScience — one thin tool - -1. `compute_status` (shipped) to learn the mode. -2. `GET /api/compute/options` → the agent picks a SKU. **Selection is the agent's job**, so no client-side - resolver and no wrapper around `compute:up`. -3. Submit `{provider, sku, budget_cents, volume_id?}`. **Refuse to launch without a verdict from Atlas.** -4. On `402`, surface `affordable_budget_cents` and stop. **Never auto-retry at a smaller budget** — a truncated - training run is not a cheaper result, it is a discarded one. -5. On `429` (concurrency cap), surface rather than retry. -6. Release on request. **No client-side deadline timer.** - -## The bounds that remain - -| Bound | Owner | Fires when | Status | -| ------------------- | -------------- | ---------------------------------------- | ----------------------- | -| `hard_cap_cents` | billing tick | approved money is spent | made functional (ch. 1) | -| Rolling window cap | lease creation | cumulative spend hits the ceiling | new (ch. 3) | -| Wallet exhaustion | billing tick | money actually runs out | already works | -| Plan TTL (24h) | billing sweep | anything has run absurdly long | already works | -| Heartbeat staleness | lease reaper | an _agent-spawned_ lease stops reporting | narrowed (ch. 0) | - -All server-side; none client-influenceable. Change 0 _removes_ a bound from user leases, which is safe precisely -because the other four apply and necessary because it is one those leases cannot satisfy. - -Billing ticks every 60s, so a budget can overrun by up to a minute of rate (~$0.12 on an H100). Approved budgets -are ceilings-plus-a-minute and must never be described as exact. - ---- - -## What we deliberately did not build - -Design exploration reached a persistent runner daemon on the box, a command queue, a result stream, scoped -storage credentials and a possible PTY relay — a remote-execution platform, designed before a single box had -been leased through the agent, to fix what was at that point a false string. - -The test that settled it: **a productivity feature is admissible only if its absence changes nothing about -enforcement.** Volumes pass (files move off the box before anything fails). Optional extension passes. -A command relay, per-lease `expires_at`, client-side timers and auto-extension all fail — each needs something -alive to act on a signal. - -Two things from that exploration survive and are folded in above: enforcement belongs in Atlas, and RunPod is -the right managed default. - ---- - -## Known defects, owned elsewhere - -- **The `compute:up` SKU race.** Verified by running the CLI from source against production: it fetches - options, picks, then estimates — and Vast's offer IDs churn in between. Since Vast supplies 204 of 292 - options it is nearly always cheapest, so **the default path and `--gpu h100` fail** with a raw - `HTTP 400: Unknown SKU`, while `--provider lambda` and `--provider runpod` succeed. No retry exists. Fix by - re-resolving once on a 400, or by moving selection server-side (which also makes fetch-and-lease atomic). -- **CLI usability.** It prints the one-time SSH private key and never saves it, so the `ssh_command` it prints - cannot work. No file transfer, no exec, no compute tests. The resolver is genuinely good; everything around - it is unfinished. -- **CLI unpublished** — a release, not code. -- **Vast and Prime Intellect leak** one public key per lease into the operator account, forever. Lambda's - delete-on-release is the fix. -- **Stale skill names in agent prompts.** `research.txt`'s appendix and `ml.txt` name ~35 skills by directory - rather than frontmatter `name` (`vllm` → `serving-llms-vllm`, `peft` → `peft-fine-tuning`, …), so the agent is - told it has skills it cannot load. `skills/scholar-evaluation/SKILL.md` has no `name:` at all. -- **`budget_cents` already exists** on the spawn path defaulting to `500`, display-only. Change 1 makes caps - real, silently giving every shipped spawn a hard $5 kill. **Not a no-op.** - -## Out of scope - -Roadmap **61** (per-job secrets), **56** (checkpointing — the follow-on change 4 enables), **4** (BYOK provider -API clients), **52** (`bun:sqlite`). Per-lease `expires_at`, client-side deadline timers, client-side price -tables, auto-extension. Any change to how `billing.llm` resolves. - -## Testing - -Atlas follows `backend/tests/test_compute_billing.py` — `_FakeProvider`, `aiosqlite` + `run_migrations`, plain -`pytest` before any deploy. OpenScience stubs `globalThis.fetch` and exercises the real tool; no mocks, no -network. - -Cases that must be covered: - -- **A budget of $B at $R/h lasts ≈ B/R hours**, asserted on elapsed billable duration. This is the headline - property and the one the previous attempt omitted from both its tests and its criteria — a $10 budget dying at - 25.8 minutes passes every "release happened" assertion while being off by 3×. -- A lease **without** a runner token survives past `HEARTBEAT_STALE_SECONDS`; one with a token is still reaped. - Without this, every budget test silently measures a 10-minute reap. -- Money-path writes are idempotent — a replayed tick does not double-charge. -- The rolling cap rejects an N+1th lease even when each individual budget is affordable. -- `volume_id` mounts, and **release does not delete the volume**. -- Extension raises the cap, is clamped, and **exhaustion proceeds normally when none arrives**. -- A tick exceeding `hard_cap_cents` releases; one that fits does not. `spent_cents` accumulates across ticks. -- No `budget_cents` → today's behaviour exactly. BYOK ignores it. Plan TTL still fires independently. - -Every new assertion must be shown failing against the specific mutation it guards — ideally the _deletion_ of -the logic, not its inversion. On this branch alone, five plan-authored test defects were caught in review, -including an assertion that compared whole tool outputs and so passed even with all guidance collapsed to one -string. - -## Acceptance criteria (Part 2) - -0. A lease with no runner token is not reaped for heartbeat staleness; one with a token still is. -1. A budget of $B at rate $R/h lasts ≈ B/R hours, asserted on elapsed billable duration. -2. The money path is idempotent under a replayed tick. -3. A rolling cap bounds spend across sequential leases. -4. `volume_id` attaches a volume that survives lease release. -5. Extension raises the cap when affordable, refuses with a structured 402 when not, never fires automatically. -6. The billing tick re-debits the grant; `spent_cents` accumulates. -7. A tick exceeding the cap releases via the existing path. -8. `POST /leases` accepts `budget_cents`; omitting it preserves today's behaviour exactly. -9. A budget exceeding the wallet is clamped, and the response reports the effective cap. -10. BYOK ignores `budget_cents`. Plan TTL fires independently. -11. The OpenScience tool refuses to launch without a verdict, surfaces 402/429 without retrying, and holds no - pricing or approval logic. -12. `pytest` and `bun test` pass with no network; changes 0 and 1 are each their own commit. - ---- - -## Verification discipline - -Four conclusions in this investigation came from reading source and were wrong about deployed reality: the -CLI's contents, whether the prompt was broken, whether managed compute was reachable, and then the parked -banner's own claim that reselling was off — **the fourth made by a correction to the third.** A document -written to warn about this failure repeated it one section later. - -Claims in this document are labelled: the production-reality section and Part 1's verification table were -checked against running services. The billing tick's debit behaviour, grant accounting, and the reaper's exact -reap path are **read from source only** — confirm against a test before relying on them. diff --git a/docs/specs/compute-mode-detection-design.md b/docs/specs/compute-mode-detection-design.md deleted file mode 100644 index ec9f1a38..00000000 --- a/docs/specs/compute-mode-detection-design.md +++ /dev/null @@ -1,391 +0,0 @@ -# Compute mode detection — design - -> **SUPERSEDED 2026-07-31 by [`compute-management-design.md`](./compute-management-design.md), which is the -> single current spec for compute. This document is kept as the historical record of the shipped work and is -> no longer maintained.** -> -> Two things below are **wrong** and were corrected during implementation — do not build from them: -> -> - **The "key AND skill" rule was overruled.** A credential alone makes a provider usable; a capable agent -> drives a documented cloud API from a bare key. The conjunction only produced a false `none`. -> - **The skill-name table matches nothing.** Real frontmatter names are `modal-serverless-gpu`, -> `modal-ml-training`, `modal-research-gpu`, `lambda-labs-gpu-cloud`, `tensorpool-gpu-cloud`, -> `prime-intellect-lab` — not `cloud-compute/modal` etc. -> -> Also stale: this document assumes managed compute is unavailable. **It is live in production.** - -Status: proposed, for review -Date: 2026-07-30 -Scope: `openscience` only. **No Atlas changes required.** -Roadmap: contributes to **5** (fix existing compute gaps); unblocks honest messaging for **55**/**103** - -## Summary - -OpenScience decides how GPU work gets paid for using a config value that never checks whether the user -actually has any provider keys. This replaces that with **runtime detection**: if provider credentials are -present, we're in BYOK mode; if not, managed; and if neither is usable, we say so instead of guessing. - -The change is small and self-contained. Its value is that it makes one currently-unrepresentable state — -_"there is no compute available"_ — expressible, which is the state we handle worst today. - -## Problem - -### 1. The mode is a config value that can contradict reality - -```ts -// src/session/billing-gate.ts:34 -export async function computeBillingMode(): Promise { - return (await Config.get()).billing?.compute ?? "byok" -} -``` - -It never inspects the environment. A brand-new user with zero provider keys resolves to `"byok"` — claiming -BYOK with nothing to BYOK with. - -Note the asymmetry with LLM billing, which already does this correctly. From the config schema -(`src/config/config.ts`): - -- `billing.llm` — _"Unset or null = **auto-detect** from the resolved credential."_ Backed by - `resolveCredentialSource(providerID, modelID)`, which inspects the actual credential and returns - `byok | managed | oauth-free`. -- `billing.compute` — _"**Unset = byok**."_ A static default. No detection function exists. - -**This design makes compute behave the way LLM already does.** - -### 2. The "no compute available" state cannot be expressed - -`BillingMode` is `"managed" | "byok"`. There is no third value, so the case _"the user has no keys **and** -managed compute isn't available"_ has nowhere to live. Today it silently resolves to one of the two working -modes and the agent is told to use a path that cannot work. - -### 3. The prompt is only injected when the mode is explicitly set — and is wrong when it is - -```ts -// src/session/prompt.ts:1553 -if (COMPUTE_AGENTS.has(input.agent.name) && (await Config.get()).billing?.compute) { -``` - -`COMPUTE_AGENTS` is `research`, `biology`, `physics`, `ml`. Two distinct problems: - -**When `billing.compute` is unset (the default), no guidance is injected at all.** The agent receives no -information about how compute is funded and picks an approach from the skill catalog with no idea whether the -user's keys exist. - -**When it is set to `managed`, the injected text is inaccurate:** - -> _"Run GPU/training work through the bundled `atlas compute` CLI (e.g. `atlas compute:up`), which bills -> Credits. Do not fall back to the user's own GPU providers unless `atlas doctor` reports managed compute -> unavailable."_ - -Three things in that sentence do not hold: - -- **`atlas compute:up` is not in the published CLI.** `@synsci/atlas@0.13.2` on npm contains no `compute:` - command. The Atlas repo's `cli/` does (`commands.mjs:922`), at the _same version number_ — so the source - and the published artifact disagree and the pin `^0.13.2` resolves to the one without it. -- **`atlas doctor` reports nothing about compute.** Verified against a live run: the keys are - `config_path`, `profile`, `base_url`, `auth`, `backend`, `package.skills`, `integrations`, `spool`, - `warnings`, `ok`. There is no compute field, so the stated condition is unobservable. -- **Managed compute is off by default server-side.** Atlas gates it on `COMPUTE_RESELL_ENABLED`, which - defaults to `false` (`backend/app/config.py:383`), plus a configured operator key. Without both, every - provider reports `funding: "unavailable"`. - -So an agent in managed mode runs an unknown command, cannot check the sanctioned availability signal, and is -pointed at _"the user's own GPU providers"_ as the remedy — which in managed mode is precisely the set of keys -that doesn't exist. Where keys _do_ exist, that fallback spends on the user's own uncapped provider account. - -## Design - -### Three states, detected at runtime - -```ts -export type ComputeSource = "byok" | "managed" | "none" -``` - -| Usable provider? | Managed available? | Resolved | Agent is told | -| ---------------- | ------------------ | --------- | ---------------------------------------------------------------- | -| yes | — | `byok` | use the user's connected providers via the cloud-compute skills | -| no | yes | `managed` | use managed compute, billed to the wallet | -| no | no | `none` | **no compute is available — connect a provider key in Settings** | - -BYOK wins when a usable provider is present. It is free to the user, it works today, and it needs nothing from -Atlas. "Usable" means a key **and** a skill — see below, because two providers have a key and no skill. - -### `billing.compute` becomes an override, not the source of truth - -Detection supplies the default; the existing setting still lets a user force a mode. This keeps the config -meaningful without letting it assert something false. - -- unset → use detection -- `"byok"` → force BYOK. If no _usable_ provider is present, resolve to `none` rather than pretending. -- `"managed"` → force managed. If managed is unavailable, resolve to `none`. - -An override may narrow the outcome to `none`; it may never manufacture a capability that isn't there. - -### What counts as a usable provider: a key **and** a skill - -A key alone is not enough. The agent runs GPU work by loading a provider's skill, so a provider with a -credential but no skill gives the agent nothing to act on. - -From `PROVIDER_ENV` (`src/server/routes/settings/compute.ts:170`) plus Modal's pair, cross-referenced against -the skill tree: - -| Provider | Env vars | Skill | Usable | -| --------------- | ------------------------------------------------------------- | -------------------------------------------------------- | ------ | -| Modal | `MODAL_TOKEN_ID` **and** `MODAL_TOKEN_SECRET` (both required) | `cloud-compute/modal`, `cloud-compute/modal-ml-training` | yes | -| Lambda | `LAMBDA_API_KEY` or `LAMBDA_LABS_API_KEY` | `cloud-compute/lambda-labs` | yes | -| TensorPool | `TENSORPOOL_KEY` or `TENSORPOOL_API_KEY` | `cloud-compute/tensorpool` | yes | -| Prime Intellect | `PRIME_API_KEY` or `PRIME_INTELLECT_API_KEY` | `ml-training/prime-intellect-lab` | yes | -| **RunPod** | `RUNPOD_API_KEY` | **none** | **no** | -| **Vast** | `VAST_API_KEY` | **none** | **no** | - -**Rule: a provider is BYOK-usable only when it has both.** So `byok` requires at least one provider with a key -_and_ a skill. A user whose only credential is RunPod resolves to `managed`/`none` with an honest message, -rather than to `byok` with an empty toolbox. - -Modal is the only credential pair — a half-pasted Modal token maps to nothing and must not count, mirroring -`compute.ts:185`. - -This rule surfaces roadmap item **5** rather than causing it: RunPod and Vast keys inject with no consumer -today, and `RUNPOD_API_KEY` is even named to the model in all six session prompts. Detection makes that gap -visible instead of silent. Either write those two skills or stop offering the providers — see open questions. - -### Filtering the skill catalog - -`SkillTool` (`src/tool/skill.ts:32`) is defined with an async init that builds its catalog and already filters -it — today by `PermissionNext.evaluate("skill", skill.name, agent.permission)`. Crucially, -`registry.ts:187` calls `await t.init({ agent })` inside `tools()`, so **that init runs per request**. - -That makes it the right seam, for two reasons: - -- **Freshness is free.** The catalog is rebuilt every turn, so a credential connected mid-session appears on - the next turn with no cache to invalidate. -- **Ordering is guaranteed by construction.** By the time a turn is served, every env injection at - `src/index.ts:102` and `:106` has long since run, so detection cannot observe a half-initialised environment. - -**Filter the catalog; do not auto-load the markdown.** Only the skills of usable providers are listed, so the -agent picks the right one because it is the only one offered. Auto-injecting a provider's markdown would fight -the existing mechanism — `tool/skill.ts` exists precisely so content is pulled on demand — and these files are -large enough that unprompted injection is expensive on turns that have nothing to do with compute. - -In `managed` and `none`, no BYOK provider skill is listed at all. - -### Ordering requirement — the main footgun - -Keys reach `process.env` from three places, all legitimate BYOK: - -1. The user's shell or `.env` -2. The **Credentials** settings panel — injected by `applyCredentialEnv()` at `src/index.ts:102` -3. The **Compute** settings panel — injected by `ComputeSettings.applyComputeEnv()` at `src/index.ts:106` - -**Detection must run after line 106.** Both injections are wrapped in `.catch(() => {})` and fail silently, so -detecting too early reports `none` for a user who has keys configured through the UI. - -The robust way to guarantee that is not to order boot steps carefully — it is to **resolve on demand and never -at startup**, at either of the two points that already run per request: `SkillTool`'s init when the catalog is -built, and `compute_status` when the agent calls it. By then every injection has run, so the constraint cannot -be violated and cannot silently regress if someone reorders `src/index.ts` later. - -**Resolution must be a single shared function** used by both call sites. Two independent implementations of -"which providers are usable" would drift, and the failure would be quiet: a catalog listing a provider the -status tool says is unavailable, or the reverse. - -### Determining whether managed is available - -One authenticated call to `GET /api/compute/options`, which already annotates each provider with `funding` -(`managed` when reselling is on and an operator key exists, else `unavailable`). If no provider reports -`managed`, managed is unavailable. - -Treat a failed, unauthenticated, or timed-out call as **unavailable** — failing toward `none` produces an honest -"connect a key" message, whereas failing toward `managed` reproduces today's bug of promising a capability we -haven't confirmed. - -**This is only reached when no usable provider is present**, so a BYOK user never pays the network call. - -**Caching:** a short in-process TTL (single-digit seconds) is fine to stop a chatty agent hammering the endpoint -within one turn, but it must not be a startup-time or process-lifetime cache. The whole reason this is a tool -rather than a prompt injection is that the answer changes mid-session — a long cache reintroduces exactly the -staleness the tool exists to avoid. Key detection itself reads `process.env` and needs no cache at all. - -### How the agent learns the mode: a tool, not a prompt injection - -**The agent pulls the mode from a tool. Nothing is injected per turn.** - -An earlier draft injected mode guidance into every turn for `COMPUTE_AGENTS`. That was wrong for a reason that -matters more than token cost: **the mode can change mid-session.** A user connects a Modal key in -Settings → Compute while a session is running, and a reminder injected at turn 3 is false by turn 12. A tool -returns the state at the moment it is asked. - -It also composes with action. The agent needs the mode only because it is about to run GPU work, so a call at -that moment can return the mode _and_ the specifics worth having — which providers are configured, whether -managed is available, and the balance and rates when it is. Injected prose is information divorced from the -decision, and it does not scale: adding rates or balance to an every-turn injection is expensive, while adding -them to a tool result is free. - -This also matches the pattern this codebase already treats as correct. `tool/science.ts` exposes three tools -over 42 connectors rather than 42 tool definitions — capability discovered on demand, tool count flat. - -#### `compute_status` - -No parameters. Returns the resolved mode plus what the agent needs to act on it: - -```ts -{ - mode: "byok" | "managed" | "none", - providers: string[], // configured BYOK providers, e.g. ["modal", "runpod"] - managed_available: boolean, - guidance: string, // the mode-specific rule, see below - balance_usd?: number // managed only -} -``` - -`guidance` carries the behavioural rule, delivered at the point of relevance: - -| mode | guidance | -| --------- | -------------------------------------------------------------------------------------------------------------------------------------------------- | -| `byok` | Run GPU work on the user's connected providers via the cloud-compute skills. Do not launch managed leases. | -| `managed` | Run GPU work through managed compute, billed to Credits. Do not use the user's own provider keys — they are not funded here. | -| `none` | No compute is available. Do not attempt GPU work. Tell the user to connect a provider key in Settings → Compute, or to top up for managed compute. | - -#### The tool description is the prompt - -Constraints have to reach the agent _before_ it starts down a path, which is the one thing an injection did -well. The tool **description** does that job at no extra per-turn cost, because tool definitions are in every -request regardless: _"Check how GPU compute is funded before running any GPU work. Returns byok, managed, or -none, with the providers available and the rule that applies."_ - -So the description constrains and the result informs. Nothing needs injecting. - -#### Prompt changes - -Delete the `atlas compute:up` instruction and the `atlas doctor` condition from `prompt.ts:1554-1562`. Both are -false today, independent of this design, and mode resolution now answers the question the `doctor` check was -reaching for. - -Whether to keep a minimal pointer in the prompt is left open — see the open questions. The default position is -no injection at all. - -#### Relationship to roadmap 51/2 - -This is the read half of the agent-facing compute tool. `compute_status` now; a `compute_submit` alongside it -when managed compute is actually switched on and the budget cap from -`docs/specs/compute-guardrails-design.md` is sound. Same seam, so the later work adds a tool rather than -reshaping this one. - -## Testing - -House pattern: no mocks, exercise the real resolver, no network in tests. - -- Each provider **that has a skill**, in isolation, resolves to `byok`. -- **A RunPod-only or Vast-only environment does NOT resolve to `byok`** — key without skill is not usable. -- **Modal with only `MODAL_TOKEN_ID` and no other provider** does not resolve to `byok` — it falls through to - the managed/none branch exactly as if no key were set. -- No keys plus managed available → `managed`. -- No keys plus managed unavailable → `none`. -- No keys plus a failed availability call → `none`. -- Override `"byok"` with no keys → `none`. -- Override `"managed"` with managed unavailable → `none`. -- Keys present → the availability call is **not** made. -- Detection reflects a key injected by `applyComputeEnv()` after startup (the ordering guarantee). -- **A key connected mid-session changes the answer on the next call** — the staleness property that motivated a - tool over an injection. Resolve, inject a key, resolve again, assert the mode changed. - -For the tool, following the house pattern of stubbing `globalThis.fetch` and exercising the real tool: - -- `compute_status` returns each of the three modes with matching `guidance` text. -- `byok` lists the usable providers in `providers`. -- `managed` includes `balance_usd`; `byok` and `none` do not. -- No prompt in `session/prompt/*.txt` or `prompt.ts` references `compute:up` or `atlas doctor` for compute - availability. - -For the catalog filtering, exercising the real `SkillTool.init`: - -- With only a Modal credential, the catalog lists the Modal skills and **not** `lambda-labs`, `tensorpool`, or - `prime-intellect-lab`. -- With a RunPod-only credential, **no** provider skill is listed. -- In `managed` and in `none`, no BYOK provider skill is listed. -- A credential added between two `init()` calls changes the catalog on the second — the per-turn freshness - property, and the reason this lives in init rather than at startup. -- Non-compute skills are unaffected by mode in every case. -- `SkillTool.init` and `compute_status` never disagree about which providers are usable (they call the same - resolver). - -Every new assertion must be demonstrated failing against the specific mutation it guards — ideally the -_deletion_ of the logic, not merely its inversion. On the preceding `science_fetch` branch seven assertion -defects were found and all seven were in plan-authored test code; the ones that held up were the ones proven -against deletion. - -## Out of scope - -- **The managed-compute budget cap** (roadmap 55/103). Parked in - `docs/specs/compute-guardrails-design.md`, which an Opus review found unsound; it also cannot be validated - until managed compute is actually switched on somewhere. -- **Turning managed compute on** — an Atlas deployment decision (`COMPUTE_RESELL_ENABLED` plus operator keys). -- **Publishing `compute:up`**, and the source/npm version divergence at `0.13.2`. Worth an independent fix: - a consumer pinning `^0.13.2` cannot tell which artifact they'll get. -- **Roadmap 61** (per-job secrets, never into logs) — the local runner `compute/jobs.ts`, unrelated to billing. -- Any change to how `billing.llm` resolves. - -## Acceptance criteria - -1. A resolver returns `byok | managed | none` from the runtime environment, not from a static default, and is - the single shared implementation used by both `SkillTool.init` and `compute_status`. -2. A provider counts toward `byok` only with both a key and a skill; a half-configured Modal credential does - not count, and a RunPod-only or Vast-only environment does not resolve to `byok`. -3. The skill catalog lists provider skills only for usable providers, and lists none in `managed` or `none`. - Non-compute skills are unaffected. -4. Provider markdown is never auto-injected — the agent still loads it through the `skill` tool. -5. With no keys and managed unavailable — including when the availability check fails — the result is `none`. -6. `billing.compute` can narrow the result to `none` but can never assert an unavailable capability. -7. A key injected by either settings panel at boot is detected (resolution happens after `src/index.ts:106`). -8. The availability call is skipped entirely when a usable provider is present. -9. A `compute_status` tool returns the mode, the usable providers, and mode-specific `guidance`, resolving - on each call rather than from a value cached at startup. -10. A credential connected mid-session is reflected on the next `compute_status` call without a restart. -11. Nothing is injected into the prompt per turn for compute mode; the tool's description carries the - "check before running GPU work" instruction. -12. No prompt references `atlas compute:up` or `atlas doctor` for compute availability. -13. In `none`, the tool's `guidance` tells the agent not to attempt GPU work and how the user can enable it. -14. `bun test` passes with no network access. - -## Open questions for review - -1. **Should a BYOK user with keys for one provider but asking about another get `byok` or a - partial answer?** This design says `byok` if any provider is configured, and leaves provider choice to the - agent and the skills. A per-provider resolution would be more precise and more complex. -2. **What do we do about RunPod and Vast?** Both accept a key in Settings, inject env vars, and have no skill - for the agent to load — so under this design they never make a user BYOK-usable. Three options: write the - two skills, remove the providers from the Compute panel, or keep them and show "key stored — skill coming". - Doing nothing means a user can connect RunPod, see it accepted, and still be told no compute is available. - This is roadmap item **5**; it is listed here because this design is what makes it user-visible. -3. **Should `none` be a hard block or a warning?** The tool's `guidance` tells the agent not to attempt GPU - work, but nothing enforces it — it still has `bash` and the cloud-compute skills. Enforcement would mean - gating those skills, which is a larger change. Worth deciding explicitly rather than by omission. -4. **Should the prompt keep a one-line pointer to the tool?** The default position here is no injection at all, - on the grounds that the tool description already carries the instruction. The risk is an agent that never - calls the tool and reaches for `bash` directly. A single line — _"call `compute_status` before GPU work"_ — - would cost a handful of tokens per turn and close that gap. This is the one place where the tool-versus-prompt - trade-off is genuinely unresolved. -5. **Does the `billing.compute` description need updating** in the config schema? It currently says - _"Unset = byok"_, which this change makes false. -6. **How long may `compute_status` block?** In `none`/`managed` it makes one authenticated call to - `/api/compute/options`. A slow or hanging Atlas would stall the agent mid-turn, so it needs a short timeout - with `none` as the timeout result — but "short" should be a stated number, not left to the implementer. - -## Appendix: what was verified, and how - -Every claim above was checked against code or a live run rather than inferred. Recorded because three earlier -conclusions in this investigation were wrong, and the corrections are the reason this design exists. - -| Claim | Verified by | -| ------------------------------------------------------ | ------------------------------------------------------------------------------------------------- | -| `computeBillingMode()` reads config only | `src/session/billing-gate.ts:34` | -| Prompt injects only when explicitly set | `src/session/prompt.ts:1553` | -| `COMPUTE_AGENTS` = research, biology, physics, ml | `src/session/prompt.ts:75` | -| Env injection order and silent failure | `src/index.ts:102,106` | -| Provider env var names | `src/server/routes/settings/compute.ts:170-176,185` | -| `compute:up` absent from published npm 0.13.2 | `npm pack @synsci/atlas`, grep of the tarball | -| `compute:up` present in Atlas repo at the same version | `atlas` `origin/main:cli/src/atlas-runtime/commands.mjs:922`; `cli/package.json` version `0.13.2` | -| `atlas doctor` reports no compute field | live `atlas doctor` output | -| Managed compute off by default | `atlas` `origin/main:backend/app/config.py:383` | -| `funding` derivation | `atlas` `origin/main:backend/app/routes/compute.py:117-122` | -| LLM billing already auto-detects | `src/config/config.ts` `billing.llm`; `billing-gate.ts:63` | From 68753c81f36079842d90f3862b49849a099e83a6 Mon Sep 17 00:00:00 2001 From: KB Date: Fri, 31 Jul 2026 21:13:40 +0530 Subject: [PATCH 30/56] spec(compute): correct three claims about the Atlas lease endpoints MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Self-review checked the endpoints instead of assuming them, and the draft was wrong three times: - The SSH private key is not one-time and not unrecoverable. Atlas stores it encrypted on the lease row and GET /leases/{id}/connection decrypts it for the owner (routes/compute.py:450-508), added so the Compute tab could re-offer it after a reload. The .pem is therefore a cache, not a record — OpenScience keeps no durable compute state at all, and a deleted key no longer strands a paid box. - ssh_port was missing from the flow. RunPod NATs SSH to a high port and Vast uses an ssh-proxy port, so the connect string the spec showed would time out on the two providers that matter most. - GET /leases returns every lease for the user, not the running ones (compute_repo.py:446). compute_list has to filter by status. Also softens the Atlas CLI defect: printing the key without saving it is a usability gap, not data loss, since /connection re-serves it. --- docs/specs/compute-design.md | 78 +++++++++++++++++++++++++----------- 1 file changed, 54 insertions(+), 24 deletions(-) diff --git a/docs/specs/compute-design.md b/docs/specs/compute-design.md index 051fee1b..8bba1048 100644 --- a/docs/specs/compute-design.md +++ b/docs/specs/compute-design.md @@ -103,11 +103,14 @@ Atlas resolve cheapest live offer matching gpu + count + max_hourly_cents check wallet funds 1h AND rolling window has headroom clamp budget to effective balance mint Ed25519 pair · size grant to effective cap · launch pod - ← { lease_id, ip, ssh_user, private_key, effective_cap_cents, hourly_cents, provider, sku } -OS write private_key → ~/.config/openscience/compute/.pem (0600) - return { lease_id, ip, ssh_user, key_path, effective_cap_cents, hourly_cents } — no key material -agent bash: ssh -i @ … scp results back + ← { lease_id, ssh_host, ssh_port, ssh_user, private_key, effective_cap_cents, hourly_cents, + provider, sku } +OS write private_key → ~/.config/openscience/compute/.pem (0600, a cache) + return { lease_id, ssh_host, ssh_port, ssh_user, key_path, … } — no key material +agent bash: ssh -i -p @ … scp results back agent compute_release { lease_id } → Atlas terminates · OS deletes the .pem + + (.pem missing? re-fetch from GET /leases/{id}/connection and rewrite it — Atlas holds it encrypted) ``` ## Why budget, not balance @@ -280,11 +283,11 @@ closed: ### Three verbs -| Tool | Input | Output | -| ----------------- | ----------------------------------------------------------------- | ------------------------------------------------------------------------------- | -| `compute_launch` | `gpu`, `count`, `budget_cents`, `max_hourly_cents?`, `volume_id?` | `lease_id`, `ip`, `ssh_user`, `key_path`, `effective_cap_cents`, `hourly_cents` | -| `compute_list` | — | running leases: id, provider, sku, ip, spent, cap | -| `compute_release` | `lease_id` | released | +| Tool | Input | Output | +| ----------------- | ----------------------------------------------------------------- | ------------------------------------------------------------------------------------------------- | +| `compute_launch` | `gpu`, `count`, `budget_cents`, `max_hourly_cents?`, `volume_id?` | `lease_id`, `ssh_host`, `ssh_port`, `ssh_user`, `key_path`, `effective_cap_cents`, `hourly_cents` | +| `compute_list` | — | unfinished leases: `lease_id`, `provider`, `sku`, `status`, `ssh_host`, `ssh_port`, rate, cap | +| `compute_release` | `lease_id` | released | Plus `compute_status` (shipped, unchanged). Each verb maps 1:1 to an Atlas endpoint. Separate tools rather than one `action` parameter, so the permission rule can ask on launch and allow on list. @@ -303,18 +306,30 @@ Behaviour: ### The agent never holds key material -`compute_launch` writes the one-time private key to `~/.config/openscience/compute/.pem` at -`0600` and returns only `key_path`. The key stays out of the transcript, out of compaction, and out of -session storage. `compute_release` deletes it. +`compute_launch` writes the private key to `~/.config/openscience/compute/.pem` at `0600` and +returns only `key_path`. The key stays out of the transcript, out of compaction, and out of session +storage. `compute_release` deletes it. + +**`-p ` is required, not optional.** RunPod NATs SSH to a high public port and Vast routes +through an ssh-proxy port; a connect built as `ssh -i key user@host` simply times out on both. The tool +returns `ssh_port` and the guidance must include it. + +### Atlas is the truth for everything, including the key -This is also the fix for the existing Atlas CLI defect: it prints the private key and never saves it, so -the `ssh_command` it prints cannot work. +The private key is **not** one-time and **not** local state. Atlas stores it encrypted on the lease row +(`compute_leases.ssh_key`, via `secret_store`) and +`GET /api/compute/leases/{lease_id}/connection` decrypts it for the authenticated owner — +`routes/compute.py:450-508`, added so the Compute tab could offer a reliable download after a page +reload. `GET /api/compute/leases` redacts the blob (`_redact_lease:424`); the connection endpoint is the +one that returns it. -### Atlas is the truth for what is running +So the `.pem` on disk is a **cache, not a record**. If it is missing — new machine, cleaned config dir, +another session — re-fetch it from `/connection` and rewrite it. **OpenScience keeps no durable local +state for compute at all**, which removes the last thing that could drift out of sync with Atlas. -`compute_list` calls `GET /api/compute/leases` rather than reading a local ledger, so there is nothing to -drift and nothing to orphan on crash. The `.pem` is the only local state, because it is the only value -that cannot be re-fetched. +`compute_list` calls `GET /api/compute/leases`, which returns `SELECT *` over every lease for the user, +newest first (`compute_repo.py:446-453`). It returns **terminated leases too**, so the tool filters to +non-terminal status rather than presenting the raw list as "what is running". ### The approval gate @@ -379,8 +394,9 @@ creation stays in the workspace UI rather than becoming a fourth tool. - **Atlas CLI unpublished.** `@synsci/atlas@0.13.2` on npm carries 155 command specs and zero `compute:`; `3e1d1ca` removed them, `205bbc0` re-added them, no version bump followed. Source and artifact disagree at an identical version. A release, not code. -- **CLI usability.** Prints the one-time private key and never saves it; no file transfer, no exec, no - compute tests. +- **CLI usability.** Prints the private key and never saves it, so the `ssh_command` it prints cannot + work as shown. Recoverable — `/leases/{id}/connection` re-serves the key — but the CLI does not call it. + No file transfer, no exec, no compute tests. - **Vast / Prime Intellect SSH key leaks** into the operator account, unbounded. - **`budget_cents` already exists** on the agent-spawn path defaulting to `500`, display-only. Change 1 makes caps real, silently giving every shipped spawn a hard $5 kill. **Not a no-op** — raise the default @@ -416,8 +432,10 @@ mocks, no network. - Extension raises the cap, is clamped, and **exhaustion proceeds normally when none arrives**. - A tick exceeding `hard_cap_cents` releases; one that fits does not. `spent_cents` accumulates. - No `budget_cents` → today's behaviour exactly. BYOK ignores it. Plan TTL still fires independently. -- OpenScience: the key is written `0600` and **never appears in the tool result**; release deletes it; - `402`/`429` surface without retry; launch without a verdict is refused. +- OpenScience: the key is written `0600` and **never appears in the tool result**; release deletes it; a + missing `.pem` is re-fetched from `/connection` instead of failing; the connect string carries + `-p ` off port 22; `compute_list` filters terminated leases; `402`/`429` surface without + retry; launch without a verdict is refused. Every new assertion must be shown failing against the specific mutation it guards — ideally the _deletion_ of the logic, not its inversion. On this branch alone, five plan-authored test defects were caught in @@ -442,8 +460,11 @@ collapsed to one string. 11. `compute_launch` refuses to launch without a verdict, surfaces `402`/`429` without retrying, holds no pricing or selection logic, writes the key `0600`, and never returns key material. 12. `compute_launch` prompts by default and is silenced only by explicit config. -13. `compute_list` reflects Atlas, not local state — a lease released out-of-band disappears from it. -14. `pytest` and `bun test` pass with no network; changes 0 and 1 are each their own commit. +13. `compute_list` reflects Atlas, not local state — a lease released out-of-band disappears from it, and + terminated leases are filtered out. +14. A deleted `.pem` is re-fetched from `/connection` rather than stranding the lease. +15. The connect string carries `-p ` whenever the port is not 22. +16. `pytest` and `bun test` pass with no network; changes 0 and 1 are each their own commit. --- @@ -467,3 +488,12 @@ Labels in this document: A source-read claim is not a deployed-behaviour claim. Confirm the money path against `pytest` before relying on it. + +Three claims in the first draft of **this** document were wrong and were caught by checking the +endpoints rather than assuming them — recorded so the pattern stays visible: + +- The private key was described as one-time and unrecoverable. It is neither; `/connection` re-serves it. + That deleted a whole category of local state from the design. +- `ssh_port` was omitted from the flow entirely, which would have produced a connect string that times + out on both RunPod and Vast. +- `GET /leases` was assumed to return running leases. It returns all of them. From f3a35c9c588ae55f0875d7e9761fe860be7e36e1 Mon Sep 17 00:00:00 2001 From: KB Date: Fri, 31 Jul 2026 22:14:31 +0530 Subject: [PATCH 31/56] spec(compute): fix five defects an adversarial review found MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A fresh review of the spec against both repos found three findings that were load-bearing. All five below were re-verified independently before this rewrite. Change 0 named the wrong reaper branch. Branch 3 explicitly skips provisioning leases (lease_reaper.py:139-141), and GPU leases never leave provisioning: the only two writers that flip a lease to ready are _reconcile_active_cpu_leases (CPU-only, lease_manager.py:270) and get_lease_status, which has no production caller (:690). So user leases die at branch 2, provisioning_timeout. The prerequisite fix and its acceptance criterion would both have shipped green while every lease still died at ten minutes. Change 0 is now two parts, ordered. The launch response cannot carry SSH coordinates. Every provider returns ssh_port 22 hardcoded and no ssh_host at acquire (runpod_provider.py :183-191 and peers); the real values exist only in connection(). compute_launch now polls /connection, and the guarding criterion asserts against a real connection payload rather than a launch one, where the port is always 22 and the test passed vacuously. The approval gate had no endpoint to source its numbers from. /estimate requires an explicit {provider, sku}, which a {gpu, count} proposal does not have, and there is no dry-run. Adds change 4, a quote endpoint, advisory and never reused so it cannot reintroduce the stale-offer race. "Cheapest offer" contradicted "RunPod is the managed default" two sections apart — Vast is 204 of 292 options, so cheapest-first would have made the key-leaking provider the norm. The resolver now ranks within an allow-list of providers whose release cleans up. :209 is unreachable on the managed path: hold_id is initialised None and never assigned (lease_manager.py:456, :519-525), so reconcile_managed_ hold returns at :120. The correction this spec made to a predecessor was itself the error. The acquire debit is never refunded, so the grant debit must be cumulative rather than an increment. Also: atomicity for the rolling cap and wallet clamp (concurrency defaults to 2, and the current check is a bare read); change 8 for releases that mark a row released after provider teardown failed; Bun.write has no mode option and would produce 0644, which ssh rejects; ctx.ask is required for the permission gate to fire at all; and lease_manager.py:563 corrected to :508. --- docs/specs/compute-design.md | 547 ++++++++++++++++++++++++----------- 1 file changed, 379 insertions(+), 168 deletions(-) diff --git a/docs/specs/compute-design.md b/docs/specs/compute-design.md index 8bba1048..1caae6c6 100644 --- a/docs/specs/compute-design.md +++ b/docs/specs/compute-design.md @@ -1,7 +1,7 @@ # Compute — design Status: **Mode detection shipped. Managed leases to build.** -Date: 2026-07-31 · single current compute spec +Date: 2026-07-31 · single current compute spec · revised after adversarial review Roadmap: **5**, **51/2**, **55**, **103**; unblocks **56** How OpenScience gets a GPU: who provisions it, who pays, and what stops it. Spans two repos — @@ -25,6 +25,17 @@ These are genuinely different mechanisms, not one flow with a funding flag. BYOK provider account, which Atlas neither meters nor bills, so there is nothing for Atlas to decide and no reason for it to custody a provider credential. **Everything below Part A is `managed` only.** +**Two different things are called BYOK, and they do not coincide.** OpenScience's `byok` means a provider +credential is in `process.env` (`src/compute/mode.ts:70-72`). Atlas has its own, independent notion: a +provider key stored server-side, gated to paid plans (`lease_manager.py:479-484`, +`routes/compute.py:109-123`). A user with an Atlas-stored key and no local env var resolves to `managed` +client-side while Atlas funds the lease as `byok` — rate 0, never debited, `budget_cents` ignored. + +Consequence for the gate below: **the quote is the authority on price, not the client's mode.** A prompt +that quotes an hourly rate the user is never charged is a lie in the safe direction, but it is still a +lie; the quote endpoint (change 4) returns the funding classification so the prompt can say "billed to +your own provider account" instead of a price. + --- # Part A — Mode detection (shipped) @@ -60,14 +71,28 @@ is also why a BYOK user never pays for the availability network call. credential exists else `none`, `"managed"` → managed if available else `none`. **An override may narrow to `none`; it may never manufacture a capability.** -Resolution happens per request, never at startup, at two points that already run per turn: `SkillTool.init` -and the `compute_status` tool. Credentials reach `process.env` from the shell, the Credentials panel and -the Compute panel — and, as the developer's own machine proved, sometimes from dashboard sync → +**Only `compute_status` resolves.** `SkillTool.init` calls `ComputeMode.offered()` (`src/tool/skill.ts:56`), +which deliberately never reaches the availability probe — it is answerable from credentials alone, so a +BYOK or keyless user pays no per-turn network cost (`src/compute/mode.ts:205-224`). Resolution is still +per request rather than at startup, because credentials arrive at unpredictable times: the shell, the +Credentials panel, the Compute panel, and — as the developer's own machine proved — dashboard sync → `synced-env.json` → `preload-env` replay. Startup detection would have reported `none`. -Availability is one authenticated `GET /api/compute/options`, 3s timeout, 5s TTL cache. A failed, -unauthenticated or timed-out call resolves to **unavailable** — failing toward `managed` would reproduce -the original bug of promising an unconfirmed capability. +`session/prompt.ts:1549-1562` injects a stateless per-turn pointer at `compute_status` for +`COMPUTE_AGENTS`. **It must not enumerate tools.** Part B adds three more; a reminder listing them goes +stale the moment the set changes, and the tool descriptions already ride in every request. + +Availability is one authenticated `GET /api/compute/options`, 3s timeout (`mode.ts:101`), 5s TTL cache +(`:107`). A failed, unauthenticated or timed-out call resolves to **unavailable** (`:139-148`) — failing +toward `managed` would reproduce the original bug of promising an unconfirmed capability. + +**That probe is expensive on the Atlas side and is not cached there.** `_catalog` fans out over all ten +`RESELL_PROVIDERS` with `asyncio.gather` (`routes/compute.py:34-45`, `:186-189`), each a live provider API +call (`:129-132`), and per-provider exceptions are swallowed into an empty option list. A paying managed +user therefore resolves to `none` whenever the aggregate exceeds the client's 3s budget. Fail-closed is +still the right default, but **the catalog needs a server-side cache** — and change 3 makes this urgent, +because a transiently-erroring Vast silently removes most of the catalog and changes which provider is +"cheapest" with no signal to anyone. `ComputeMode.offered()` filters the skill catalog and is non-empty **only in `byok`**. **This is a listing filter, not a gate:** a hidden skill remains loadable by exact name and the agent still has `bash`. Gating @@ -95,24 +120,36 @@ and obeys a verdict. ## The flow ``` -agent compute_status → mode=managed, balance_usd -agent compute_launch { gpu: "h100", count: 1, budget_cents: 3000, max_hourly_cents?, volume_id? } - └─ permission gate (default ask): - "RunPod H100 · $2.79/hr · cap $30.00 · balance $198.83" -Atlas resolve cheapest live offer matching gpu + count + max_hourly_cents - check wallet funds 1h AND rolling window has headroom - clamp budget to effective balance - mint Ed25519 pair · size grant to effective cap · launch pod - ← { lease_id, ssh_host, ssh_port, ssh_user, private_key, effective_cap_cents, hourly_cents, - provider, sku } -OS write private_key → ~/.config/openscience/compute/.pem (0600, a cache) - return { lease_id, ssh_host, ssh_port, ssh_user, key_path, … } — no key material +agent compute_status → mode=managed, balance_usd + +agent compute_launch { gpu:"h100", count:1, budget_cents:3000, max_hourly_cents?, volume_id? } + +OS → POST /api/compute/quote { gpu, count, max_hourly_cents, budget_cents } (change 4) + ← { provider, sku, hourly_cents, effective_cap_cents, balance_cents, funding } ADVISORY + └─ permission gate (default ask): + "RunPod H100 · $2.79/hr · cap $30.00 · balance $198.83" + +OS → POST /api/compute/leases { gpu, count, max_hourly_cents, budget_cents, volume_id? } +Atlas RE-RESOLVE — the quote is never trusted or reused + reserve against wallet + rolling cap ATOMICALLY · clamp budget to effective balance + mint Ed25519 · size grant to effective cap · launch pod + ← { lease_id, private_key, effective_cap_cents, hourly_cents, provider, sku } + (no ssh_host yet — the pod is still `provisioning`) + +OS → poll GET /api/compute/leases/{id}/connection until status is running AND ssh_host is non-empty + (bounded; on timeout, RELEASE and report — never leave a paid box the agent cannot reach) + write private_key → /compute/.pem (0600) + ← { lease_id, ssh_host, ssh_port, ssh_user, key_path, effective_cap_cents, hourly_cents } + agent bash: ssh -i -p @ … scp results back -agent compute_release { lease_id } → Atlas terminates · OS deletes the .pem - (.pem missing? re-fetch from GET /leases/{id}/connection and rewrite it — Atlas holds it encrypted) +agent compute_release { lease_id } → Atlas terminates · OS deletes the .pem + (.pem missing? re-fetch from /connection and rewrite it — Atlas holds it encrypted) ``` +**`compute_launch` is three Atlas calls, not one.** An earlier draft claimed each verb maps 1:1 to an +endpoint; that was wrong in a way that hid two missing pieces — see changes 4 and 0. + ## Why budget, not balance The obvious design is to check the wallet each hour and stop the box when it can no longer fund another @@ -133,80 +170,142 @@ authorised rather than what was asked for. ## Why Atlas resolves the SKU -The agent states requirements (`gpu`, `count`, optional `max_hourly_cents`); Atlas picks the cheapest -matching live offer and leases it in one call. Three reasons, in order of weight: +The agent states requirements (`gpu`, `count`, optional `max_hourly_cents`); Atlas picks a matching live +offer and leases it in the same request. Three reasons, in order of weight: -1. **It fixes the offer-ID race by construction.** `compute:up` fetches options, picks, then estimates — - and Vast's SKUs are ephemeral marketplace offer IDs that churn in between. Vast supplies 204 of 292 - live options and is therefore almost always the cheapest pick, so the default path fails with a raw - `HTTP 400: Unknown SKU`. Making fetch-and-lease atomic inside Atlas removes the window. +1. **It narrows the offer-ID race.** `compute:up` fetches options, picks, then estimates — and Vast's SKUs + are ephemeral marketplace offer IDs that churn in between, so the default path fails with a raw + `HTTP 400: Unknown SKU`. 2. **292 options never enter the context window.** -3. **It keeps every price decision server-side**, which is what the trust boundary already required. +3. **It keeps every price decision server-side**, which the trust boundary already required. -The cost is an Atlas resolver that does not exist yet. The alternative — agent picks, client retries on -400 — puts ranking logic in OpenScience, which is exactly what the boundary forbids. +**It does not eliminate the race, and an earlier draft claiming "by construction" was wrong.** +`create_lease` already re-fetches the catalog and re-validates the SKU server-side immediately before +`provider.acquire` (`routes/compute.py:327-333`), so a server-internal window exists today and change 3 +does not close it. No transaction can span a third-party marketplace. **The resolver must therefore +re-resolve and retry on a provider `400`, bounded to N attempts against a re-fetched catalog**, and give +up with a structured error rather than looping. + +### The resolver's provider policy is not "cheapest" + +By this document's own numbers, Vast supplies 204 of 292 options and is almost always cheapest — so a +naive cheapest-first resolver picks the provider the next section argues against, on nearly every launch, +and makes the per-lease key leak the norm rather than the exception. That contradiction was real and is +resolved here: + +**Rank by price within an allow-list of providers whose release cleans up after itself.** Today that is +RunPod and Lambda. Vast and Prime Intellect are excluded until their key-cleanup defect is fixed, at which +point they are added and the resolver changes nothing else. The allow-list is server config, not a client +concern. ## Why RunPod is the managed default Every provider generates a fresh Ed25519 keypair per lease and shows the provider only the public half, so the returned key opens exactly one box everywhere. They differ in what they leave behind: -| Provider | Key attachment | Account artifact | Cleaned up | -| --------------- | ---------------------------------------------- | ---------------- | ---------------------------------- | -| **RunPod** | `PUBLIC_KEY` env var consumed on boot | **none** | nothing to clean | -| Lambda | account key registry | yes | yes — on release and failed launch | -| Vast | account `/ssh/` **and** `/instances/{id}/ssh/` | yes | **no** | -| Prime Intellect | `POST /ssh_keys/` → `sshKeyId` | yes | **no** | +| Provider | Key attachment | Account artifact | Cleaned up on release | +| --------------- | ---------------------------------------------- | ---------------- | ---------------------------------------------- | +| **RunPod** | `PUBLIC_KEY` env var consumed on boot | **none** | nothing to clean | +| Lambda | account key registry | yes | yes (`lambda_provider.py:248-268`) | +| Vast | account `/ssh/` **and** `/instances/{id}/ssh/` | yes | **no** (`vast_provider.py:307-325`) | +| Prime Intellect | `POST /ssh_keys/` → `sshKeyId` | yes | **no** (`prime_intellect_provider.py:306-325`) | RunPod is the only one leaving no account-level trace, which is the right property when Atlas owns the box lifecycle. Its creation body also takes an arbitrary `env` dict, so anything Atlas later wants running on boot needs no SSH bootstrap. -Vast is cheapest and deepest but is interruptible spot and leaks a key per lease. **Vast and Prime -Intellect leak one public key per lease into the operator account, forever** — Lambda's delete-on-release -is the fix. Separate ticket. +**Vast and Prime Intellect leak one public key per lease into the operator account, forever.** Lambda's +delete-on-release is the fix. Separate ticket — and until it lands, the resolver allow-list above keeps +the leak from scaling with usage. --- ## Atlas changes -### Change 0 — scope the lease reaper _(prerequisite, live defect)_ +### Change 0 — make a lease reach `ready`, then scope the reaper _(prerequisite, two parts)_ + +This is the prerequisite everything else waits on, and **the first draft of this spec got it wrong** — +it named the heartbeat branch, which is the one branch that cannot be killing these leases. The +correction matters because the acceptance criterion it implied would have passed green while every user +lease still died at ten minutes. + +**Part (a) — GPU leases never leave `provisioning`.** `RunPodProvider.acquire` returns +`status: "provisioning"` (`runpod_provider.py:183-191`) and `create_lease` persists it. Only two writers +ever flip a lease to `ready`, and neither runs for a managed GPU lease: + +- `lease_manager.py:270`, inside `_reconcile_active_cpu_leases`, which iterates `list_active_cpu_leases` + — **CPU only**. +- `lease_manager.py:690`, inside `LeaseManager.get_lease_status` — which has **no production caller**. + `grep` finds only `backend/tests/test_compute_lease_manager.py:192` and `:603`; there is no status route + in `routes/compute.py`. -`lease_reaper.sweep_once` branch 3 (`backend/app/jobs/lease_reaper.py:141-144`) applies -`HEARTBEAT_STALE_SECONDS` = 600 (`config.py:69`) to everything returned by -`compute_repo.list_unfinished_leases`, whose own docstring reads _"category-agnostic"_ -(`compute_repo.py:456-462`). `create_lease` mints no runner token, and the telemetry endpoint requires -one. +So the lease dies at **branch 2, `provisioning_timeout`** (`lease_reaper.py:133-136`, +`PROVISION_TIMEOUT_SECONDS` = 600, `config.py:70`) — not branch 3. Branch 3 explicitly excludes +provisioning leases and says so in its own comment (`lease_reaper.py:139-141`). -**A user lease cannot prove liveness and is destroyed ~10 minutes after creation**, with provisioning -eating several of those minutes. A $30 budget lease dies having spent about $1.17. No budget can bind -until this is fixed. +The fix is a GPU reconcile pass that polls the provider and flips `provisioning` → `ready`, mirroring +what `_reconcile_active_cpu_leases` already does for CPU. `update_lease_status` already accepts +`ssh_host`/`ssh_port` (`compute_repo.py:397-405`) and nothing in production passes them — **the same pass +must persist them**, which is what makes `compute_list` and the `/connection` poll useful. -Scope the heartbeat check to leases holding a runner token. User leases stay bounded by plan TTL, wallet -exhaustion, explicit release, and the budget cap. The provider-terminal (`:123`) and provisioning-timeout -(`:133`) branches continue to apply to everything. +**Part (b) — then the heartbeat branch becomes the killer.** Once a lease reaches `ready`, branch 3 +(`lease_reaper.py:141-144`) applies `HEARTBEAT_STALE_SECONDS` = 600 (`config.py:69`) to it, over +`list_unfinished_leases`, whose docstring reads _"category-agnostic"_ (`compute_repo.py:456-462`). +`create_lease` mints no runner token and the telemetry endpoint requires one, so **a user lease cannot +prove liveness**. Scope the heartbeat check to leases holding a runner token. -**Ships first, alone, with its own test.** +Order matters: (b) alone changes nothing, because branch 2 kills the lease first. (a) alone moves the +death from 10 minutes to 10 minutes. **Both, or neither.** + +User leases then stay bounded by plan TTL, wallet exhaustion, explicit release, and the budget cap. The +provider-terminal branch (`:117-131`) continues to apply to everything. + +**Ships first, with its own test, before any budget work.** ### Change 1 — make `hard_cap_cents` a real running cap -The column exists (`migrations.py:522`, `pg_migrations.py:775`) and the atomic ceiling exists +The column exists (`migrations.py:522`, `pg_migrations.py:775`) and an atomic ceiling exists (`compute_repo.py:196` — `AND (spent_cents + ?) <= hard_cap_cents`). But `debit_grant` is called in exactly four places, all in `lease_manager.py`: acquire for one hour (`:529`), wallet-insufficient -rollback (`:545`), reservation undo (`:77`), and settle true-up estimate→actual (`:209`). +rollback (`:545`), pre-provisioning failure undo (`:77`), and a settle true-up (`:209`). **`compute_billing_service.tick_once` never calls it** — only `usage_service.charge` and `mark_billed`. So `spent_cents` freezes at hour one and the ceiling is never re-evaluated. -**The billing tick re-debits the grant by the same delta it charges**, releasing the lease when the debit -would exceed the cap — reusing the path that already fires on wallet exhaustion. +**The billing tick must re-debit the grant**, releasing the lease when the debit would exceed the cap — +reusing the path that already fires on wallet exhaustion. **This is not a double charge.** The wallet is money; the grant is an authorisation envelope drawn against it. An implementer who "de-duplicates" these has removed the cap. -_Known trap:_ the acquire-time debit is reversed on failure and trued up at settle, but **not before the -first tick**. A naive re-debit therefore double-counts hour one — a $10 budget at $6.99/h would die at -25.8 minutes instead of ~1.4 hours. (A predecessor spec stated the debit "is never rolled back", which is -wrong; the rollback paths are `:77` and `:209`. The trap is real, the reasoning for it was not.) +**The acquire-time debit is never refunded on the managed path.** `:209` is unreachable there: it sits in +`reconcile_managed_hold`'s `hold_id`-present branch, and the managed GPU path never sets `hold_id` — it is +initialised `None` at `:456`, never assigned, and the code says so at `:519-525` ("`hold_id` stays None"). +`reconcile_managed_hold` returns from `if not hold_id:` at `:120`. Only `:77` fires, and only when +provisioning fails before the box exists. + +_(A predecessor spec said the debit "is never rolled back". A correction in the first draft of this +document called that wrong and cited `:77` and `:209`. On the happy managed path the predecessor was +closer to right, and the correction was the error — the fourth-order instance of the failure this +document's last section is about.)_ + +**Make the grant debit cumulative, mirroring the tick's own idempotency.** The tick is already replay-safe +because it charges `wall_clock_cents(rate, elapsed) - total_spent_cents` +(`compute_billing_service.py:144-148`) — a replay yields `delta <= 0` and skips. `debit_grant` is an +increment (`compute_repo.py:193`), which is not replay-safe and would double-count hour one. Add a +set-to-total variant — `spent_cents = + `, guarded by the same atomic +`WHERE … <= hard_cap_cents AND status = 'active'` — so a replayed tick is a no-op by construction and the +acquire debit is counted exactly once. + +Two things the implementer must decide explicitly, because both are money: + +- **Write order.** Charge → `mark_billed` → grant debit. A crash between them must not double-charge; the + cumulative form is what guarantees that. +- **The final increment.** When the debit would exceed the cap, **charge the elapsed time, then release.** + The user consumed it and the operator owes the provider for it. Skipping the charge loses real money. + +_Latent, not live:_ `debit_grant`'s predicate includes `status = 'active'` (`compute_repo.py:195`). +`expire_grants_by_session` (`:204`) is defined and never called, so no grant expires today — but under +this change, any future grant expiry silently becomes "release the lease". ### Change 2 — accept a budget on lease creation @@ -215,40 +314,79 @@ POST /api/compute/leases { provider?, sku?, gpu?, count?, max_hourly_cents?, region?, node_id?, budget_cents?, volume_id? } ``` -`budget_cents` is optional; absent preserves today's behaviour exactly, which matters because the Atlas -dashboard and `compute:up` both call this endpoint without it. Rejection reuses the structured `402`, -extended with `affordable_budget_cents`. +`budget_cents` is optional. Rejection reuses the structured `402`, extended with +`affordable_budget_cents`. **A budget larger than the wallet is clamped, not rejected**, and the response +reports the **effective** cap. **Managed only** — BYOK ignores it and is never debited. -**A budget larger than the wallet is clamped, not rejected.** The response reports the **effective** cap. +**"Absent preserves today's behaviour" is false once change 1 lands, and an earlier draft claimed it was +true.** `routes/compute.py:359-362` already sizes every user grant to +`max(charge_raw * ttl_hours, charge_raw, 1)` with `ttl_hours = 24` on every plan tier (`config.py:468`, +`:481`, `:494`). That number is inert today because nothing enforces it. After change 1 it becomes a live +ceiling — and because the acquire debit consumes hour one up front and is never refunded, **a no-budget +lease dies at ~23h instead of the 24h plan TTL**, for every existing caller including the dashboard and +`compute:up`. -**Managed only.** BYOK ignores `budget_cents` and is never debited. +Fix it deliberately: size the default grant to `rate * (ttl + 1)`, or have the cumulative debit account +for the acquire debit so the two do not stack. **Assert unchanged runtime, not merely an accepted +request.** ### Change 3 — resolve a SKU from requirements -Accept `{gpu, count, max_hourly_cents?}` in place of an explicit `sku`, resolve to the cheapest live -matching offer, and lease it in the same transaction. Explicit `provider`/`sku` continues to work. +Accept `{gpu, count, max_hourly_cents?}` in place of an explicit `sku`; rank by price within the provider +allow-list; lease in the same request. Explicit `provider`/`sku` continues to work. + +Retry on a provider `400` (stale offer) against a re-fetched catalog, bounded, then fail with a structured +error. `GET /api/compute/options` (`routes/compute.py:214`) already does the read. + +### Change 4 — quote a proposal without spending + +``` +POST /api/compute/quote { gpu, count, max_hourly_cents?, budget_cents } +→ { provider, sku, hourly_cents, effective_cap_cents, balance_cents, funding } +``` + +**Without this the permission gate cannot exist.** The gate must show provider, SKU, rate and effective +cap _before_ money moves, and the client is forbidden from computing any of them. Nothing today can +supply them: `POST /api/compute/estimate` requires an explicit `{provider, sku}` (`EstimateRequest`, +`routes/compute.py:238-241`), which a `{gpu, count}` proposal does not have, and there is no dry-run flag +anywhere in `routes/compute.py`. + +**The quote is advisory and is never reused.** `POST /leases` re-resolves from scratch; a quote token +carried into the lease call would reintroduce exactly the stale-offer race change 3 exists to narrow. The +agent may therefore be shown a rate that differs by cents from the one billed — acceptable, and the launch +response's `effective_cap_cents` is what the user is told they authorised. -`GET /api/compute/options` (`routes/compute.py:214`) and `POST /api/compute/estimate` (`:244`) already do -the reads; what is new is doing them atomically with the lease. +`funding` lets the prompt distinguish an operator-billed lease from an Atlas-BYOK one, which is charged +at rate 0. -### Change 4 — bound cumulative spend +### Change 5 — bound cumulative spend, atomically -The cap is per-grant and a grant is per-lease, so release-and-reacquire is unbounded. What exists today is -`MANAGED_GPU_CONCURRENT` (`lease_manager.py:563`), which bounds concurrent boxes, not total cost. A $30 -budget honoured twenty times is $600. +The cap is per-grant and a grant is per-lease, so release-and-reacquire is unbounded. What exists is +`MANAGED_GPU_CONCURRENT` (read at `lease_manager.py:508`, default **2** at `config.py:117`), which bounds +concurrent boxes, not total cost. A $30 budget honoured twenty times is $600. Add a **rolling window cap** at lease creation, with window and ceiling as plan config alongside `gpu_sandbox_max_ttl_hours`, and a distinct `error` code so clients can tell "this box is too expensive" from "you have spent enough today". -Design it in now — retrofitting changes the meaning of a number users already trust. If `compute_grants` -is not indexed by `(user_id, created_at)`, that index is the migration. +**It must be atomic, and so must the wallet clamp.** Today's wallet check is a bare read-then-compare with +no reservation (`lease_manager.py:540-553`); nothing is held. With a default concurrency of 2, two +simultaneous launches on a $3 wallet both pass — and change 2's clamp makes it worse, because each clamps +to the _full_ effective balance and authorises 2× the wallet. A `SUM` over `compute_grants` followed by an +`INSERT` races identically. **The only atomic primitive in the money path today is `debit_grant` +(`compute_repo.py:187-201`); express the window cap the same way** — a single conditional write — or +serialise per user. -### Change 5 — attach a persistent volume +`compute_grants` is indexed on `user_id` and `session_id` only (`migrations.py:586-587`, +`pg_migrations.py:1062-1063`). `(user_id, created_at)` is the migration. + +Design it in now — retrofitting changes the meaning of a number users already trust. + +### Change 6 — attach a persistent volume Atlas already has `POST /api/compute/volumes` (`routes/compute.py:565`), `list_volumes` and -`delete_volume`. **Leases do not use them**, and the RunPod provider passes `volumeInGb: 20` — a -pod-scoped volume destroyed with the pod. +`delete_volume`. **Leases do not use them**, and the RunPod provider passes `volumeInGb: 20` +(`runpod_provider.py:159`) — a pod-scoped volume destroyed with the pod. Add `volume_id?` to the lease request, pass it to RunPod as `networkVolumeId` mounted at `/workspace`. **Releasing a lease must not cascade a volume delete.** @@ -260,7 +398,7 @@ dollars per GPU-hour. unless it checkpointed to `/workspace`. The volume is the substrate roadmap **56** needs, not a substitute for it. -### Change 6 — extend a live budget +### Change 7 — extend a live budget ``` POST /api/compute/leases/{lease_id}/budget { additional_cents } @@ -277,6 +415,19 @@ closed: - **No warning event required.** An 80% notification is worth adding for humans, but it is advice; the cap must never depend on anyone reading it. +### Change 8 — release must not report success it did not achieve + +`LeaseManager.release_lease` swallows a provider teardown failure into +`provider_result = {"warning": …}` (`lease_manager.py:801-802`) and then **unconditionally** marks the row +released (`:808`). A released row leaves `list_active_leases` (`compute_repo.py:470-487`) so billing +stops, and leaves `list_unfinished_leases` (`:463-464`, `status NOT IN ('released','failed')`) so the +reaper never revisits it. **The box runs indefinitely on the operator's account with nothing metering +it**, and the route returns 200. + +Add a distinct terminal-pending status the reaper re-sweeps, or at minimum surface `provider_result` +so the caller knows teardown failed. Until then, "explicit release works" is only true when the provider +call succeeds. + --- ## OpenScience changes @@ -287,10 +438,10 @@ closed: | ----------------- | ----------------------------------------------------------------- | ------------------------------------------------------------------------------------------------- | | `compute_launch` | `gpu`, `count`, `budget_cents`, `max_hourly_cents?`, `volume_id?` | `lease_id`, `ssh_host`, `ssh_port`, `ssh_user`, `key_path`, `effective_cap_cents`, `hourly_cents` | | `compute_list` | — | unfinished leases: `lease_id`, `provider`, `sku`, `status`, `ssh_host`, `ssh_port`, rate, cap | -| `compute_release` | `lease_id` | released | +| `compute_release` | `lease_id` | released, plus any provider teardown warning | -Plus `compute_status` (shipped, unchanged). Each verb maps 1:1 to an Atlas endpoint. Separate tools rather -than one `action` parameter, so the permission rule can ask on launch and allow on list. +Plus `compute_status` (shipped, unchanged). Separate tools rather than one `action` parameter, so the +permission rule can ask on launch and allow on list. The agent does the actual work with plain `ssh`/`scp` from `bash`. **No relay, no exec wrapper, no file-transfer helper** — that path is a remote-execution platform, and it was already designed once and @@ -298,34 +449,58 @@ rejected. Behaviour: -- **Refuse to launch without a verdict from Atlas.** - On `402`, surface `affordable_budget_cents` and stop. **Never auto-retry at a smaller budget** — a truncated training run is not a cheaper result, it is a discarded one. -- On `429` (concurrency cap), surface rather than retry. +- On `429` (concurrency cap) and `409` (already released, `routes/compute.py:531-535`), surface rather + than retry. +- A non-2xx or malformed launch response **writes no `.pem` and reports no lease**. +- If the readiness poll times out, **release the lease** and report. A paid box the agent cannot reach is + worse than no box. - **No client-side deadline timer**, price table, or SKU ranking. +### SSH coordinates do not exist at launch + +Every provider's `acquire()` returns a hardcoded `ssh_port: 22` and **no `ssh_host`** — +`runpod_provider.py:183-191`, `lambda_provider.py:190-198`, `vast_provider.py:257`, +`prime_intellect_provider.py:273`. The real values live only in `provider.connection()` +(`runpod_provider.py:214-232` → `ssh_host: pod.publicIp`, `ssh_port: _ssh_port(pod)`), reachable through +`GET /leases/{id}/connection`. + +So `compute_launch` **must poll `/connection`** before returning. A tool that returns the launch payload +directly hands the agent `ssh_host: null, ssh_port: 22` — and a test asserting "`-p` appears when the port +is not 22" passes vacuously, because the port at launch is always 22. + +**`-p ` is required, not optional.** RunPod NATs SSH to a high public port and Vast routes +through an ssh-proxy port; `ssh -i key user@host` times out on both. Assert it against a real +RunPod-shaped `/connection` payload, never a launch payload. + ### The agent never holds key material -`compute_launch` writes the private key to `~/.config/openscience/compute/.pem` at `0600` and -returns only `key_path`. The key stays out of the transcript, out of compaction, and out of session +`compute_launch` writes the private key under `Global.Path.config` (`src/global/index.ts:46` — **not a +hardcoded `~/.config`**, which is wrong whenever `XDG_CONFIG_HOME` is set) at `compute/.pem`, +and returns only `key_path`. The key stays out of the transcript, out of compaction, and out of session storage. `compute_release` deletes it. -**`-p ` is required, not optional.** RunPod NATs SSH to a high public port and Vast routes -through an ssh-proxy port; a connect built as `ssh -i key user@host` simply times out on both. The tool -returns `ssh_port` and the guidance must include it. +Modes are load-bearing in both directions — `ssh` refuses a group-readable key outright, so this is a +functional requirement as much as a security one: + +- `mkdir(dir, { recursive: true, mode: 0o700 })` +- `writeFile(path, key, { mode: 0o600 })` **followed by an explicit `chmod`** — `mode` is ignored when the + file already exists, so a re-fetch over a stale loose-permission `.pem` would stay loose. +- **`Bun.write()` has no `mode` option.** The house style prefers Bun APIs; here it produces `0644` and a + broken feature. Use `node:fs/promises`. ### Atlas is the truth for everything, including the key The private key is **not** one-time and **not** local state. Atlas stores it encrypted on the lease row -(`compute_leases.ssh_key`, via `secret_store`) and -`GET /api/compute/leases/{lease_id}/connection` decrypts it for the authenticated owner — -`routes/compute.py:450-508`, added so the Compute tab could offer a reliable download after a page -reload. `GET /api/compute/leases` redacts the blob (`_redact_lease:424`); the connection endpoint is the -one that returns it. +(`compute_leases.ssh_key`, via `secret_store`) and `GET /api/compute/leases/{lease_id}/connection` +decrypts it for the authenticated owner (`routes/compute.py:450-508`), added so the Compute tab could +offer a reliable download after a page reload. `GET /api/compute/leases` redacts the blob +(`_redact_lease:424`); the connection endpoint is the one that returns it. So the `.pem` on disk is a **cache, not a record**. If it is missing — new machine, cleaned config dir, -another session — re-fetch it from `/connection` and rewrite it. **OpenScience keeps no durable local -state for compute at all**, which removes the last thing that could drift out of sync with Atlas. +another session — re-fetch and rewrite it. **OpenScience keeps no durable local state for compute at +all.** `compute_list` calls `GET /api/compute/leases`, which returns `SELECT *` over every lease for the user, newest first (`compute_repo.py:446-453`). It returns **terminated leases too**, so the tool filters to @@ -333,10 +508,14 @@ non-terminal status rather than presenting the raw list as "what is running". ### The approval gate -`compute_launch` goes through the standard permission path. `PermissionNext.evaluate` already defaults to -`ask` when no rule matches (`src/permission/next.ts:237`) and `Permission` has a `.catchall` -(`src/config/config.ts:642`), so the gate is on by default and needs no schema change. The prompt shows -provider, SKU, hourly rate, proposed cap and current balance. +**There is no generic per-tool permission gate.** `PermissionNext.evaluate`'s `ask` default +(`src/permission/next.ts:237`) is only consulted when a tool explicitly calls `ctx.ask` +(`src/tool/tool.ts:25`; `src/tool/bash.ts:148,157` is the precedent). A `compute_launch` that omits +`ctx.ask` prompts for nothing and never reaches `evaluate`. + +So: `compute_launch` **must call `ctx.ask({ permission: "compute_launch", … })`** with the quote's +provider, SKU, hourly rate, effective cap and balance. `evaluate` then defaults to `ask`, and `Permission` +has a `.catchall` (`src/config/config.ts:642`), so no config schema change is needed. Settings ▸ Compute exposes it as `permission.compute_launch`, so a user who wants unattended overnight runs sets `allow` deliberately. **This is UX, not enforcement** — a fork can delete it. The server caps @@ -348,21 +527,22 @@ are what actually bind. Verified against the Atlas checkout at HEAD `7b0e9b6`, source-read (not a running deploy). -| Bound | Owner | Fires when | Today | -| ------------------- | -------------- | ------------------------------------- | ------------------------------------------- | -| `hard_cap_cents` | billing tick | approved money is spent | **column only, does not enforce** — ch. 1 | -| Rolling window cap | lease creation | cumulative spend hits the ceiling | **does not exist** — ch. 4 | -| Wallet exhaustion | billing tick | money actually runs out | **works** (`tick_once:152/225 → :172/239`) | -| Plan TTL (24h) | billing sweep | anything has run absurdly long | **works** (`_release_stale_gpu_leases:303`) | -| Explicit release | agent/user | asked | **works** (`routes/compute.py:513`) | -| Heartbeat staleness | lease reaper | an _agent-spawned_ lease stops report | **fires on user leases too** — ch. 0 | +| Bound | Owner | Fires when | Today | +| -------------------- | -------------- | --------------------------------- | ------------------------------------------- | +| `hard_cap_cents` | billing tick | approved money is spent | **column only, does not enforce** — ch. 1 | +| Rolling window cap | lease creation | cumulative spend hits the ceiling | **does not exist** — ch. 5 | +| Wallet exhaustion | billing tick | money actually runs out | **works, but races** (`tick_once:152→:172`) | +| Plan TTL (24h) | billing sweep | anything has run absurdly long | **works** (`_release_stale_gpu_leases:303`) | +| Explicit release | agent/user | asked | **works only if teardown succeeds** — ch. 8 | +| Provisioning timeout | lease reaper | a lease never boots | **fires on every user lease** — ch. 0(a) | +| Heartbeat staleness | lease reaper | a booted lease stops reporting | **will fire once 0(a) lands** — ch. 0(b) | -All server-side; none client-influenceable. Change 0 _removes_ a bound from user leases, which is safe -precisely because the others apply and necessary because it is one those leases cannot satisfy. +All server-side; none client-influenceable. Change 0 _removes_ two bounds from user leases, which is safe +precisely because the others apply and necessary because they are bounds those leases cannot satisfy. -`COMPUTE_BILLING_TICK_SECONDS` defaults to 60 (`compute_billing_service.py:44`), so a budget can overrun -by up to a minute of rate (~$0.12 on an H100). **Approved budgets are ceilings-plus-a-minute and must -never be described as exact.** +`COMPUTE_BILLING_TICK_SECONDS` defaults to 60 (`compute_billing_service.py:44`) and +`FIRST_BILL_GRACE_SECONDS` is 30 (`:52`), so a budget can overrun by up to ~90s of rate (~$0.17 on an +H100). **Approved budgets are ceilings-plus-90-seconds and must never be described as exact.** --- @@ -384,26 +564,27 @@ creation stays in the workspace UI rather than becoming a fourth tool. ## What you are accepting - **A run killed mid-epoch that was not checkpointing is gone.** No server-side mechanism can fix that. - Change 5 converts "you lost the job" into "you lost the GPU"; roadmap **56** is what makes it good. + Change 6 converts "you lost the job" into "you lost the GPU"; roadmap **56** is what makes it good. - **Time is unbounded within a budget.** A cheap CPU lease could run for days inside a small budget. The 24-hour plan TTL is the only backstop, deliberately. +- **The quoted rate is advisory.** The lease re-resolves, so the billed rate can differ by cents. - **`none` is guidance, not enforcement.** The agent still has `bash`. ## Known defects, owned elsewhere - **Atlas CLI unpublished.** `@synsci/atlas@0.13.2` on npm carries 155 command specs and zero `compute:`; - `3e1d1ca` removed them, `205bbc0` re-added them, no version bump followed. Source and artifact disagree - at an identical version. A release, not code. + `3e1d1ca` removed them, `205bbc0` re-added them, no version bump followed. A release, not code. - **CLI usability.** Prints the private key and never saves it, so the `ssh_command` it prints cannot work as shown. Recoverable — `/leases/{id}/connection` re-serves the key — but the CLI does not call it. No file transfer, no exec, no compute tests. - **Vast / Prime Intellect SSH key leaks** into the operator account, unbounded. -- **`budget_cents` already exists** on the agent-spawn path defaulting to `500`, display-only. Change 1 - makes caps real, silently giving every shipped spawn a hard $5 kill. **Not a no-op** — raise the default - deliberately or exempt spawn-path grants. -- **Stale skill names in agent prompts.** `research.txt`'s appendix and `ml.txt` name ~35 skills by - directory rather than frontmatter `name` (`vllm` → `serving-llms-vllm`), so the agent is told it has - skills it cannot load. `skills/scholar-evaluation/SKILL.md` has no `name:` at all. +- **The options catalog is uncached** and fans out to ten live provider APIs per call. +- **`budget_cents` already exists** on the agent-spawn path defaulting to `500` (`agent_tools.py:1386`, + `models/agent.py:101`, `spawn_queue_service.py:69` → `create_grant` at `:1501`), display-only. Change 1 + makes caps real, silently giving every shipped spawn a hard $5 kill. **Not a no-op.** +- **Stale skill names in agent prompts.** `research.txt:353` and `ml.txt:192` name skills by directory + rather than frontmatter `name` (`vllm` → `serving-llms-vllm`), so the agent is told it has skills it + cannot load. `skills/scholar-evaluation/SKILL.md` has no frontmatter at all. ## Out of scope @@ -419,23 +600,29 @@ Atlas follows `backend/tests/test_compute_billing.py` — `_FakeProvider`, `aios plain `pytest` before any deploy. OpenScience stubs `globalThis.fetch` and exercises the real tool; no mocks, no network. -- **A budget of $B at $R/h lasts ≈ B/R hours**, asserted on elapsed billable duration. This is the - headline property and the one the previous attempt omitted from both its tests and its criteria — a $10 - budget dying at 25.8 minutes passes every "release happened" assertion while being off by 3×. -- A lease **without** a runner token survives past `HEARTBEAT_STALE_SECONDS`; one with a token is still - reaped. Without this, every budget test silently measures a 10-minute reap. -- Money-path writes are idempotent — a replayed tick does not double-charge. +- **A budget of $B at $R/h lasts B/R hours ± 90s of rate**, asserted on elapsed billable duration. The + tolerance is not a detail: it is `COMPUTE_BILLING_TICK_SECONDS` + `FIRST_BILL_GRACE_SECONDS`, and + leaving it unstated lets an implementer pick a tolerance that hides the acquire-debit double-count. +- A user lease survives past **both** `PROVISION_TIMEOUT_SECONDS` **and** `HEARTBEAT_STALE_SECONDS`; a + runner-token lease is still reaped for heartbeat staleness. Testing only the second passes green while + the box dies at ten minutes. +- The GPU reconcile pass flips `provisioning` → `ready` **and persists `ssh_host`/`ssh_port`**. +- A replayed tick does not double-charge and does not double-debit the grant. +- **Two concurrent launches against a wallet that funds only one**: exactly one succeeds. - The rolling cap rejects an N+1th lease even when each individual budget is affordable. -- SKU resolution picks the cheapest offer honouring `max_hourly_cents`, and leases atomically — a stale - offer ID cannot appear between resolve and lease. +- Resolution leases the offer it ranked; a provider `400` triggers at most N re-resolves against a + re-fetched catalog, then a structured error. Resolution never picks a provider outside the allow-list. +- The quote spends nothing and creates no lease row. - `volume_id` mounts, and **release does not delete the volume**. - Extension raises the cap, is clamped, and **exhaustion proceeds normally when none arrives**. -- A tick exceeding `hard_cap_cents` releases; one that fits does not. `spent_cents` accumulates. -- No `budget_cents` → today's behaviour exactly. BYOK ignores it. Plan TTL still fires independently. -- OpenScience: the key is written `0600` and **never appears in the tool result**; release deletes it; a - missing `.pem` is re-fetched from `/connection` instead of failing; the connect string carries - `-p ` off port 22; `compute_list` filters terminated leases; `402`/`429` surface without - retry; launch without a verdict is refused. +- A tick exceeding `hard_cap_cents` charges the elapsed time and then releases; one that fits does not. +- A no-budget lease still runs the **full** plan TTL after change 1 — asserted on runtime, not on the + request being accepted. +- A release whose provider teardown fails does not silently present as success. +- OpenScience: the key is written `0600` in a `0700` directory and **never appears in the tool result**; a + re-fetch over an existing loose-mode file tightens it; a malformed launch response writes no `.pem`; a + readiness timeout releases the lease; the connect string carries `-p ` against a real + `/connection` payload; `compute_list` filters terminated leases; `402`/`429`/`409` surface without retry. Every new assertion must be shown failing against the specific mutation it guards — ideally the _deletion_ of the logic, not its inversion. On this branch alone, five plan-authored test defects were caught in @@ -444,27 +631,35 @@ collapsed to one string. ## Acceptance criteria -0. A lease with no runner token is not reaped for heartbeat staleness; one with a token still is. -1. A budget of $B at rate $R/h lasts ≈ B/R hours, asserted on elapsed billable duration. -2. The money path is idempotent under a replayed tick. -3. The billing tick re-debits the grant; `spent_cents` accumulates; a tick exceeding the cap releases via - the existing path. -4. `POST /leases` accepts `budget_cents`; omitting it preserves today's behaviour exactly. +0. A user lease survives past both `PROVISION_TIMEOUT_SECONDS` and `HEARTBEAT_STALE_SECONDS`; a + runner-token lease is still reaped for heartbeat staleness; the reconcile pass persists + `ssh_host`/`ssh_port`. +1. A budget of $B at rate $R/h lasts B/R hours ± 90s of rate, asserted on elapsed billable duration. +2. A replayed tick neither double-charges the wallet nor double-debits the grant. +3. The billing tick re-debits the grant cumulatively; a tick exceeding the cap charges elapsed time and + then releases. +4. A lease created without `budget_cents` runs the full plan TTL — unchanged **runtime**, not merely an + accepted request. 5. A budget exceeding the wallet is clamped, and the response reports the effective cap. -6. `{gpu, count, max_hourly_cents}` resolves to the cheapest matching live offer and leases atomically. -7. A rolling cap bounds spend across sequential leases. -8. `volume_id` attaches a volume that survives lease release. -9. Extension raises the cap when affordable, refuses with a structured `402` when not, never fires - automatically. -10. BYOK ignores `budget_cents`. Plan TTL fires independently. -11. `compute_launch` refuses to launch without a verdict, surfaces `402`/`429` without retrying, holds no - pricing or selection logic, writes the key `0600`, and never returns key material. -12. `compute_launch` prompts by default and is silenced only by explicit config. -13. `compute_list` reflects Atlas, not local state — a lease released out-of-band disappears from it, and - terminated leases are filtered out. -14. A deleted `.pem` is re-fetched from `/connection` rather than stranding the lease. -15. The connect string carries `-p ` whenever the port is not 22. -16. `pytest` and `bun test` pass with no network; changes 0 and 1 are each their own commit. +6. Two concurrent launches against a wallet funding one: exactly one succeeds. Same for the rolling cap. +7. `{gpu, count, max_hourly_cents}` resolves within the provider allow-list, retries a provider `400` + against a re-fetched catalog at most N times, then fails with a structured error. +8. `POST /quote` returns provider, SKU, rate, effective cap, balance and funding, and **spends nothing**. +9. `volume_id` attaches a volume that survives lease release. +10. Extension raises the cap when affordable, refuses with a structured `402` when not, never fires + automatically. +11. BYOK ignores `budget_cents`. Plan TTL fires independently. +12. A release whose provider teardown fails is not reported as a clean release. +13. `compute_launch` on a non-2xx or malformed response writes no `.pem` and reports no lease; on a + readiness timeout it releases the lease; it never returns key material; it holds no pricing or + selection logic. +14. The key is written `0600` inside a `0700` directory, under `Global.Path.config`, and a re-fetch over + an existing loose-mode file tightens it. +15. `compute_launch` calls `ctx.ask` and prompts by default; `permission.compute_launch: "allow"` silences + it. +16. The connect string carries `-p ` when a real `/connection` payload reports a non-22 port. +17. `compute_list` filters terminated leases and reflects Atlas, not local state. +18. `pytest` and `bun test` pass with no network; change 0 and change 1 are each their own commit. --- @@ -480,20 +675,36 @@ Labels in this document: vast / prime_intellect operator-funded with 292 launchable options; `/compute/estimate` returns `funding: "managed"` with a real rate and runway; the published npm artifact contains no `compute:` command; `compute:up`'s default path fails on the Vast SKU race while `--provider lambda|runpod` - succeeds. + succeeds. **The 204-of-292 Vast share is load-bearing for the resolver's allow-list and comes from this + production check, not from source.** - **Verified against the Atlas checkout at HEAD `7b0e9b6` (source-read, 2026-07-31):** every `file:line` - citation in Part B — the reaper's branch structure, `debit_grant`'s four call sites and the tick's - absence from them, the 60s tick default, the 24h GPU TTL sweep, the absence of any rolling cap, and the - existing endpoint set. + citation in Part B, re-checked after review. +- **Not verified:** `compute:up`'s internal fetch→pick→estimate sequence. The Atlas CLI source is in + neither repo; the commit chronology corroborates the shape but the sequence itself is inferred. A source-read claim is not a deployed-behaviour claim. Confirm the money path against `pytest` before relying on it. -Three claims in the first draft of **this** document were wrong and were caught by checking the -endpoints rather than assuming them — recorded so the pattern stays visible: - -- The private key was described as one-time and unrecoverable. It is neither; `/connection` re-serves it. - That deleted a whole category of local state from the design. -- `ssh_port` was omitted from the flow entirely, which would have produced a connect string that times - out on both RunPod and Vast. -- `GET /leases` was assumed to return running leases. It returns all of them. +### What review caught in this document + +Recorded because the pattern is the point. The first draft was written the same way the predecessors +were, and an adversarial pass over the same source found: + +- **Change 0 named the wrong reaper branch.** Branch 3 explicitly skips `provisioning` leases; GPU leases + never leave `provisioning` because both status writers are CPU-only or uncalled. The prerequisite fix + and its acceptance criterion would both have shipped green while every user lease still died at ten + minutes. +- **The launch response cannot carry SSH coordinates.** Providers return `ssh_port: 22` and no host at + acquire. The criterion guarding this passed vacuously, since the port at launch is always 22. +- **The approval gate had no endpoint to source its numbers from.** No dry-run, and `/estimate` needs an + explicit SKU. The gate was specified as the UX centrepiece and could not have been built. +- **"Cheapest offer" contradicted "RunPod is the default"** two sections apart, and would have made the + key-leaking providers the norm. +- **`:209` is unreachable on the managed path**, so a correction this document made to a predecessor was + itself the error — the same failure, one generation on. +- **One plain citation miss** (`lease_manager.py:563` for `:508`), against a section claiming every + citation had been verified. + +Two lessons, both cheap to apply: a citation that is literally correct can still fail to support the claim +built on it, and the acceptance criteria are where a wrong mechanism hides — a criterion that cannot fail +is worse than no criterion. From 3fd0c0b15be2cf21a718782d7a3a32bdfc5b06e7 Mon Sep 17 00:00:00 2001 From: KB Date: Fri, 31 Jul 2026 22:25:21 +0530 Subject: [PATCH 32/56] spec(compute): cheapest box wins, across every operator provider MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reverses the provider allow-list. Atlas ranks purely by price; there is no default provider and no preference. Ranking is well-defined — every option carries a normalised price_cents_per_hour set uniformly for all providers (routes/compute.py:158-163) — but needs a canonical GPU-model map, since providers spell the same card differently. Four consequences, all now in scope rather than deferred: - Vast supplies 204 of 292 options, so it wins most launches, so its per-lease SSH key leak grows one key per launch forever. Promoted from a background ticket to change 9, shipping with the resolver. Follows Lambda's delete-on-release-and-on-failed-launch pattern. - Vast's SKUs are the ephemeral ones, so the offer-ID race is now on the common path. Changes 3 and 4 become mandatory; there is no variant where the agent picks a SKU and the default path still works. - The retry path rebuilds an uncached ten-provider catalog per attempt, so caching it is a prerequisite rather than a nicety. - Volumes are worse than the spec claimed: create_volume writes a DB row and calls no provider API, so change 6 is "make volumes real", per provider. Resolved by treating volume_id as a requirement — the pool narrows to volume-capable providers and cheapest still wins within it. Also corrects an inherited error: Vast is not spot. list_options queries type: on-demand (vast_provider.py:110-115), so cheapest-first buys no preemption risk. It also runs a second premium-tier query by GPU name, so an H100 request reaches real H100 offers. --- docs/specs/compute-design.md | 169 ++++++++++++++++++++++++----------- 1 file changed, 119 insertions(+), 50 deletions(-) diff --git a/docs/specs/compute-design.md b/docs/specs/compute-design.md index 1caae6c6..3ee37928 100644 --- a/docs/specs/compute-design.md +++ b/docs/specs/compute-design.md @@ -126,8 +126,8 @@ agent compute_launch { gpu:"h100", count:1, budget_cents:3000, max_hourly_cents OS → POST /api/compute/quote { gpu, count, max_hourly_cents, budget_cents } (change 4) ← { provider, sku, hourly_cents, effective_cap_cents, balance_cents, funding } ADVISORY - └─ permission gate (default ask): - "RunPod H100 · $2.79/hr · cap $30.00 · balance $198.83" + └─ permission gate (default ask) — names the provider, because cheapest-first means it varies: + "Vast H100 · $1.94/hr · cap $30.00 · balance $198.83" OS → POST /api/compute/leases { gpu, count, max_hourly_cents, budget_cents, volume_id? } Atlas RE-RESOLVE — the quote is never trusted or reused @@ -186,37 +186,55 @@ does not close it. No transaction can span a third-party marketplace. **The reso re-resolve and retry on a provider `400`, bounded to N attempts against a re-fetched catalog**, and give up with a structured error rather than looping. -### The resolver's provider policy is not "cheapest" +### Cheapest wins, across every operator provider -By this document's own numbers, Vast supplies 204 of 292 options and is almost always cheapest — so a -naive cheapest-first resolver picks the provider the next section argues against, on nearly every launch, -and makes the per-lease key leak the norm rather than the exception. That contradiction was real and is -resolved here: +**The resolver ranks purely by price.** No provider allow-list, no default provider, no preference — the +cheapest live offer matching the requirements is the one leased. That is the product decision, and the +rest of this document conforms to it. -**Rank by price within an allow-list of providers whose release cleans up after itself.** Today that is -RunPod and Lambda. Vast and Prime Intellect are excluded until their key-cleanup defect is fixed, at which -point they are added and the resolver changes nothing else. The allow-list is server config, not a client -concern. +Ranking is well-defined: every catalogued option carries a normalised `price_cents_per_hour` — the +provider's exact pass-through rate, set uniformly for all six providers at `routes/compute.py:158-163`. +Match on GPU model and `count`, honour `max_hourly_cents` if given, then take the minimum. The one real +implementation detail is GPU-name normalisation: options expose `name`, `gpu_ram_gb` and `upstream`, and +providers spell the same card differently, so the resolver needs a canonical model map rather than a +substring match. -## Why RunPod is the managed default +An earlier draft of this section proposed an allow-list restricted to providers whose release cleans up +after itself, which would have excluded Vast — and Vast supplies 204 of 292 live options. **That is +overruled.** Two consequences follow, and both are now in scope rather than deferred: + +- **The key leak stops being rare and becomes the norm** — see below. Change 9 fixes it, and it is no + longer somebody else's ticket. +- **The offer-ID race is live on the common path**, because Vast's SKUs are the ephemeral ones. Changes 3 + and 4 are therefore mandatory, not optional. There is no version of this design where the agent picks a + SKU itself and the default path still works. + +**Vast is not spot, and a predecessor claim that it was is wrong.** `VastProvider.list_options` queries +`"type": "on-demand"` (`vast_provider.py:110-115`), so cheapest-first does not buy preemption risk. It +also runs two queries — cheapest-first alone never surfaces the datacenter cards, so the premium tier is +fetched by name (`:105-121`) — which means an H100 request reaches real H100 offers rather than bottoming +out in consumer GPUs. + +### What cheapest-first obliges us to fix Every provider generates a fresh Ed25519 keypair per lease and shows the provider only the public half, so the returned key opens exactly one box everywhere. They differ in what they leave behind: | Provider | Key attachment | Account artifact | Cleaned up on release | | --------------- | ---------------------------------------------- | ---------------- | ---------------------------------------------- | -| **RunPod** | `PUBLIC_KEY` env var consumed on boot | **none** | nothing to clean | +| RunPod | `PUBLIC_KEY` env var consumed on boot | **none** | nothing to clean | | Lambda | account key registry | yes | yes (`lambda_provider.py:248-268`) | | Vast | account `/ssh/` **and** `/instances/{id}/ssh/` | yes | **no** (`vast_provider.py:307-325`) | | Prime Intellect | `POST /ssh_keys/` → `sshKeyId` | yes | **no** (`prime_intellect_provider.py:306-325`) | -RunPod is the only one leaving no account-level trace, which is the right property when Atlas owns the -box lifecycle. Its creation body also takes an arbitrary `env` dict, so anything Atlas later wants running -on boot needs no SSH bootstrap. +RunPod leaves no account-level trace and its creation body takes an arbitrary `env` dict, so boot-time +setup needs no SSH bootstrap. That makes it the **easiest** provider to operate — it is not the default, +and nothing in this design prefers it. -**Vast and Prime Intellect leak one public key per lease into the operator account, forever.** Lambda's -delete-on-release is the fix. Separate ticket — and until it lands, the resolver allow-list above keeps -the leak from scaling with usage. +`VastProvider.release` deletes only `/instances/{lease_id}/` (`vast_provider.py:307-325`); Prime +Intellect's deletes only the pod. **Under cheapest-first, that is one public key leaked into the operator +account per launch, unbounded and permanent.** Lambda's delete-on-release (`lambda_provider.py:248-268`) +is the pattern. This is change 9, and it ships alongside the resolver rather than after it. --- @@ -330,13 +348,21 @@ Fix it deliberately: size the default grant to `rate * (ttl + 1)`, or have the c for the acquire debit so the two do not stack. **Assert unchanged runtime, not merely an accepted request.** -### Change 3 — resolve a SKU from requirements +### Change 3 — resolve the cheapest SKU from requirements _(mandatory)_ -Accept `{gpu, count, max_hourly_cents?}` in place of an explicit `sku`; rank by price within the provider -allow-list; lease in the same request. Explicit `provider`/`sku` continues to work. +Accept `{gpu, count, max_hourly_cents?}` in place of an explicit `sku`; rank **every** operator provider's +options by `price_cents_per_hour`; lease the cheapest match in the same request. Explicit `provider`/`sku` +continues to work for the dashboard and the CLI. -Retry on a provider `400` (stale offer) against a re-fetched catalog, bounded, then fail with a structured -error. `GET /api/compute/options` (`routes/compute.py:214`) already does the read. +Needs a canonical GPU-model map — providers spell the same card differently, and a substring match on +`name` will silently mis-rank. + +**The retry is not optional here.** Vast supplies most of the catalog and its SKUs are ephemeral offer IDs, +so the cheapest pick is usually the raciest one. On a provider `400`, re-resolve against a re-fetched +catalog, bounded to N attempts, then fail with a structured error rather than looping. +`GET /api/compute/options` (`routes/compute.py:214`) already does the read — but it is uncached and fans +out to ten live provider APIs, so N retries is N full catalog rebuilds. **Cache the catalog server-side +before shipping this**, or the retry path costs more than the lease. ### Change 4 — quote a proposal without spending @@ -382,14 +408,25 @@ serialise per user. Design it in now — retrofitting changes the meaning of a number users already trust. -### Change 6 — attach a persistent volume +### Change 6 — attach a persistent volume _(larger than it looks, and cheapest-first makes it harder)_ + +Atlas has `POST /api/compute/volumes` (`routes/compute.py:565`), `list_volumes` and `delete_volume` — but +**they provision nothing.** `create_volume` clamps a size and writes a `compute_volume_repo` row; no +provider API is called anywhere. The only volume in the compute providers is RunPod's `volumeInGb: 20` +(`runpod_provider.py:159`), which is pod-scoped and destroyed with the pod. So this is not "pass an +existing volume through" — it is "make volumes real", per provider. -Atlas already has `POST /api/compute/volumes` (`routes/compute.py:565`), `list_volumes` and -`delete_volume`. **Leases do not use them**, and the RunPod provider passes `volumeInGb: 20` -(`runpod_provider.py:159`) — a pod-scoped volume destroyed with the pod. +**Cheapest-first turns that into a per-provider matrix.** The volume has to exist wherever the resolver +lands, and the four operator providers do not share a network-volume primitive with the same semantics. -Add `volume_id?` to the lease request, pass it to RunPod as `networkVolumeId` mounted at `/workspace`. -**Releasing a lease must not cascade a volume delete.** +Resolution: **`volume_id` is a requirement, not a preference.** When the request carries one, the resolver +ranks only providers with real network-volume support and takes the cheapest of those. That is still +cheapest-first — a volume is a constraint like `gpu` or `count`, not an override of the pricing rule — and +it degrades honestly: a user who wants durable storage pays whatever the cheapest volume-capable provider +costs, and is told which one. + +Start with RunPod (`networkVolumeId`, mounted at `/workspace`) and add providers as their volume APIs are +wired. **Releasing a lease must not cascade a volume delete.** Budget exhaustion then costs the compute, not the work — a network volume is cents per GB-month against dollars per GPU-hour. @@ -428,7 +465,19 @@ Add a distinct terminal-pending status the reaper re-sweeps, or at minimum surfa so the caller knows teardown failed. Until then, "explicit release works" is only true when the provider call succeeds. ---- +### Change 9 — stop Vast and Prime Intellect leaking a key per lease _(forced by cheapest-first)_ + +`VastProvider.release` deletes only `/instances/{lease_id}/` (`vast_provider.py:307-325`) and Prime +Intellect's deletes only the pod (`prime_intellect_provider.py:306-325`). The per-lease public key stays +in the operator account permanently. + +This was a background annoyance while RunPod was the default. **Cheapest-first makes Vast the usual +winner, so the leak now grows one key per launch, forever.** Follow Lambda +(`lambda_provider.py:248-268`): delete the registered key on release **and** on failed launch, so an +acquire that dies after key registration does not leak either. + +Ships with change 3. A resolver that makes Vast the common path without this is a resolver that turns a +known defect into a scaling one. ## OpenScience changes @@ -568,6 +617,11 @@ creation stays in the workspace UI rather than becoming a fourth tool. - **Time is unbounded within a budget.** A cheap CPU lease could run for days inside a small budget. The 24-hour plan TTL is the only backstop, deliberately. - **The quoted rate is advisory.** The lease re-resolves, so the billed rate can differ by cents. +- **The provider varies per launch, and so does the box.** Cheapest-first means image, disk, region and + network differ run to run. The prompt names the provider for exactly this reason. A user who needs a + specific one passes `provider`/`sku` explicitly, which still works. +- **Durable storage narrows the pool.** Asking for a volume means paying the cheapest volume-capable + provider, not the cheapest provider. - **`none` is guidance, not enforcement.** The agent still has `bash`. ## Known defects, owned elsewhere @@ -577,8 +631,8 @@ creation stays in the workspace UI rather than becoming a fourth tool. - **CLI usability.** Prints the private key and never saves it, so the `ssh_command` it prints cannot work as shown. Recoverable — `/leases/{id}/connection` re-serves the key — but the CLI does not call it. No file transfer, no exec, no compute tests. -- **Vast / Prime Intellect SSH key leaks** into the operator account, unbounded. -- **The options catalog is uncached** and fans out to ten live provider APIs per call. +- **The options catalog is uncached** and fans out to ten live provider APIs per call. Change 3's retry + path makes this urgent rather than merely wasteful. - **`budget_cents` already exists** on the agent-spawn path defaulting to `500` (`agent_tools.py:1386`, `models/agent.py:101`, `spawn_queue_service.py:69` → `create_grant` at `:1501`), display-only. Change 1 makes caps real, silently giving every shipped spawn a hard $5 kill. **Not a no-op.** @@ -610,8 +664,15 @@ mocks, no network. - A replayed tick does not double-charge and does not double-debit the grant. - **Two concurrent launches against a wallet that funds only one**: exactly one succeeds. - The rolling cap rejects an N+1th lease even when each individual budget is affordable. +- Resolution picks the **globally cheapest** matching offer across all operator providers — asserted with + a fake catalog where the cheapest match is deliberately not the first provider polled, and again where + it is not the provider the previous test picked. A resolver that always returns one provider must fail. +- GPU-model matching is canonical, not substring: an `h100` request does not match `h100`-shaped names + from a different card, and does match the same card spelled differently across providers. - Resolution leases the offer it ranked; a provider `400` triggers at most N re-resolves against a - re-fetched catalog, then a structured error. Resolution never picks a provider outside the allow-list. + re-fetched catalog, then a structured error. +- Vast and Prime Intellect release **deletes the registered public key** — on normal release and on failed + launch. Asserted on the provider's key list, not on the release return value. - The quote spends nothing and creates no lease row. - `volume_id` mounts, and **release does not delete the volume**. - Extension raises the cap, is clamped, and **exhaustion proceeds normally when none arrives**. @@ -642,24 +703,29 @@ collapsed to one string. accepted request. 5. A budget exceeding the wallet is clamped, and the response reports the effective cap. 6. Two concurrent launches against a wallet funding one: exactly one succeeds. Same for the rolling cap. -7. `{gpu, count, max_hourly_cents}` resolves within the provider allow-list, retries a provider `400` - against a re-fetched catalog at most N times, then fails with a structured error. +7. `{gpu, count, max_hourly_cents}` resolves to the **globally cheapest** matching offer across all + operator providers — proven against a catalog where the winner is neither the first provider polled nor + the same provider twice — honours `max_hourly_cents`, matches GPU models canonically rather than by + substring, retries a provider `400` against a re-fetched catalog at most N times, then fails with a + structured error. 8. `POST /quote` returns provider, SKU, rate, effective cap, balance and funding, and **spends nothing**. -9. `volume_id` attaches a volume that survives lease release. -10. Extension raises the cap when affordable, refuses with a structured `402` when not, never fires +9. Vast and Prime Intellect delete the registered public key on release **and** on failed launch, asserted + against the provider's key list rather than the release return value. +10. `volume_id` attaches a volume that survives lease release. +11. Extension raises the cap when affordable, refuses with a structured `402` when not, never fires automatically. -11. BYOK ignores `budget_cents`. Plan TTL fires independently. -12. A release whose provider teardown fails is not reported as a clean release. -13. `compute_launch` on a non-2xx or malformed response writes no `.pem` and reports no lease; on a +12. BYOK ignores `budget_cents`. Plan TTL fires independently. +13. A release whose provider teardown fails is not reported as a clean release. +14. `compute_launch` on a non-2xx or malformed response writes no `.pem` and reports no lease; on a readiness timeout it releases the lease; it never returns key material; it holds no pricing or selection logic. -14. The key is written `0600` inside a `0700` directory, under `Global.Path.config`, and a re-fetch over +15. The key is written `0600` inside a `0700` directory, under `Global.Path.config`, and a re-fetch over an existing loose-mode file tightens it. -15. `compute_launch` calls `ctx.ask` and prompts by default; `permission.compute_launch: "allow"` silences +16. `compute_launch` calls `ctx.ask` and prompts by default; `permission.compute_launch: "allow"` silences it. -16. The connect string carries `-p ` when a real `/connection` payload reports a non-22 port. -17. `compute_list` filters terminated leases and reflects Atlas, not local state. -18. `pytest` and `bun test` pass with no network; change 0 and change 1 are each their own commit. +17. The connect string carries `-p ` when a real `/connection` payload reports a non-22 port. +18. `compute_list` filters terminated leases and reflects Atlas, not local state. +19. `pytest` and `bun test` pass with no network; change 0 and change 1 are each their own commit. --- @@ -675,8 +741,9 @@ Labels in this document: vast / prime_intellect operator-funded with 292 launchable options; `/compute/estimate` returns `funding: "managed"` with a real rate and runway; the published npm artifact contains no `compute:` command; `compute:up`'s default path fails on the Vast SKU race while `--provider lambda|runpod` - succeeds. **The 204-of-292 Vast share is load-bearing for the resolver's allow-list and comes from this - production check, not from source.** + succeeds. **The 204-of-292 Vast share is load-bearing — it is why cheapest-first makes Vast the common + path, and therefore why change 9 is in scope — and it comes from this production check, not from + source.** - **Verified against the Atlas checkout at HEAD `7b0e9b6` (source-read, 2026-07-31):** every `file:line` citation in Part B, re-checked after review. - **Not verified:** `compute:up`'s internal fetch→pick→estimate sequence. The Atlas CLI source is in @@ -698,8 +765,10 @@ were, and an adversarial pass over the same source found: acquire. The criterion guarding this passed vacuously, since the port at launch is always 22. - **The approval gate had no endpoint to source its numbers from.** No dry-run, and `/estimate` needs an explicit SKU. The gate was specified as the UX centrepiece and could not have been built. -- **"Cheapest offer" contradicted "RunPod is the default"** two sections apart, and would have made the - key-leaking providers the norm. +- **"Cheapest offer" contradicted "RunPod is the default"** two sections apart. Settled by product + decision in favour of cheapest, which makes the key-leaking providers the common path — and so promotes + the leak fix from a background ticket into change 9, and makes changes 3 and 4 mandatory rather than + optional. - **`:209` is unreachable on the managed path**, so a correction this document made to a predecessor was itself the error — the same failure, one generation on. - **One plain citation miss** (`lease_manager.py:563` for `:508`), against a section claiming every From deae90d50e1c46a63a0799d0676dbd91781a3e35 Mon Sep 17 00:00:00 2001 From: KB Date: Fri, 31 Jul 2026 22:37:01 +0530 Subject: [PATCH 33/56] spec(compute): rank on measured reliability, pin the image, canonicalise GPUs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds what a comparable multi-provider aggregator does that this design did not, plus two gaps its page exposed. Change 10 — measure boot and availability, rank on it. Cheapest per hour inverts: boot is billed wall-clock, and published 7-day distributions put RunPod near a 59s median with a tight spread against Vast's ~1m9s median with a tail past six minutes. Cheapest-first sends most launches to Vast. The dataset costs nothing: compute_leases.ready_at already exists (migrations.py:555) and update_lease_status already stamps it (compute_repo.py:425-427) — nothing populates it only because nothing flips a GPU lease to ready, which change 0(a) fixes. Ranking becomes cheapest above a floor, degrading to pure price with no history. That same tail makes PROVISION_TIMEOUT_SECONDS=600 a hazard rather than a formality: one global constant across a 6x spread will reap legitimate Vast provisions. Now per-provider, derived from measured p99. The client's readiness poll must outlive the server's timeout and defer to its verdict — a shorter client bound kills launches that were about to succeed. Change 11 — pin the image. Nothing said what is on the box, and the answer varies: RunPod pins a CUDA devel image, Vast launches pytorch/pytorch:latest (vast_provider.py:220), Lambda and Prime set none. A floating tag means the same experiment run a month apart gets a different toolchain with no record. For a reproducible-research tool that is a correctness bug. Defines a minimum environment contract, pins per provider, records the resolved image on the lease. Canonical GPU map replaces substring matching, at SXM/PCIe/NVL granularity — the three H100 variants differ in throughput and price, so "h100" is not a usable resolver input. Unmappable options are excluded from ranking rather than guessed at. --- docs/specs/compute-design.md | 150 ++++++++++++++++++++++++++++++----- 1 file changed, 131 insertions(+), 19 deletions(-) diff --git a/docs/specs/compute-design.md b/docs/specs/compute-design.md index 3ee37928..18973e7d 100644 --- a/docs/specs/compute-design.md +++ b/docs/specs/compute-design.md @@ -122,7 +122,8 @@ and obeys a verdict. ``` agent compute_status → mode=managed, balance_usd -agent compute_launch { gpu:"h100", count:1, budget_cents:3000, max_hourly_cents?, volume_id? } +agent compute_launch { gpu:"H100-SXM", count:1, budget_cents:3000, max_hourly_cents?, volume_id? } + (gpu is a canonical model id — "h100" is too coarse, interconnect changes price and throughput) OS → POST /api/compute/quote { gpu, count, max_hourly_cents, budget_cents } (change 4) ← { provider, sku, hourly_cents, effective_cap_cents, balance_cents, funding } ADVISORY @@ -278,6 +279,20 @@ death from 10 minutes to 10 minutes. **Both, or neither.** User leases then stay bounded by plan TTL, wallet exhaustion, explicit release, and the budget cap. The provider-terminal branch (`:117-131`) continues to apply to everything. +**Part (a) also produces the boot-time dataset** change 10 ranks on, because flipping to `ready` is what +stamps `ready_at`. + +#### `PROVISION_TIMEOUT_SECONDS` = 600 is a guess, and cheapest-first tests it + +It is a single global number (`config.py:70`) applied to every provider. Measured 7-day boot distributions +put RunPod's median near 59s but Vast's tail past **6 minutes** — so under cheapest-first, which sends +most launches to Vast, a slice of legitimate provisions runs close to the limit and some will exceed it. +A box reaped mid-provision is a launch the user paid for and never received. + +**Make the timeout per-provider and set it from the p99 of measured boot times** (change 10), not from a +round number. Until that data exists, raise it for the providers whose observed tail demands it rather +than leaving one value covering a 6× spread. + **Ships first, with its own test, before any budget work.** ### Change 1 — make `hard_cap_cents` a real running cap @@ -354,8 +369,27 @@ Accept `{gpu, count, max_hourly_cents?}` in place of an explicit `sku`; rank **e options by `price_cents_per_hour`; lease the cheapest match in the same request. Explicit `provider`/`sku` continues to work for the dashboard and the CLI. -Needs a canonical GPU-model map — providers spell the same card differently, and a substring match on -`name` will silently mis-rank. +Ranking is **cheapest above a reliability floor**, not cheapest outright — see change 10. + +#### The canonical GPU map + +Providers spell the same card differently, and a substring match on `name` will silently mis-rank. +`"h100"` is too coarse to be an input: **interconnect is part of the model identity**, and the three H100 +variants differ in both throughput and price. + +``` +A10 · A40 · A100-40GB-PCIe · A100-40GB-SXM · A100-80GB-PCIe · A100-80GB-SXM +H100-PCIe · H100-NVL · H100-SXM · H200-NVL · H200-SXM · B200 +L4 · L40 · L40S · RTX-3090 · RTX-4090 · RTX-5090 · RTX-6000-Ada +RTX-A6000 · RTX-PRO-6000 · RTX-PRO-6000-WK +``` + +The resolver takes a canonical id from this set plus `count`. Each provider module maps its own naming +into it, and an option that cannot be mapped is **excluded from ranking rather than guessed at** — a +mis-mapped card is a wrong machine at the wrong price, silently. + +This taxonomy is the one a comparable aggregator settled on, which is a reasonable signal that it is the +right granularity rather than over-specification. **The retry is not optional here.** Vast supplies most of the catalog and its SKUs are ephemeral offer IDs, so the cheapest pick is usually the raciest one. On a provider `400`, re-resolve against a re-fetched @@ -479,6 +513,62 @@ acquire that dies after key registration does not leak either. Ships with change 3. A resolver that makes Vast the common path without this is a resolver that turns a known defect into a scaling one. +### Change 10 — measure boot and availability, and rank on it + +**Cheapest per hour is a proxy that inverts.** Boot time is billed wall-clock: a six-minute provision on a +$2.16/h H100 spends $0.22 before any work starts, and a box that boots six times slower is not the cheap +one. A comparable aggregator publishes exactly this measurement across providers, and its 7-day +distribution is stark — RunPod a ~59s median with a tight spread, Vast a ~1m9s median with a tail running +past **6m** and a long scatter across the whole range. Vast is worse on the median and far worse on the +variance, and cheapest-first sends most launches to Vast. + +**The dataset is free once change 0(a) lands.** `compute_leases.ready_at` already exists +(`migrations.py:555`, `pg_migrations.py:806`) and `update_lease_status` already stamps it on the +transition to ready (`compute_repo.py:425-427`) — nothing populates it today only because nothing flips a +GPU lease to `ready`. So `ready_at - created_at` per lease is the boot-time series, and reaped or failed +launches are the availability series. No new instrumentation, no new table required to start. + +Aggregate per `(provider, canonical_gpu)` over a rolling window and use it two ways: + +- **A floor.** Exclude offers from a provider whose recent failure rate or p95 boot time is beyond + threshold, then rank the survivors by price. This keeps the product rule — cheapest wins — while + measuring "cheapest" correctly. +- **Timeouts** (below). + +**Cold start must degrade to today's behaviour.** With no history a provider is not penalised; ranking is +pure price until enough leases exist to say otherwise. A floor that silently excludes every provider on +day one is worse than no floor. + +Vendor-published signals are worth folding in where they exist and cost nothing — +`VastProvider.list_options` currently discards `reliability2`, `dlperf_per_dphtotal` and `inet_down` from +every offer it reads. `inet_down` matters more than it looks: pulling a 200 GB dataset at 50 Mbit instead +of 5 Gbit is hours of GPU time billed for waiting. **But measured beats published** — vendor scores +describe the host, our telemetry describes what actually happened to our leases. + +### Change 11 — pin the image, per provider + +**Nothing in this design says what is on the box**, and the answer today is inconsistent in a way that +breaks reproducibility: + +| Provider | Image | +| --------------- | ------------------------------------------------------------------------------- | +| RunPod | `runpod/pytorch:2.4.0-py3.11-cuda12.4.1-devel-ubuntu22.04` (pinned, has `nvcc`) | +| Vast | `pytorch/pytorch:latest` (`vast_provider.py:220`) — **a floating tag** | +| Lambda | none set — whatever the provider defaults to | +| Prime Intellect | none set | + +Under cheapest-first the provider varies per launch, so **the environment varies per launch too** — and +`latest` means the same experiment run a month apart gets a different toolchain with no record of it. +For a tool whose purpose is reproducible research, that is a correctness bug, not an ergonomics one. + +Define a **minimum environment contract** the agent may rely on and nothing more — a pinned CUDA runtime, +a pinned Python, and a package manager — and pin an image satisfying it for every provider the resolver +can select. Record the resolved image on the lease row so a run can be reproduced later. + +Anything beyond the contract the agent installs itself, and **that install is billed at GPU rates out of +the user's budget**, which is the argument for keeping the contract small and the image warm rather than +bootstrapping from bare Ubuntu on every launch. + ## OpenScience changes ### Three verbs @@ -503,8 +593,12 @@ Behaviour: - On `429` (concurrency cap) and `409` (already released, `routes/compute.py:531-535`), surface rather than retry. - A non-2xx or malformed launch response **writes no `.pem` and reports no lease**. -- If the readiness poll times out, **release the lease** and report. A paid box the agent cannot reach is - worse than no box. +- **The readiness poll outlives the server's provisioning timeout, and defers to it.** Poll until Atlas + reports a terminal status, bounded at that provider's `PROVISION_TIMEOUT_SECONDS` plus a margin. A + client bound set _shorter_ than the server's is the bug to avoid: measured Vast boots run past six + minutes, so a three-minute client timeout would kill a meaningful share of launches that were about to + succeed. On timeout with the lease still live, **release it** — a paid box the agent cannot reach is + worse than no box — but a lease the server has already reaped needs reporting, not releasing. - **No client-side deadline timer**, price table, or SKU ranking. ### SSH coordinates do not exist at launch @@ -622,6 +716,10 @@ creation stays in the workspace UI rather than becoming a fourth tool. specific one passes `provider`/`sku` explicitly, which still works. - **Durable storage narrows the pool.** Asking for a volume means paying the cheapest volume-capable provider, not the cheapest provider. +- **Boot time is billed and varies by minutes across providers.** The floor bounds how bad it gets; it + does not make a marketplace box boot like a datacenter one. +- **The environment is a contract, not a guarantee of parity.** Two providers satisfying the same pinned + CUDA/Python contract are still different machines. - **`none` is guidance, not enforcement.** The agent still has `bash`. ## Known defects, owned elsewhere @@ -667,8 +765,15 @@ mocks, no network. - Resolution picks the **globally cheapest** matching offer across all operator providers — asserted with a fake catalog where the cheapest match is deliberately not the first provider polled, and again where it is not the provider the previous test picked. A resolver that always returns one provider must fail. -- GPU-model matching is canonical, not substring: an `h100` request does not match `h100`-shaped names - from a different card, and does match the same card spelled differently across providers. +- GPU-model matching is canonical, not substring: `H100-SXM`, `H100-PCIe` and `H100-NVL` are three + distinct targets and never satisfy each other; the same card spelled differently across providers maps + to one id; an unmappable option is excluded rather than guessed. +- The reliability floor excludes a provider whose recent record is beyond threshold **and** leaves ranking + on pure price when there is no history — a cold-start floor that excludes everything must fail. +- Boot telemetry accumulates: `ready_at` is stamped on the transition to ready, and the rolling aggregate + reflects it. +- A provision that exceeds a short global timeout but fits that provider's measured p99 is **not** reaped. +- The resolved image is recorded on the lease row, and no provider is launched on a floating tag. - Resolution leases the offer it ranked; a provider `400` triggers at most N re-resolves against a re-fetched catalog, then a structured error. - Vast and Prime Intellect release **deletes the registered public key** — on normal release and on failed @@ -711,21 +816,28 @@ collapsed to one string. 8. `POST /quote` returns provider, SKU, rate, effective cap, balance and funding, and **spends nothing**. 9. Vast and Prime Intellect delete the registered public key on release **and** on failed launch, asserted against the provider's key list rather than the release return value. -10. `volume_id` attaches a volume that survives lease release. -11. Extension raises the cap when affordable, refuses with a structured `402` when not, never fires +10. Ranking applies a reliability floor derived from measured boot and failure history, and falls back to + pure price when there is no history. `ready_at` is stamped on the transition to ready. +11. `PROVISION_TIMEOUT_SECONDS` is per-provider and derived from measured boot times; a provision that + exceeds a short global constant but fits its provider's p99 is not reaped. +12. Every launchable provider has a pinned image satisfying the environment contract, no floating tags, + and the resolved image is recorded on the lease. +13. `volume_id` attaches a volume that survives lease release, and narrows the resolver to volume-capable + providers rather than failing. +14. Extension raises the cap when affordable, refuses with a structured `402` when not, never fires automatically. -12. BYOK ignores `budget_cents`. Plan TTL fires independently. -13. A release whose provider teardown fails is not reported as a clean release. -14. `compute_launch` on a non-2xx or malformed response writes no `.pem` and reports no lease; on a - readiness timeout it releases the lease; it never returns key material; it holds no pricing or - selection logic. -15. The key is written `0600` inside a `0700` directory, under `Global.Path.config`, and a re-fetch over +15. BYOK ignores `budget_cents`. Plan TTL fires independently. +16. A release whose provider teardown fails is not reported as a clean release. +17. `compute_launch` on a non-2xx or malformed response writes no `.pem` and reports no lease; its + readiness poll outlives the server's provisioning timeout and defers to the server's verdict; it never + returns key material; it holds no pricing or selection logic. +18. The key is written `0600` inside a `0700` directory, under `Global.Path.config`, and a re-fetch over an existing loose-mode file tightens it. -16. `compute_launch` calls `ctx.ask` and prompts by default; `permission.compute_launch: "allow"` silences +19. `compute_launch` calls `ctx.ask` and prompts by default; `permission.compute_launch: "allow"` silences it. -17. The connect string carries `-p ` when a real `/connection` payload reports a non-22 port. -18. `compute_list` filters terminated leases and reflects Atlas, not local state. -19. `pytest` and `bun test` pass with no network; change 0 and change 1 are each their own commit. +20. The connect string carries `-p ` when a real `/connection` payload reports a non-22 port. +21. `compute_list` filters terminated leases and reflects Atlas, not local state. +22. `pytest` and `bun test` pass with no network; change 0 and change 1 are each their own commit. --- From 6df6c051753201a309ab2640471245448adf6d6f Mon Sep 17 00:00:00 2001 From: KB Date: Fri, 31 Jul 2026 23:06:46 +0530 Subject: [PATCH 34/56] =?UTF-8?q?spec(compute):=20second=20review=20?= =?UTF-8?q?=E2=80=94=20one=20live=20security=20defect,=20four=20wrong=20cl?= =?UTF-8?q?aims?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A second adversarial pass found more than the first. Every finding below was re-verified independently against source before this rewrite. SECURITY, live in production today. VastProvider.acquire posts each lease's public key to the shared operator ACCOUNT (vast_provider.py :206-211), and the module docstring states the purpose: "registers the public key on the Vast account so new instances pick it up" (:10-16). Managed leases all run on one operator credential (:78). So one user's private key opens another user's box. The spec called this a leak into the operator account — hygiene — and proposed delete-on-release, which cannot fix it: concurrently live leases have their keys on the account by construction. Now change 9(a), blocking, and the fix is to drop the account POST in favour of the per-instance attach already at :244-248. Change 9 was also unimplementable as written. Prime returns the pod name where the key id belongs (prime_intellect_provider.py:274) and that name is identical across all of a user's leases; Vast keeps no identifier; there is no column to store one; and "on failed launch" has no hook outside each provider's acquire. Change 10's "free dataset" was wrong three ways, all from correct citations. ready_at is stamped at poll time (compute_repo.py:408, :426-428) under a 60s sweep (config.py:68), against a signal that is a 10s difference in medians. The sample is censored at the timeout it was meant to derive. And no availability series exists: nothing ever writes 'failed' to compute_leases. Cheapest-first ranked on the wrong column. price_cents_per_hour is the raw provider rate on both funding paths; what the user pays is price_cents_per_hour_display, zero on BYOK (routes/compute.py:161-162). The resolver would have preferred a billed offer over a free one. Criterion 22 mandated shipping a regression: change 1 alone IS the ~23h regression, since the default-grant fix lives in change 2. They now land together. Also: catalog cache and orphan-lease reap promoted to changes 12 and 13 (both were prerequisites with no ticket); image pinning is impossible on Lambda and Prime, so non-compliant providers are excluded from ranking the way volume_id already narrows it; change 8's release_pending status would have kept billing and holding a concurrency slot; the readiness poll had no status vocabulary to poll on and no endpoint returning the timeout it bounds on; the fan-out is 5 requests not 10; global/index.ts :46 is the cache dir, not config — the cited path would have written a private key under a cache path. --- docs/specs/compute-design.md | 498 +++++++++++++++++++++++++++-------- 1 file changed, 391 insertions(+), 107 deletions(-) diff --git a/docs/specs/compute-design.md b/docs/specs/compute-design.md index 18973e7d..c6d2ba07 100644 --- a/docs/specs/compute-design.md +++ b/docs/specs/compute-design.md @@ -71,7 +71,7 @@ is also why a BYOK user never pays for the availability network call. credential exists else `none`, `"managed"` → managed if available else `none`. **An override may narrow to `none`; it may never manufacture a capability.** -**Only `compute_status` resolves.** `SkillTool.init` calls `ComputeMode.offered()` (`src/tool/skill.ts:56`), +**Only `compute_status` resolves.** `SkillTool.init` calls `ComputeMode.offered()` (`src/tool/skill.ts:63`), which deliberately never reaches the availability probe — it is answerable from credentials alone, so a BYOK or keyless user pays no per-turn network cost (`src/compute/mode.ts:205-224`). Resolution is still per request rather than at startup, because credentials arrive at unpredictable times: the shell, the @@ -86,11 +86,19 @@ Availability is one authenticated `GET /api/compute/options`, 3s timeout (`mode. (`:107`). A failed, unauthenticated or timed-out call resolves to **unavailable** (`:139-148`) — failing toward `managed` would reproduce the original bug of promising an unconfirmed capability. -**That probe is expensive on the Atlas side and is not cached there.** `_catalog` fans out over all ten -`RESELL_PROVIDERS` with `asyncio.gather` (`routes/compute.py:34-45`, `:186-189`), each a live provider API -call (`:129-132`), and per-provider exceptions are swallowed into an empty option list. A paying managed -user therefore resolves to `none` whenever the aggregate exceeds the client's 3s budget. Fail-closed is -still the right default, but **the catalog needs a server-side cache** — and change 3 makes this urgent, +**That probe is expensive on the Atlas side and is not cached there.** `_catalog` gathers over +`RESELL_PROVIDERS` (`routes/compute.py:34-45`, `:186-189`) and per-provider exceptions are swallowed into +an empty option list (`:129-132`). + +Precisely: **five HTTP requests, not ten.** Six of the ten entries are `ScaffoldProvider`s with no +credential, and `:126-127` returns before any network call — their `list_options` is a pure-Python +sentinel stripped by the `sku` filter at `:133`. Only lambda / runpod / vast / prime_intellect are wired +(`main.py:144-151`), and Vast issues two requests (`vast_provider.py:122-134`). Five requests at 15–20s +provider timeouts still blows the client's 3s budget, so the conclusion holds — but the number matters, +because it is the quantitative argument for change 12. + +A paying managed user therefore resolves to `none` whenever the aggregate exceeds 3s. Fail-closed is still +right, but **the catalog needs a server-side cache** (change 12) — and change 3 makes it load-bearing, because a transiently-erroring Vast silently removes most of the catalog and changes which provider is "cheapest" with no signal to anyone. @@ -98,6 +106,19 @@ because a transiently-erroring Vast silently removes most of the catalog and cha filter, not a gate:** a hidden skill remains loadable by exact name and the agent still has `bash`. Gating the load path was considered and declined. +### The gap Part A left, which Part B must answer + +**RunPod and Vast have no catalogued skills** (`mode.ts:56`, `:60`). A user whose only credential is +`RUNPOD_API_KEY` therefore resolves to `byok`, is told to use the cloud-compute skills, is offered none — +and because `byok` wins whenever a credential exists, managed is suppressed at the same time (`:186`). +They have a key, no skills, and no managed path. + +Part B makes this sharper rather than fixing it: `ComputeTools` is registered unconditionally +(`src/tool/registry.ts:135`), so `compute_launch` stays callable while `compute_status` tells the agent +not to launch managed leases (`src/tool/compute.ts:22`). **The three new tools must state their behaviour +in `byok` and `none`** — refuse with the reason, rather than attempting a managed lease the mode says is +unavailable. No criterion covered this before. + --- # Part B — Managed leases (to build) @@ -193,12 +214,18 @@ up with a structured error rather than looping. cheapest live offer matching the requirements is the one leased. That is the product decision, and the rest of this document conforms to it. -Ranking is well-defined: every catalogued option carries a normalised `price_cents_per_hour` — the -provider's exact pass-through rate, set uniformly for all six providers at `routes/compute.py:158-163`. -Match on GPU model and `count`, honour `max_hourly_cents` if given, then take the minimum. The one real -implementation detail is GPU-name normalisation: options expose `name`, `gpu_ram_gb` and `upstream`, and -providers spell the same card differently, so the resolver needs a canonical model map rather than a -substring match. +**Rank on `price_cents_per_hour_display`, not `price_cents_per_hour`.** This distinction is the whole +correctness of the rule. `price_cents_per_hour` is the provider's raw pass-through rate and is set +identically on both funding paths (`routes/compute.py:161`); the column reflecting what the **user** pays +is `price_cents_per_hour_display`, which is `0` when that provider resolves to BYOK (`:162`). Funding is +decided per provider (`:117-123`), so a paid-plan user holding Atlas-stored keys for some providers and +not others has a genuinely mixed catalog — and ranking on the raw rate would prefer a **$1.90/h billed** +offer over a **$2.00/h free** one. Cheapest means cheapest _to the user_. + +Match on GPU model and `count`, honour `max_hourly_cents` if given, then take the minimum display rate. +The one real implementation detail is GPU-name normalisation: options expose `name`, `gpu_ram_gb` and +`upstream`, and providers spell the same card differently, so the resolver needs a canonical model map +rather than a substring match. An earlier draft of this section proposed an allow-list restricted to providers whose release cleans up after itself, which would have excluded Vast — and Vast supplies 204 of 292 live options. **That is @@ -218,8 +245,9 @@ out in consumer GPUs. ### What cheapest-first obliges us to fix -Every provider generates a fresh Ed25519 keypair per lease and shows the provider only the public half, -so the returned key opens exactly one box everywhere. They differ in what they leave behind: +Atlas generates a fresh Ed25519 keypair per lease and shows the provider only the public half. An earlier +draft concluded from that "the returned key opens exactly one box everywhere." **That is false on Vast, +and the consequence is cross-tenant.** | Provider | Key attachment | Account artifact | Cleaned up on release | | --------------- | ---------------------------------------------- | ---------------- | ---------------------------------------------- | @@ -232,10 +260,26 @@ RunPod leaves no account-level trace and its creation body takes an arbitrary `e setup needs no SSH bootstrap. That makes it the **easiest** provider to operate — it is not the default, and nothing in this design prefers it. -`VastProvider.release` deletes only `/instances/{lease_id}/` (`vast_provider.py:307-325`); Prime -Intellect's deletes only the pod. **Under cheapest-first, that is one public key leaked into the operator -account per launch, unbounded and permanent.** Lambda's delete-on-release (`lambda_provider.py:248-268`) -is the pattern. This is change 9, and it ships alongside the resolver rather than after it. +**Vast's account key registration is a live cross-tenant exposure, today, independent of anything in this +document.** `VastProvider.acquire` posts the per-lease public key to the **account** endpoint +(`vast_provider.py:206-211`), and the module docstring states the purpose plainly: _"registers the public +key on the Vast account (`POST /ssh/`) **so new instances pick it up**"_ (`:10-12`). Under managed funding +every user's lease runs on the single operator credential (`_headers_for` falls back to +`config.VAST_API_KEY`, `:78`). So one user's private key opens another user's box, for as long as the +first key sits on the account when the second instance is created. + +The docstring's justification — the account key is _"keyed by content and harmless to reuse"_ (`:15-16`) — +is true for a single-tenant account and false for a reseller. + +**Delete-on-release cannot fix this.** Concurrently live leases have their keys on the account by +construction, and `MANAGED_GPU_CONCURRENT` defaults to 2 per user (`config.py:117`) with no global bound. +The fix is to stop registering account-level keys at all: `acquire` already attaches the key to the +instance directly (`:244-248`), described in the code as the fallback _"so SSH works even if the account +key wasn't applied at launch"_. **Verify the per-instance attach alone suffices, then drop the account +`POST /ssh/`.** + +This is change 9, and it is a **blocking prerequisite** for making Vast reachable by default — not +something that ships alongside the resolver. --- @@ -273,6 +317,17 @@ must persist them**, which is what makes `compute_list` and the `/connection` po `create_lease` mints no runner token and the telemetry endpoint requires one, so **a user lease cannot prove liveness**. Scope the heartbeat check to leases holding a runner token. +**Pin that predicate to a column, because two different credentials are called "the runner token".** The +lease row carries `runner_api_key_id` (`migrations.py:568`), a `thk_*` value set only by the spawn path. +The token the telemetry endpoint authenticates is a separate `thrk_*` value in `runner_tokens` +(`agent_repo.py:371-381`, `routes/agent.py:1829-1840`). The predicate is `runner_api_key_id IS NOT NULL`; +leaving it as prose invites the implementation to guess. + +**Part (c) — normalise the status field.** `routes/compute.py:500` returns +`conn.get("status") or lease.get("status")`, mixing four provider vocabularies with the DB's own. The +client's readiness poll cannot be written against it as-is; see the OpenScience section. Normalising it +belongs here because change 0(a) is what introduces the `ready` transition in the first place. + Order matters: (b) alone changes nothing, because branch 2 kills the lease first. (a) alone moves the death from 10 minutes to 10 minutes. **Both, or neither.** @@ -363,11 +418,22 @@ Fix it deliberately: size the default grant to `rate * (ttl + 1)`, or have the c for the acquire debit so the two do not stack. **Assert unchanged runtime, not merely an accepted request.** +**Therefore change 1 must not ship alone.** An earlier draft required changes 0 and 1 to be separate +commits, which is right for attribution — but shipping change 1 by itself _is_ shipping the ~23h +regression to every existing caller, because the default-grant fix lives here in change 2. **Changes 1 +and 2 land together**, in that order, with criterion 4 gating the pair. Change 0 remains its own commit. + ### Change 3 — resolve the cheapest SKU from requirements _(mandatory)_ Accept `{gpu, count, max_hourly_cents?}` in place of an explicit `sku`; rank **every** operator provider's -options by `price_cents_per_hour`; lease the cheapest match in the same request. Explicit `provider`/`sku` -continues to work for the dashboard and the CLI. +options by `price_cents_per_hour_display` — the funding-adjusted rate, not the raw one; lease the cheapest +match in the same request. Explicit `provider`/`sku` continues to work for the dashboard and the CLI. + +**`count > 1` is thinner than the catalog size suggests.** RunPod hardcodes `"gpu": 1` in its options and +`"gpuCount": 1` on acquire (`runpod_provider.py:124`, `:156`), Vast dedups to one row per +`(gpu_name, count)` (`:142-156`), and Prime to one per `(gpuType, upstream)` +(`prime_intellect_provider.py:138-156`). A resolver tested only at `count = 1` will not exercise the path +most multi-GPU training needs. Ranking is **cheapest above a reliability floor**, not cheapest outright — see change 10. @@ -394,9 +460,9 @@ right granularity rather than over-specification. **The retry is not optional here.** Vast supplies most of the catalog and its SKUs are ephemeral offer IDs, so the cheapest pick is usually the raciest one. On a provider `400`, re-resolve against a re-fetched catalog, bounded to N attempts, then fail with a structured error rather than looping. -`GET /api/compute/options` (`routes/compute.py:214`) already does the read — but it is uncached and fans -out to ten live provider APIs, so N retries is N full catalog rebuilds. **Cache the catalog server-side -before shipping this**, or the retry path costs more than the lease. +`GET /api/compute/options` (`routes/compute.py:214`) already does the read — but it is uncached and costs +five provider requests, so N retries is N full catalog rebuilds and change 4 adds another per launch. +**Change 12 (catalog cache) ships before this**, or the retry path costs more than the lease. ### Change 4 — quote a proposal without spending @@ -429,13 +495,24 @@ Add a **rolling window cap** at lease creation, with window and ceiling as plan `gpu_sandbox_max_ttl_hours`, and a distinct `error` code so clients can tell "this box is too expensive" from "you have spent enough today". -**It must be atomic, and so must the wallet clamp.** Today's wallet check is a bare read-then-compare with -no reservation (`lease_manager.py:540-553`); nothing is held. With a default concurrency of 2, two -simultaneous launches on a $3 wallet both pass — and change 2's clamp makes it worse, because each clamps -to the _full_ effective balance and authorises 2× the wallet. A `SUM` over `compute_grants` followed by an -`INSERT` races identically. **The only atomic primitive in the money path today is `debit_grant` -(`compute_repo.py:187-201`); express the window cap the same way** — a single conditional write — or -serialise per user. +**It must be atomic, and so must the wallet clamp — but they need different mechanisms, and conflating +them is a vacuous-pass trap.** + +_The window cap_ can follow `debit_grant` (`compute_repo.py:187-201`), the one atomic primitive in the +money path: express it as a single conditional write rather than a `SUM` followed by an `INSERT`. + +_The wallet clamp cannot._ The balance lives behind `usage_service.effective_balance`, read with a bare +read-then-compare at `lease_manager.py:540-543` — and read **after** the grant was already debited at +`:529`. Two concurrent launches create two separate grants (`routes/compute.py:365`), each sized +`rate × 24h`, so `debit_grant` succeeds for both and the entire race sits in the wallet read. With a +default concurrency of 2 (`config.py:117`), two simultaneous launches on a $3 wallet both pass, and change +2's clamp makes it worse because each clamps to the _full_ effective balance and authorises 2× the wallet. + +The one reservation primitive that exists — the pre-auth `hold_id` path (`lease_manager.py:180-216`) — is +deliberately unused on the managed GPU path (`:456`, `:519-525`). So the wallet clamp needs either a real +hold or per-user serialisation, decided explicitly. **A `debit_grant`-shaped window cap satisfies +criterion 6's first half while the wallet still double-authorises**, which is exactly the failure mode the +closing section of this document is about. `compute_grants` is indexed on `user_id` and `session_id` only (`migrations.py:586-587`, `pg_migrations.py:1062-1063`). `(user_id, created_at)` is the migration. @@ -499,19 +576,55 @@ Add a distinct terminal-pending status the reaper re-sweeps, or at minimum surfa so the caller knows teardown failed. Until then, "explicit release works" is only true when the provider call succeeds. -### Change 9 — stop Vast and Prime Intellect leaking a key per lease _(forced by cheapest-first)_ - -`VastProvider.release` deletes only `/instances/{lease_id}/` (`vast_provider.py:307-325`) and Prime -Intellect's deletes only the pod (`prime_intellect_provider.py:306-325`). The per-lease public key stays -in the operator account permanently. - -This was a background annoyance while RunPod was the default. **Cheapest-first makes Vast the usual -winner, so the leak now grows one key per launch, forever.** Follow Lambda -(`lambda_provider.py:248-268`): delete the registered key on release **and** on failed launch, so an -acquire that dies after key registration does not leak either. - -Ships with change 3. A resolver that makes Vast the common path without this is a resolver that turns a -known defect into a scaling one. +**A new non-terminal status is not free, and the spec must choose.** Every gate is +`status NOT IN ('released','failed')` — `list_active_leases` (`compute_repo.py:481`, which drives billing +in `tick_once`), `count_active_managed_gpu_leases` (`:347`, which drives `MANAGED_GPU_CONCURRENT`), and +`list_unfinished_leases` (`:464`). So a `release_pending` status lands inside all three: **the user who +asked to release keeps being charged and keeps burning one of their two managed slots**, indefinitely, +because the retry that would clear it is the one that keeps failing. + +Decide both explicitly: **stop billing at the release request** (the user asked; the operator's continued +exposure is an operator problem, not theirs) and **free the concurrency slot**, while keeping the row +sweepable for teardown retry. That means the retry set cannot be `list_unfinished_leases` — it needs its +own query. + +### Change 9 — SSH key lifecycle _(security prerequisite, ships before change 3)_ + +Two distinct problems. The first is a live cross-tenant exposure; the second is hygiene that becomes +unbounded under cheapest-first. + +**9(a) — stop Vast registering account-level keys.** As above: `POST /ssh/` (`vast_provider.py:206-211`) +puts every lease's public key on the shared operator account so that new instances pick it up. Confirm the +per-instance attach at `:244-248` is sufficient on its own, then remove the account registration. If it is +not sufficient, the account key must be deleted **immediately after instance creation**, not on release — +deleting on release leaves it live for the whole lease, which is exactly the window that matters. + +Until 9(a) ships, managed Vast leases are mutually reachable. This is true of production today. + +**9(b) — delete the registered key on release and on failed launch.** Lambda is the pattern +(`lambda_provider.py:248-268`), but **it cannot simply be copied**, because Lambda works only through +machinery the other two lack: + +- Lambda's `acquire` persists `ssh_key_name = key_name` (`lambda_provider.py:196`), `_delete_ssh_key` + looks it up by that name (`:230-246`), and `release_lease` forwards it (`lease_manager.py:798-799`). +- **Prime Intellect persists the wrong string.** The key is registered under `key_name` + (`prime_intellect_provider.py:220`, `:228-230`) and the API returns an `id` (`:203`) used as `sshKeyId` + (`:245`) — but `acquire` returns `"ssh_key_name": name` (`:274`), the **pod** name, which is + `atlas-{user_id[:8]}` (`routes/compute.py:384`) and therefore identical across every lease that user + ever creates. A name-keyed delete would either miss or delete all of that user's keys. The `sshKeyId` is + discarded. +- **Vast has no identifier at all.** The key is posted as `{"ssh_key": public_openssh}` with no name + (`:207-211`) and the response is swallowed by a bare `except` (`:212-213`). +- **There is nowhere to store one.** `compute_leases` carries `ssh_key_name` and `ssh_public_key` only + (`migrations.py:549-553`, `pg_migrations.py:800-804`). +- **"On failed launch" has no hook.** `lease_manager.py:573-584` re-raises and + `_release_managed_reservation` (`:63-79`) touches only the grant and hold. Cleanup must live inside each + provider's `acquire`, as Lambda's does at `:186`. + +So 9(b) is: fix `prime_intellect_provider.py:274` to return the key id, add a nullable provider-key-id +column, capture Vast's response instead of discarding it, and put failure cleanup inside each `acquire`. +**Test against the provider's key listing, not against the release return value** — a release that returns +success having deleted nothing is the defect. ### Change 10 — measure boot and availability, and rank on it @@ -522,18 +635,48 @@ distribution is stark — RunPod a ~59s median with a tight spread, Vast a ~1m9s past **6m** and a long scatter across the whole range. Vast is worse on the median and far worse on the variance, and cheapest-first sends most launches to Vast. -**The dataset is free once change 0(a) lands.** `compute_leases.ready_at` already exists -(`migrations.py:555`, `pg_migrations.py:806`) and `update_lease_status` already stamps it on the -transition to ready (`compute_repo.py:425-427`) — nothing populates it today only because nothing flips a -GPU lease to `ready`. So `ready_at - created_at` per lease is the boot-time series, and reaped or failed -launches are the availability series. No new instrumentation, no new table required to start. - -Aggregate per `(provider, canonical_gpu)` over a rolling window and use it two ways: +**An earlier draft claimed this dataset was free once change 0(a) landed** — `compute_leases.ready_at` +exists (`migrations.py:555`, `pg_migrations.py:806`) and `update_lease_status` stamps it on the transition +to ready (`compute_repo.py:425-427`), so `ready_at - created_at` looked like a boot-time series for +nothing. **Every one of those citations is correct and the conclusion does not follow, three times over.** + +**It is quantised to the sweep interval, and the signal is smaller than the interval.** `ready_at` is +`_now()` evaluated when the reconcile _poll_ observes readiness (`compute_repo.py:408`, `:426-428`), not +when the box became ready. Change 0(a) mirrors `_reconcile_active_cpu_leases`, and every existing sweeper +ticks at 60s (`REAPER_TICK_SECONDS`, `config.py:68`; `COMPUTE_BILLING_TICK_SECONDS`, +`compute_billing_service.py:44`). The discrimination this change exists to make is a ~59s median against a +~1m9s median — **a 10s difference under 60s quantisation, biased upward on both sides.** As specified the +measurement cannot resolve the signal it ranks on. + +**It is censored at the timeout, which makes the timeout criterion circular.** A boot exceeding +`PROVISION_TIMEOUT_SECONDS` is reaped (`lease_reaper.py:133-136`) and never gets a `ready_at`, so the +sample is truncated at exactly the value we wanted to derive from it. The p99 of the surviving sample can +never exceed the timeout that produced the sample. + +**The availability series does not exist.** A reaped lease and a user-released lease write the same row: +`_reap` → `release_lease` → `compute_repo.release_lease` sets `status = 'released'` for both +(`compute_repo.py:549-556`). **Nothing ever writes `'failed'` to `compute_leases`** — the string appears +only inside `NOT IN` filters. The reap reason exists only on an `agent_telemetry` row +(`lease_reaper.py:79-85`), an unmentioned join, and compute-route leases have a `NULL` `node_id` +(`routes/compute.py:303`). + +So change 10 needs real instrumentation, and its cost must be priced rather than assumed away: + +- **Record boot duration explicitly**, not as a difference of two poll-quantised timestamps. Either poll + provisioning leases on a tighter cadence than the 60s sweep — which is a provider API call per + provisioning lease per tick, a cost this document must state — or take the provider's own ready + timestamp where it reports one. +- **Record a reap reason on the lease**, so released-by-user and reaped-for-timeout are distinguishable + without joining telemetry that compute leases do not write. +- **Derive timeouts from a deliberately uncensored window** — a period with the timeout raised well past + the expected tail — rather than from the post-reap sample. + +Then aggregate per `(provider, canonical_gpu)` over a rolling window and use it two ways: - **A floor.** Exclude offers from a provider whose recent failure rate or p95 boot time is beyond - threshold, then rank the survivors by price. This keeps the product rule — cheapest wins — while - measuring "cheapest" correctly. -- **Timeouts** (below). + threshold, then rank the survivors. This keeps the product rule — cheapest wins — while measuring + "cheapest" correctly. +- **Timeouts** (change 0). **Cold start must degrade to today's behaviour.** With no history a provider is not penalised; ranking is pure price until enough leases exist to say otherwise. A floor that silently excludes every provider on @@ -550,24 +693,68 @@ describe the host, our telemetry describes what actually happened to our leases. **Nothing in this design says what is on the box**, and the answer today is inconsistent in a way that breaks reproducibility: -| Provider | Image | -| --------------- | ------------------------------------------------------------------------------- | -| RunPod | `runpod/pytorch:2.4.0-py3.11-cuda12.4.1-devel-ubuntu22.04` (pinned, has `nvcc`) | -| Vast | `pytorch/pytorch:latest` (`vast_provider.py:220`) — **a floating tag** | -| Lambda | none set — whatever the provider defaults to | -| Prime Intellect | none set | +| Provider | Image | Pinnable? | +| --------------- | ------------------------------------------------------------------------------- | ----------------------------- | +| RunPod | `runpod/pytorch:2.4.0-py3.11-cuda12.4.1-devel-ubuntu22.04` (pinned, has `nvcc`) | yes — `imageName` in the body | +| Vast | `pytorch/pytorch:latest` (`vast_provider.py:220`) — **a floating tag** | yes — `image` in the body | +| Lambda | none set — whatever the provider defaults to | **no image parameter exists** | +| Prime Intellect | none set | **no image parameter exists** | Under cheapest-first the provider varies per launch, so **the environment varies per launch too** — and `latest` means the same experiment run a month apart gets a different toolchain with no record of it. For a tool whose purpose is reproducible research, that is a correctness bug, not an ergonomics one. Define a **minimum environment contract** the agent may rely on and nothing more — a pinned CUDA runtime, -a pinned Python, and a package manager — and pin an image satisfying it for every provider the resolver -can select. Record the resolved image on the lease row so a run can be reproduced later. +a pinned Python, and a package manager. Anything beyond it the agent installs itself, and **that install +is billed at GPU rates out of the user's budget**, which is the argument for keeping the contract small +and the image warm rather than bootstrapping from bare Ubuntu on every launch. + +**Only two of the four providers can be pinned at all.** Lambda's launch body is +`{region_name, instance_type_name, name, quantity, ssh_key_names}` (`lambda_provider.py:174-180`) and +Prime's pod body is `{name, cloudId, gpuType, socket, gpuCount, …}` (`prime_intellect_provider.py:231-249`) +— neither takes an image, and Prime fronts heterogeneous upstreams. An earlier draft demanded pinning for +"every provider the resolver can select" while its own table said two were unset; that criterion could not +have passed. + +Resolve it the same way change 6 resolves volumes: **satisfying the environment contract is a +requirement**, so a provider that cannot be pinned is excluded from ranking until its image story is +understood — Lambda ships a fixed OS image that may already satisfy the contract, in which case it is +declared compliant rather than pinned, and that determination is the work. This narrows the pool exactly +like `volume_id` does, and for the same reason: cheapest wins **among boxes that can do the job**. + +**Needs a migration.** `compute_leases` has no image column (`migrations.py:532-584`). Record the resolved +image on the lease so a run can be reproduced later. + +### Change 12 — cache the options catalog _(prerequisite for 3 and 4)_ + +`_catalog` is uncached, and every consumer rebuilds it: the availability probe on every `compute_status`, +`/quote` on every launch, `POST /leases` on every launch, and once more per resolver retry. Five provider +requests each at 15–20s timeouts. -Anything beyond the contract the agent installs itself, and **that install is billed at GPU rates out of -the user's budget**, which is the argument for keeping the contract small and the image warm rather than -bootstrapping from bare Ubuntu on every launch. +Two earlier sections say this must ship before change 3, and an earlier draft then filed it under "known +defects, owned elsewhere" — no number, no owner, no criterion. **It is a numbered change because two +others depend on it.** + +Cache per `(user_id, byok_eligible)` with a short TTL, and make the resolver's retry re-fetch +deliberately rather than reusing a cached miss — the retry exists precisely because the cached offer went +stale. + +### Change 13 — reap orphaned user leases + +Change 0 removes both reaper branches that currently apply to user leases, and `create_lease` mints no +runner token — `set_runner_api_key` has exactly one caller, the spawn path (`agent_tools.py:1755`) — so a +user lease can **never** be heartbeat-reaped by design. That is correct: it is a bound those leases cannot +satisfy. + +But it leaves a hole the rest of the design assumes is closed. If the OpenScience process dies between +`POST /leases` and the readiness poll's release — the exact case the release-on-timeout rule exists for — +**nothing releases the box.** `budget_cents` is optional (change 2) and the default grant is `rate × 24h`, +so the worst case is a full-TTL H100 the user never reached and whose key was never written to disk. + +Two options, and the spec picks the first: **make `budget_cents` mandatory on the tool path**, so an +orphan is bounded by the budget rather than the TTL. Second, additionally: reap a lease with no runner +token whose `/connection` has not been fetched in N minutes — liveness a user lease _can_ answer, unlike +telemetry. ## OpenScience changes @@ -593,14 +780,41 @@ Behaviour: - On `429` (concurrency cap) and `409` (already released, `routes/compute.py:531-535`), surface rather than retry. - A non-2xx or malformed launch response **writes no `.pem` and reports no lease**. -- **The readiness poll outlives the server's provisioning timeout, and defers to it.** Poll until Atlas - reports a terminal status, bounded at that provider's `PROVISION_TIMEOUT_SECONDS` plus a margin. A - client bound set _shorter_ than the server's is the bug to avoid: measured Vast boots run past six - minutes, so a three-minute client timeout would kill a meaningful share of launches that were about to - succeed. On timeout with the lease still live, **release it** — a paid box the agent cannot reach is - worse than no box — but a lease the server has already reaped needs reporting, not releasing. +- **The readiness poll outlives the server's provisioning timeout, and defers to it.** A client bound set + _shorter_ than the server's is the bug to avoid: measured Vast boots run past six minutes, so a + three-minute client timeout would kill launches that were about to succeed. On timeout with the lease + still live, **release it** — a paid box the agent cannot reach is worse than no box — but a lease the + server has already reaped needs reporting, not releasing. - **No client-side deadline timer**, price table, or SKU ranking. +Two prerequisites for that poll, neither of which exists today. + +**The bound is a server constant no endpoint returns.** `PROVISION_TIMEOUT_SECONDS` is server env config +(`config.py:70`) and change 0 makes it per-provider. It appears in no response — not `/options`, not +`/estimate`, not the proposed `/quote`, not the launch payload — and under cheapest-first the client does +not know which provider it got until the launch returns. Telling the client to bound at "that provider's +timeout" while exposing neither the provider nor the timeout means hardcoding a duplicate of a server +constant that silently goes stale. **Change 2's response must carry `provider` and +`provisioning_timeout_seconds`.** + +**There is no status vocabulary to poll on.** `routes/compute.py:500` returns +`conn.get("status") or lease.get("status")` — two disjoint vocabularies in one field. On the provider +path RunPod maps `running|stopped|terminated|unknown` (`runpod_provider.py:39-43`), Vast +`running|provisioning|stopped|unknown` (`:55-65`), Prime +`provisioning|active|stopped|error|terminating|terminated|unknown` (`prime_intellect_provider.py:42-51`), +and **Lambda passes the raw upstream string through unmapped** (`lambda_provider.py:214`, `:224`). When +the provider call throws it falls back to the DB status: `provisioning|ready|released`. + +So `"running"` is unreachable on the DB path, `"ready"` — what change 0(a) writes — is unreachable on the +provider path, Prime says `"active"`, Lambda says whatever Lambda says, and a dead RunPod pod maps to +`"unknown"`. **A client written against either vocabulary breaks on at least two of the four providers.** +Normalise the field as part of change 0 and enumerate the accepted values here; "poll until running" and +"poll until terminal" are both unimplementable as written. + +**Polling is not free either.** `get_lease_connection` calls `provider.connection()` on every poll +(`lease_manager.py:742-753`) — an uncached provider HTTP GET. A six-minute Vast boot polled every 5s is +~70 operator-key provider calls per launch. Specify the interval and a backoff. + ### SSH coordinates do not exist at launch Every provider's `acquire()` returns a hardcoded `ssh_port: 22` and **no `ssh_host`** — @@ -619,8 +833,10 @@ RunPod-shaped `/connection` payload, never a launch payload. ### The agent never holds key material -`compute_launch` writes the private key under `Global.Path.config` (`src/global/index.ts:46` — **not a -hardcoded `~/.config`**, which is wrong whenever `XDG_CONFIG_HOME` is set) at `compute/.pem`, +`compute_launch` writes the private key at `/compute/.pem` +(`src/global/index.ts:82`). **Not a hardcoded `~/.config`**, which is wrong whenever `XDG_CONFIG_HOME` is +set — and **not `:46`**, which an earlier draft cited and which is the _cache_ directory (`:47` is +config). A private key under a cache path is the wrong answer twice over. and returns only `key_path`. The key stays out of the transcript, out of compaction, and out of session storage. `compute_release` deletes it. @@ -642,8 +858,14 @@ offer a reliable download after a page reload. `GET /api/compute/leases` redacts (`_redact_lease:424`); the connection endpoint is the one that returns it. So the `.pem` on disk is a **cache, not a record**. If it is missing — new machine, cleaned config dir, -another session — re-fetch and rewrite it. **OpenScience keeps no durable local state for compute at -all.** +another session — re-fetch and rewrite it. OpenScience keeps no durable local state it would have to +reconcile against Atlas. + +**But a cache with no eviction is not a cache.** `compute_release` is the only deletion path, and every +_server-side_ termination bypasses it: wallet exhaustion (`compute_billing_service.py:172`), plan TTL +(`:342`), the reaper (`lease_reaper.py:58`). Each leaves a `0600` private key on disk permanently. Sweep +on `compute_list` — any local `.pem` whose lease Atlas reports as terminal gets deleted — so the eviction +rides on a call the agent already makes. `compute_list` calls `GET /api/compute/leases`, which returns `SELECT *` over every lease for the user, newest first (`compute_repo.py:446-453`). It returns **terminated leases too**, so the tool filters to @@ -729,8 +951,8 @@ creation stays in the workspace UI rather than becoming a fourth tool. - **CLI usability.** Prints the private key and never saves it, so the `ssh_command` it prints cannot work as shown. Recoverable — `/leases/{id}/connection` re-serves the key — but the CLI does not call it. No file transfer, no exec, no compute tests. -- **The options catalog is uncached** and fans out to ten live provider APIs per call. Change 3's retry - path makes this urgent rather than merely wasteful. +- **Six of the ten `RESELL_PROVIDERS` are `ScaffoldProvider` stubs** — registered, `operator: false`, zero + options, no network call. Scaffolding, not integrations. - **`budget_cents` already exists** on the agent-spawn path defaulting to `500` (`agent_tools.py:1386`, `models/agent.py:101`, `spawn_queue_service.py:69` → `create_grant` at `:1501`), display-only. Change 1 makes caps real, silently giving every shipped spawn a hard $5 kill. **Not a no-op.** @@ -760,24 +982,41 @@ mocks, no network. the box dies at ten minutes. - The GPU reconcile pass flips `provisioning` → `ready` **and persists `ssh_host`/`ssh_port`**. - A replayed tick does not double-charge and does not double-debit the grant. -- **Two concurrent launches against a wallet that funds only one**: exactly one succeeds. +- **Two concurrent launches against a wallet that funds only one**: exactly one succeeds. Asserted + separately from the rolling cap, because a `debit_grant`-shaped window cap passes the cap half while the + wallet still double-authorises. - The rolling cap rejects an N+1th lease even when each individual budget is affordable. - Resolution picks the **globally cheapest** matching offer across all operator providers — asserted with a fake catalog where the cheapest match is deliberately not the first provider polled, and again where it is not the provider the previous test picked. A resolver that always returns one provider must fail. +- Ranking is on the **funding-adjusted** rate: a catalog holding a cheaper billed offer and a dearer BYOK + offer resolves to the BYOK one. +- Resolution is exercised at `count > 1`, not only `count = 1`. - GPU-model matching is canonical, not substring: `H100-SXM`, `H100-PCIe` and `H100-NVL` are three distinct targets and never satisfy each other; the same card spelled differently across providers maps to one id; an unmappable option is excluded rather than guessed. - The reliability floor excludes a provider whose recent record is beyond threshold **and** leaves ranking on pure price when there is no history — a cold-start floor that excludes everything must fail. -- Boot telemetry accumulates: `ready_at` is stamped on the transition to ready, and the rolling aggregate - reflects it. +- Boot telemetry has **resolution finer than the difference it must detect** — a recorded duration, not a + difference of 60s-quantised poll timestamps — and a reaped lease is distinguishable from a + released-by-user one without joining `agent_telemetry`. - A provision that exceeds a short global timeout but fits that provider's measured p99 is **not** reaped. -- The resolved image is recorded on the lease row, and no provider is launched on a floating tag. +- The resolved image is recorded on the lease row, and no provider is launched on a floating tag. A + provider that cannot satisfy the environment contract is excluded from ranking, not launched anyway. - Resolution leases the offer it ranked; a provider `400` triggers at most N re-resolves against a re-fetched catalog, then a structured error. -- Vast and Prime Intellect release **deletes the registered public key** — on normal release and on failed - launch. Asserted on the provider's key list, not on the release return value. +- **Vast registers no account-level SSH key** — asserted against the account key listing after a launch, + which is the cross-tenant property, not the hygiene one. Then: a second user's instance created while + the first user's lease is live is **not** reachable with the first user's private key. +- Release deletes the provider-side key for Vast and Prime Intellect, on normal release and on failed + launch, asserted against the provider's key listing. Prime's stored identifier is the key id, not the + pod name — a test that passes with `name` in that field is testing nothing. +- The launch response carries `provider` and `provisioning_timeout_seconds`. +- The `/connection` status field is normalised: every provider's terminal and ready states map to one + documented vocabulary, including Lambda's unmapped upstream strings. +- A terminated lease's `.pem` is deleted on the next `compute_list`, including when the termination was + server-side. +- `compute_launch` in `byok` or `none` refuses with the reason rather than attempting a managed lease. - The quote spends nothing and creates no lease row. - `volume_id` mounts, and **release does not delete the volume**. - Extension raises the cap, is clamped, and **exhaustion proceeds normally when none arrives**. @@ -807,37 +1046,51 @@ collapsed to one string. 4. A lease created without `budget_cents` runs the full plan TTL — unchanged **runtime**, not merely an accepted request. 5. A budget exceeding the wallet is clamped, and the response reports the effective cap. -6. Two concurrent launches against a wallet funding one: exactly one succeeds. Same for the rolling cap. +6. **Separately asserted:** (a) two concurrent launches against a wallet funding one — exactly one + succeeds; (b) the rolling cap rejects an N+1th lease. One test covering both passes while the wallet + still double-authorises. 7. `{gpu, count, max_hourly_cents}` resolves to the **globally cheapest** matching offer across all - operator providers — proven against a catalog where the winner is neither the first provider polled nor - the same provider twice — honours `max_hourly_cents`, matches GPU models canonically rather than by - substring, retries a provider `400` against a re-fetched catalog at most N times, then fails with a - structured error. + operator providers, ranked on the **funding-adjusted** rate — a cheaper billed offer never beats a + dearer BYOK one — proven against a catalog where the winner is neither the first provider polled nor + the same provider twice, exercised at `count > 1`, honouring `max_hourly_cents`, matching GPU models + canonically, retrying a provider `400` against a re-fetched catalog at most N times, then failing with + a structured error. 8. `POST /quote` returns provider, SKU, rate, effective cap, balance and funding, and **spends nothing**. -9. Vast and Prime Intellect delete the registered public key on release **and** on failed launch, asserted - against the provider's key list rather than the release return value. -10. Ranking applies a reliability floor derived from measured boot and failure history, and falls back to - pure price when there is no history. `ready_at` is stamped on the transition to ready. -11. `PROVISION_TIMEOUT_SECONDS` is per-provider and derived from measured boot times; a provision that - exceeds a short global constant but fits its provider's p99 is not reaped. -12. Every launchable provider has a pinned image satisfying the environment contract, no floating tags, - and the resolved image is recorded on the lease. +9. **(a)** A launch registers no account-level SSH key on Vast, and a second user's instance created while + another user's lease is live is not reachable with that user's private key. **(b)** Release deletes the + provider-side key on Vast and Prime Intellect, on normal release and on failed launch, asserted against + the provider's key listing; Prime stores the key id, not the pod name. +10. Boot telemetry resolves finer than the difference it must detect, and a reaped lease is + distinguishable from a released-by-user one without joining `agent_telemetry`. Ranking applies a + reliability floor and falls back to pure price with no history. +11. `PROVISION_TIMEOUT_SECONDS` is per-provider and derived from an **uncensored** measurement window; a + provision that exceeds a short global constant but fits its provider's p99 is not reaped. +12. Every provider the resolver can select satisfies the environment contract with a non-floating image, + the resolved image is recorded on the lease, and a provider that cannot be pinned or declared + compliant is excluded from ranking. 13. `volume_id` attaches a volume that survives lease release, and narrows the resolver to volume-capable providers rather than failing. 14. Extension raises the cap when affordable, refuses with a structured `402` when not, never fires automatically. 15. BYOK ignores `budget_cents`. Plan TTL fires independently. -16. A release whose provider teardown fails is not reported as a clean release. -17. `compute_launch` on a non-2xx or malformed response writes no `.pem` and reports no lease; its - readiness poll outlives the server's provisioning timeout and defers to the server's verdict; it never - returns key material; it holds no pricing or selection logic. -18. The key is written `0600` inside a `0700` directory, under `Global.Path.config`, and a re-fetch over - an existing loose-mode file tightens it. -19. `compute_launch` calls `ctx.ask` and prompts by default; `permission.compute_launch: "allow"` silences +16. A release whose provider teardown fails is not reported as a clean release, **stops billing**, and + **frees the concurrency slot** while remaining sweepable for teardown retry. +17. An orphaned lease — client dead between creation and the first poll — is bounded by its budget, not + only by the 24h TTL. +18. `compute_launch` on a non-2xx or malformed response writes no `.pem` and reports no lease; its + readiness poll is bounded by the `provisioning_timeout_seconds` the launch response carried, against a + normalised status vocabulary; it never returns key material; it holds no pricing or selection logic. +19. The key is written `0600` inside a `0700` directory, under `Global.Path.config` (**not the cache + dir**), and a re-fetch over an existing loose-mode file tightens it. +20. A terminated lease's `.pem` is deleted on the next `compute_list`, including on server-side + termination. +21. `compute_launch` calls `ctx.ask` and prompts by default; `permission.compute_launch: "allow"` silences it. -20. The connect string carries `-p ` when a real `/connection` payload reports a non-22 port. -21. `compute_list` filters terminated leases and reflects Atlas, not local state. -22. `pytest` and `bun test` pass with no network; change 0 and change 1 are each their own commit. +22. The connect string carries `-p ` when a real `/connection` payload reports a non-22 port. +23. `compute_list` filters terminated leases and reflects Atlas, not local state. +24. `compute_launch` in `byok` or `none` refuses with the reason rather than attempting a managed lease. +25. `pytest` and `bun test` pass with no network. Change 0 is its own commit; **changes 1 and 2 land + together** — shipping 1 alone ships the ~23h regression. --- @@ -856,8 +1109,10 @@ Labels in this document: succeeds. **The 204-of-292 Vast share is load-bearing — it is why cheapest-first makes Vast the common path, and therefore why change 9 is in scope — and it comes from this production check, not from source.** -- **Verified against the Atlas checkout at HEAD `7b0e9b6` (source-read, 2026-07-31):** every `file:line` - citation in Part B, re-checked after review. +- **Verified against the Atlas checkout at HEAD `7b0e9b6` (source-read, 2026-07-31):** the `file:line` + citations in Part B, re-checked after two adversarial reviews. The claim "every citation verified" + has now been falsified twice; treat it as "checked, not guaranteed", and check the one you are about to + build on. - **Not verified:** `compute:up`'s internal fetch→pick→estimate sequence. The Atlas CLI source is in neither repo; the commit chronology corroborates the shape but the sequence itself is inferred. @@ -889,3 +1144,32 @@ were, and an adversarial pass over the same source found: Two lessons, both cheap to apply: a citation that is literally correct can still fail to support the claim built on it, and the acceptance criteria are where a wrong mechanism hides — a criterion that cannot fail is worse than no criterion. + +**A second adversarial pass, after that rewrite, found more — including one live security defect.** + +- **Vast registers every lease's public key on the shared operator account** so that new instances pick it + up (`vast_provider.py:206-211`, docstring `:10-16`). One user's private key opens another user's box. + The draft called this "leaked into the operator account", i.e. hygiene, and proposed delete-on-release — + which cannot fix it, because concurrently live leases have their keys on the account by construction. + **This is true of production today and is now change 9(a), a blocking prerequisite.** +- **Change 9 was unimplementable as "follow Lambda".** Prime stores the pod name where the key id belongs + (`prime_intellect_provider.py:274`, identical across all of a user's leases), Vast keeps no identifier at + all, there is no column to store one, and "on failed launch" has no hook outside each provider's + `acquire`. +- **"The dataset is free" was wrong three ways.** `ready_at` is stamped at poll time under a 60s sweep, + against a signal that is a 10s difference; the sample is censored at the timeout it was meant to derive; + and no availability series exists, because **nothing ever writes `'failed'`** to `compute_leases`. All + three citations were literally correct — the same failure as the first pass, one layer deeper. +- **Cheapest-first ranked on the wrong column.** `price_cents_per_hour` is the raw provider rate on both + funding paths; what the user pays is `price_cents_per_hour_display`, zero on BYOK. The resolver would + have preferred a billed offer over a free one. +- **Criterion 22 mandated shipping a regression** — change 1 as its own commit _is_ the ~23h regression, + because the fix lives in change 2. +- **Three criteria could pass while broken:** the concurrency one (satisfied by the window cap while the + wallet races), the resolver one (silent on funding), the readiness one (no status vocabulary exists, and + the timeout it bounds on is returned by no endpoint). +- **A citation miss with teeth:** `global/index.ts:46` is the **cache** directory, not config. Following it + writes a private key to a cache path. + +The pattern is now three-for-three: **every round, the errors are in claims built on correct citations, +and in criteria that cannot fail.** Both are cheap to check and neither is caught by re-reading. From b677b4536178eba8750e3d2d88e28a18953aa910 Mon Sep 17 00:00:00 2001 From: KB Date: Fri, 31 Jul 2026 23:34:13 +0530 Subject: [PATCH 35/56] =?UTF-8?q?plan(compute):=20lease=20prerequisites=20?= =?UTF-8?q?=E2=80=94=20Vast=20key=20exposure=20and=20the=20reaper?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four tasks in the atlas repo, scoped to what every other compute change waits on: the cross-tenant SSH key on Vast, and the two reaper branches that kill a user lease ten minutes after it is created. Deliberately excludes budgets, the resolver, volumes and the OpenScience tools. Until a lease survives, none of them can be observed to work. --- .../2026-07-31-compute-lease-prerequisites.md | 763 ++++++++++++++++++ 1 file changed, 763 insertions(+) create mode 100644 docs/plans/2026-07-31-compute-lease-prerequisites.md diff --git a/docs/plans/2026-07-31-compute-lease-prerequisites.md b/docs/plans/2026-07-31-compute-lease-prerequisites.md new file mode 100644 index 00000000..028c02fc --- /dev/null +++ b/docs/plans/2026-07-31-compute-lease-prerequisites.md @@ -0,0 +1,763 @@ +# Compute lease prerequisites — implementation plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development to implement +> this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Close the live cross-tenant SSH exposure on Vast, and make a user-launched GPU lease survive long +enough to be usable — the two things every other compute change waits on. + +**Architecture:** All four tasks are in the **Atlas** repo (`~/codes/InkVell/atlas`), Python/FastAPI. Task 1 +removes an account-level SSH key registration. Tasks 2–3 fix the lease reaper: today a managed GPU lease +never leaves `provisioning` status, so it is reaped as a provisioning timeout ~10 minutes after creation. +Task 4 normalises the status vocabulary the OpenScience client will later poll on. + +**Tech Stack:** Python 3.12, FastAPI, aiosqlite, pytest + pytest-asyncio, respx (HTTP mocking), httpx. + +**Spec:** `~/codes/InkVell/openscience/docs/specs/compute-design.md` — changes 9(a), 0(a), 0(b), 0(c). + +## Global Constraints + +- **Repo:** `~/codes/InkVell/atlas`. All paths below are relative to that repo root. +- **Run tests with:** `cd ~/codes/InkVell/atlas/backend && .venv/bin/python -m pytest -q` + There is no activated virtualenv; always invoke `.venv/bin/python` explicitly. +- **Branch:** work on `feat/compute-lease-prerequisites`, cut from `main`. The repo is currently on + `feat/managed-catalog-opus5-frontier`, which is unrelated work — **do not commit onto it.** +- **Never add a `Co-Authored-By:` trailer or any AI attribution to commits.** Organisation rule. +- **No mocks of our own code.** Stub HTTP at the transport boundary with `respx`; use the real repo and + provider objects. This matches the existing suite. +- **Every new assertion must be shown failing first.** Run the test before writing the implementation and + paste the failure. A test that has never failed is not evidence. +- **Ignore `.claude/worktrees/`** — other branches, will mislead greps. +- Baseline before starting: `.venv/bin/python -m pytest tests/test_lease_reaper.py -q` → **7 passed**. + +--- + +### Task 0: Branch setup + +- [ ] **Step 1: Cut the branch from main** + +```bash +cd ~/codes/InkVell/atlas +git fetch origin +git checkout -b feat/compute-lease-prerequisites origin/main +``` + +- [ ] **Step 2: Confirm the baseline suite passes** + +Run: `cd ~/codes/InkVell/atlas/backend && .venv/bin/python -m pytest tests/test_lease_reaper.py tests/test_compute_providers.py -q` +Expected: all pass. If anything fails here, stop and report — it is pre-existing and not yours to fix. + +--- + +### Task 1: Vast must not register account-level SSH keys + +**Why:** `VastProvider.acquire` posts every lease's public key to the **shared operator account** +(`POST /ssh/`), and the module docstring says the purpose is "so new instances pick it up" +(`app/compute/vast_provider.py:10-12`). Under managed funding all users share one operator credential +(`:78`), so one user's private key opens another user's instance. The per-instance attach at `:244-248` +already exists and its comment describes it as the path that works "even if the account key wasn't applied +at launch" — i.e. it is the reliable one. + +**Files:** + +- Modify: `backend/app/compute/vast_provider.py:202-213` (remove the account POST), `:10-16` (docstring) +- Create: `backend/tests/test_compute_vast_provider_http.py` + +**Interfaces:** + +- Consumes: nothing from earlier tasks. +- Produces: nothing later tasks depend on. Standalone security fix. + +- [ ] **Step 1: Write the failing test** + +Create `backend/tests/test_compute_vast_provider_http.py`: + +```python +"""HTTP-contract tests for the Vast provider. + +Regression anchor: ``acquire`` must NOT register the per-lease public key on +the Vast *account*. Managed leases all run on one operator credential, so an +account-level key is readable by every subsequently created instance — one +user's private key would open another user's box. The per-instance attach is +the only key path. +""" + +from __future__ import annotations + +import httpx +import respx + +from app.compute.vast_provider import VAST_API, VastProvider + +CREDS = {"secret": "vast-test-key"} + + +@respx.mock +async def test_acquire_registers_no_account_ssh_key(): + account = respx.post(f"{VAST_API}/ssh/").mock( + return_value=httpx.Response(200, json={}) + ) + respx.put(f"{VAST_API}/asks/9999/").mock( + return_value=httpx.Response(200, json={"new_contract": 4242}) + ) + instance = respx.post(f"{VAST_API}/instances/4242/ssh/").mock( + return_value=httpx.Response(200, json={}) + ) + + lease = await VastProvider().acquire("9999", "us", user_credentials=CREDS) + + assert lease["lease_id"] == "4242" + assert instance.called, "the per-instance key attach is the only key path" + assert not account.called, "account-level key registration is cross-tenant readable" + + +@respx.mock +async def test_acquire_still_returns_the_private_key(): + respx.put(f"{VAST_API}/asks/9999/").mock( + return_value=httpx.Response(200, json={"new_contract": 4242}) + ) + respx.post(f"{VAST_API}/instances/4242/ssh/").mock( + return_value=httpx.Response(200, json={}) + ) + + lease = await VastProvider().acquire("9999", "us", user_credentials=CREDS) + + assert lease["ssh_private_key"].startswith("-----BEGIN") + assert lease["ssh_public_key"].startswith("ssh-ed25519") + assert lease["status"] == "provisioning" +``` + +- [ ] **Step 2: Run the test to verify it fails** + +Run: `cd ~/codes/InkVell/atlas/backend && .venv/bin/python -m pytest tests/test_compute_vast_provider_http.py -q` +Expected: `test_acquire_registers_no_account_ssh_key` FAILS on +`assert not account.called`. The second test should already pass — it pins behaviour we must not break. +**Paste the failure output into your report.** + +- [ ] **Step 3: Remove the account-level registration** + +In `backend/app/compute/vast_provider.py`, delete this block from `acquire` (currently `:203-213`): + +```python + # Register the public key on the account so the new instance + # picks it up; best-effort (a content-identical key may already + # exist, which Vast tolerates). + try: + await client.post( + f"{VAST_API}/ssh/", + headers=headers, + json={"ssh_key": public_openssh}, + ) + except Exception: # noqa: BLE001 + pass +``` + +Then update the per-instance attach comment (currently `:240-241`) to say it is the only key path: + +```python + # Attach the per-lease key to the instance. This is the ONLY key + # path: the account-level POST /ssh/ that used to run here made the + # key readable by every instance created afterwards on the same + # (shared, operator) account. +``` + +- [ ] **Step 4: Correct the module docstring** + +Replace lines 10-16 of `backend/app/compute/vast_provider.py`: + +```python +SSH: Atlas generates a fresh keypair per lease and attaches the *public* key +to the instance (``POST /instances/{id}/ssh/``). It is deliberately NOT +registered on the account (``POST /ssh/``): managed leases share one operator +credential, so an account key is picked up by every instance created after it, +which would let one user's private key open another user's box. +``` + +- [ ] **Step 5: Run the tests to verify they pass** + +Run: `cd ~/codes/InkVell/atlas/backend && .venv/bin/python -m pytest tests/test_compute_vast_provider_http.py tests/test_compute_providers.py -q` +Expected: all PASS. + +- [ ] **Step 6: Commit** + +```bash +cd ~/codes/InkVell/atlas +git add backend/app/compute/vast_provider.py backend/tests/test_compute_vast_provider_http.py +git commit -m "fix(vast): stop registering per-lease SSH keys on the shared account + +acquire posted each lease's public key to the operator account so that new +instances would pick it up. Managed leases all run on one operator credential, +so every instance created afterwards accepted that key -- one user's private +key opened another user's box. + +The per-instance attach already present is the only key path needed; its own +comment described it as working even when the account key was not applied." +``` + +- [ ] **Step 7: Flag the deploy gate in your report** + +This cannot be fully verified without a live Vast launch. **State in your report** that before this +deploys, someone must lease one real Vast box and confirm SSH still works with the returned key. The +code comment suggests the per-instance attach is sufficient; that is an inference, not a measurement. + +--- + +### Task 2: Scope heartbeat reaping to leases that can answer it + +**Why:** `sweep_once` branch 3 reaps any non-`provisioning` lease whose telemetry is stale past +`HEARTBEAT_STALE_SECONDS` (600). `create_lease` mints no runner token — `set_runner_api_key` has exactly +one caller, the agent-spawn path — so a user lease can never emit telemetry and can never pass this check. +Task 3 makes leases reach a non-`provisioning` status, which is what would expose this; do it first so the +two land in either order safely. + +The predicate is the **`runner_api_key_id` column** (`backend/app/db/migrations.py:567`). Do not confuse it +with the `thrk_*` runner token in the `runner_tokens` table — different credential, different table. + +**Files:** + +- Modify: `backend/app/jobs/lease_reaper.py:138-144` +- Test: `backend/tests/test_lease_reaper.py` (append) + +**Interfaces:** + +- Consumes: nothing. +- Produces: nothing. Behaviour change only. + +- [ ] **Step 1: Write the failing tests** + +Append to `backend/tests/test_lease_reaper.py`: + +```python +@pytest.mark.asyncio +async def test_user_lease_without_runner_token_is_not_heartbeat_reaped(test_db, monkeypatch): + """A user-launched lease has no runner token, so it can never emit + telemetry. Reaping it for heartbeat staleness destroys a box the user is + paying for, ~10 minutes after it becomes ready.""" + db = test_db["db"]; uid = test_db["user_id"] + await compute_repo.create_lease(db, lease_id="Luser", status="ready", + user_id=uid, provider="runpod", requested_sku="h100", region="us", + hourly_rate_cents=279, category="gpu") + # No telemetry at all, and created long ago -> branch 3 would fire today. + old = _iso(datetime.now(timezone.utc) - timedelta(seconds=9999)) + await db.execute( + "UPDATE compute_leases SET created_at = ?, started_at = ? WHERE lease_id = 'Luser'", + (old, old)) + await db.commit() + + class _P: + async def status(self, _id): return {"status": "running"} + monkeypatch.setattr(reaper, "get_provider", lambda _name: _P()) + async def _noop_release(self, _db, _lease): return {"status": "released"} + monkeypatch.setattr(reaper.LeaseManager, "release_lease", _noop_release) + + n = await reaper.sweep_once(db, now=datetime.now(timezone.utc)) + + assert n == 0 + row = await compute_repo.get_lease(db, "Luser") + assert row["status"] == "ready" + + +@pytest.mark.asyncio +async def test_agent_lease_with_runner_token_is_still_heartbeat_reaped(test_db, monkeypatch): + """The narrowing must not disable the check for leases that CAN report.""" + db = test_db["db"]; uid = test_db["user_id"] + await compute_repo.create_lease(db, lease_id="Lagent", status="ready", + user_id=uid, provider="modal", requested_sku="cpu-small", region="us", + hourly_rate_cents=0, category="cpu", node_id="na") + old = _iso(datetime.now(timezone.utc) - timedelta(seconds=9999)) + await db.execute( + "UPDATE compute_leases SET created_at = ?, started_at = ?, " + "runner_api_key_id = 'thk_test' WHERE lease_id = 'Lagent'", (old, old)) + await db.commit() + + class _P: + async def status(self, _id): return {"status": "running"} + monkeypatch.setattr(reaper, "get_provider", lambda _name: _P()) + reaped = [] + async def _fake_reconcile(_db, *, lease_id, node_id, forced_state=None, reason=None): + reaped.append((lease_id, reason)); return {"outcome": "failed"} + monkeypatch.setattr(reaper.run_reconcile_service, "reconcile_completed_run", _fake_reconcile) + async def _noop_release(self, _db, _lease): return {"status": "released"} + monkeypatch.setattr(reaper.LeaseManager, "release_lease", _noop_release) + + n = await reaper.sweep_once(db, now=datetime.now(timezone.utc)) + + assert n == 1 + assert reaped[0] == ("Lagent", "heartbeat_timeout") +``` + +- [ ] **Step 2: Run to verify the first fails and the second passes** + +Run: `cd ~/codes/InkVell/atlas/backend && .venv/bin/python -m pytest tests/test_lease_reaper.py -q` +Expected: `test_user_lease_without_runner_token_is_not_heartbeat_reaped` FAILS (`assert 1 == 0`). +`test_agent_lease_with_runner_token_is_still_heartbeat_reaped` PASSES already — it pins the behaviour the +narrowing must preserve. **Paste the failure.** + +- [ ] **Step 3: Narrow branch 3** + +In `backend/app/jobs/lease_reaper.py`, replace the branch-3 block: + +```python + # 3. Heartbeat staleness — only for leases that have actually booted + # AND are supposed to report. A still-`provisioning` lease legitimately + # has no telemetry yet (PROVISION_TIMEOUT_SECONDS governs it), and a + # user-launched lease is issued no runner token at all, so it can never + # emit telemetry — reaping it would destroy a box the user is paying + # for. `runner_api_key_id` is set only by the agent-spawn path. + if ( + reason is None + and lease.get("status") != "provisioning" + and lease.get("runner_api_key_id") + ): + latest = _parse_iso(live["latest_at"]) or _lease_started(lease) + if latest and (now - latest).total_seconds() > config.HEARTBEAT_STALE_SECONDS: + reason = "heartbeat_timeout" +``` + +- [ ] **Step 4: Run the tests to verify they pass** + +Run: `cd ~/codes/InkVell/atlas/backend && .venv/bin/python -m pytest tests/test_lease_reaper.py -q` +Expected: 9 passed. If `test_sweep_reaps_heartbeat_stale` (the pre-existing one) now fails, its fixture +lease has no `runner_api_key_id` — **fix the fixture, not the predicate**: add +`runner_api_key_id = 'thk_test'` to that lease, since it models an agent-spawned lease. + +- [ ] **Step 5: Commit** + +```bash +cd ~/codes/InkVell/atlas +git add backend/app/jobs/lease_reaper.py backend/tests/test_lease_reaper.py +git commit -m "fix(reaper): heartbeat staleness only applies to leases with a runner token + +create_lease mints no runner token, so a user-launched lease can never emit +telemetry and can never satisfy the heartbeat check. It stayed bounded by plan +TTL, wallet exhaustion and explicit release; it does not need a liveness probe +it has no way to answer." +``` + +--- + +### Task 3: Promote a provisioning lease to ready and persist its SSH coordinates + +**Why:** two facts combine into the defect. `RunPodProvider.acquire` returns `status: "provisioning"` and +`create_lease` persists it; and the only two writers that flip a lease to `ready` are +`_reconcile_active_cpu_leases` (CPU-only, `lease_manager.py:270`) and `LeaseManager.get_lease_status`, +which has **no production caller** (`:690`; only two test callers). So a managed GPU lease stays +`provisioning` forever and branch 2 reaps it at `PROVISION_TIMEOUT_SECONDS` = 600. + +Separately, `acquire` returns a hardcoded `ssh_port: 22` and no `ssh_host` on every provider — the real +values live only in `provider.connection()`. `update_lease_status` already accepts them +(`compute_repo.py:397-405`) and nothing passes them. + +Do both in the reaper sweep, which already runs every 60s and **already calls `provider.status()` for +every unfinished lease** at branch 1 — so the promotion check costs no extra provider call. The +`connection()` call happens once, on the transition only. + +**Files:** + +- Modify: `backend/app/jobs/lease_reaper.py` (add `_PROVIDER_READY`, add promotion between branches 1 and 2) +- Test: `backend/tests/test_lease_reaper.py` (append) + +**Interfaces:** + +- Consumes: nothing. +- Produces: leases now reach `status = "ready"` with `ssh_host` / `ssh_port` / `ready_at` populated. Task 4 + and all later compute work depend on this. + +- [ ] **Step 1: Write the failing tests** + +Append to `backend/tests/test_lease_reaper.py`: + +```python +@pytest.mark.asyncio +async def test_sweep_promotes_provisioning_lease_and_persists_ssh(test_db, monkeypatch): + """A GPU lease is created 'provisioning' and nothing in production ever + moves it. The sweep must promote it once the provider says it is up, and + record the SSH coordinates, which acquire() cannot know.""" + db = test_db["db"]; uid = test_db["user_id"] + await compute_repo.create_lease(db, lease_id="Lprom", status="provisioning", + user_id=uid, provider="runpod", requested_sku="h100", region="us", + hourly_rate_cents=279, category="gpu") + + class _P: + async def status(self, _id): return {"status": "running"} + async def connection(self, _id, **kw): + return {"status": "running", "ssh_host": "194.68.245.162", + "ssh_port": 22065, "ssh_user": "root"} + monkeypatch.setattr(reaper, "get_provider", lambda _name: _P()) + + n = await reaper.sweep_once(db, now=datetime.now(timezone.utc)) + + assert n == 0 # promotion is not a reap + row = await compute_repo.get_lease(db, "Lprom") + assert row["status"] == "ready" + assert row["ssh_host"] == "194.68.245.162" + assert row["ssh_port"] == 22065 + assert row["ready_at"] is not None + + +@pytest.mark.asyncio +async def test_sweep_does_not_promote_a_lease_still_booting(test_db, monkeypatch): + db = test_db["db"]; uid = test_db["user_id"] + await compute_repo.create_lease(db, lease_id="Lboot", status="provisioning", + user_id=uid, provider="runpod", requested_sku="h100", region="us", + hourly_rate_cents=279, category="gpu") + + class _P: + async def status(self, _id): return {"status": "provisioning"} + async def connection(self, _id, **kw): + raise AssertionError("connection() must not be called before the provider is up") + monkeypatch.setattr(reaper, "get_provider", lambda _name: _P()) + + await reaper.sweep_once(db, now=datetime.now(timezone.utc)) + + row = await compute_repo.get_lease(db, "Lboot") + assert row["status"] == "provisioning" + assert row["ready_at"] is None + + +@pytest.mark.asyncio +async def test_promotion_survives_a_connection_failure(test_db, monkeypatch): + """Provider is up but connection() errors: still promote, so the lease is + not reaped as a provisioning timeout. Coordinates arrive on a later sweep.""" + db = test_db["db"]; uid = test_db["user_id"] + await compute_repo.create_lease(db, lease_id="Lconn", status="provisioning", + user_id=uid, provider="runpod", requested_sku="h100", region="us", + hourly_rate_cents=279, category="gpu") + + class _P: + async def status(self, _id): return {"status": "running"} + async def connection(self, _id, **kw): raise RuntimeError("provider 503") + monkeypatch.setattr(reaper, "get_provider", lambda _name: _P()) + + await reaper.sweep_once(db, now=datetime.now(timezone.utc)) + + row = await compute_repo.get_lease(db, "Lconn") + assert row["status"] == "ready" + assert row["ssh_host"] is None + + +@pytest.mark.asyncio +async def test_promoted_lease_is_not_reaped_at_the_provisioning_timeout(test_db, monkeypatch): + """The headline property: a user lease created 20 minutes ago survives. + Today it dies at PROVISION_TIMEOUT_SECONDS = 600.""" + db = test_db["db"]; uid = test_db["user_id"] + await compute_repo.create_lease(db, lease_id="Llive", status="provisioning", + user_id=uid, provider="runpod", requested_sku="h100", region="us", + hourly_rate_cents=279, category="gpu") + old = _iso(datetime.now(timezone.utc) - timedelta(seconds=1200)) + await db.execute( + "UPDATE compute_leases SET created_at = ?, started_at = ? WHERE lease_id = 'Llive'", + (old, old)) + await db.commit() + + class _P: + async def status(self, _id): return {"status": "running"} + async def connection(self, _id, **kw): + return {"ssh_host": "1.2.3.4", "ssh_port": 20095, "ssh_user": "root"} + monkeypatch.setattr(reaper, "get_provider", lambda _name: _P()) + async def _noop_release(self, _db, _lease): return {"status": "released"} + monkeypatch.setattr(reaper.LeaseManager, "release_lease", _noop_release) + + n = await reaper.sweep_once(db, now=datetime.now(timezone.utc)) + + assert n == 0, "a live 20-minute-old lease must not be reaped" + row = await compute_repo.get_lease(db, "Llive") + assert row["status"] == "ready" +``` + +- [ ] **Step 2: Run to verify all four fail** + +Run: `cd ~/codes/InkVell/atlas/backend && .venv/bin/python -m pytest tests/test_lease_reaper.py -q -k "promot or booting or conn or reaped_at_the"` +Expected: `test_sweep_promotes...` FAILS (`status == 'provisioning'`), +`test_promotion_survives_a_connection_failure` FAILS, `test_promoted_lease_is_not_reaped...` FAILS +(`assert 1 == 0` — it was reaped). `test_sweep_does_not_promote_a_lease_still_booting` passes already. +**Paste the failures.** + +- [ ] **Step 3: Add the ready-state set** + +In `backend/app/jobs/lease_reaper.py`, below `_PROVIDER_TERMINAL` (line 34): + +```python +# Provider vocabularies for "the box is up". RunPod and Vast say "running", +# Prime Intellect says "active", Lambda passes its upstream string through. +_PROVIDER_READY = {"running", "active", "ready"} +``` + +- [ ] **Step 4: Add the promotion helper** + +In `backend/app/jobs/lease_reaper.py`, above `sweep_once`: + +```python +async def _promote_to_ready(db, provider, lease: dict) -> None: + """A provisioning lease whose provider reports it up becomes `ready`, with + the SSH coordinates acquire() could not know. + + Without this a managed GPU lease stays `provisioning` forever — the only + other writers of `ready` are CPU-scoped or have no production caller — and + branch 2 reaps it at PROVISION_TIMEOUT_SECONDS. Coordinates are best-effort: + promotion must happen even if connection() fails, or the lease is reaped + while the box is alive. A later sweep fills them in. + """ + ssh_host = ssh_port = None + try: + conn = (await provider.connection(lease["lease_id"])) or {} + ssh_host = conn.get("ssh_host") or None + ssh_port = conn.get("ssh_port") or None + except Exception: + log.exception("reaper: connection lookup failed for %s", lease["lease_id"]) + await compute_repo.update_lease_status( + db, lease["lease_id"], "ready", ssh_host=ssh_host, ssh_port=ssh_port, + ) + lease["status"] = "ready" +``` + +- [ ] **Step 5: Call it from the sweep** + +In `sweep_once`, insert immediately after branch 1's `if provider is not None:` block and **before** +branch 2 (currently line 132). The provider status was already fetched at branch 1; reuse it: + +```python + # 1b. Promotion. `acquire` returns status='provisioning' and nothing + # else in production advances it, so without this every managed GPU + # lease is reaped by branch 2 at PROVISION_TIMEOUT_SECONDS. + if ( + reason is None + and lease.get("status") == "provisioning" + and provider is not None + and (pstat.get("status") or "").lower() in _PROVIDER_READY + ): + await _promote_to_ready(db, provider, lease) +``` + +`pstat` is bound inside `if provider is not None:` at branch 1. Hoist its initialisation so it is always +defined — change the top of branch 1 from `if provider is not None:` to: + +```python + pstat: dict = {} + provider = get_provider((lease.get("provider") or "").lower()) + if provider is not None: +``` + +and delete the now-redundant `pstat = {}` assignments inside the `try`/`except` — keep +`pstat = (await provider.status(lease_id)) or {}` in the `try` and `pstat = {}` in the `except`. + +- [ ] **Step 6: Run the tests to verify they pass** + +Run: `cd ~/codes/InkVell/atlas/backend && .venv/bin/python -m pytest tests/test_lease_reaper.py -q` +Expected: 13 passed. + +- [ ] **Step 7: Run the wider compute suite for regressions** + +Run: `cd ~/codes/InkVell/atlas/backend && .venv/bin/python -m pytest tests/ -q -k "compute or lease or telemetry"` +Expected: all pass. Report any failure rather than fixing it if it looks unrelated. + +- [ ] **Step 8: Commit** + +```bash +cd ~/codes/InkVell/atlas +git add backend/app/jobs/lease_reaper.py backend/tests/test_lease_reaper.py +git commit -m "fix(reaper): promote provisioning leases to ready and record SSH coordinates + +A managed GPU lease was created 'provisioning' and nothing in production ever +advanced it: the only writers of 'ready' are the CPU-scoped reconcile pass and +get_lease_status, which has no production caller. So every user lease was +reaped as a provisioning timeout ~10 minutes after creation. + +The sweep already polls provider.status for each unfinished lease, so the +promotion check is free. connection() runs once on the transition, and its +failure must not block promotion -- a lease left provisioning gets reaped while +the box is alive. This also populates ssh_host/ssh_port, which acquire() cannot +know: every provider returns a hardcoded port 22 and no host." +``` + +--- + +### Task 4: Normalise the lease status vocabulary + +**Why:** `GET /api/compute/leases/{id}/connection` returns +`conn.get("status") or lease.get("status")` (`backend/app/routes/compute.py:500`) — one field carrying five +vocabularies. RunPod maps `running|stopped|terminated|unknown`, Vast `running|provisioning|stopped|unknown`, +Prime `provisioning|active|stopped|error|terminating|terminated|unknown`, Lambda passes its raw upstream +string through unmapped, and when the provider call throws it falls back to the DB's +`provisioning|ready|released`. A client cannot poll on that. Add a normalised field; **do not change +`status`**, which the dashboard consumes. + +**Files:** + +- Modify: `backend/app/routes/compute.py:450-508` (add `state` to the response) +- Create: `backend/app/compute/lease_state.py` +- Test: `backend/tests/test_compute_lease_state.py` + +**Interfaces:** + +- Consumes: Task 3's `ready` status. +- Produces: `normalise_state(raw: str | None) -> str` in `app/compute/lease_state.py`, returning one of + `"provisioning" | "ready" | "terminated" | "unknown"`. The `/connection` response gains a `state` field + with that value. The OpenScience `compute_launch` readiness poll will consume it. + +- [ ] **Step 1: Write the failing test** + +Create `backend/tests/test_compute_lease_state.py`: + +```python +"""One vocabulary for lease state. + +/connection returns `conn.get("status") or lease.get("status")`, which mixes +four provider vocabularies with the DB's own. A client polling for readiness +cannot be written against that: "running" never appears on the DB path, +"ready" never appears on the provider path, Prime says "active", and Lambda +passes its upstream string through unmapped. +""" + +from __future__ import annotations + +import pytest + +from app.compute.lease_state import normalise_state + + +@pytest.mark.parametrize("raw", ["running", "active", "ready", "RUNNING", "Active"]) +def test_up_states_normalise_to_ready(raw): + assert normalise_state(raw) == "ready" + + +@pytest.mark.parametrize( + "raw", ["terminated", "released", "stopped", "failed", "error", "TERMINATED"] +) +def test_dead_states_normalise_to_terminated(raw): + assert normalise_state(raw) == "terminated" + + +@pytest.mark.parametrize("raw", ["provisioning", "terminating", "PROVISIONING"]) +def test_in_flight_states_normalise_to_provisioning(raw): + assert normalise_state(raw) == "provisioning" + + +@pytest.mark.parametrize("raw", [None, "", "unknown", "some-lambda-string"]) +def test_unrecognised_states_are_unknown_not_ready(raw): + """Fail toward 'unknown'. A client must never read an unmapped Lambda + string as readiness and try to SSH into a box that is still booting.""" + assert normalise_state(raw) == "unknown" +``` + +- [ ] **Step 2: Run to verify it fails** + +Run: `cd ~/codes/InkVell/atlas/backend && .venv/bin/python -m pytest tests/test_compute_lease_state.py -q` +Expected: FAIL — `ModuleNotFoundError: No module named 'app.compute.lease_state'`. **Paste it.** + +- [ ] **Step 3: Write the implementation** + +Create `backend/app/compute/lease_state.py`: + +```python +"""Normalise the five lease-status vocabularies into one. + +`/connection` returns `conn.get("status") or lease.get("status")`, so the same +field carries whichever provider answered plus the DB's own values. Callers +that need to act on readiness use `state`, not `status`. + +`status` is left untouched: the dashboard renders it. +""" + +from __future__ import annotations + +_READY = {"running", "active", "ready"} +_TERMINATED = {"terminated", "released", "stopped", "failed", "error"} +_PROVISIONING = {"provisioning", "terminating", "pending", "starting"} + + +def normalise_state(raw: str | None) -> str: + """Map a provider or DB status onto provisioning | ready | terminated | + unknown. + + Unrecognised input is `unknown`, never `ready` — Lambda passes its upstream + string through unmapped, and a client that read an unknown string as + readiness would SSH into a box that is still booting. + """ + value = (raw or "").strip().lower() + if value in _READY: + return "ready" + if value in _TERMINATED: + return "terminated" + if value in _PROVISIONING: + return "provisioning" + return "unknown" +``` + +- [ ] **Step 4: Run to verify it passes** + +Run: `cd ~/codes/InkVell/atlas/backend && .venv/bin/python -m pytest tests/test_compute_lease_state.py -q` +Expected: 18 passed. + +- [ ] **Step 5: Expose it on the connection endpoint** + +In `backend/app/routes/compute.py`, add the import near the other local imports inside +`lease_connection`, and add one key to the returned dict (after `"status"`): + +```python + from app.compute.lease_state import normalise_state +``` + +```python + "status": conn.get("status") or lease.get("status"), + # Normalised: provisioning | ready | terminated | unknown. Poll on this, + # not on `status`, which carries whichever provider vocabulary answered. + "state": normalise_state(conn.get("status") or lease.get("status")), +``` + +- [ ] **Step 6: Verify the route still passes its suite** + +Run: `cd ~/codes/InkVell/atlas/backend && .venv/bin/python -m pytest tests/ -q -k "compute"` +Expected: all pass. + +- [ ] **Step 7: Commit** + +```bash +cd ~/codes/InkVell/atlas +git add backend/app/compute/lease_state.py backend/tests/test_compute_lease_state.py backend/app/routes/compute.py +git commit -m "feat(compute): normalise lease state into one vocabulary + +/connection's status field carries whichever of four provider vocabularies +answered, or the DB's own when the provider call throws. 'running' never +appears on the DB path, 'ready' never on the provider path, Prime says +'active', and Lambda passes its upstream string through unmapped -- so a +readiness poll cannot be written against it. + +Adds a `state` field: provisioning | ready | terminated | unknown. Unmapped +input is 'unknown', never 'ready'. `status` is unchanged; the dashboard reads +it." +``` + +--- + +## Whole-branch verification + +- [ ] **Step 1: Full compute suite** + +Run: `cd ~/codes/InkVell/atlas/backend && .venv/bin/python -m pytest tests/ -q -k "compute or lease or reaper or telemetry"` +Expected: all pass, no network access required. + +- [ ] **Step 2: Confirm the headline property holds end to end** + +Run: `cd ~/codes/InkVell/atlas/backend && .venv/bin/python -m pytest tests/test_lease_reaper.py -q -v` +Confirm by name that both of these pass: +`test_promoted_lease_is_not_reaped_at_the_provisioning_timeout` and +`test_user_lease_without_runner_token_is_not_heartbeat_reaped`. +**Together they are the deliverable:** a user lease now survives both reaper branches that killed it. + +- [ ] **Step 3: Report the deploy gates** + +Two things this plan cannot verify, both of which must be stated in the final report: + +1. **Task 1 needs a live Vast launch** to confirm SSH works without the account key. +2. **Nothing here has run against a real provider.** These are source-level fixes with unit coverage; the + spec's own record is that four source-read conclusions were wrong about deployed behaviour. + +--- + +## Out of scope for this plan + +Budget caps (spec changes 1–2), the resolver and quote endpoint (3–4), rolling window cap (5), volumes (6), +budget extension (7), release honesty (8), the SSH key-id lifecycle (9b), boot telemetry (10), image +pinning (11), catalog cache (12), orphan reap (13), and all three OpenScience tools. Each needs this plan +landed first — until a lease survives, none of them can be observed to work. From e84bc9137de58c179dcf6cc7a1f674bc919403f8 Mon Sep 17 00:00:00 2001 From: KB Date: Sat, 1 Aug 2026 10:19:49 +0530 Subject: [PATCH 36/56] plan(compute): close the four known lease defects BYOK leases cannot be promoted so they still die at 600s; Vast's release claims terminated regardless of what happened; Prime leaks an account SSH key per lease and stores the pod name where the key id belongs; Lambda's booting status reads as unknown. Volumes and the catalog cache are excluded: both are full spec changes needing design decisions, not defects. --- .../plans/2026-08-01-compute-lease-defects.md | 483 ++++++++++++++++++ 1 file changed, 483 insertions(+) create mode 100644 docs/plans/2026-08-01-compute-lease-defects.md diff --git a/docs/plans/2026-08-01-compute-lease-defects.md b/docs/plans/2026-08-01-compute-lease-defects.md new file mode 100644 index 00000000..65e4ca2f --- /dev/null +++ b/docs/plans/2026-08-01-compute-lease-defects.md @@ -0,0 +1,483 @@ +# Compute lease defects — implementation plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development to implement +> this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Close the four known-unfixed defects in the managed-lease path that live testing and review +surfaced but the prerequisites branch deliberately left alone. + +**Architecture:** All four tasks are in the **Atlas** repo (`~/codes/InkVell/atlas`), continuing on the +`feat/compute-lease-prerequisites` branch. Two are provider-level correctness (Vast release honesty, Prime +key lifecycle), one restores a whole lease class to the promotion path (BYOK), one is a status-mapping +gap. + +**Tech Stack:** Python 3.12, FastAPI, aiosqlite, pytest + pytest-asyncio, respx, httpx. + +**Spec:** `~/codes/InkVell/openscience/docs/specs/compute-design.md` — change 9(b), plus defects recorded +in `.superpowers/sdd/2026-07-31-compute-lease-prerequisites/progress.md`. + +## Global Constraints + +- **Repo:** `~/codes/InkVell/atlas`, branch `feat/compute-lease-prerequisites` (already checked out). + Do not create or switch branches. +- **Run tests with:** `cd ~/codes/InkVell/atlas/backend && .venv/bin/python -m pytest -q`. + No activated virtualenv — always invoke `.venv/bin/python` explicitly. +- **Never add a `Co-Authored-By:` trailer or any AI/assistant attribution to commit messages.** +- **No mocks of our own code.** Stub HTTP at the transport boundary with `respx`; use real repo functions + and the real `sweep_once`, matching `backend/tests/test_lease_reaper.py` and + `backend/tests/test_compute_vast_provider_http.py`. +- Provider HTTP tests use `@respx.mock` only — **no** `@pytest.mark.asyncio` (pytest-asyncio is in auto + mode). +- **Every new assertion must be shown failing first.** Paste the failure into your report. +- Baseline: full suite is **1610 passed, 1 skipped**. It must be green when each task finishes. +- Ignore `.claude/worktrees/`. + +--- + +### Task 1: BYOK GPU leases can never be promoted, so they still die at 600s + +**Why:** `sweep_once` polls providers with no user credentials — the module's own comment concedes it +(`backend/app/jobs/lease_reaper.py`, branch 1: _"status is polled WITHOUT BYOK user_credentials, so +GPU/BYOK leases can't be classified here"_). For a BYOK lease `provider.status()` either raises or returns +`not_configured`, so `pstat` never matches `_PROVIDER_READY`, promotion never runs, the lease stays +`provisioning`, and branch 2 reaps it at `PROVISION_TIMEOUT_SECONDS`. `release_lease` **does** use BYOK +credentials, so the user's own paid instance is genuinely destroyed. + +`LeaseManager` already has the helper: `_credentials_for_lease(db, lease)` +(`backend/app/compute/lease_manager.py`), which returns BYOK creds for a stored lease or `None`. + +**Files:** + +- Modify: `backend/app/jobs/lease_reaper.py` — branch 1's `status()` call, and the `connection()` call + inside `_promote_to_ready` / `_backfill_coordinates` +- Test: `backend/tests/test_lease_reaper.py` (append) + +**Interfaces:** + +- Consumes: `_credentials_for_lease(db, lease) -> dict | None` from `app.compute.lease_manager`. +- Produces: nothing later tasks depend on. + +- [ ] **Step 1: Read the current call sites** + +Read `sweep_once`'s branch 1 and both coordinate helpers. Note how `_promote_to_ready` and +`_backfill_coordinates` obtain a provider and call `connection()`. All three provider calls need the same +credentials. + +- [ ] **Step 2: Write the failing test** + +Append to `backend/tests/test_lease_reaper.py`: + +```python +@pytest.mark.asyncio +async def test_byok_lease_is_promoted_using_the_users_credentials(test_db, monkeypatch): + """A BYOK lease runs on the user's own provider account. The reaper polls + with no credentials, so it could never see the box as up, never promoted, + and branch 2 destroyed the user's own paid instance at 600s.""" + db = test_db["db"]; uid = test_db["user_id"] + await compute_repo.create_lease(db, lease_id="Lbyok", status="provisioning", + user_id=uid, provider="runpod", requested_sku="h100", region="us", + hourly_rate_cents=279, category="gpu", funding="byok", + ssh_key_name="atlas-byok") + old = _iso(datetime.now(timezone.utc) - timedelta(seconds=1200)) + await db.execute( + "UPDATE compute_leases SET created_at = ?, started_at = ? WHERE lease_id = 'Lbyok'", + (old, old)) + await db.commit() + + seen = {"status": None, "connection": None} + + class _P: + async def status(self, _id, **kw): + seen["status"] = kw.get("user_credentials") + if not kw.get("user_credentials"): + return {"status": "not_configured"} + return {"status": "running"} + + async def connection(self, _id, **kw): + seen["connection"] = kw.get("user_credentials") + if not kw.get("user_credentials"): + return {} + return {"ssh_host": "1.2.3.4", "ssh_port": 20095, "ssh_user": "root"} + + monkeypatch.setattr(reaper, "get_provider", lambda _name: _P()) + + async def _creds(_db, _lease): + return {"secret": "user-byok-key"} + monkeypatch.setattr(reaper, "_credentials_for_lease", _creds, raising=False) + + async def _noop_release(self, _db, _lease): return {"status": "released"} + monkeypatch.setattr(reaper.LeaseManager, "release_lease", _noop_release) + + n = await reaper.sweep_once(db, now=datetime.now(timezone.utc)) + + assert n == 0, "a live BYOK lease must not be reaped" + row = await compute_repo.get_lease(db, "Lbyok") + assert row["status"] == "ready" + assert row["ssh_host"] == "1.2.3.4" + assert seen["status"] == {"secret": "user-byok-key"} + assert seen["connection"] == {"secret": "user-byok-key"} +``` + +- [ ] **Step 3: Run it and confirm it fails** + +Run: `cd ~/codes/InkVell/atlas/backend && .venv/bin/python -m pytest tests/test_lease_reaper.py -q -k byok` +Expected: FAIL — the lease is reaped (`n == 1`) or stays `provisioning`, and `seen["status"]` is `None`. +Paste the failure. + +- [ ] **Step 4: Thread credentials through the three provider calls** + +Import `_credentials_for_lease` into `lease_reaper.py` at module level, alongside the existing +`from app.compute.lease_manager import LeaseManager`: + +```python +from app.compute.lease_manager import LeaseManager, _credentials_for_lease +``` + +In `sweep_once`, resolve credentials once per lease before branch 1 and pass them to `status()`: + +```python + creds = await _credentials_for_lease(db, lease) + kw = {"user_credentials": creds} if creds is not None else {} +``` + +Pass `**kw` to `provider.status(lease_id, **kw)` in branch 1, and thread `creds` into +`_promote_to_ready` and `_backfill_coordinates` so their `connection()` calls use it too. Update those +two helpers' signatures to accept the credentials, and update every call site. + +**Keep the failure handling as it is** — a raising `status()` must still leave `pstat = {}` and fall +through to the timeout branches. This change adds credentials; it must not change what happens when a +provider call fails. + +Correct the now-stale comment on branch 1 that says BYOK leases cannot be classified here. + +- [ ] **Step 5: Run the tests** + +Run: `cd ~/codes/InkVell/atlas/backend && .venv/bin/python -m pytest tests/test_lease_reaper.py -q` +Expected: all pass, including every pre-existing test. Then +`cd ~/codes/InkVell/atlas/backend && .venv/bin/python -m pytest tests/ -q` → 1611 passed / 1 skipped. + +- [ ] **Step 6: Commit** + +```bash +cd ~/codes/InkVell/atlas +git add backend/app/jobs/lease_reaper.py backend/tests/test_lease_reaper.py +git commit -m "fix(reaper): poll BYOK leases with the user's own credentials + +The sweep polled every provider with no credentials, so a BYOK lease could +never be seen as up: status() returned not_configured or raised, promotion +never ran, and branch 2 destroyed the user's own paid instance at the +provisioning timeout. release_lease already used BYOK credentials, so the +teardown worked even though the liveness check could not. + +_credentials_for_lease already existed on LeaseManager; the reaper now uses it +for status() and for the connection() lookups behind promotion and backfill." +``` + +--- + +### Task 2: `VastProvider.release` reports success it never verified + +**Why:** the release wraps its DELETE in `try/except: pass` and then returns +`{"lease_id": lease_id, "status": "terminated"}` **unconditionally**. A 4xx, a network error, or a +half-completed teardown all return the same "terminated". Observed live: the return value is not evidence +of anything. + +This is the provider half of spec change 8. The `LeaseManager.release_lease` half — which marks the row +released regardless — stays out of scope here; this task makes the provider able to tell the truth. + +**Files:** + +- Modify: `backend/app/compute/vast_provider.py` — the `release` method +- Test: `backend/tests/test_compute_vast_provider_http.py` (append) + +**Interfaces:** + +- Produces: `VastProvider.release` returns `status: "terminated"` only on a 2xx; otherwise + `status: "unknown"` with a `warning` key describing what happened. Callers that only read `status` are + unaffected on the success path. + +- [ ] **Step 1: Write the failing tests** + +Append to `backend/tests/test_compute_vast_provider_http.py`: + +```python +@respx.mock +async def test_release_reports_failure_instead_of_claiming_terminated(): + respx.delete(f"{VAST_API}/instances/4242/").mock( + return_value=httpx.Response(403, text="forbidden") + ) + out = await VastProvider().release("4242", user_credentials=CREDS) + assert out["status"] != "terminated", "a refused delete must not report success" + assert "warning" in out + assert "403" in str(out["warning"]) + + +@respx.mock +async def test_release_reports_failure_on_transport_error(): + respx.delete(f"{VAST_API}/instances/4242/").mock( + side_effect=httpx.ConnectError("boom") + ) + out = await VastProvider().release("4242", user_credentials=CREDS) + assert out["status"] != "terminated" + assert "warning" in out + + +@respx.mock +async def test_release_reports_terminated_on_success(): + respx.delete(f"{VAST_API}/instances/4242/").mock( + return_value=httpx.Response(200, json={"success": True}) + ) + out = await VastProvider().release("4242", user_credentials=CREDS) + assert out["status"] == "terminated" + assert "warning" not in out +``` + +- [ ] **Step 2: Run and confirm the first two fail** + +Run: `cd ~/codes/InkVell/atlas/backend && .venv/bin/python -m pytest tests/test_compute_vast_provider_http.py -q -k release` +Expected: the first two FAIL (`status` is `"terminated"` regardless), the third passes. Paste the failure. + +- [ ] **Step 3: Make the release honest** + +Rewrite the body of `release` so it inspects the response and reports what actually happened. Keep it +**non-fatal** — it must not start raising, because callers rely on release being best-effort — and log at +warning level, matching the logging added to the key-attach path in the same module: + +```python + try: + async with httpx.AsyncClient(timeout=30.0) as client: + resp = await client.delete( + f"{VAST_API}/instances/{lease_id}/", headers=headers + ) + if resp.status_code >= 400: + detail = (resp.text or "")[:200] + logger.warning( + "vast.release: HTTP %s destroying instance %s: %s", + resp.status_code, lease_id, detail, + ) + return { + "lease_id": lease_id, + "status": "unknown", + "warning": f"HTTP {resp.status_code}: {detail}", + } + except Exception as exc: # noqa: BLE001 + logger.exception("vast.release: destroying instance %s failed", lease_id) + return {"lease_id": lease_id, "status": "unknown", "warning": str(exc)[:200]} + return {"lease_id": lease_id, "status": "terminated"} +``` + +Update the method's docstring to say the return value now distinguishes a confirmed teardown from an +unconfirmed one. + +- [ ] **Step 4: Run the tests** + +Run: `cd ~/codes/InkVell/atlas/backend && .venv/bin/python -m pytest tests/test_compute_vast_provider_http.py -q` +Expected: all pass. Then the full suite → 1614 passed / 1 skipped. + +- [ ] **Step 5: Check for callers that assumed the old contract** + +Grep for callers of `release(` on providers and confirm none of them break when `status` is `"unknown"`. +`LeaseManager.release_lease` already treats the provider result as advisory. **Report what you find** — +if any caller keys off `status == "terminated"` to decide something important, say so rather than +changing it. + +- [ ] **Step 6: Commit** + +```bash +cd ~/codes/InkVell/atlas +git add backend/app/compute/vast_provider.py backend/tests/test_compute_vast_provider_http.py +git commit -m "fix(vast): release reports what happened instead of always 'terminated' + +The DELETE was wrapped in try/except: pass and the method returned +status='terminated' unconditionally, so a 403, a transport error and a real +teardown were indistinguishable to the caller. Still best-effort and still +non-raising; it just tells the truth now." +``` + +--- + +### Task 3: Prime Intellect leaks an account SSH key per lease, and stores the wrong identifier + +**Why:** `PrimeIntellectProvider.acquire` registers the public key with `_register_ssh_key`, which returns +the provider's key **id** — used as `sshKeyId` on the pod. But `acquire` then returns +`"ssh_key_name": name`, the **pod** name (`atlas-{user_id[:8]}`), which is identical for every lease that +user ever creates. The key id is discarded, `release` deletes nothing, and every lease leaves a key in the +operator account permanently. + +Vast no longer leaks (the account registration was removed there), so Prime is the remaining case. This is +spec change 9(b), scoped to the one provider that still needs it. + +There is **no column** to store a provider-side key id — `compute_leases` carries `ssh_key_name` and +`ssh_public_key` only. Follow the additive-column pattern already used for `runner_api_key_id` in +`backend/app/db/migrations.py` (see the `if "runner_api_key_id" not in existing:` guard) and mirror it in +`backend/app/db/pg_migrations.py`. + +**Files:** + +- Modify: `backend/app/db/migrations.py`, `backend/app/db/pg_migrations.py` (add `provider_key_id TEXT`) +- Modify: `backend/app/db/repos/compute_repo.py` (`create_lease` accepts and persists it) +- Modify: `backend/app/compute/lease_manager.py` (pass it from the acquire result into `create_lease`; + forward it to `release`) +- Modify: `backend/app/compute/prime_intellect_provider.py` (return the key id; delete on release and on + failed launch) +- Test: `backend/tests/test_compute_prime_provider_http.py` (append) + +**Interfaces:** + +- Consumes: nothing from earlier tasks. +- Produces: `compute_leases.provider_key_id`, a nullable TEXT column. `acquire` results may carry + `provider_key_id`; `release` accepts `provider_key_id=` and deletes it when present. + +- [ ] **Step 1: Read the existing shapes first** + +Read `PrimeIntellectProvider._register_ssh_key`, `acquire` and `release`; `LambdaProvider._delete_ssh_key` +and how `release_lease` forwards `ssh_key_name` (`lease_manager.py`); and the `runner_api_key_id` +migration guard. **Report the Prime delete endpoint you find** — if the module does not already know how +to delete a key, find it in the provider's API surface before writing code. + +- [ ] **Step 2: Write the failing tests** + +Append to `backend/tests/test_compute_prime_provider_http.py`, matching that file's existing respx style +(read it first — reuse its base-URL constant and credentials fixture rather than inventing new ones): + +1. `acquire` returns `provider_key_id` equal to the id `POST /ssh_keys/` returned, and **not** the pod + name. +2. `release` called with that `provider_key_id` issues a delete for that key. +3. `release` with no `provider_key_id` does not attempt a key delete and still terminates the pod. +4. A launch that fails **after** key registration deletes the key it registered (no leak on the error + path). + +- [ ] **Step 3: Run and confirm they fail** + +Run: `cd ~/codes/InkVell/atlas/backend && .venv/bin/python -m pytest tests/test_compute_prime_provider_http.py -q` +Paste the failures. + +- [ ] **Step 4: Implement** + +Work in this order so each piece is independently runnable: + +1. Migration: `provider_key_id TEXT` on `compute_leases`, both sqlite and Postgres, additive and guarded + exactly like `runner_api_key_id`. +2. `compute_repo.create_lease`: accept `provider_key_id: str | None = None`, persist it, include it in the + returned dict. +3. `prime_intellect_provider.acquire`: return `"provider_key_id": ssh_key_id` alongside the existing keys. + **Leave `ssh_key_name` as it is** — other code reads it, and this task is not a rename. +4. `prime_intellect_provider.acquire`: wrap the pod-creation call so a failure after key registration + deletes the registered key before re-raising. +5. `prime_intellect_provider.release`: accept `provider_key_id: str | None = None` and delete that key + when present, best-effort and logged, never raising. +6. `lease_manager`: pass the acquire result's `provider_key_id` into `create_lease`, and forward the + stored `provider_key_id` into `provider.release(...)` the same way `ssh_key_name` is already forwarded. + +- [ ] **Step 5: Run the tests** + +Run: `cd ~/codes/InkVell/atlas/backend && .venv/bin/python -m pytest tests/ -q -k "prime or compute or lease"` +then the full suite. Both green. + +- [ ] **Step 6: Commit** + +```bash +cd ~/codes/InkVell/atlas +git add -A backend/app backend/tests +git commit -m "fix(prime): delete the per-lease SSH key instead of leaking it + +acquire registered a key, used its id as sshKeyId, then returned the POD name +in ssh_key_name and discarded the id -- so release had nothing correct to +delete and every lease left a key in the operator account forever. The pod name +is also identical across all of a user's leases, so a name-keyed delete would +have missed or over-deleted. + +Adds a nullable provider_key_id column, returns the real id from acquire, +deletes on release and on a launch that fails after registration." +``` + +--- + +### Task 4: Lambda's in-flight status reads as `unknown` + +**Why:** `LambdaProvider.status` returns Lambda's raw upstream string unmapped +(`backend/app/compute/lambda_provider.py`), and `normalise_state`'s `_PROVISIONING` set contains only +`"provisioning"`. So a booting Lambda box reports `state: "unknown"` for its whole boot window. Safe — +`unknown` never reads as ready — but a client polling for progress cannot distinguish "coming up" from +"something is wrong". + +`booting` is named as an in-flight status by this repo's own frontend +(`frontend/src/components/compute/InstancesTab.tsx:29`), which is the in-repo evidence that was missing +when this was first deferred. + +**Files:** + +- Modify: `backend/app/compute/lease_state.py` +- Test: `backend/tests/test_compute_lease_state.py` + +**Interfaces:** + +- Consumes: `normalise_state` from Task-4-of-the-previous-plan. Unchanged signature. + +- [ ] **Step 1: Add the failing case** + +In `backend/tests/test_compute_lease_state.py`, add `"booting"` to the parametrisation of +`test_in_flight_states_normalise_to_provisioning`, and **remove** it from any unknown-case list if present. + +- [ ] **Step 2: Run and confirm it fails** + +Run: `cd ~/codes/InkVell/atlas/backend && .venv/bin/python -m pytest tests/test_compute_lease_state.py -q` +Expected: FAIL — `normalise_state("booting")` returns `"unknown"`. Paste it. + +- [ ] **Step 3: Map it** + +Add `"booting"` to `_PROVISIONING` in `backend/app/compute/lease_state.py`, with a comment naming Lambda +as the emitter and the frontend file as the evidence. + +**Do not add `unhealthy`.** Lambda's enum is not documented in this repo, and `unhealthy` is genuinely +ambiguous between "degraded but alive" and "dead" — mapping it wrong is worse than leaving it `unknown`. +Say so in the comment. + +- [ ] **Step 4: Run the tests** + +Run: `cd ~/codes/InkVell/atlas/backend && .venv/bin/python -m pytest tests/test_compute_lease_state.py -q` +then the full suite. Both green. + +- [ ] **Step 5: Commit** + +```bash +cd ~/codes/InkVell/atlas +git add backend/app/compute/lease_state.py backend/tests/test_compute_lease_state.py +git commit -m "fix(compute): map Lambda's booting status to provisioning + +Lambda passes its upstream status string through unmapped, so a booting box +reported state='unknown' for its whole boot window -- safe, since unknown never +reads as ready, but a client could not tell 'coming up' from 'broken'. The +repo's own frontend already treats booting as an in-flight status. + +unhealthy is deliberately left unmapped: it is ambiguous between degraded and +dead, and guessing is worse than unknown." +``` + +--- + +## Whole-branch verification + +- [ ] Full suite: `cd ~/codes/InkVell/atlas/backend && .venv/bin/python -m pytest tests/ -q` — green, no + network. +- [ ] Confirm by name that the previous plan's deliverable tests still pass: + `test_promoted_lease_is_not_reaped_at_the_provisioning_timeout`, + `test_user_lease_without_runner_token_is_not_heartbeat_reaped`, + `test_modal_cpu_sandbox_promotes_without_ssh_host`, + `test_status_of_destroyed_instance_is_terminated_not_provisioning`. +- [ ] Report which defects remain open (see below) so nothing is assumed closed. + +## Deliberately out of scope + +Two items from the same defect list are **full spec changes, not defects**, and each needs its own plan: + +- **Volumes provision nothing** (spec change 6). `create_volume` writes a DB row and calls no provider + API; no provider except RunPod has any volume concept. This is "make volumes real, per provider", and + under cheapest-first it also narrows the resolver pool. A design task, not a fix. +- **The options catalog is uncached** (spec change 12). Five provider requests per call behind a 3s client + timeout. It needs a cache key, a TTL, and a decision about how the resolver's retry interacts with it — + all design choices the spec has not made. + +Also still open and recorded, not addressed here: the `LeaseManager.release_lease` half of change 8 (marks +a row released even when provider teardown failed), and `PROVISION_TIMEOUT_SECONDS` being a single global +constant. From 784633ee51f3c7e8fe11817a3cfbb874b92058bd Mon Sep 17 00:00:00 2001 From: KB Date: Sat, 1 Aug 2026 15:25:35 +0530 Subject: [PATCH 37/56] fix(compute): stop claiming managed compute with a zero balance /api/compute/options reports a managed provider whenever reselling is on and an operator key exists, independent of wallet balance. Live testing against a deployed backend with a zero wallet showed compute_status telling the agent to run GPU work through managed compute while also forbidding the only fallback (the user's own keys) -- every lease acquire in that state returns HTTP 402 insufficient_cli_credit. Same defect class Part A removed from mode resolution, one layer up in the guidance text. Make the managed guidance a function of balance: unaffordable (balance exactly 0) now says the wallet is empty, launches will be refused, and to top up or connect a provider key. The resolved mode is untouched -- managed capability genuinely exists, only the funds don't, and that distinction needs different user-facing advice than "none" would give. --- backend/cli/src/tool/compute.ts | 39 ++++++++++++++++++-- backend/cli/test/tool/compute-status.test.ts | 31 ++++++++++++++++ 2 files changed, 66 insertions(+), 4 deletions(-) diff --git a/backend/cli/src/tool/compute.ts b/backend/cli/src/tool/compute.ts index 33ec9384..10dbae34 100644 --- a/backend/cli/src/tool/compute.ts +++ b/backend/cli/src/tool/compute.ts @@ -18,13 +18,44 @@ import { ComputeMode } from "@/compute/mode" * expensive; adding them here is free. */ -const GUIDANCE: Record = { +const GUIDANCE: Record, string> = { byok: "Run GPU work on the user's connected providers via the cloud-compute skills. Do not launch managed leases — they bill Credits and are not the funded path here.", - managed: - "Run GPU work through managed compute, billed to Credits. Do not use the user's own provider keys — they are not funded here.", none: "No compute is available. Do not attempt GPU work. Tell the user to connect a provider key in Settings ▸ Compute, or to top up for managed compute.", } +/** + * `managed` guidance is a function of balance, not a fixed string, because + * `GET /api/compute/options` reports a provider as managed whenever reselling + * is on and an operator key exists — availability and affordability are + * independent there. Live testing against a deployed backend with a zero + * wallet found the old fixed string sent the agent down a path that cannot + * work (every lease acquire returns HTTP 402 insufficient_cli_credit) while + * simultaneously forbidding the only fallback that would (the user's own + * provider keys). That is the same "claims a capability with nothing behind + * it" defect Part A removed from mode resolution itself (see ComputeMode's + * doc comment) — just one layer up: the mode really is `managed`, the + * capability exists, but the guidance text assumed `managed` meant funded. + * + * This does NOT change `state.mode`. Managed is genuinely configured; an + * empty wallet is missing funds, not a missing capability, and the two need + * different advice — top up vs. connect a key. Collapsing them would destroy + * that distinction, which is exactly the mistake the override rule in + * `ComputeMode.resolve` (narrow, never manufacture) exists to prevent. + * + * Only `balance === 0` counts as unaffordable. Acquiring a lease requires one + * hour of the chosen SKU's rate up front, and rates span cents to dollars an + * hour depending on the catalog Atlas holds and this tool never sees — any + * non-zero cutoff here would be a guess. `balance === undefined` (the probe + * succeeded but the response carried no balance field) is left alone too: + * that is missing information, not a zero balance, and treating it as empty + * would be its own honesty bug. + */ +function managedGuidance(balance: number | undefined): string { + if (balance === 0) + return "Managed compute is configured, but the wallet is empty — every lease attempt will be refused (HTTP 402). Tell the user to top up in Settings ▸ Compute, or to connect their own provider key to run BYOK instead." + return "Run GPU work through managed compute, billed to Credits. Do not use the user's own provider keys — they are not funded here." +} + export const ComputeStatusTool = Tool.define("compute_status", { description: [ "Check how GPU compute is funded before running any GPU, training, or cluster work.", @@ -40,7 +71,7 @@ export const ComputeStatusTool = Tool.define("compute_status", { `**managed available**: ${state.managed ? "yes" : "no"}`, ] if (state.balance !== undefined) lines.push(`**balance**: $${state.balance.toFixed(2)}`) - lines.push("", GUIDANCE[state.mode]) + lines.push("", state.mode === "managed" ? managedGuidance(state.balance) : GUIDANCE[state.mode]) return { title: `Compute: ${state.mode}`, diff --git a/backend/cli/test/tool/compute-status.test.ts b/backend/cli/test/tool/compute-status.test.ts index e28ba3a9..48365450 100644 --- a/backend/cli/test/tool/compute-status.test.ts +++ b/backend/cli/test/tool/compute-status.test.ts @@ -47,6 +47,11 @@ const MANAGED_ON = { cli_effective_balance_cents: 4200, } const MANAGED_OFF = { providers: [], resell_enabled: false, cli_effective_balance_cents: 0 } +const MANAGED_ZERO = { + providers: [{ provider: "lambda", funding: "managed", has_byok: false, has_operator: true, count: 2 }], + resell_enabled: true, + cli_effective_balance_cents: 0, +} async function run(skills: string[], fn?: () => Promise) { await using tmp = await tmpdir({ @@ -107,6 +112,32 @@ describe("compute_status", () => { expect(calls.filter((url) => url.includes("/api/compute/options")).length).toBe(1) }) + test("managed with a zero balance stops telling the agent to spend it", async () => { + // Live testing against a deployed Atlas backend with a zero wallet found + // /api/compute/options reports a "managed" provider regardless of + // balance — availability and affordability are independent. Every lease + // attempt in this state returns HTTP 402 insufficient_cli_credit, so the + // old guidance sent the agent down a path that cannot work while also + // forbidding the only fallback (the user's own keys). + stub(MANAGED_ZERO) + const result = await run([]) + expect(result.output.toLowerCase()).not.toContain("run gpu work through managed compute") + expect(result.output.toLowerCase()).toContain("top up") + }) + + test("managed with a positive balance keeps today's guidance", async () => { + stub(MANAGED_ON) + const result = await run([]) + expect(result.output.toLowerCase()).toContain("run gpu work through managed compute") + }) + + test("a zero balance narrows guidance only — mode stays managed, balance stays reported", async () => { + stub(MANAGED_ZERO) + const result = await run([]) + expect(result.metadata.mode).toBe("managed") + expect(result.metadata.balance_usd).toBe(0) + }) + test("none tells the agent not to attempt GPU work and how to enable it", async () => { stub(MANAGED_OFF) const result = await run([]) From 8431583c409d8ac3f170f1f60cc6acf6794a206d Mon Sep 17 00:00:00 2001 From: KB Date: Sat, 1 Aug 2026 16:05:55 +0530 Subject: [PATCH 38/56] chore(compute): drop the throwaway guardrail prototype from src/ package.json publishes "src", so both PROTOTYPE-guardrail-*.ts files (~492 lines) shipped to npm despite their own header saying "throwaway, not wired into the product, delete or lift, don't ship". The repl also read ATLAS_TOKEN and made live authenticated fetches, and both files were typechecked on every build. Nothing imports them; e12e486 keeps them in history if the state machine is ever lifted. --- backend/cli/package.json | 3 +- .../src/compute/PROTOTYPE-guardrail-model.ts | 232 ---------------- .../src/compute/PROTOTYPE-guardrail-repl.ts | 260 ------------------ 3 files changed, 1 insertion(+), 494 deletions(-) delete mode 100644 backend/cli/src/compute/PROTOTYPE-guardrail-model.ts delete mode 100644 backend/cli/src/compute/PROTOTYPE-guardrail-repl.ts diff --git a/backend/cli/package.json b/backend/cli/package.json index 0db00965..49b6680e 100644 --- a/backend/cli/package.json +++ b/backend/cli/package.json @@ -11,8 +11,7 @@ "typecheck": "tsgo --noEmit", "test": "bun test --timeout 15000", "build": "bun run script/build.ts", - "dev": "bun run --conditions=browser ./src/index.ts", - "prototype:guardrail": "bun run ./src/compute/PROTOTYPE-guardrail-repl.ts" + "dev": "bun run --conditions=browser ./src/index.ts" }, "bin": { "openscience": "./bin/openscience" diff --git a/backend/cli/src/compute/PROTOTYPE-guardrail-model.ts b/backend/cli/src/compute/PROTOTYPE-guardrail-model.ts deleted file mode 100644 index 720c975d..00000000 --- a/backend/cli/src/compute/PROTOTYPE-guardrail-model.ts +++ /dev/null @@ -1,232 +0,0 @@ -/** - * PROTOTYPE — throwaway. Not wired into the product. Delete or lift, don't ship. - * - * ── The question ──────────────────────────────────────────────────────────── - * We want managed GPU compute where the AGENT PROPOSES a duration and ATLAS - * DECIDES whether it can be afforded — because OpenScience is open-source, so a - * decision made client-side is a decision a fork can delete. - * - * Does that state machine hold together? Four sub-questions: - * (a) what must an agent see to propose a sensible duration? - * (b) what does a rejection look like, and can the agent act on it? - * (c) what happens at the deadline, and who actually enforces it? - * (d) what does the client do when release fails or can't be confirmed? - * - * ── What is real vs modelled ──────────────────────────────────────────────── - * Atlas today (read from ~/codes/InkVell/atlas, origin/main): - * - POST /api/compute/estimate returns a RATE only. No duration, no total. - * - POST /api/compute/leases has NO duration field and NO expires_at column. - * It derives a flat 24h TTL from the user's plan, and verifies the wallet - * can fund ONE HOUR (402 otherwise). - * - The only server clocks are that 24h TTL and a 10-min heartbeat check. - * - All four leasable providers silently DISCARD the timeout argument, so - * provider-native auto-termination does not exist. Something must call - * release, or the VM runs on. - * - * That last group is why this prototype exists: it lets you toggle each - * assumption and watch the money. - * - * NOTE: the Atlas code above was read but NOT verified at runtime, so this - * models its apparent behaviour, not confirmed behaviour. - * ──────────────────────────────────────────────────────────────────────────── - */ - -/** What the agent asks for. Untrusted — it only ever proposes. */ -export interface Proposal { - provider: string - sku: string - /** The agent's own guess at how long the work needs. */ - hours: number -} - -/** What Atlas returns from /estimate today: a rate, and nothing about duration. */ -export interface Quote { - funding: "managed" | "byok" | "unavailable" - rateCentsPerHour: number - balanceCents: number -} - -/** - * Which affordability rule the server applies. - * - "first-hour" is what Atlas does TODAY: can you fund one hour? - * - "total" is the proposed change: can you fund rate x approved_hours? - * Toggling this is the point — see what "first-hour" approves. - */ -export type GateMode = "first-hour" | "total" - -export interface DecideInput { - proposal: Proposal - quote: Quote - /** Server-side ceiling. Atlas uses plan.gpu_sandbox_max_ttl_hours (24, all plans). */ - planTtlHours: number - mode: GateMode -} - -export type Verdict = - | { - ok: true - /** May be less than proposed — the plan TTL clamps it. */ - approvedHours: number - clampedFrom?: number - costCents: number - /** Free for BYOK; the wallet is never touched. */ - billed: boolean - } - | { - ok: false - /** Mirrors the shapes Atlas actually returns. */ - code: "insufficient_credit" | "unavailable" | "invalid" - message: string - /** The actionable part: what WOULD be approved. This is (b). */ - remedy?: { affordableHours: number; neededCents: number; availableCents: number } - } - -export function quoteTotal(rateCentsPerHour: number, hours: number): number { - return Math.round(rateCentsPerHour * hours) -} - -/** - * THE decision. Runs server-side in the real design, which is the whole point: - * a fork of the open-source client cannot reach it. - */ -export function decide(input: DecideInput): Verdict { - const { proposal, quote, planTtlHours, mode } = input - - if (quote.funding === "unavailable") - return { ok: false, code: "unavailable", message: `${proposal.sku} is not available on ${proposal.provider}.` } - - if (!Number.isFinite(proposal.hours) || proposal.hours <= 0) - return { ok: false, code: "invalid", message: "Proposed duration must be a positive number of hours." } - - const approvedHours = Math.min(proposal.hours, planTtlHours) - const clampedFrom = approvedHours < proposal.hours ? proposal.hours : undefined - - // BYOK runs on the user's own provider account. We never bill, so there is - // nothing to gate on affordability. - if (quote.funding === "byok") - return { ok: true, approvedHours, clampedFrom, costCents: 0, billed: false } - - const total = quoteTotal(quote.rateCentsPerHour, approvedHours) - const required = mode === "first-hour" ? quote.rateCentsPerHour : total - - if (required > quote.balanceCents) { - const affordableHours = quote.rateCentsPerHour > 0 ? quote.balanceCents / quote.rateCentsPerHour : 0 - return { - ok: false, - code: "insufficient_credit", - message: `Needs ${(required / 100).toFixed(2)} USD, wallet has ${(quote.balanceCents / 100).toFixed(2)} USD.`, - remedy: { - affordableHours: Math.floor(affordableHours * 10) / 10, - neededCents: required, - availableCents: quote.balanceCents, - }, - } - } - - return { ok: true, approvedHours, clampedFrom, costCents: total, billed: true } -} - -// ── The running lease ─────────────────────────────────────────────────────── - -export type Phase = - | "running" - | "past-deadline" - | "released" - /** Nobody is left to call release. This is the financial exposure. */ - | "orphaned" - -export interface Lease { - id: string - rateCentsPerHour: number - approvedHours: number - startedAtMs: number - /** Absent when the server has no deadline column — i.e. Atlas as it stands. */ - expiresAtMs?: number - phase: Phase - /** Wall-clock cost so far. Atlas meters per second, never rounding up to an hour. */ - spentCents: number - releaseAttempts: number - note?: string -} - -export interface World { - /** Does the SERVER hold the deadline and enforce it in its sweep? */ - serverEnforcesDeadline: boolean - /** Is our process still alive to run its own timer? */ - clientAlive: boolean - /** Simulate release calls failing (provider flake, network, auth). */ - releaseFails: boolean - /** Atlas's absolute backstop: plan.gpu_sandbox_max_ttl_hours. */ - planTtlHours: number -} - -export function spend(lease: Lease, nowMs: number): number { - const secs = Math.max(0, (nowMs - lease.startedAtMs) / 1000) - return Math.round((lease.rateCentsPerHour * secs) / 3600) -} - -export function overrunHours(lease: Lease, nowMs: number): number { - const elapsed = (nowMs - lease.startedAtMs) / 3_600_000 - return Math.max(0, elapsed - lease.approvedHours) -} - -/** What SHOULD happen next, given the world. Pure — the caller applies it. */ -export type Action = - | { kind: "none" } - | { kind: "server-releases"; why: string } - | { kind: "client-should-release"; why: string } - | { kind: "nobody-will-release"; why: string; exposureCents: number } - -export function evaluate(lease: Lease, nowMs: number, world: World): Action { - if (lease.phase === "released" || lease.phase === "orphaned") return { kind: "none" } - - const elapsedHours = (nowMs - lease.startedAtMs) / 3_600_000 - const pastApproved = elapsedHours >= lease.approvedHours - const pastPlanTtl = elapsedHours >= world.planTtlHours - - // The absolute backstop fires regardless of anything else. - if (pastPlanTtl) return { kind: "server-releases", why: `plan TTL of ${world.planTtlHours}h reached` } - - if (!pastApproved) return { kind: "none" } - - // Past the approved duration. Who notices? - if (world.serverEnforcesDeadline && lease.expiresAtMs !== undefined) - return { kind: "server-releases", why: "server-side expires_at reached" } - - if (world.clientAlive) return { kind: "client-should-release", why: "client deadline timer fired" } - - // Nobody is watching. This is the case the design has to prevent. - const untilBackstopH = Math.max(0, world.planTtlHours - elapsedHours) - return { - kind: "nobody-will-release", - why: `no server deadline and the client is gone; billing until the ${world.planTtlHours}h backstop`, - exposureCents: Math.round(lease.rateCentsPerHour * untilBackstopH), - } -} - -export type ReleaseResult = - | { ok: true; already: boolean } - | { ok: false; retryable: boolean; message: string } - -/** - * Release, modelling the outcomes Atlas actually returns. 409 (already - * released) is SUCCESS for our purposes — the VM is gone either way. - */ -export function attemptRelease(lease: Lease, world: World): ReleaseResult { - if (lease.phase === "released") return { ok: true, already: true } - if (world.releaseFails) - return { ok: false, retryable: true, message: "release failed (network/provider); VM may still be running" } - return { ok: true, already: false } -} - -/** Fail closed: never assume cleanup happened. Give up only after N tries. */ -export const MAX_RELEASE_ATTEMPTS = 3 - -export function formatCents(cents: number): string { - return `$${(cents / 100).toFixed(2)}` -} - -export function formatHours(h: number): string { - if (h < 1) return `${Math.round(h * 60)}m` - return `${h.toFixed(h < 10 ? 1 : 0)}h` -} diff --git a/backend/cli/src/compute/PROTOTYPE-guardrail-repl.ts b/backend/cli/src/compute/PROTOTYPE-guardrail-repl.ts deleted file mode 100644 index 9e893417..00000000 --- a/backend/cli/src/compute/PROTOTYPE-guardrail-repl.ts +++ /dev/null @@ -1,260 +0,0 @@ -#!/usr/bin/env bun -/** - * PROTOTYPE — throwaway TUI. Run: cd backend/cli && bun run prototype:guardrail - * - * Drives the managed-compute guardrail state machine in - * PROTOTYPE-guardrail-model.ts. See that file for the question this answers. - * - * POST /leases and /release are FAKED — the real ones provision GPUs and bill - * real money. GET /options and POST /estimate are free, so [o] hits them for - * real to verify response shapes (needs ATLAS_TOKEN + optional ATLAS_BASE). - * - * This shell is disposable. The model next door is the liftable part. - */ -import { - decide, - evaluate, - attemptRelease, - spend, - overrunHours, - quoteTotal, - formatCents, - formatHours, - MAX_RELEASE_ATTEMPTS, - type Proposal, - type Quote, - type GateMode, - type Lease, - type World, - type Verdict, -} from "./PROTOTYPE-guardrail-model" - -const B = "\x1b[1m" -const D = "\x1b[2m" -const R = "\x1b[0m" -const G = "\x1b[32m" -const Y = "\x1b[33m" -const RED = "\x1b[31m" -const C = "\x1b[36m" -const INV = "\x1b[7m" - -const HOUR = 3_600_000 - -// A plausible managed H100 rate, from the Modal table in the Atlas source. -let quote: Quote = { funding: "managed", rateCentsPerHour: 699, balanceCents: 1500 } -let proposal: Proposal = { provider: "lambda", sku: "gpu_1x_h100_pcie", hours: 4 } -let mode: GateMode = "first-hour" -let world: World = { - serverEnforcesDeadline: false, - clientAlive: true, - releaseFails: false, - planTtlHours: 24, -} -let verdict: Verdict | null = null -let lease: Lease | null = null -let now = Date.now() -let realProbe: string[] = [] -let log: string[] = [] - -function say(line: string) { - log.unshift(line) - log = log.slice(0, 6) -} - -function launch() { - verdict = decide({ proposal, quote, planTtlHours: world.planTtlHours, mode }) - if (!verdict.ok) { - say(`${RED}REJECTED${R} ${verdict.code} — ${verdict.message}`) - if (verdict.remedy) - say( - ` ${D}remedy the agent can act on:${R} propose ≤ ${formatHours(verdict.remedy.affordableHours)} ` + - `(needs ${formatCents(verdict.remedy.neededCents)}, has ${formatCents(verdict.remedy.availableCents)})`, - ) - lease = null - return - } - lease = { - id: `lease-${Math.abs(now % 100000)}`, - rateCentsPerHour: quote.funding === "byok" ? 0 : quote.rateCentsPerHour, - approvedHours: verdict.approvedHours, - startedAtMs: now, - // The server only records a deadline if we build that column. - expiresAtMs: world.serverEnforcesDeadline ? now + verdict.approvedHours * HOUR : undefined, - phase: "running", - spentCents: 0, - releaseAttempts: 0, - } - const clamp = verdict.clampedFrom ? ` ${Y}(clamped from ${formatHours(verdict.clampedFrom)})${R}` : "" - say( - `${G}APPROVED${R} ${formatHours(verdict.approvedHours)}${clamp} — ` + - `${verdict.billed ? formatCents(verdict.costCents) : "free (BYOK)"}`, - ) -} - -function tick(hours: number) { - now += hours * HOUR - if (!lease || lease.phase === "released" || lease.phase === "orphaned") return - lease.spentCents = spend(lease, now) - const action = evaluate(lease, now, world) - if (action.kind === "server-releases") { - lease.phase = "released" - lease.note = action.why - say(`${G}server released${R} — ${action.why} · billed ${formatCents(lease.spentCents)}`) - } else if (action.kind === "client-should-release") { - lease.phase = "past-deadline" - say(`${Y}deadline passed${R} — ${action.why}. Press [r] to release.`) - } else if (action.kind === "nobody-will-release") { - lease.phase = "orphaned" - lease.note = action.why - say(`${RED}${INV} ORPHANED ${R} ${action.why}`) - say(` ${RED}projected extra spend: ${formatCents(action.exposureCents)}${R}`) - } -} - -function release() { - if (!lease || lease.phase === "released") return say(`${D}nothing to release${R}`) - lease.releaseAttempts++ - const res = attemptRelease(lease, world) - if (res.ok) { - lease.phase = "released" - lease.spentCents = spend(lease, now) - say(`${G}released${R}${res.already ? " (409 already — still success)" : ""} · billed ${formatCents(lease.spentCents)}`) - return - } - say(`${RED}release failed${R} (attempt ${lease.releaseAttempts}/${MAX_RELEASE_ATTEMPTS}) — ${res.message}`) - if (lease.releaseAttempts >= MAX_RELEASE_ATTEMPTS) { - lease.phase = "orphaned" - lease.note = "release failed repeatedly — fail closed, surface loudly" - say(`${RED}${INV} FAIL CLOSED ${R} gave up after ${MAX_RELEASE_ATTEMPTS} — must alert, never assume cleanup`) - } -} - -async function probeReal() { - const base = process.env["ATLAS_BASE"] || "https://app.syntheticsciences.ai" - const token = process.env["ATLAS_TOKEN"] - realProbe = [`${D}base ${base}${R}`] - if (!token) { - realProbe.push(`${Y}set ATLAS_TOKEN=thk_… to probe the real (free) endpoints${R}`) - return - } - for (const [label, path, body] of [ - ["GET /api/compute/options", "/api/compute/options", null], - ["POST /api/compute/estimate", "/api/compute/estimate", { provider: proposal.provider, sku: proposal.sku }], - ] as const) { - try { - const res = await fetch(`${base}${path}`, { - method: body ? "POST" : "GET", - headers: { Authorization: `Bearer ${token}`, "Content-Type": "application/json" }, - body: body ? JSON.stringify(body) : undefined, - }) - const text = (await res.text()).slice(0, 160).replace(/\s+/g, " ") - realProbe.push(`${res.ok ? G : RED}${res.status}${R} ${label} ${D}${text}${R}`) - } catch (err) { - realProbe.push(`${RED}ERR${R} ${label} ${D}${err instanceof Error ? err.message : String(err)}${R}`) - } - } -} - -function render() { - console.clear() - console.log(`${B}managed-compute guardrail — state-machine prototype${R}`) - console.log(`${D}Q: does agent-proposes / Atlas-decides hold up? who enforces the deadline?${R}\n`) - - const total = quoteTotal(quote.rateCentsPerHour, Math.min(proposal.hours, world.planTtlHours)) - console.log( - `${B}PROPOSAL${R} ${proposal.sku} ${D}on${R} ${proposal.provider} ` + - `${C}${formatHours(proposal.hours)}${R} ${D}→ ${formatCents(total)} at ${formatCents(quote.rateCentsPerHour)}/h${R}`, - ) - console.log( - `${B}WALLET${R} ${formatCents(quote.balanceCents)} ${B}FUNDING${R} ${quote.funding} ` + - `${B}PLAN TTL${R} ${world.planTtlHours}h`, - ) - console.log( - `${B}GATE${R} ${mode === "first-hour" ? `${Y}first-hour (Atlas today)${R}` : `${G}total (proposed)${R}`}` + - ` ${B}SERVER DEADLINE${R} ${world.serverEnforcesDeadline ? `${G}yes${R}` : `${RED}no (Atlas today)${R}`}` + - ` ${B}CLIENT${R} ${world.clientAlive ? `${G}alive${R}` : `${RED}dead${R}`}` + - ` ${B}RELEASE${R} ${world.releaseFails ? `${RED}failing${R}` : `${G}ok${R}`}\n`, - ) - - if (lease) { - const el = (now - lease.startedAtMs) / HOUR - const over = overrunHours(lease, now) - const phase = - lease.phase === "running" - ? `${G}running${R}` - : lease.phase === "past-deadline" - ? `${Y}past-deadline${R}` - : lease.phase === "released" - ? `${D}released${R}` - : `${RED}${INV} ORPHANED ${R}` - console.log(`${B}LEASE${R} ${lease.id} ${phase}`) - console.log( - ` ${D}approved${R} ${formatHours(lease.approvedHours)} ${D}elapsed${R} ${formatHours(el)}` + - (over > 0 ? ` ${RED}overrun ${formatHours(over)}${R}` : "") + - ` ${D}billed${R} ${formatCents(spend(lease, now))}`, - ) - if (lease.note) console.log(` ${D}${lease.note}${R}`) - } else console.log(`${D}no lease — press [enter] to submit the proposal${R}`) - - if (realProbe.length) { - console.log(`\n${B}REAL API PROBE${R} ${D}(free endpoints only)${R}`) - for (const l of realProbe) console.log(` ${l}`) - } - - if (log.length) { - console.log(`\n${B}LOG${R}`) - for (const l of log) console.log(` ${l}`) - } - - console.log( - `\n${D}[h/H] hours -/+ [b/B] wallet -/+ [g] gate mode [s] server deadline [x] kill client` + - `\n[f] release failure [enter] submit [t] +1h [T] +6h [r] release [o] probe real [n] reset [q] quit${R}`, - ) -} - -function reset() { - now = Date.now() - lease = null - verdict = null - log = [] - say(`${D}reset${R}`) -} - -async function main() { - if (!process.stdin.isTTY || typeof process.stdin.setRawMode !== "function") { - console.error("This prototype is interactive — run it in a terminal:\n bun run prototype:guardrail\n") - process.exit(1) - } - process.stdin.setRawMode(true) - process.stdin.resume() - render() - for await (const chunk of process.stdin) { - const k = chunk.toString() - if (k === "q" || k === "") break - if (k === "h") proposal.hours = Math.max(0.5, proposal.hours - 0.5) - else if (k === "H") proposal.hours = Math.min(48, proposal.hours + 0.5) - else if (k === "b") quote.balanceCents = Math.max(0, quote.balanceCents - 500) - else if (k === "B") quote.balanceCents += 500 - else if (k === "g") mode = mode === "first-hour" ? "total" : "first-hour" - else if (k === "s") world.serverEnforcesDeadline = !world.serverEnforcesDeadline - else if (k === "x") world.clientAlive = !world.clientAlive - else if (k === "f") world.releaseFails = !world.releaseFails - else if (k === "\r" || k === "\n") launch() - else if (k === "t") tick(1) - else if (k === "T") tick(6) - else if (k === "r") release() - else if (k === "n") reset() - else if (k === "o") { - say(`${D}probing real endpoints…${R}`) - render() - await probeReal() - } - render() - } - process.stdin.setRawMode(false) - console.clear() - console.log("prototype exited\n") - process.exit(0) -} - -main() From 9431408bb5075029bb70ed4defd1c74e2ef27d81 Mon Sep 17 00:00:00 2001 From: KB Date: Sat, 1 Aug 2026 16:07:28 +0530 Subject: [PATCH 39/56] fix(compute): record WHY a mode was resolved, and stop asserting an unmeasured probe MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Resolution gains `origin` — "environment", "config:byok" or "config:managed". Without it a caller cannot tell "managed because the environment says so", where connecting a provider key flips the next call to byok, from "managed because billing.compute pins it", where connecting a key changes neither resolve() (funded() never reads providers) nor offered() (empty under a managed override). Those two states need opposite advice, and the guidance layer has been giving the first state's advice to both. The override VALUE is carried rather than a forced/not-forced bit because "none" is reachable from both overrides. `managed` becomes optional. Both byok arms returned `managed: false` without probing — the skip is deliberate (a byok user never pays for the round trip), so the answer was never measured. Absent now means "not checked". --- backend/cli/src/compute/mode.ts | 39 ++++++++++++++---- backend/cli/test/compute/mode.test.ts | 58 +++++++++++++++++++++++++++ 2 files changed, 90 insertions(+), 7 deletions(-) diff --git a/backend/cli/src/compute/mode.ts b/backend/cli/src/compute/mode.ts index fcd8ab5e..c325ccea 100644 --- a/backend/cli/src/compute/mode.ts +++ b/backend/cli/src/compute/mode.ts @@ -86,13 +86,36 @@ export namespace ComputeMode { return Object.keys(PROVIDERS).filter((id) => keyed(PROVIDERS[id].env)) } + /** + * WHY `mode` is what it is — the one thing a caller cannot reconstruct from + * `mode` alone, and the thing that decides what advice is even actionable. + * "managed" reached from the environment means the user holds no provider + * credential, so connecting one flips the next call to byok. "managed" + * reached from `billing.compute` means the setting pins it: connecting a + * credential changes neither `resolve()` (`funded()` never consults + * `providers`) nor `offered()` (empty under a managed override), so telling + * the user to connect one is advice the setting itself defeats. The override + * value is carried, not just a forced/not-forced bit, because "none" is + * reachable from BOTH overrides and they need opposite advice. + */ + export type Origin = "environment" | "config:byok" | "config:managed" + export interface Resolution { mode: Source /** Credentialed BYOK providers, in PROVIDERS declaration order. */ providers: string[] - managed: boolean + /** + * Whether managed compute is available — `undefined` when nothing measured + * it. The byok arms return before `available()` runs (that skip is the + * point: a byok user never pays for the round trip), so they have no + * verdict to report. Absence means "not checked", never "no": a hardcoded + * `false` there would be a fact the code never established, which is the + * exact defect this module exists to remove. + */ + managed?: boolean /** Wallet balance in USD. Present only when mode === "managed". */ balance?: number + origin: Origin } /** Hard ceiling on how long resolution may block an agent turn. Atlas's own @@ -154,12 +177,13 @@ export namespace ComputeMode { * `available()` completely and must never fall back to "byok" just because * a credential happens to be present (that would silently defeat the * managed override — see the "managed with a usable provider" test). */ - function funded(providers: string[], state: { managed: boolean; balance?: number }): Resolution { + function funded(providers: string[], state: { managed: boolean; balance?: number }, origin: Origin): Resolution { return { mode: state.managed ? "managed" : "none", providers, managed: state.managed, balance: state.managed ? state.balance : undefined, + origin, } } @@ -173,19 +197,20 @@ export namespace ComputeMode { const override = (await Config.get()).billing?.compute if (override === "byok") { - return { mode: providers.length ? "byok" : "none", providers, managed: false } + return { mode: providers.length ? "byok" : "none", providers, origin: "config:byok" } } if (override === "managed") { - return funded(providers, await available()) + return funded(providers, await available(), "config:managed") } // BYOK wins when a credentialed provider is present: it is free to the user, // it works today, and it needs nothing from Atlas. This is also why a BYOK - // user never pays for the availability call. - if (providers.length) return { mode: "byok", providers, managed: false } + // user never pays for the availability call — and why `managed` is left + // unset here rather than false (see Resolution.managed). + if (providers.length) return { mode: "byok", providers, origin: "environment" } - return funded(providers, await available()) + return funded(providers, await available(), "environment") } /** diff --git a/backend/cli/test/compute/mode.test.ts b/backend/cli/test/compute/mode.test.ts index 25996029..4646a3fe 100644 --- a/backend/cli/test/compute/mode.test.ts +++ b/backend/cli/test/compute/mode.test.ts @@ -277,6 +277,41 @@ describe("ComputeMode.resolve", () => { expect(calls.filter((url) => url.includes(OPTIONS_URL))).toEqual([]) }) + test("a byok resolution leaves managed availability UNMEASURED, not false", async () => { + // The byok arms deliberately skip the availability probe — that skip is the + // performance win. `managed: false` there would be an unmeasured claim, and + // the tool prints it as a fact ("managed available: no"). + await signIn() + stubOptions(MANAGED_ON) + process.env["LAMBDA_API_KEY"] = "k" + const result = await withSkills(["lambda-labs-gpu-cloud"], () => ComputeMode.resolve()) + expect(result.mode).toBe("byok") + expect(result.managed).toBeUndefined() + expect(calls.filter((url) => url.includes(OPTIONS_URL))).toEqual([]) + }) + + test("a probed resolution reports availability as a measured boolean", async () => { + await signIn() + stubOptions(MANAGED_ON) + expect((await withSkills([], () => ComputeMode.resolve())).managed).toBe(true) + ComputeMode.invalidate() + stubOptions(MANAGED_OFF) + expect((await withSkills([], () => ComputeMode.resolve())).managed).toBe(false) + }) + + test("with no override, every arm records origin 'environment'", async () => { + await signIn() + stubOptions(MANAGED_ON) + process.env["LAMBDA_API_KEY"] = "k" + expect((await withSkills(["lambda-labs-gpu-cloud"], () => ComputeMode.resolve())).origin).toBe("environment") + clearEnv() + ComputeMode.invalidate() + expect((await withSkills([], () => ComputeMode.resolve())).origin).toBe("environment") + ComputeMode.invalidate() + stubOptions(MANAGED_OFF) + expect((await withSkills([], () => ComputeMode.resolve())).origin).toBe("environment") + }) + test("the availability answer is cached within the TTL", async () => { await signIn() stubOptions(MANAGED_ON) @@ -381,6 +416,29 @@ describe("ComputeMode.resolve override", () => { expect(result.providers).toEqual(["lambda"]) }) + test("an override stamps the origin with the setting that forced the mode", async () => { + // A caller cannot otherwise tell "managed because the environment says so" + // (where connecting a key flips to byok next call) from "managed because + // billing.compute pins it" (where connecting a key changes nothing) — and + // those two states need opposite advice. + await signIn() + stubOptions(MANAGED_ON) + process.env["LAMBDA_API_KEY"] = "k" + expect((await withOverride("byok", ["lambda-labs-gpu-cloud"], () => ComputeMode.resolve())).origin).toBe( + "config:byok", + ) + ComputeMode.invalidate() + expect((await withOverride("managed", ["lambda-labs-gpu-cloud"], () => ComputeMode.resolve())).origin).toBe( + "config:managed", + ) + // Narrowed to "none" the origin still has to name the setting that narrowed it. + ComputeMode.invalidate() + stubOptions(MANAGED_OFF) + const narrowed = await withOverride("managed", ["lambda-labs-gpu-cloud"], () => ComputeMode.resolve()) + expect(narrowed.mode).toBe("none") + expect(narrowed.origin).toBe("config:managed") + }) + test("override managed beats a usable provider when managed IS available", async () => { await signIn() stubOptions(MANAGED_ON) From 05ef093f44d2f5e9be24811bf21c334de6db363c Mon Sep 17 00:00:00 2001 From: KB Date: Sat, 1 Aug 2026 16:10:26 +0530 Subject: [PATCH 40/56] fix(compute): stop compute_status naming capabilities the client does not have MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three strings claimed something untrue. Managed with a positive balance said "Run GPU work through managed compute" and "Do not use the user's own provider keys". There is no managed launch mechanism here: ComputeTools is [ComputeStatusTool], the only /api/compute call in the product is mode.ts's read-only /options probe, and compute_launch/list/ release are Part B, unbuilt. A signed-in keyless user with a funded wallet — the default path — was told to do the impossible and forbidden the only fallback. It now says managed is funded, that OpenScience cannot launch it, and that connecting a provider key is the working path. `none` offered a top-up. No path to none is balance-related — probe() returns managed:false only for no session, non-2xx, a network/parse failure, or no provider with funding "managed", and Atlas reports managed regardless of balance — so topping up could never move a user out of none. Both remedies assumed the user could act. Under an explicit billing.compute override they cannot, so the advice now switches on the new `origin` and names the setting instead. Availability is printed tri-state. The byok arms never probe, so "managed available: no" was an unmeasured assertion; it now reads "not checked" and metadata.managed_available is absent rather than false. --- backend/cli/src/tool/compute.ts | 111 +++++++++++++------ backend/cli/test/tool/compute-status.test.ts | 91 +++++++++++++-- 2 files changed, 164 insertions(+), 38 deletions(-) diff --git a/backend/cli/src/tool/compute.ts b/backend/cli/src/tool/compute.ts index 10dbae34..bbe133d4 100644 --- a/backend/cli/src/tool/compute.ts +++ b/backend/cli/src/tool/compute.ts @@ -16,44 +16,93 @@ import { ComputeMode } from "@/compute/mode" * are in every request regardless, so it costs nothing extra. The RESULT carries * the specifics. Adding rates or a balance to an every-turn injection would be * expensive; adding them here is free. + * + * Every string below is bound by one rule: never name a capability that does not + * exist. That is what mode resolution enforces for `mode` itself, and the + * guidance text sits one layer above it, where the same defect keeps reappearing + * in a different shape. */ -const GUIDANCE: Record, string> = { - byok: "Run GPU work on the user's connected providers via the cloud-compute skills. Do not launch managed leases — they bill Credits and are not the funded path here.", - none: "No compute is available. Do not attempt GPU work. Tell the user to connect a provider key in Settings ▸ Compute, or to top up for managed compute.", +/** + * The two ways out of a mode that cannot run GPU work, and they are mutually + * exclusive. `origin` decides which one is true (see ComputeMode.Origin): a + * config-pinned "managed" makes CONNECT a lie, because `funded()` never reads + * `providers` and `offered()` returns empty under that override — the key would + * be connected, ignored, and its skills still hidden. + */ +const CONNECT = + "Tell the user to connect a provider key in Settings ▸ Compute — a key connected there is picked up on the next call, no restart, and BYOK is the only path that runs GPU work today." +const PINNED = + '`billing.compute` is pinned to "managed" in the config, so connecting a provider key will not switch this session to byok — the setting has to be changed or removed first.' + +function escape(origin: ComputeMode.Origin): string { + return origin === "config:managed" ? PINNED : CONNECT } +const BYOK = + "Run GPU work on the user's connected providers via the cloud-compute skills. Do not launch managed leases — they bill Credits and are not the funded path here." + /** - * `managed` guidance is a function of balance, not a fixed string, because - * `GET /api/compute/options` reports a provider as managed whenever reselling - * is on and an operator key exists — availability and affordability are - * independent there. Live testing against a deployed backend with a zero - * wallet found the old fixed string sent the agent down a path that cannot - * work (every lease acquire returns HTTP 402 insufficient_cli_credit) while - * simultaneously forbidding the only fallback that would (the user's own - * provider keys). That is the same "claims a capability with nothing behind - * it" defect Part A removed from mode resolution itself (see ComputeMode's - * doc comment) — just one layer up: the mode really is `managed`, the - * capability exists, but the guidance text assumed `managed` meant funded. + * `managed` guidance is a function of balance AND origin, not a fixed string. * - * This does NOT change `state.mode`. Managed is genuinely configured; an - * empty wallet is missing funds, not a missing capability, and the two need - * different advice — top up vs. connect a key. Collapsing them would destroy - * that distinction, which is exactly the mistake the override rule in - * `ComputeMode.resolve` (narrow, never manufacture) exists to prevent. + * Balance, because `GET /api/compute/options` reports a provider as managed + * whenever reselling is on and an operator key exists — availability and + * affordability are independent there, and live testing against a deployed + * backend with a zero wallet confirmed it (every lease acquire returns HTTP 402 + * insufficient_cli_credit). Only `balance === 0` counts as unaffordable: + * acquiring a lease requires an hour of the chosen SKU's rate up front, and + * rates span cents to dollars an hour depending on a catalog this tool never + * sees, so any non-zero cutoff would be a guess. `balance === undefined` (the + * probe succeeded but carried no balance field) is missing information, not an + * empty wallet, and is left alone. * - * Only `balance === 0` counts as unaffordable. Acquiring a lease requires one - * hour of the chosen SKU's rate up front, and rates span cents to dollars an - * hour depending on the catalog Atlas holds and this tool never sees — any - * non-zero cutoff here would be a guess. `balance === undefined` (the probe - * succeeded but the response carried no balance field) is left alone too: - * that is missing information, not a zero balance, and treating it as empty - * would be its own honesty bug. + * Origin, because the way out differs — see `escape`. + * + * Neither branch tells the agent to run the work. THERE IS NO MANAGED LAUNCH + * MECHANISM IN THIS CLIENT: `ComputeTools` is `[ComputeStatusTool]`, the only + * `/api/compute` call in the product is mode.ts's read-only `/options` probe, + * and `compute_launch`/`list`/`release` are Part B, unbuilt (docs/specs/ + * compute-design.md, "The gap Part A exposed"). A funded, keyless user — the + * default for anyone signed in — was previously told to do the one thing that + * cannot be done and forbidden the one thing that can. + * + * This does NOT change `state.mode`. Managed is genuinely configured; an empty + * wallet is missing funds and an unbuilt launch path is a missing tool, and + * neither is a missing capability at the resolution layer. Collapsing them + * would destroy distinctions the override rule in `ComputeMode.resolve` + * (narrow, never manufacture) exists to preserve. */ -function managedGuidance(balance: number | undefined): string { +function managedGuidance(balance: number | undefined, origin: ComputeMode.Origin): string { if (balance === 0) - return "Managed compute is configured, but the wallet is empty — every lease attempt will be refused (HTTP 402). Tell the user to top up in Settings ▸ Compute, or to connect their own provider key to run BYOK instead." - return "Run GPU work through managed compute, billed to Credits. Do not use the user's own provider keys — they are not funded here." + return `Managed compute is configured, but the wallet is empty — every lease attempt would be refused (HTTP 402), and OpenScience cannot launch it in any case: this client has no managed-lease command yet, only this status tool. Do not launch GPU work through managed compute. Topping up in Settings ▸ Compute fixes the wallet but not the missing launch path. ${escape(origin)}` + return `Managed compute is funded, but OpenScience cannot launch it — this client has no managed-lease command yet, only this status tool. Do not launch GPU work through managed compute. ${escape(origin)}` +} + +/** + * No path to `none` is balance-related — `probe()` returns `managed: false` + * only for no session, a non-2xx response, a network/parse failure, or no + * provider with `funding: "managed"`, and Atlas reports managed regardless of + * balance. So "top up for managed compute" was a remedy that could never move a + * user out of this mode. + */ +function noneGuidance(origin: ComputeMode.Origin): string { + return `No compute is available. Do not attempt GPU work. ${escape(origin)}` +} + +/** + * Tri-state on purpose. Both byok arms return before the availability probe + * runs — that skip is the performance win, so there is no verdict to print and + * "no" would be an unmeasured claim. + */ +function availability(managed: boolean | undefined): string { + if (managed === undefined) return "not checked (byok takes precedence, so availability was never probed)" + return managed ? "yes" : "no" +} + +function guidance(state: ComputeMode.Resolution): string { + if (state.mode === "byok") return BYOK + if (state.mode === "managed") return managedGuidance(state.balance, state.origin) + return noneGuidance(state.origin) } export const ComputeStatusTool = Tool.define("compute_status", { @@ -68,10 +117,10 @@ export const ComputeStatusTool = Tool.define("compute_status", { const lines = [ `**mode**: ${state.mode}`, `**providers**: ${state.providers.length ? state.providers.join(", ") : "none configured"}`, - `**managed available**: ${state.managed ? "yes" : "no"}`, + `**managed available**: ${availability(state.managed)}`, ] if (state.balance !== undefined) lines.push(`**balance**: $${state.balance.toFixed(2)}`) - lines.push("", state.mode === "managed" ? managedGuidance(state.balance) : GUIDANCE[state.mode]) + lines.push("", guidance(state)) return { title: `Compute: ${state.mode}`, diff --git a/backend/cli/test/tool/compute-status.test.ts b/backend/cli/test/tool/compute-status.test.ts index 48365450..c22a097d 100644 --- a/backend/cli/test/tool/compute-status.test.ts +++ b/backend/cli/test/tool/compute-status.test.ts @@ -53,7 +53,9 @@ const MANAGED_ZERO = { cli_effective_balance_cents: 0, } -async function run(skills: string[], fn?: () => Promise) { +/** `override` writes billing.compute into the project's openscience.json, the + * only way to reach a config-forced origin. */ +async function run(skills: string[], override?: "byok" | "managed") { await using tmp = await tmpdir({ git: true, init: async (dir) => { @@ -63,12 +65,14 @@ async function run(skills: string[], fn?: () => Promise) { `---\nname: ${name}\ndescription: Fixture ${name}.\ncategory: cloud-compute\n---\n\n# ${name}\n`, ) } + if (override) { + await Bun.write(path.join(dir, "openscience.json"), JSON.stringify({ billing: { compute: override } })) + } }, }) return Instance.provide({ directory: tmp.path, fn: async () => { - await fn?.() const tool = await ComputeStatusTool.init({}) return tool.execute({}, CTX as never) }, @@ -106,7 +110,7 @@ describe("compute_status", () => { expect(result.metadata.mode).toBe("managed") expect(result.metadata.balance_usd).toBe(42) expect(result.output).toContain("42") - expect(result.output.toLowerCase()).toContain("credits") + expect(result.output.toLowerCase()).toContain("managed compute is funded") // balance_usd must come from the SAME /api/compute/options response that // decided managed availability, never a second round trip. expect(calls.filter((url) => url.includes("/api/compute/options")).length).toBe(1) @@ -122,13 +126,45 @@ describe("compute_status", () => { stub(MANAGED_ZERO) const result = await run([]) expect(result.output.toLowerCase()).not.toContain("run gpu work through managed compute") - expect(result.output.toLowerCase()).toContain("top up") + expect(result.output.toLowerCase()).toContain("wallet is empty") + expect(result.output.toLowerCase()).toContain("topping up") }) - test("managed with a positive balance keeps today's guidance", async () => { + test("a funded managed wallet does not claim OpenScience can launch managed compute", async () => { + // There is no managed launch mechanism in this client: ComputeTools is + // [ComputeStatusTool] and the only /api/compute call anywhere is mode.ts's + // read-only /options probe. Telling a funded, keyless user (the default + // path for anyone signed in) to "run GPU work through managed compute" + // while forbidding the only fallback left the agent with nothing that works. stub(MANAGED_ON) const result = await run([]) - expect(result.output.toLowerCase()).toContain("run gpu work through managed compute") + const output = result.output.toLowerCase() + expect(output).not.toContain("run gpu work through managed compute") + expect(output).not.toContain("do not use the user's own provider keys") + expect(output).toContain("cannot launch it") + expect(output).toContain("settings ▸ compute") + }) + + test("a managed override names billing.compute instead of advice that setting blocks", async () => { + // Under an explicit override, connecting a key flips nothing: funded() + // never consults `providers` and offered() returns empty, so the skills + // stay hidden too. "Connect a key to run BYOK instead" is advice the + // setting itself defeats. + stub(MANAGED_ZERO) + const result = await run([], "managed") + expect(result.metadata.mode).toBe("managed") + expect(result.output).toContain("billing.compute") + expect(result.output.toLowerCase()).toContain("will not switch") + }) + + test("managed resolved from the environment still says to connect a key", async () => { + // The mirror of the override case: with no override, mode is managed only + // because the user holds no credential, so connecting one really does flip + // the next call to byok. + stub(MANAGED_ZERO) + const result = await run([]) + expect(result.output).not.toContain("billing.compute") + expect(result.output).toContain("Settings ▸ Compute") }) test("a zero balance narrows guidance only — mode stays managed, balance stays reported", async () => { @@ -147,6 +183,47 @@ describe("compute_status", () => { expect(result.metadata.balance_usd).toBeUndefined() }) + test("none never offers a top-up, which cannot move a user out of none", async () => { + // No path to `none` is balance-related: probe() returns managed:false only + // for no session, non-2xx, a network/parse failure, or no provider with + // funding "managed" — and Atlas reports managed regardless of balance. + stub(MANAGED_OFF) + const result = await run([]) + expect(result.output.toLowerCase()).not.toContain("top up") + }) + + test("a none narrowed by a managed override names the setting that narrowed it", async () => { + stub(MANAGED_OFF) + process.env["LAMBDA_API_KEY"] = "k" + const result = await run(["lambda-labs-gpu-cloud"], "managed") + expect(result.metadata.mode).toBe("none") + expect(result.output).toContain("billing.compute") + }) + + test("byok reports managed availability as unchecked, never as 'no'", async () => { + // The byok arms skip the availability probe by design, so "managed + // available: no" would be a fact the tool never measured — harmless today, + // load-bearing the moment Part B reads managed_available from metadata. + stub(MANAGED_ON) + process.env["LAMBDA_API_KEY"] = "k" + const result = await run(["lambda-labs-gpu-cloud"]) + expect(result.output).toContain("**managed available**: not checked") + expect(result.metadata.managed_available).toBeUndefined() + expect(calls.filter((url) => url.includes("/api/compute/options"))).toEqual([]) + }) + + test("a probed mode still reports availability as measured yes/no", async () => { + stub(MANAGED_ON) + const managed = await run([]) + expect(managed.output).toContain("**managed available**: yes") + expect(managed.metadata.managed_available).toBe(true) + ComputeMode.invalidate() + stub(MANAGED_OFF) + const none = await run([]) + expect(none.output).toContain("**managed available**: no") + expect(none.metadata.managed_available).toBe(false) + }) + test("a provider with a key but no skill is still reported as usable byok", async () => { stub(MANAGED_OFF) process.env["RUNPOD_API_KEY"] = "rpa_x" @@ -180,7 +257,7 @@ describe("compute_status", () => { // in that mode's output and ONLY that mode's output. const PHRASE = { byok: "do not launch managed", - managed: "do not use the user's own provider keys", + managed: "cannot launch it", none: "do not attempt gpu work", } const output = { From 6f03e02e49b1db3077dfb73784d8b2e3a0b498cb Mon Sep 17 00:00:00 2001 From: KB Date: Sat, 1 Aug 2026 16:12:04 +0530 Subject: [PATCH 41/56] fix(prompt): make research.txt's Stage 5 compute skills conditional MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stage 5 gated skill loading on compute_status and then, two lines later, told the agent unconditionally to load `modal-research-gpu` and `modal-serverless-gpu` — precisely the names ComputeMode.SKILLS hides when Modal is not credentialed. The unconditional lines won. They are folded into the byok branch, and the `managed` branch no longer says to run the work through managed compute, which the client cannot do. The guard test is written over ComputeMode.SKILLS and every COMPUTE_AGENTS prompt rather than over research.txt, so a sibling prompt cannot reintroduce it. --- backend/cli/src/agent/prompt/research.txt | 13 ++-- .../cli/test/session/compute-prompt.test.ts | 61 +++++++++++++++++++ 2 files changed, 69 insertions(+), 5 deletions(-) diff --git a/backend/cli/src/agent/prompt/research.txt b/backend/cli/src/agent/prompt/research.txt index a27feee4..7940edb8 100644 --- a/backend/cli/src/agent/prompt/research.txt +++ b/backend/cli/src/agent/prompt/research.txt @@ -254,11 +254,14 @@ Execute computational work. If `methodology.md` exists, follow the pipeline defi - Call the `compute_status` tool before launching any GPU work. It reports how compute is funded right now — `byok`, `managed`, or `none` — which providers are usable, and the rule that applies. This is the only compute-availability signal — CLI auth status is not one. -- If it returns `byok`, load the cloud-compute skill for one of the providers it lists. - If `managed`, run the work through managed compute. If `none`, do not launch GPU work — - tell the user to connect a provider key in Settings ▸ Compute. -- Load: `modal-research-gpu` for GPU-accelerated scientific computing -- Load: `modal-serverless-gpu` for general serverless GPU (inference, serving) +- If it returns `byok`, load the cloud-compute skill for one of the providers it lists — + when Modal is one of them, `modal-research-gpu` for GPU-accelerated scientific computing + or `modal-serverless-gpu` for general serverless GPU (inference, serving). Never load a + compute skill for a provider the tool did not list. +- If it returns `managed`, do not launch GPU work either — OpenScience cannot start a + managed lease yet. Follow the rule the tool returns and tell the user what it says. +- If it returns `none`, do not launch GPU work — tell the user to connect a provider key + in Settings ▸ Compute. - Load: domain libraries as needed (see Scientific Computing skills) - Present cost estimate and get approval before launching jobs - Run computations, monitor progress, collect outputs diff --git a/backend/cli/test/session/compute-prompt.test.ts b/backend/cli/test/session/compute-prompt.test.ts index a0746139..1c921c07 100644 --- a/backend/cli/test/session/compute-prompt.test.ts +++ b/backend/cli/test/session/compute-prompt.test.ts @@ -16,7 +16,68 @@ async function sources() { return Promise.all(files.map(async (file) => [file, await Bun.file(file).text()] as const)) } +/** Every primary agent that can send a user down a compute path. `sources()` + * globs the whole prompt tree but the assertions below used to hardcode + * research.txt, which made them structurally unable to catch the same defect + * in a sibling prompt. */ +const COMPUTE_AGENTS = ["research", "biology", "physics", "ml"] + +async function agents() { + return Promise.all( + COMPUTE_AGENTS.map(async (name) => { + const file = path.join(root, "agent", "prompt", `${name}.txt`) + return [file, await Bun.file(file).text()] as const + }), + ) +} + +/** Markdown bullets with continuation lines folded in, so a rule about what an + * instruction says sees the whole instruction rather than its first line. */ +function bullets(text: string) { + const out: string[] = [] + let open = false + for (const line of text.split("\n")) { + if (/^\s*[-*] /.test(line)) { + out.push(line.trim()) + open = true + continue + } + if (open && /^\s+\S/.test(line)) { + out[out.length - 1] += " " + line.trim() + continue + } + open = false + } + return out +} + describe("compute prompt text", () => { + test("every COMPUTE_AGENTS prompt exists and is non-empty", async () => { + // The bans below are only worth as much as the file set they run over — a + // renamed prompt must fail loudly, not silently drop out of coverage. + const loaded = await agents() + expect(loaded.map(([file]) => path.relative(root, file))).toEqual( + COMPUTE_AGENTS.map((name) => path.join("agent", "prompt", `${name}.txt`)), + ) + for (const [file, text] of loaded) expect(text.length, file).toBeGreaterThan(100) + }) + + test("no agent prompt tells the agent to load a mode-gated compute skill unconditionally", async () => { + // ComputeMode.SKILLS names are hidden from the catalog unless the provider + // is credentialed. An unconditional "Load: `modal-research-gpu`" both + // overrides whatever compute_status just returned and points at a skill the + // filter may have removed, so any bullet naming one has to be gated on byok. + const gated = [...ComputeMode.SKILLS] + const hits = (await agents()).flatMap(([file, text]) => + bullets(text) + .filter((bullet) => /\bload\b/i.test(bullet)) + .filter((bullet) => gated.some((skill) => bullet.includes(skill))) + .filter((bullet) => !bullet.includes("byok")) + .map((bullet) => `${path.relative(root, file)}: ${bullet}`), + ) + expect(hits).toEqual([]) + }) + test("no prompt or session source references atlas compute:up", async () => { const hits = (await sources()).filter(([, text]) => text.includes("compute:up")) expect(hits.map(([file]) => path.relative(root, file))).toEqual([]) From 7125d6b052bba9e5375c727d6b033f3b0a9f5939 Mon Sep 17 00:00:00 2001 From: KB Date: Sat, 1 Aug 2026 16:13:15 +0530 Subject: [PATCH 42/56] fix(prompt): stop telling users to restart to pick up a credential MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit research.txt and biology.txt both said "then restart openscience", which contradicts this branch's central claim and the test that asserts it (compute-status.test.ts, "a credential connected between two calls changes the answer, no restart"). The Compute and Credentials panels call applyComputeEnv/applyCredentialEnv on save, and a key added in the hosted dashboard lands via refreshIfStale's background sync on the next message. Both lines now also point GPU provider keys at Settings ▸ Compute, the surface compute_status's own guidance names. The bans (compute:up, restart) now run over every COMPUTE_AGENTS prompt instead of research.txt alone. Widening them caught a third file the review had not listed: ml.txt's "in Settings → Credentials and restart". Its skill-by-directory names (`modal`, `tinker`, `tensorpool`) are a separate known defect and are untouched. --- backend/cli/src/agent/prompt/biology.txt | 3 ++- backend/cli/src/agent/prompt/ml.txt | 3 ++- backend/cli/src/agent/prompt/research.txt | 3 ++- .../cli/test/session/compute-prompt.test.ts | 25 +++++++++++++++++++ 4 files changed, 31 insertions(+), 3 deletions(-) diff --git a/backend/cli/src/agent/prompt/biology.txt b/backend/cli/src/agent/prompt/biology.txt index 2bb8848b..34aab5f8 100644 --- a/backend/cli/src/agent/prompt/biology.txt +++ b/backend/cli/src/agent/prompt/biology.txt @@ -56,7 +56,8 @@ Before using any external service, verify credentials: [ -n "$VAR_NAME" ] && echo "set" || echo "not set" If not connected, tell the user: - "Connect [service] at https://app.syntheticsciences.ai -> Services, then restart openscience." + "Connect [service] at https://app.syntheticsciences.ai -> Services (GPU provider keys: + Settings ▸ Compute). A newly connected key is picked up on the next message — no restart." ## Native Database & Analysis Tools diff --git a/backend/cli/src/agent/prompt/ml.txt b/backend/cli/src/agent/prompt/ml.txt index f6fcf30a..4ff811ca 100644 --- a/backend/cli/src/agent/prompt/ml.txt +++ b/backend/cli/src/agent/prompt/ml.txt @@ -57,7 +57,8 @@ Before using any external service, verify credentials are set: ``` Common: `TINKER_API_KEY`, `MODAL_TOKEN_ID`+`MODAL_TOKEN_SECRET`, `HF_TOKEN`, `WANDB_API_KEY`, `PRIME_API_KEY`, `TENSORPOOL_KEY`. If a service is not connected, tell the user to connect it -in Settings → Credentials and restart, then continue with what you can do. +in Settings → Credentials (GPU provider keys: Settings → Compute); it is picked up on the +next message, no restart. Continue with what you can do meanwhile. ## CRITICAL: Environment Setup Before running ANY Python, set up an isolated environment in the working directory: diff --git a/backend/cli/src/agent/prompt/research.txt b/backend/cli/src/agent/prompt/research.txt index 7940edb8..0b3ea22c 100644 --- a/backend/cli/src/agent/prompt/research.txt +++ b/backend/cli/src/agent/prompt/research.txt @@ -42,7 +42,8 @@ Before using any external service, verify credentials: [ -n "$VAR_NAME" ] && echo "set" || echo "not set" If not connected, tell the user: - "Connect [service] at https://app.syntheticsciences.ai -> Services, then restart openscience." + "Connect [service] at https://app.syntheticsciences.ai -> Services (GPU provider keys: + Settings ▸ Compute). A newly connected key is picked up on the next message — no restart." ## CRITICAL: Convergence & Anti-Loop Your job is to finish the task IN THIS SESSION, not to hand it off. diff --git a/backend/cli/test/session/compute-prompt.test.ts b/backend/cli/test/session/compute-prompt.test.ts index 1c921c07..2edb2b15 100644 --- a/backend/cli/test/session/compute-prompt.test.ts +++ b/backend/cli/test/session/compute-prompt.test.ts @@ -62,6 +62,31 @@ describe("compute prompt text", () => { for (const [file, text] of loaded) expect(text.length, file).toBeGreaterThan(100) }) + /** + * Instructions no compute-capable prompt may carry. + * + * The restart pattern is phrase-shaped, not word-shaped, on purpose: "Do not + * restart on a missing tool" is a legitimate instruction and so is stating + * that a newly connected key needs NO restart. What is banned is telling the + * user to restart to pick a credential up — which contradicts a tested design + * claim (compute-status.test.ts, "a credential connected between two calls + * changes the answer, no restart"): the Compute and Credentials panels call + * applyComputeEnv/applyCredentialEnv on save, and a key added in the hosted + * dashboard lands via refreshIfStale's background sync on the next message. + */ + const BANNED: Array<[string, RegExp]> = [ + ["atlas compute:up", /compute:up/], + ["telling the user to restart", /(?:then|and)\s+restart|restart\s+openscience|restart\s+the\s+(?:cli|session)/i], + ] + + test("every compute-capable agent prompt is free of the banned instructions", async () => { + const loaded = await agents() + const hits = BANNED.flatMap(([label, pattern]) => + loaded.filter(([, text]) => pattern.test(text)).map(([file]) => `${path.relative(root, file)}: ${label}`), + ) + expect(hits).toEqual([]) + }) + test("no agent prompt tells the agent to load a mode-gated compute skill unconditionally", async () => { // ComputeMode.SKILLS names are hidden from the catalog unless the provider // is credentialed. An unconditional "Load: `modal-research-gpu`" both From a732396d669435f31f8a9990287c63d1cf1df1d8 Mon Sep 17 00:00:00 2001 From: KB Date: Sat, 1 Aug 2026 19:36:01 +0530 Subject: [PATCH 43/56] fix(sandbox): tmpfs the XDG cache dir so bwrap stops read-only-fs errors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit bubblewrapArgs mounts the whole fs read-only and deliberately refuses $HOME as a writable root (tooBroadToConfine), so any tool that writes to ~/.cache on startup — zsh's compdump/history lock, pip, npm, uv — fails with "Read-only file system" inside the sandbox. Mount a tmpfs over the resolved XDG cache dir (XDG_CACHE_HOME, else ~/.cache) after the root ro-bind and before the policy.writable binds: writes succeed so the tools stop erroring, but nothing persists to the real home, so the containment tooBroadToConfine enforces is untouched. --- backend/cli/src/sandbox/sandbox.ts | 10 ++++++++++ backend/cli/test/sandbox/sandbox.test.ts | 18 ++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/backend/cli/src/sandbox/sandbox.ts b/backend/cli/src/sandbox/sandbox.ts index 2f261ffc..23883d47 100644 --- a/backend/cli/src/sandbox/sandbox.ts +++ b/backend/cli/src/sandbox/sandbox.ts @@ -259,6 +259,16 @@ export namespace Sandbox { // Whole fs read-only, a fresh /dev and /proc, and a throwaway writable /tmp; // then re-mount the bits that must be writable on top. const args = ["--ro-bind", "/", "/", "--dev", "/dev", "--proc", "/proc", "--tmpfs", "/tmp"] + // $HOME is deliberately refused as a writable root (tooBroadToConfine), which + // leaves the whole-fs --ro-bind covering the XDG cache dir too. Startup tools + // that write there unconditionally (zsh's compdump/history lock, pip/npm/uv + // caches, ...) then fail with "Read-only file system". A tmpfs — not a bind — + // is the fix: writes succeed so those tools stop erroring, but nothing here + // persists to the real home, so the containment tooBroadToConfine enforces + // stays intact. Mounted before the writable binds below, so an explicitly + // writable path under the cache dir still wins. + const cache = process.env.XDG_CACHE_HOME || path.join(os.homedir(), ".cache") + args.push("--tmpfs", cache) for (const p of dedupe(policy.writable)) { // Skip only the /tmp mount root itself — it is provided as a fresh tmpfs and // re-binding host /tmp would defeat it. A workspace that lives *under* /tmp diff --git a/backend/cli/test/sandbox/sandbox.test.ts b/backend/cli/test/sandbox/sandbox.test.ts index 503f7c6f..9d0e939e 100644 --- a/backend/cli/test/sandbox/sandbox.test.ts +++ b/backend/cli/test/sandbox/sandbox.test.ts @@ -1,5 +1,6 @@ import { describe, expect, test } from "bun:test" import os from "os" +import path from "path" import { Sandbox } from "../../src/sandbox/sandbox" const shell = "/bin/sh" @@ -66,6 +67,23 @@ describe("Sandbox.bubblewrapArgs", () => { test("unshares the PID namespace so /proc escape vectors are closed", () => { expect(Sandbox.bubblewrapArgs({ writable: ["/w"], network: true })).toContain("--unshare-pid") }) + + test("mounts a tmpfs over the XDG cache dir, after the root ro-bind and before writable binds", () => { + // $HOME is deliberately not writable (tooBroadToConfine), so tools that touch + // the XDG cache on startup (zsh compdump/history lock, pip/npm/uv caches, …) + // hit a read-only $HOME and fail — this is the regression this test pins. + const args = Sandbox.bubblewrapArgs({ writable: ["/work/project"], network: true }) + const cache = process.env.XDG_CACHE_HOME ?? path.join(os.homedir(), ".cache") + const cacheIdx = args.indexOf(cache) + expect(cacheIdx).toBeGreaterThan(-1) + expect(args[cacheIdx - 1]).toBe("--tmpfs") + + const roIdx = args.indexOf("--ro-bind") + expect(cacheIdx).toBeGreaterThan(roIdx) // after the whole-fs read-only mount + + const bindIdx = args.indexOf("--bind-try") + expect(cacheIdx).toBeLessThan(bindIdx) // before the explicit writable binds + }) }) describe("Sandbox.backend/describe", () => { From 2e56c614791760f095f913be78812ba1ba791537 Mon Sep 17 00:00:00 2001 From: KB Date: Sat, 1 Aug 2026 20:10:24 +0530 Subject: [PATCH 44/56] plan(compute): make the budget cap bind, and accept a budget Changes 1 and 2, which must land together: making the cap real turns the existing rate x 24h grant sizing into a live ceiling, and the one-hour debit taken at acquire would kill a no-budget lease an hour early. Central decision recorded: the grant update is a set-to-total, not an increment, mirroring the tick's own cumulative charge. That gets replay safety for free, supersedes the acquire debit rather than double-counting it, and preserves the full plan TTL. --- docs/plans/2026-08-01-compute-budget-cap.md | 383 ++++++++++++++++++++ 1 file changed, 383 insertions(+) create mode 100644 docs/plans/2026-08-01-compute-budget-cap.md diff --git a/docs/plans/2026-08-01-compute-budget-cap.md b/docs/plans/2026-08-01-compute-budget-cap.md new file mode 100644 index 00000000..7930ce0d --- /dev/null +++ b/docs/plans/2026-08-01-compute-budget-cap.md @@ -0,0 +1,383 @@ +# Managed compute budget cap — implementation plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development to implement +> this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Make `hard_cap_cents` a ceiling that actually binds, and let a caller propose a budget — so a +managed GPU lease is bounded by the money authorised for it rather than by the user's entire wallet. + +**Architecture:** All work is in the **Atlas** repo (`~/codes/InkVell/atlas`), continuing on +`feat/compute-lease-prerequisites`. Spec changes 1 and 2, which **must land together** — see below. + +**Tech Stack:** Python 3.12, FastAPI, aiosqlite, pytest + pytest-asyncio. + +**Spec:** `~/codes/InkVell/openscience/docs/specs/compute-design.md`, Part B changes 1 and 2. + +## Global Constraints + +- **Repo:** `~/codes/InkVell/atlas`, branch `feat/compute-lease-prerequisites` (already checked out). + Do not create or switch branches. +- **Run tests with:** `cd ~/codes/InkVell/atlas/backend && .venv/bin/python -m pytest -q`. + No activated virtualenv — always invoke `.venv/bin/python` explicitly. +- **Never add a `Co-Authored-By:` trailer or any AI/assistant attribution to commit messages.** +- **No mocks of our own code.** Use the real repo functions, real `tick_once`, `aiosqlite` + + `run_migrations`, and a `_FakeProvider` at the boundary — the pattern in + `backend/tests/test_compute_billing.py`. +- **Every new assertion must be shown failing first.** Paste the failure into your report. +- Baseline: full suite is **1670 passed, 1 skipped**. It must be green when each task finishes. +- Ignore `.claude/worktrees/`. + +## Established facts — do not re-derive these + +Verified in the source, and the last two against production: + +1. **`tick_once`'s managed branch is cumulative and replay-safe** + (`backend/app/services/compute_billing_service.py`, the `if funding == "managed":` branch): + ```python + elapsed_total = (now - started).total_seconds() # started = started_at or created_at + already = int(lease.get("total_spent_cents") or 0) + delta_cents = wall_clock_cents(hourly, elapsed_total) - already + if delta_cents <= 0: skip + ``` + A replayed tick yields `delta <= 0` and skips. **The grant debit must mirror this shape.** +2. `wall_clock_cents(hourly, secs) = round(hourly * secs / 3600)` — `app/compute/provider_registry.py:10`. +3. **BYOK leases are skipped by the tick entirely** (`if funding == "byok": continue`), so they can never + be charged and must never be grant-debited. +4. `compute_repo.debit_grant` is an **increment** with an atomic ceiling: + `SET spent_cents = spent_cents + ? … WHERE grant_id = ? AND status = 'active' AND (spent_cents + ?) <= hard_cap_cents`. +5. **Acquire debits exactly one hour up front** and never refunds it — + `first_hour_cents = int(price_cents_per_hour)` then `debit_grant(db, grant_id, first_hour_cents)` in + `lease_manager.acquire_lease`. +6. The grant is sized at lease creation to `max_spend = max(charge_raw * ttl_hours, charge_raw, 1)` with + `ttl_hours = gpu_sandbox_max_ttl_hours` = **24** on every plan — `app/routes/compute.py`. +7. `LeaseRequest` is `{provider, sku, region?, node_id?}` — no budget field today. +8. **Measured in production (2026-08-01):** a live 34¢/hr lease had `grant.hard_cap_cents = 816` + (= 34 × 24), `grant.spent_cents = 34` **frozen**, while `lease.total_spent_cents` climbed 0 → 2 → 3 → + 5 → 6. The tick charges the wallet and never touches the grant, so `spent_cents` can never approach + `hard_cap_cents`. **The cap is decorative.** That is the bug this plan fixes. + +## The central design decision + +The grant update is a **set-to-total, not an increment**: + +``` +grant.spent_cents := wall_clock_cents(hourly, elapsed_total) +``` + +Three things follow, and they are why an increment is wrong: + +- **Replay safety comes for free.** It mirrors the tick's own cumulative model (fact 1). An increment is + not replay-safe: a crash between the charge and the debit either double-counts on the next tick or + loses the debit. +- **It supersedes the acquire-time one-hour debit instead of double-counting it.** This is the trap the + spec records: a naive re-debit makes a $10 budget at $6.99/h die at 25.8 minutes instead of ~1.4 hours. + A set writes the truth and the pre-tick placeholder simply disappears. +- **It preserves the full plan TTL.** At exactly 24h, `spent = 24 × rate = hard_cap`, and the guard is + `<=`, so the lease survives its TTL and the next tick releases it. Adding the acquire debit on top + (`wall_clock + rate`) would kill a no-budget lease at ~23h — a silent regression for the dashboard and + `compute:up`, which is precisely why changes 1 and 2 must land together. + +**Before the first tick the acquire debit still does its job**, bounding an un-ticked lease at one hour of +grant. Only once the tick runs does wall-clock become the truth. + +--- + +### Task 1: An atomic set-to-total for grant spend + +**Files:** + +- Modify: `backend/app/db/repos/compute_repo.py` (add `set_grant_spend` beside `debit_grant`) +- Test: `backend/tests/test_compute_grant_spend.py` (new) + +**Interfaces:** + +- Produces: `async def set_grant_spend(db, grant_id: str, total_cents: int) -> bool` — sets + `spent_cents` to `total_cents` when the grant is active and `total_cents <= hard_cap_cents`; returns + `False` without writing when it would exceed the cap or the grant is not active. `debit_grant` is left + untouched — other callers still use it. + +- [ ] **Step 1: Write the failing tests** + +Create `backend/tests/test_compute_grant_spend.py`. Follow the fixture style of +`backend/tests/test_compute_billing.py` (`aiosqlite` + `run_migrations` against a tmp DB). Cover: + +1. a set below the cap writes the exact total and returns `True` +2. a set **equal** to the cap succeeds — the guard is `<=`, and this is what preserves the full TTL +3. a set above the cap returns `False` **and leaves `spent_cents` unchanged** (assert the row, not just + the return value) +4. a set on a non-`active` grant returns `False` +5. it is idempotent — calling it twice with the same total leaves the same row + +- [ ] **Step 2: Run and confirm they fail** + +Run: `cd ~/codes/InkVell/atlas/backend && .venv/bin/python -m pytest tests/test_compute_grant_spend.py -q` +Expected: `ImportError` / `AttributeError` — `set_grant_spend` does not exist. Paste it. + +- [ ] **Step 3: Implement** + +In `backend/app/db/repos/compute_repo.py`, beside `debit_grant`: + +```python +async def set_grant_spend(db, grant_id: str, total_cents: int) -> bool: + """Set a grant's cumulative spend, refusing to exceed its ceiling. + + A SET rather than an increment, mirroring the billing tick, which charges + ``wall_clock_cents(rate, elapsed_since_started) - total_spent_cents``. Both + sides then derive from the same wall-clock truth, so a replayed tick is a + no-op instead of a double count, and the one-hour debit ``acquire_lease`` + takes up front is superseded rather than added to. + + Returns False without writing when the total would exceed ``hard_cap_cents`` + or the grant is no longer active — the caller releases the lease. + """ + now = _now() + cursor = await db.execute( + """ + UPDATE compute_grants + SET spent_cents = ?, updated_at = ? + WHERE grant_id = ? + AND status = 'active' + AND ? <= hard_cap_cents + """, + (total_cents, now, grant_id, total_cents), + ) + await db.commit() + return cursor.rowcount > 0 +``` + +- [ ] **Step 4: Run and confirm they pass** + +Run: `cd ~/codes/InkVell/atlas/backend && .venv/bin/python -m pytest tests/test_compute_grant_spend.py -q` +Then the full suite. Both green. + +- [ ] **Step 5: Commit** + +```bash +cd ~/codes/InkVell/atlas +git add backend/app/db/repos/compute_repo.py backend/tests/test_compute_grant_spend.py +git commit -m "feat(compute): an atomic set-to-total for grant spend + +debit_grant increments, which cannot mirror a billing tick that charges +cumulatively from started_at. set_grant_spend writes the total instead, so a +replayed tick is a no-op and the one-hour debit taken at acquire is superseded +rather than added to. Refuses above the ceiling without writing." +``` + +--- + +### Task 2: Make the cap bind + +**Files:** + +- Modify: `backend/app/services/compute_billing_service.py` (the `funding == "managed"` branch of + `tick_once`) +- Test: `backend/tests/test_compute_billing.py` (append) + +**Interfaces:** + +- Consumes: `compute_repo.set_grant_spend` from Task 1. +- Produces: a managed lease is released once its grant ceiling is reached. + +- [ ] **Step 1: Write the failing tests** + +Append to `backend/tests/test_compute_billing.py`. **The headline test is the first one** — the spec +records that both previous attempts at this change omitted it from their tests _and_ their acceptance +criteria, and that a $10 budget dying at 25.8 minutes passes every "a release happened" assertion while +being wrong by 3×. + +1. **A budget of $B at $R/h lasts ≈ B/R hours.** Seed a grant with `hard_cap_cents = B`, a managed lease + at `hourly_rate_cents = R`, drive `tick_once` with an injected `now` advancing in steps, and assert on + the **elapsed billable duration at the moment of release** — not merely that a release occurred. + Allow ±90s of rate: `COMPUTE_BILLING_TICK_SECONDS` (60) + `FIRST_BILL_GRACE_SECONDS` (30). +2. A tick whose new total fits under the cap does **not** release, and `grant.spent_cents` now equals + `wall_clock_cents(rate, elapsed_total)` — proving it tracks rather than freezing at the acquire debit. +3. A tick whose new total would exceed the cap **charges the elapsed time first, then releases.** The + user consumed that time and the operator owes the provider for it; skipping the charge loses real + money. Assert both the wallet movement and the release. +4. A replayed tick (same injected `now`) neither double-charges the wallet nor moves `grant.spent_cents`. +5. **A BYOK lease is never grant-debited** — it is skipped before this code runs. +6. **A no-budget lease still runs its full plan TTL.** With `hard_cap = rate × 24`, the lease survives to + 24h. This is the regression guard for the acquire-debit interaction; if the implementation adds the + acquire debit on top of wall-clock instead of superseding it, this test dies at ~23h. + +- [ ] **Step 2: Run and confirm they fail** + +Run: `cd ~/codes/InkVell/atlas/backend && .venv/bin/python -m pytest tests/test_compute_billing.py -q` +Paste the failures. + +- [ ] **Step 3: Implement** + +In the `funding == "managed"` branch, after the existing `mark_billed` call and before `continue`, set the +grant to the same wall-clock total the charge was derived from, and release when the ceiling refuses it: + +```python + grant_id = lease.get("grant_id") + if grant_id: + # Mirror the charge: the tick bills wall-clock-since-started + # minus what was already billed, so the grant's cumulative + # spend is that same wall-clock total. A set, not an increment — + # see set_grant_spend. The one-hour debit taken at acquire is + # superseded here, which is what keeps a no-budget lease alive + # for its full plan TTL instead of dying an hour early. + within = await compute_repo.set_grant_spend( + db, grant_id, wall_clock_cents(hourly, elapsed_total), + ) + if not within: + # The approved money is spent. The charge above already + # captured the time actually used, which the user consumed + # and the operator owes the provider for. + logger.info( + "compute_billing: grant ceiling reached - releasing lease=%s", + lease_id, + ) + await _safe_release(db, lease) + released += 1 + continue +``` + +Import `wall_clock_cents` if it is not already in scope — it lives in `app.compute.provider_registry`. + +**Order matters and is load-bearing:** charge → `mark_billed` → `set_grant_spend` → release on refusal. +Charging after the ceiling check would drop the final increment; setting the grant before the charge +would authorise money that was never taken. + +- [ ] **Step 4: Run and confirm they pass** + +Run the file, then `cd ~/codes/InkVell/atlas/backend && .venv/bin/python -m pytest tests/ -q`. Both green. + +- [ ] **Step 5: Commit** + +```bash +cd ~/codes/InkVell/atlas +git add backend/app/services/compute_billing_service.py backend/tests/test_compute_billing.py +git commit -m "fix(billing): make hard_cap_cents a ceiling that actually binds + +The tick charged the wallet and never touched the grant, so spent_cents froze +at the one-hour debit taken at acquire and could never approach hard_cap_cents. +Measured in production: a 34c/hr lease with hard_cap 816c sat at spent 34c +while the lease accrued past it. The ceiling was decorative and the wallet was +the only real bound. + +The tick now sets the grant to the same wall-clock total it charges from, and +releases when that total is refused." +``` + +--- + +### Task 3: Accept a budget on lease creation + +**Files:** + +- Modify: `backend/app/routes/compute.py` (`LeaseRequest`, and the grant sizing in `create_lease`) +- Test: `backend/tests/test_compute_resell_routes.py` (append) + +**Interfaces:** + +- Consumes: the binding cap from Task 2. +- Produces: `POST /api/compute/leases` accepts optional `budget_cents`; the response carries the + **effective** cap actually authorised. + +- [ ] **Step 1: Write the failing tests** + +Append to `backend/tests/test_compute_resell_routes.py`, following its existing route-test style: + +1. **Omitting `budget_cents` preserves today's behaviour exactly** — the grant is still sized to + `rate × ttl_hours`. Assert the grant row, since the dashboard and `compute:up` both call this endpoint + without the field. +2. `budget_cents` present sizes the grant to it. +3. **A budget larger than the wallet is clamped, not rejected** — the wallet is always the outer bound — + and the response reports the **effective** cap, not the asked-for one. +4. A budget that cannot fund the first hour is refused with the existing structured `402`, extended with + `affordable_budget_cents`. +5. **BYOK ignores `budget_cents`** and is never grant-debited. + +- [ ] **Step 2: Run and confirm they fail** + +Run: `cd ~/codes/InkVell/atlas/backend && .venv/bin/python -m pytest tests/test_compute_resell_routes.py -q` +Paste the failures. + +- [ ] **Step 3: Implement** + +Add `budget_cents: int | None = None` to `LeaseRequest`. In `create_lease`, replace the fixed sizing: + +```python + max_spend = max(charge_raw * ttl_hours, charge_raw, 1) +``` + +with a budget-aware version that keeps the wallet as the outer bound. Read the current code around it +before editing — `charge_raw`, `ttl_hours` and the effective balance lookup are already in scope, and the +existing `402` for insufficient credit is the one to extend rather than duplicate. + +Report the effective cap on the response so a caller can tell the user what was actually authorised. + +- [ ] **Step 4: Run and confirm they pass** + +Run the file, then the full suite. Both green. + +- [ ] **Step 5: Commit** (message in the same style; state that absent `budget_cents` is unchanged + behaviour, and that the wallet remains the outer bound) + +--- + +### Task 4: The agent-spawn default becomes a hard kill + +**Why:** `budget_cents` already exists on the agent-spawn path with a default of `500` +(`app/services/agent_tools.py`, plus `models/agent.py` and `spawn_queue_service.py`), where it is +**display-only** today. Task 2 makes caps real, so every already-shipped spawn silently acquires a hard +$5 ceiling. The spec calls this out explicitly: _"Not a no-op, and the previous draft called it one."_ + +**Files:** to be determined by Step 1 — do not guess. + +- [ ] **Step 1: Investigate and report before changing anything** + +Trace how the spawn path's `budget_cents` reaches `create_grant`, what `hard_cap_cents` it produces +today, and what a typical spawn actually costs. **Report:** + +- the value a spawn's grant is currently created with +- whether $5 is above or below a realistic spawn cost, with a number +- whether spawn grants are the same `compute_grants` rows the billing tick now enforces against + +- [ ] **Step 2: Choose and implement, having reported** + +Two options the spec names — pick one on the evidence from Step 1 and say why: + +- **raise the default deliberately** to a value chosen with enforcement in mind, or +- **exempt spawn-path grants** until their budgets are set with enforcement in mind. + +Whichever you choose, add a test proving a spawn is not killed at a budget nobody chose. + +- [ ] **Step 3: Commit** + +--- + +## Whole-branch verification + +- [ ] Full suite: `cd ~/codes/InkVell/atlas/backend && .venv/bin/python -m pytest tests/ -q` — green, no + network. +- [ ] Confirm by name that the prior branch's deliverables still pass: + `test_promoted_lease_is_not_reaped_at_the_provisioning_timeout`, + `test_user_lease_without_runner_token_survives`, + `test_class_4_managed_gpu_with_owner_key_never_uses_it`, + `test_out_of_credit_release_of_a_managed_lease_uses_the_operator_key`. +- [ ] Confirm the headline property test exists and asserts on **elapsed billable duration**, not on the + fact that a release happened. + +## Acceptance criteria + +1. A budget of $B at rate $R/h lasts B/R hours ± 90s of rate, asserted on elapsed billable duration. +2. `grant.spent_cents` tracks wall-clock spend instead of freezing at the acquire-time debit. +3. A tick exceeding the ceiling charges the elapsed time and then releases; one that fits does not. +4. A replayed tick neither double-charges the wallet nor moves `grant.spent_cents`. +5. A lease created without `budget_cents` runs its **full** plan TTL — asserted on runtime, not on the + request being accepted. +6. `budget_cents` sizes the grant; a budget above the wallet is clamped and the response reports the + effective cap; one that cannot fund the first hour is refused with a structured `402`. +7. BYOK leases ignore `budget_cents` and are never grant-debited. +8. An agent spawn is not killed at a budget nobody chose. +9. `pytest` passes with no network access. + +## Out of scope + +Rolling window cap across sequential leases (change 5), the resolver and quote endpoint (changes 3, 4, +12), volumes (6), budget extension (7), and the three OpenScience tools. A budget bounds one lease; it +does not yet bound release-and-reacquire. From b1214615ef90d06c9c04187d2145ee38fa4f03e3 Mon Sep 17 00:00:00 2001 From: KB Date: Sat, 1 Aug 2026 21:10:23 +0530 Subject: [PATCH 45/56] spec(compute): changes 1 and 2 shipped, and change 1's mechanism was wrong MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The spec prescribed "the billing tick must re-debit the grant". An increment cannot mirror a tick that charges cumulatively, and following that literally rebuilds the double-count it warned about: measured by mutation, a $10 budget at $6.99/h dies at 26.0 minutes instead of ~1.4 hours, with grant.spent_cents reading 300 where wall-clock is 180. The correction — a set-to-total — is now recorded above the original text rather than replacing it, since the reasoning is the useful part. The premise was confirmed in production before the fix (a 34c/hr lease with hard_cap 816 and spent frozen at 34) and the property confirmed after it (release at 5160s against a theoretical 5150s). The "confirm the money path against pytest" caveat is discharged for change 1; it still stands for the resolver, volumes and the rolling cap. The spawn-path known defect is closed, and was worse than recorded: 500c flat for every SKU, killing a 4-hour A100 at 1.38h and refusing an H100 at acquire outright. --- docs/specs/compute-design.md | 72 ++++++++++++++++++++++++++++++++---- 1 file changed, 64 insertions(+), 8 deletions(-) diff --git a/docs/specs/compute-design.md b/docs/specs/compute-design.md index c6d2ba07..f562e722 100644 --- a/docs/specs/compute-design.md +++ b/docs/specs/compute-design.md @@ -1,6 +1,7 @@ # Compute — design -Status: **Mode detection shipped. Managed leases to build.** +Status: **Mode detection shipped. Lease prerequisites and the budget cap shipped. Selection and the +agent-facing tools to build.** Date: 2026-07-31 · single current compute spec · revised after adversarial review Roadmap: **5**, **51/2**, **55**, **103**; unblocks **56** @@ -121,7 +122,7 @@ unavailable. No criterion covered this before. --- -# Part B — Managed leases (to build) +# Part B — Managed leases (changes 0, 1, 2, 8, 9 shipped; the rest to build) ## The gap Part A exposed @@ -350,7 +351,32 @@ than leaving one value covering a 6× spread. **Ships first, with its own test, before any budget work.** -### Change 1 — make `hard_cap_cents` a real running cap +### Change 1 — make `hard_cap_cents` a real running cap — **SHIPPED** + +> **Shipped** as `0a9da07` + `a4a79ca`. **The mechanism this section originally prescribed was wrong** and +> is corrected below — read the correction before touching this code. +> +> **The premise was confirmed in production before the fix.** A live 34¢/hr lease had +> `hard_cap_cents = 816` (= 34 × 24) with `spent_cents` **frozen at 34** while `lease.total_spent_cents` +> climbed 0 → 2 → 3 → 5 → 6. The ceiling was decorative and the wallet was the only real bound. This +> section previously carried a "read from source only — confirm against a test" caveat; that is now +> discharged, against both a test and a running deployment. +> +> **The correction: the grant update is a _set-to-total_, not a re-debit.** `debit_grant` increments, and +> an increment cannot mirror a tick that charges cumulatively +> (`wall_clock_cents(rate, now - started_at) - total_spent_cents`). Incrementing double-counts the hour +> `acquire_lease` debits up front — measured by mutation: a $10 budget at $6.99/h died at **26.0 minutes** +> instead of ~1.4 hours, and `grant.spent_cents` read 300 where wall-clock was 180. `set_grant_spend` +> writes the wall-clock total instead, so replay safety is inherited from the tick and the acquire debit +> is superseded rather than added to. +> +> **Measured after the fix:** a $10 budget at $6.99/h releases at 5160s against a theoretical 5150s — a +> 9.8s overshoot that is pure tick quantisation. +> +> A wrong implementation still passes every "a release happened" assertion. Under the mutation above, +> `released == 1`, the status was `released`, and the wallet moved. **Only an assertion on elapsed +> billable duration catches it** — which is why the spec insisted on that property and why both earlier +> attempts, which omitted it, shipped the bug. The column exists (`migrations.py:522`, `pg_migrations.py:775`) and an atomic ceiling exists (`compute_repo.py:196` — `AND (spent_cents + ?) <= hard_cap_cents`). But `debit_grant` is called in @@ -359,7 +385,8 @@ rollback (`:545`), pre-provisioning failure undo (`:77`), and a settle true-up ( **`compute_billing_service.tick_once` never calls it** — only `usage_service.charge` and `mark_billed`. So `spent_cents` freezes at hour one and the ceiling is never re-evaluated. -**The billing tick must re-debit the grant**, releasing the lease when the debit would exceed the cap — +**The billing tick must update the grant** (see the correction above — a set, not a re-debit), releasing +the lease when the new total would exceed the cap — reusing the path that already fires on wallet exhaustion. **This is not a double charge.** The wallet is money; the grant is an authorisation envelope drawn against @@ -395,7 +422,22 @@ _Latent, not live:_ `debit_grant`'s predicate includes `status = 'active'` (`com `expire_grants_by_session` (`:204`) is defined and never called, so no grant expires today — but under this change, any future grant expiry silently becomes "release the lease". -### Change 2 — accept a budget on lease creation +### Change 2 — accept a budget on lease creation — **SHIPPED** + +> **Shipped** as `31fc598`, together with change 1 as this section requires. `budget_cents` is optional +> on `POST /api/compute/leases`, clamped to the effective balance, and the response reports the effective +> cap. Absent, the grant is still sized to `rate × ttl_hours` — pinned by a test that fails if that +> number moves, because it is now a live ceiling on the dashboard and `compute:up`, which both call this +> endpoint without a budget. +> +> Two decisions taken during implementation, neither in this section's original text: +> +> - The `402` keeps one body shape but carries **two `error` values** — `insufficient_cli_credit` when the +> wallet is short, and a new `budget_below_hourly_rate` when the wallet is fine and only the budget is. +> Reusing the credit code would make a client tell a user with a full wallet to top up. +> - The budget is deliberately **not** clamped to `rate × ttl_hours`. The TTL is already a time bound, so +> a larger grant is unreachable spend, and clamping to it would shrink a stated budget for a reason that +> has nothing to do with affordability. ``` POST /api/compute/leases @@ -953,9 +995,17 @@ creation stays in the workspace UI rather than becoming a fourth tool. No file transfer, no exec, no compute tests. - **Six of the ten `RESELL_PROVIDERS` are `ScaffoldProvider` stubs** — registered, `operator: false`, zero options, no network call. Scaffolding, not integrations. -- **`budget_cents` already exists** on the agent-spawn path defaulting to `500` (`agent_tools.py:1386`, - `models/agent.py:101`, `spawn_queue_service.py:69` → `create_grant` at `:1501`), display-only. Change 1 - makes caps real, silently giving every shipped spawn a hard $5 kill. **Not a no-op.** +- ~~**`budget_cents` already exists** on the agent-spawn path defaulting to `500`, display-only. Change 1 + makes caps real, silently giving every shipped spawn a hard $5 kill.~~ **FIXED** (`71dccba`), and it was + worse than this entry claimed. Every spawn grant was `hard_cap_cents = 500` flat for **every SKU** — no + caller in either repo ever sent anything else. A spawn requests 4 hours; that is $14.60 on an + A100-40GB, $18.36 on an A100-80GB, $27.96 on an H100, so only T4 and A10G fitted under $5. Measured on + the real spawn path against the real tick: **a 4-hour A100 spawn was killed at 1.38h**, 35% of its + requested life. An H100 was **refused outright at acquire** ("need 699 cents, have 500") — that half + was pre-existing, dating to `1e49dc9`, which is itself evidence the 500 was never sized against any + GPU. `budget_cents` is now `int | None`; `None` sizes the ceiling to the spawn's own lifetime at the + SKU rate, with a one-hour floor, and an explicitly chosen budget still binds. Only CPU spawns were ever + safe, because they come out `funding=byok` at rate 0 and the tick skips them. - **Stale skill names in agent prompts.** `research.txt:353` and `ml.txt:192` name skills by directory rather than frontmatter `name` (`vllm` → `serving-llms-vllm`), so the agent is told it has skills it cannot load. `skills/scholar-evaluation/SKILL.md` has no frontmatter at all. @@ -1119,6 +1169,12 @@ Labels in this document: A source-read claim is not a deployed-behaviour claim. Confirm the money path against `pytest` before relying on it. +**The money path is now confirmed**, in both directions. Against a running deployment: the billing tick +charges the wallet correctly (34¢/hr decrementing in lockstep with `total_spent_cents`) while +`grant.spent_cents` sat frozen at the acquire debit — the defect. Against tests, after the fix: a $10 +budget at $6.99/h lasts 5160s versus a theoretical 5150s. The caveat that stood over change 1 is +discharged; the ones over the resolver, volumes and the rolling cap are not. + ### What review caught in this document Recorded because the pattern is the point. The first draft was written the same way the predecessors From 58318f5f2a3467f3e02165cf05ba212d113b8a61 Mon Sep 17 00:00:00 2001 From: KB Date: Sat, 1 Aug 2026 23:15:11 +0530 Subject: [PATCH 46/56] docs(compute): retract a wrong correction, and re-date the audit notes 06-compute-integrations.md carried a 'Correction to the initial audit' asserting that atlas compute:up is a real command in the published 0.13.2 and that this repo pins @synsci/atlas@^0.5.12. Both are false. package.json:123 pins ^0.13.2, and npm pack @synsci/atlas@latest resolves to 0.13.2 with zero files containing compute:up or compute:lease. The commands live only in the atlas repo -- 3e1d1ca removed them, 0.13.1 and 0.13.2 shipped without them, 205bbc0 re-added them with no version bump -- so source and artifact disagree at an identical version. It is a release problem, not a pinning problem. The retraction is kept in place rather than deleted: a source-read conclusion about a published artifact, checked against neither, is the useful part of the record. Path C was also overtaken. aa9b3142 merged an SSH/Slurm/PBS job dispatcher to main on 2026-07-29, two days after the 52845c3 audit baseline, so the 'store-only dead-end, no dispatch' verdict is wrong for SSH hosts and still right for model endpoints. The same commit falsifies ROADMAP notes on 2, 3, 51, 52, 58 and finding 1; those notes are corrected and dated, and no status mark is re-graded, which would need a fresh audit and a recount. ROADMAP 5, 55 and 103 gain what the Atlas guardrail work changed and, more importantly, what it does not: it sits on unmerged draft PRs and OpenScience still has no tool that can launch a lease, so nothing meets this document's own bar for DONE. --- docs/ROADMAP.md | 83 +++++++++----- docs/plans/06-compute-integrations.md | 159 ++++++++++++++++++++------ 2 files changed, 176 insertions(+), 66 deletions(-) diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 72bf433e..b65a6556 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -5,6 +5,19 @@ Ranking is **findings-first**: every item was checked against the tree at `52845c3` before being placed, so "P0" means _users hit this today_, not _it sounded important_. +> **Audit-note currency — 2026-08-01.** Statuses below are unchanged: `✅ DONE` still means _shipped and +> reachable by a user_, and nothing has been re-graded. Some **audit notes** have been corrected in place, +> each marked `[2026-08-01]`, for two reasons: +> +> - **`aa9b3142` ("feat: add managed compute jobs") merged to `main` on 2026-07-29, after the `52845c3` +> audit.** It shipped an SSH/Slurm/PBS job dispatcher, which falsifies several "no SSH client / nothing +> reads this / 0 hits for slurm" notes. The **status marks were deliberately not re-graded** — that +> needs a fresh audit at a new baseline and a recount of the table above, which is its own task. +> - **Compute guardrail work landed on two Atlas/OpenScience branches that are still unmerged draft PRs** +> (deliberately: the team holds compute PRs draft until the feature is complete). It is real, tested and +> in several cases verified against a live deployment — and it is **not reachable by a user**, so it +> cannot be `✅ DONE` under this document's own legend. Notes say where it lives; marks stay put. + ## Status at a glance | Status | Count | Meaning | @@ -46,7 +59,7 @@ built and then never connected to anything a user can reach. Ten independent ins | **`session-review.tsx`** | ~500 lines: diff view, line comments, focus | `grep SessionReview frontend/workspace/src` → **0 hits** | 12, 80 | | **Command palette** | ⌘K, mounted both routes, e2e-tested | Contains exactly 3 commands (open folder, settings, back) — zero scientific actions | 66, 75 | | **OpenTelemetry flag** | `experimental.openTelemetry` in `config.ts:1234` | No `@opentelemetry` dep, no tracer, no exporter anywhere — **the flag emits no span** | 110 | -| **SSH hosts + model endpoints** | Settings panels, persisted to disk | No SSH client in the dep tree; nothing reads either store | 58 | +| **Model endpoints** | Settings panel, persisted to disk | Nothing reads the store — no inference-routing consumer. (**SSH hosts left this list on 2026-08-01**: `aa9b3142` wired them) | 58 | | **`Run.inputs` / `Artifact.contentHash`** | Declared in `provenance/store.ts:38,47` | `ProvenanceRecordTool` exposes no parameter for either — never populated | 10, 18, 19 | | **`open-bench/`** | 2 benches, real run artifacts (`chembench`, `biomnibench`) | Untracked by git; `src/openbench/**` has **only `.pyc`, zero `.py`**; no `pyproject.toml` | 108 | | **Artifact `type` / `summary`** | Accepted by `register()` | Not persisted — `list()` hardcodes `type:"unknown"` (`artifacts.ts:61`) | 13 | @@ -70,11 +83,19 @@ Nothing on this list needs an architecture decision. 1. **`bash` cannot run anything in the background.** `tool/bash.ts` is synchronous with a `timeout` that `killTree`s on expiry. No job registry, no output persistence, no SQLite. A training run cannot survive a single tool call. This is the keystone: items 2, 3, 4, 51, 52, 56, 57, 63 are all downstream of it. + **[2026-08-01] Half of this is now false, and the important half is not.** `aa9b3142` shipped + `compute/jobs.ts`: a job registry with detached execution, persisted metadata and streamed logs. It is + **not reachable from `bash` or from any tool** — the agent still cannot start a job that outlives a tool + call. The keystone stands; what changed is that the substrate exists and needs exposing, not building. 2. **Six GPU providers are advertised; zero have an API client.** `settings/compute.ts` encrypts a key and injects an env var, then hopes a markdown skill shells out to the vendor CLI. **RunPod and Vast have no skill at all** — and `RUNPOD_API_KEY` is named to the model in **all six session prompts**, so the agent is told a capability exists that nothing implements. (Vast isn't even advertised — connecting it is a pure no-op.) Modal is stored in two panels where Credentials silently wins. + **[2026-08-01] Re-verified and still true in this repo.** All six provider prompts still name + `RUNPOD_API_KEY` at `:119`. RunPod and Vast do have real API clients in **Atlas** now, driven end to end + against a deployed backend — but that is the managed path, behind `POST /api/compute/leases` on an + unmerged draft branch, and OpenScience has no tool that calls it. The finding stands as written. 3. **Sharing is hard-disabled at three layers** (`disabled = true` in `share.ts:74` and `share-next.ts:18`, plus `Session.share` returning empty strings) while ~10 orphaned i18n strings per locale still describe the feature. We ship the vocabulary of a feature we don't have. @@ -113,26 +134,26 @@ Fifteen items: **1 done · 6 partial · 1 skill-only · 7 missing.** ## Group A — Broken or misleading in the shipped product -| # | Item | Status | Current state | -| ------- | --------------------------- | ------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| **5** | Fix existing compute gaps | ❌ | RunPod/Vast keys inject but **no consumer exists**, while `RUNPOD_API_KEY` is named to the model in all 6 `session/prompt/*.txt`; Modal double-stored vs `credentials.ts` (Credentials applies first at boot and wins); `last_used` rendered but never assigned | -| **14** | Shared sessions | 🟡 | Code written, then hard-disabled: `share.ts:74` + `share-next.ts:18` both `disabled = true`; `session/index.ts:253` returns empty strings. No permissions model. Orphaned i18n in every locale | -| **13** | Scientific artifact manager | 🟡 | `session/rlm/artifacts.ts` is a blob cache for context relief: IDs are `Date.now()+random`, **no checksums**, `list()` hardcodes `type:"unknown"` and drops your summary | -| **100** | Sandbox docs update | ✅ | **No work needed — the premise was wrong.** `sandbox.mdx` (91 lines) matches `sandbox/sandbox.ts` (509 lines) including a Limitations section; both shipped in `a737ddc`, no drift since | +| # | Item | Status | Current state | +| ------- | --------------------------- | ------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **5** | Fix existing compute gaps | ❌ | RunPod/Vast keys inject but **no consumer exists**, while `RUNPOD_API_KEY` is named to the model in all 6 `session/prompt/*.txt`; Modal double-stored vs `credentials.ts` (Credentials applies first at boot and wins); `last_used` rendered but never assigned. **[2026-08-01] Re-verified after this branch edited the prompts: the `RUNPOD_API_KEY` claim still holds** — all six provider prompts name it at `:119` (`anthropic`, `beast`, `codex_header`, `copilot-gpt-5`, `gemini`, `qwen`); `last_used` is still only defaulted and carried (`routes/settings/compute.ts:252`, `:272`). RunPod and Vast are now leasable as **managed** providers server-side in Atlas, driven end to end on a deployed backend — but behind `POST /api/compute/leases`, on an unmerged draft branch, with no OpenScience tool that calls it, so a key in this panel is still consumed by nothing | +| **14** | Shared sessions | 🟡 | Code written, then hard-disabled: `share.ts:74` + `share-next.ts:18` both `disabled = true`; `session/index.ts:253` returns empty strings. No permissions model. Orphaned i18n in every locale | +| **13** | Scientific artifact manager | 🟡 | `session/rlm/artifacts.ts` is a blob cache for context relief: IDs are `Date.now()+random`, **no checksums**, `list()` hardcodes `type:"unknown"` and drops your summary | +| **100** | Sandbox docs update | ✅ | **No work needed — the premise was wrong.** `sandbox.mdx` (91 lines) matches `sandbox/sandbox.ts` (509 lines) including a Limitations section; both shipped in `a737ddc`, no drift since | ## Group B — The job system (keystone) One workstream. Guardrails ship **with** the runner — a job system that can spend money before it can stop spending money is a liability. -| # | Item | Status | Current state | -| ------- | ------------------------------------ | ------ | ---------------------------------------------------------------------------------------------------------------------------------------------- | -| **51** | Provider-independent job abstraction | ❌ | `util/queue.ts` is an in-memory `AsyncQueue`; `scheduler/index.ts` is a 61-line `setInterval`. Start here | -| **52** | Queue/history database | ❌ | No SQLite/Drizzle/Prisma anywhere; `storage/storage.ts` is JSON files. Use `bun:sqlite` — preserves the single-binary ship | -| **2** | Real compute jobs page | ❌ | No `/jobs` route. `app.tsx` has exactly 3 routes: `/`, `/:dir`, `/:dir/session/:id?` | -| **61** | Per-job secrets, never into logs | 🟡 | Log redaction exists (`OpenScience.redactSecrets` → `bash.ts:207`). **Per-job scoping does not** — injection is global via `applyComputeEnv()` | -| **55** | Budget guardrails + kill switches | ❌ | 0 hits for `costLimit\|spendLimit\|budget` outside a compaction comment. `cli/cmd/stats.ts` reports cost after the fact | -| **103** | Cost approval gates | ❌ | `session/billing-gate.ts` only _classifies_ calls managed/BYOK/free. No pre-flight estimate, cap, or prompt | +| # | Item | Status | Current state | +| ------- | ------------------------------------ | ------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| **51** | Provider-independent job abstraction | ❌ | `util/queue.ts` is an in-memory `AsyncQueue`; `scheduler/index.ts` is a 61-line `setInterval`. Start here. **[2026-08-01] Partly overtaken by `aa9b3142`**: `src/compute/jobs.ts` (`ComputeJobs`) is a real job abstraction over two targets (`local`, `ssh`) and three schedulers (`none`, `slurm`, `pbs`), with a resource request, `apptainer` support, artifact collection, log streaming and cancel. It is **not** provider-independent in this item's sense — no managed-lease or cloud-GPU backend behind it — and **the agent cannot drive it** (`grep ComputeJobs src/tool/` → 0 hits). Read this item as "generalise the abstraction that now exists", not "start from nothing" | +| **52** | Queue/history database | ❌ | No SQLite/Drizzle/Prisma anywhere; `storage/storage.ts` is JSON files. Use `bun:sqlite` — preserves the single-binary ship. **[2026-08-01] Still accurate**, and now load-bearing: `ComputeJobs` persists job metadata as a JSON blob at mode `0600` (`compute/jobs.ts:162`), which is exactly the store this item replaces | +| **2** | Real compute jobs page | ❌ | No `/jobs` route. `app.tsx` has exactly 3 routes: `/`, `/:dir`, `/:dir/session/:id?`. **[2026-08-01] The route claim still holds — the "no page" claim does not.** `aa9b3142` shipped `frontend/workspace/src/atlas/ComputeJobs.tsx`, mounted in `RightPane.tsx:297` (rendered by the session page), backed by `/jobs`, `/jobs/completed`, `/jobs/:id/log`, `/jobs/:id/cancel`. It is a right-pane panel for SSH/local jobs, not a first-class monitoring route, and it shows nothing about managed leases or cost | +| **61** | Per-job secrets, never into logs | 🟡 | Log redaction exists (`OpenScience.redactSecrets` → `bash.ts:207`). **Per-job scoping does not** — injection is global via `applyComputeEnv()` | +| **55** | Budget guardrails + kill switches | ❌ | **In this repo, still nothing**: 0 hits for `costLimit\|spendLimit\|budget` outside a compaction comment; `cli/cmd/stats.ts` reports cost after the fact. **[2026-08-01] The server-side half now exists and binds** — `hard_cap_cents` was decorative (measured in production: a 34¢/hr lease had `hard_cap_cents = 816` with `spent_cents` frozen at 34 while the lease accrued), and the Atlas billing tick now sets the grant to its wall-clock total and releases when the ceiling refuses it. Measured after the fix: a $10 budget at $6.99/h releases at 5160s vs a theoretical 5150s. `POST /api/compute/leases` also accepts `budget_cents`, clamped to the wallet, reporting the effective cap. **Unmerged draft branch, and OpenScience cannot launch a lease to spend against it** — so the CLI-scoped claim above is the one that describes what a user has | +| **103** | Cost approval gates | ❌ | `session/billing-gate.ts` only _classifies_ calls managed/BYOK/free. No pre-flight estimate, cap, or prompt. **[2026-08-01] The cap half is built** (Atlas, unmerged draft): `budget_cents` on `POST /api/compute/leases` is clamped to the wallet and the response reports the **effective** cap, so a caller can state what was actually authorised. **Both approval halves are still missing**: no pre-flight quote (`POST /api/compute/quote` — spec change 4, unbuilt, and `/estimate` needs an explicit `{provider, sku}` a `{gpu, count}` proposal does not have) and no prompt (a `ctx.ask` gate needs a `compute_launch` tool, which does not exist) | ## Group C — Core science UX that is half-built @@ -154,15 +175,15 @@ Twenty-eight items: **0 done · 14 partial · 6 skill-only · 8 missing.** ## Compute, made real -| # | Item | Status | Current state | -| ------ | --------------------------------- | ------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| **4** | Cloud GPU backends (real clients) | 🟡 | Credential plumbing is real, encrypted, and unit-tested — but grep for `api.runpod`/`modal.com`/`lambdalabs` outside skill markdown returns **0 hits**. AWS/GCP/Azure Batch absent entirely | -| **3** | Slurm/HPC integration | ❌ | 0 hits for `sbatch\|squeue\|scancel\|apptainer\|slurm` in any source tree. Prose in skill docs only | -| **57** | Multi-node training | 📄 | Skills exist (`ray-train`, `deepspeed`, `torchtitan`, `megatron-core`). No product code. Needs **51** | -| **58** | Remote kernels over SSH/Jupyter | 🟡 | The SSH-hosts panel persists data **nothing reads**; no SSH client in the dep tree. Make it real or remove the panel | -| **59** | Interactive tunnels | ❌ | JupyterLab, TensorBoard, MLflow, W&B | -| **53** | Artifact upload/download | 📄 | No `@aws-sdk`/`@google-cloud`/`@azure`/`@huggingface` in any `package.json`. Skills shell out to `aws`/`gsutil`/`hf` CLIs | -| **64** | Pre-launch runtime health checks | ❌ | Cheapest possible way to stop burning GPU-hours on a bad environment | +| # | Item | Status | Current state | +| ------ | --------------------------------- | ------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **4** | Cloud GPU backends (real clients) | 🟡 | Credential plumbing is real, encrypted, and unit-tested — but grep for `api.runpod`/`modal.com`/`lambdalabs` outside skill markdown returns **0 hits**. AWS/GCP/Azure Batch absent entirely | +| **3** | Slurm/HPC integration | ❌ | ~~0 hits for `sbatch\|squeue\|scancel\|apptainer\|slurm` in any source tree. Prose in skill docs only~~ — **[2026-08-01] falsified by `aa9b3142`.** `compute/jobs.ts` submits with `sbatch --wait --parsable` (`:331`), cancels with `scancel --name` (`:710`), wraps in `apptainer exec` (`:275`), and probes a host for Slurm/PBS (`:510`). What is genuinely absent: `squeue` polling, job-array support, module systems, and any HPC path the **agent** can reach | +| **57** | Multi-node training | 📄 | Skills exist (`ray-train`, `deepspeed`, `torchtitan`, `megatron-core`). No product code. Needs **51** | +| **58** | Remote kernels over SSH/Jupyter | 🟡 | ~~The SSH-hosts panel persists data **nothing reads**; no SSH client in the dep tree. Make it real or remove the panel~~ — **[2026-08-01] `aa9b3142` made it real.** `ComputeJobs` reads `ssh_hosts` and dispatches over the system `ssh` binary (`compute/jobs.ts:362-366`), with a reachability probe returning latency + Python/GPU/Slurm/PBS (`routes/settings/compute.ts:394`). "No SSH client in the dep tree" is still literally true and no longer evidence — it shells out rather than adding a dep. **Remote _kernels_ are still missing**: this dispatches batch commands, not a Jupyter/IPython kernel the notebook tool can attach to | +| **59** | Interactive tunnels | ❌ | JupyterLab, TensorBoard, MLflow, W&B | +| **53** | Artifact upload/download | 📄 | No `@aws-sdk`/`@google-cloud`/`@azure`/`@huggingface` in any `package.json`. Skills shell out to `aws`/`gsutil`/`hf` CLIs | +| **64** | Pre-launch runtime health checks | ❌ | Cheapest possible way to stop burning GPU-hours on a bad environment | ## Scientific data as a first-class citizen @@ -248,13 +269,13 @@ Every pack below is 📄 or ❌. The parenthetical counts how many named tools i ## Compute depth -| # | Item | Status | Current state | -| ------ | ---------------------------------------------------- | ------ | -------------------------------------------- | -| **54** | GPU availability planner (VRAM, CUDA, region, price) | ❌ | | -| **56** | Spot/preemptible checkpointing | ❌ | Needs **51** | -| **60** | Container builder/cache with CUDA base images | 📄 | CUDA images appear only in Modal skill prose | -| **62** | Dataset locality planner | ❌ | No code, no skill, no doc | -| **63** | Workflow runners: Argo, Cromwell/WDL, Seqera | ❌ | | +| # | Item | Status | Current state | +| ------ | ---------------------------------------------------- | ------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **54** | GPU availability planner (VRAM, CUDA, region, price) | ❌ | | +| **56** | Spot/preemptible checkpointing | ❌ | Needs **51**. **[2026-08-01] Still nothing, and the compute spec sharpens why it matters**: a managed lease is now released the moment its budget is exhausted, so an un-checkpointed run dies with it. `compute-design.md` states the limit plainly — a volume preserves _files_, not _process state_. Note that cheapest-first is **not** spot: `VastProvider.list_options` queries `"type": "on-demand"`, so this is budget/TTL preemption, not marketplace preemption | +| **60** | Container builder/cache with CUDA base images | 📄 | CUDA images appear only in Modal skill prose | +| **62** | Dataset locality planner | ❌ | No code, no skill, no doc | +| **63** | Workflow runners: Argo, Cromwell/WDL, Seqera | ❌ | | ## UX parity diff --git a/docs/plans/06-compute-integrations.md b/docs/plans/06-compute-integrations.md index 2ce09ed2..9d0c5b8b 100644 --- a/docs/plans/06-compute-integrations.md +++ b/docs/plans/06-compute-integrations.md @@ -2,14 +2,27 @@ Workstream: verify each compute path and fix what's broken — BYOK GPU (confirm), cloud storage, SSH, managed compute via the Atlas CLI. Findings-first. Audited against the Atlas backend (cloned) and the installed `atlas` CLI (`0.13.1` = npm `@synsci/atlas`). Citations `file:line`. +> **Currency note — 2026-08-01.** This audit was written in late July and parts of it have been +> overtaken. Three things changed and each is marked inline below, not silently rewritten: +> +> 1. **Path C is no longer a dead end.** `aa9b3142` ("feat: add managed compute jobs", merged to +> `main` on 2026-07-29 — _after_ this audit) shipped a real SSH/Slurm/PBS dispatcher. +> 2. **Path D's "Correction to the initial audit" is false** and is retracted in place. The pin is +> not `^0.5.12` and the published CLI has no `compute:` commands at all. +> 3. **The managed backend behind Path D now works end to end** — lease → promote → SSH → release, +> on Vast and RunPod — but only on an **unmerged draft** Atlas branch, and OpenScience still +> cannot launch a lease. +> +> Everything not marked below is a July fact that has **not** been re-audited. Treat it as such. + ## Status per path -| Path | Verdict | One-line | -| ------------------------------------ | --------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| **A. BYOK GPU providers** | ✅ works (with gaps) | Key encrypt→env-injection is solid + unit-tested for the 4 providers with skills (Modal, Lambda, TensorPool, Prime). **Vast + RunPod keys inject but no skill reads them.** | -| **B. Cloud storage** | ⚠️ creds-only, unverified | No mount/rclone abstraction — AWS/GCP creds → env → whatever CLI a skill invokes. **Azure object storage is advertised but not backed.** Needs real creds+bucket to verify (FLAG). | -| **C. SSH-based compute** | ❌ dead-end | "SSH hosts" + "Model endpoints" panels persist data **nothing ever reads**. No SSH client, no dispatch, no routing. | -| **D. Managed compute via atlas CLI** | ⚠️ real, but version-gapped | The atlas CLI **does** ship a full compute suite (`compute:up`/`catalog`/`list`/`ssh`/`release` → `/api/compute/leases`) in the **published 0.13.2** — but OpenScience pins `@synsci/atlas@^0.5.12`, so the shipped CLI predates it and the prompt's `atlas compute:up` can't resolve. Resale off by default server-side; `billing.compute` prompt-only. | +| Path | Verdict | One-line | +| ------------------------------------ | ------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **A. BYOK GPU providers** | ✅ works (with gaps) | Key encrypt→env-injection is solid + unit-tested for the 4 providers with skills (Modal, Lambda, TensorPool, Prime). **Vast + RunPod keys inject but no skill reads them.** | +| **B. Cloud storage** | ⚠️ creds-only, unverified | No mount/rclone abstraction — AWS/GCP creds → env → whatever CLI a skill invokes. **Azure object storage is advertised but not backed.** Needs real creds+bucket to verify (FLAG). | +| **C. SSH-based compute** | ~~❌ dead-end~~ → **fixed** | ~~"SSH hosts" + "Model endpoints" panels persist data **nothing ever reads**. No SSH client, no dispatch, no routing.~~ **Overtaken by `aa9b3142`:** `src/compute/jobs.ts` dispatches over `ssh`, with Slurm/PBS/none schedulers and a reachability probe. **Model endpoints are still unread.** | +| **D. Managed compute via atlas CLI** | ❌ command surface unpublished | The published `@synsci/atlas@0.13.2` contains **zero** `compute:` commands. The suite exists only in the atlas **repo**, unreleased. The pin is already `^0.13.2` (`backend/cli/package.json:123`) — there is no version to bump to. Atlas's `/api/compute/leases` itself works; nothing shipped can call it. | Ground truth: `atlas doctor` on this machine is seeded (`~/.config/atlas-cli/config.json`), authed, backend reachable — so atlas **auth/config-seeding works**; only the **compute command surface** is broken. @@ -21,10 +34,18 @@ Compute panel (`Compute.tsx`, 6 cards) → `server/routes/settings/compute.ts`: - ❌ **Vast/RunPod inject but no skill reads them** (`VAST_API_KEY`/`RUNPOD_API_KEY` set, no skill) — connecting does nothing. - ⚠️ `last_used` is declared + rendered but **never written** → always "never". - ⚠️ **Modal double-stored** — also in the Credentials panel, which injects the same vars and runs first at boot (`index.ts:107` before `:111`), so a Modal key set in both panels has the Credentials value silently win. -- ⚠️ latent: the atlas-bin fallback resolver walks for `@openscience/atlas` while the dep is `@synsci/atlas` (`index.ts:225`) — dead fallback. +- ⚠️ ~~latent: the atlas-bin fallback resolver walks for `@openscience/atlas` while the dep is `@synsci/atlas` (`index.ts:225`) — dead fallback.~~ **Fixed** — `grep -r '@openscience/atlas' backend/cli/src` → 0 hits (2026-08-01). - **FLAG:** an actual job round-trip (Modal/Lambda/TensorPool/Prime) needs live provider accounts — plumbing verified, round-trip not. -**Fixes:** author Vast/RunPod skills or drop them from the catalog (interim: "key stored — skill coming"); populate or remove `last_used`; pick one home for Modal (recommend removing from Compute, Credentials owns it) or share one precedence; fix the `@openscience→@synsci` scope typo. +**Still true 2026-08-01:** `last_used` is still only defaulted and carried forward, never assigned +(`routes/settings/compute.ts:252`, `:272`). Vast/RunPod still have no catalogued skill. + +**Note on Vast/RunPod:** both are now leasable **server-side**, as managed providers in Atlas, and +were driven end to end on 2026-08-01. That does not close this gap: the capability lives behind +`POST /api/compute/leases`, on an unmerged draft branch, and OpenScience has no tool that calls it. +A user's `VAST_API_KEY`/`RUNPOD_API_KEY` in this panel is still consumed by nothing. + +**Fixes:** author Vast/RunPod skills or drop them from the catalog (interim: "key stored — skill coming"); populate or remove `last_used`; pick one home for Modal (recommend removing from Compute, Credentials owns it) or share one precedence; ~~fix the `@openscience→@synsci` scope typo~~ (done). ## Path B — Cloud storage (⚠️) @@ -37,30 +58,97 @@ Compute panel (`Compute.tsx`, 6 cards) → `server/routes/settings/compute.ts`: **Fixes:** add an Azure Storage cred or drop "Azure" from the copy; document the "creds-only, needs CLIs, no mount" contract; optionally seed an `rclone` remote from stored creds. -## Path C — SSH-based compute (❌) +## Path C — SSH-based compute (~~❌~~ → **half fixed**, 2026-08-01) Compute.tsx promises "SSH hosts" (dispatch runs over SSH) + "Model endpoints" (route inference). `compute.ts` persists `ssh_hosts` + `endpoints`. -- ❌ **store-only dead-ends** — no SSH client dep anywhere, no `ssh` spawn, `ssh_hosts` read only by its own CRUD/SDK-types/UI; `endpoints` has no inference-routing consumer. The agent cannot dispatch to a saved host or route to a saved endpoint. +**The July verdict below was overtaken by `aa9b3142` ("feat: add managed compute jobs", merged to +`main` 2026-07-29 — two days after this audit's baseline `52845c3`). Option (b) was chosen and +built for the SSH half.** What exists now: + +- ✅ **`ssh_hosts` is read and dispatched to.** `src/compute/jobs.ts` (`ComputeJobs`) spawns the + system `ssh` binary (`:362-366`, `BatchMode=yes`, honours a per-host port), supports `local` and + `ssh` targets, `none`/`slurm`/`pbs` schedulers (`sbatch --wait --parsable` at `:331`, `scancel` + at `:710`), optional `apptainer exec` (`:275`), a resource request (cpus/gpus/memory/time/ + partition), artifact collection, and job metadata persisted as JSON at mode `0600` (`:162`). +- ✅ **Reachability is probed, not assumed.** `POST /ssh/:id/test` returns latency plus + Python / NVIDIA GPU / Slurm / PBS capability flags (`routes/settings/compute.ts:394`). +- ✅ **It is user-reachable** — `frontend/workspace/src/atlas/ComputeJobs.tsx`, mounted in + `RightPane.tsx:297`, which the session page renders. Routes: `/jobs`, `/jobs/completed`, + `/jobs/:id/log`, `/jobs/:id/cancel`. +- ❌ **Model endpoints are still a store-only dead end.** `endpoints` still has no + inference-routing consumer; the panel still promises routing that nothing performs. +- ❌ **No agent tool.** `grep ComputeJobs backend/cli/src/tool/` → **0 hits**. The job system is + driven from the UI; the agent cannot start or cancel a job. +- ⚠️ **No SSH client _dependency_ was added** — it shells out to the host's `ssh`. So audits + phrased as "no SSH client in the dep tree" remain literally true and are no longer evidence of + anything. Key handling is still the user's `~/.ssh`; the store still has no key field, which is + why the security sign-off below never had to happen. - ⚠️ nuance: cloud-compute skills SSH into boxes **they provision** (lambda/tensorpool/skypilot) — real SSH, unrelated to the panel. -**Fixes (pick one):** (a) **remove** the SSH-hosts + model-endpoints sections + routes (recommended near-term — stop advertising vaporware); (b) **wire it** — a remote-exec tool reading `ssh_hosts` (needs an SSH client dep + key handling) + treat `endpoints` as selectable OpenAI-compatible targets, injected into compute-agent context. **Decision required + security sign-off** on private-key storage (the store has no key field today). +~~**Fixes (pick one):** (a) **remove** the SSH-hosts + model-endpoints sections + routes (recommended near-term — stop advertising vaporware); (b) **wire it** — a remote-exec tool reading `ssh_hosts` (needs an SSH client dep + key handling) + treat `endpoints` as selectable OpenAI-compatible targets, injected into compute-agent context. **Decision required + security sign-off** on private-key storage (the store has no key field today).~~ + +**Remaining fixes:** (a) **model endpoints** — still the original decision: remove the section or +wire it as selectable OpenAI-compatible targets. (b) Expose `ComputeJobs` to the agent, or state +that it is deliberately UI-only. ## Path D — Managed compute via the Atlas CLI (❌) Config seeding works (`ensureAtlasCliConfig`, verified by `atlas doctor`). Intended UX: "Compute spend = Managed" (`Spend.tsx`) → `billing.compute` → a `` injected by `insertReminders` (`prompt.ts:1321-1333`) telling the agent to run `atlas compute:up`. Atlas has the machinery: `POST /api/compute/leases` provisions Modal sandboxes + reseller GPU VMs, billed to the wallet (`compute.py:305-421`, `compute_billing_service.py`). -**Correction to the initial audit** (which tested the _installed 0.13.1_): the atlas CLI at the **published latest (0.13.2)** ships a real compute suite — `cli/src/atlas-runtime/commands.mjs:915-923` registers `compute:up` (aliases `launch`/`lease` → `POST /compute/leases`, _"zero flags = cheapest GPU; managed bills the wallet per hour; BYOK free"_), `compute:catalog`/`gpus`/`options` (browse GPUs → `/compute/options`), `compute:list`/`leases` (`GET /compute/leases`), `compute:ssh` (`/connection`), `compute:release`/`down` (`/release`). So `atlas compute:up` **is a real command in 0.13.2**, hitting the exact `/api/compute/leases` API — the prompt is _aspirationally correct_, not naming a phantom. +### ~~Correction to the initial audit~~ — the correction was itself wrong (retracted 2026-08-01) + +**Kept in place rather than deleted, because the mistake is the useful part of this section.** It +was a source-read conclusion about a _published artifact_, checked against neither the artifact nor +the manifest it claimed to be quoting. + +~~**Correction to the initial audit** (which tested the _installed 0.13.1_): the atlas CLI at the **published latest (0.13.2)** ships a real compute suite — `cli/src/atlas-runtime/commands.mjs:915-923` registers `compute:up` (aliases `launch`/`lease` → `POST /compute/leases`, _"zero flags = cheapest GPU; managed bills the wallet per hour; BYOK free"_), `compute:catalog`/`gpus`/`options` (browse GPUs → `/compute/options`), `compute:list`/`leases` (`GET /compute/leases`), `compute:ssh` (`/connection`), `compute:release`/`down` (`/release`). So `atlas compute:up` **is a real command in 0.13.2**, hitting the exact `/api/compute/leases` API — the prompt is _aspirationally correct_, not naming a phantom.~~ -- ⚠️ **Version gap is the core defect.** OpenScience pins `@synsci/atlas@^0.5.12` (`backend/cli/package.json`); the installed CLI is 0.13.1 (whose `--help` doesn't surface compute); **npm latest is 0.13.2** (which does). So the MANAGED prompt (`prompt.ts:1329` `atlas compute:up`) names a real command the **shipped/pinned atlas CLI predates** → it doesn't resolve for users today. +**Both of its load-bearing claims are false. Verified 2026-08-01:** + +- **The pin is not `^0.5.12`.** `backend/cli/package.json:123` reads `"@synsci/atlas": "^0.13.2"`. + There is no version gap, and nothing to bump. +- **The published artifact carries no `compute:` commands at all.** `npm pack @synsci/atlas@latest` + resolves to **0.13.2**, and **zero** files in that tarball contain `compute:up` or + `compute:lease`. Not an older surface — no surface. + +The commands _are_ real, but only in the atlas **repo**: `3e1d1ca` removed them, `0.13.1` **and** +`0.13.2` both shipped without them, and `205bbc0` re-added them **with no version bump**. So the +source tree and the published artifact disagree at an identical version number — which is precisely +why reading the repo told the first correction the opposite of what a user installs. + +**This is a release problem, not a code or pinning problem.** Until a version ships with the +commands in it, no prompt, doc or runbook may name `atlas compute:*`. + +- ✅ **The prompt no longer names it.** `12a43695` replaced the `atlas compute:up` guidance with a + `compute_status` pointer; `grep -r 'compute:up' backend/cli/src` → **0 hits**. The defect this + section was written to describe is closed on the client side. - ⚠️ **The surface has churned** — the CLI CHANGELOG shows a `compute:*` set removed then a richer one re-added; and it also describes provisioning as a **web-dashboard "Lambda Labs reseller" Compute tab**. Confirm the intended UX (CLI leasing vs web dashboard, Modal as agent-runtime-internal) is settled before wiring the prompt hard to it. - ⚠️ `exec:start` is a **separate** graph-ledger command (INSERTs a bookkeeping row, no Modal/lease call, `execution_service.py:45-87`) — not the compute path; don't conflate the two. -- ⚠️ server-side managed GPU is **off by default** — `COMPUTE_RESELL_ENABLED="false"` (`config.py:387`). +- ⚠️ ~~server-side managed GPU is **off by default**~~ — the **default** is still `"false"` + (`config.py:383-384`), but **production has it on**: `resell_enabled: true` with lambda / runpod / + vast / prime_intellect operator-funded and 292 launchable options, verified against + `thesis-synsc` on 2026-07-31. Reading the default and concluding "managed is off" was one of the + four source-read errors this workstream has now made about deployed behaviour. - ⚠️ `billing.compute` is **prompt-only** — unlike `billing.llm` (mirrors to server + resyncs), it just persists + injects the reminder. - ⚠️ substrate named 3 ways — "Daytona-backed" (`research.txt:229`) vs "Modal sandbox" (`atlas agent:run --help`) vs "Atlas-provisioned" (`config.ts:984`). -- **FLAG:** an actual lease still needs `COMPUTE_RESELL_ENABLED=true` + operator keys + a funded wallet + a Modal account to verify end-to-end. - -**Fixes:** (1) **primary — bump `@synsci/atlas` `^0.5.12`→`^0.13.2`** (align the pin + the seeded/expected version to the published CLI that has compute) and verify `atlas compute:*` resolves against the installed version; then `prompt.ts:1329`'s `atlas compute:up` is truthful. Add a prompt-time guard: if the installed `atlas` lacks `compute:*`, the MANAGED reminder falls back to BYOK rather than naming an unresolvable command. (2) enable resale (`COMPUTE_RESELL_ENABLED`) + wire `billing.compute` to reality (mirror `billing.llm`) so the managed path actually leases. (3) reconcile the CLI-leasing vs web-dashboard-reseller UX (owner decision). (4) reconcile substrate naming. (5) decide the BYOK source of truth. +- ~~**FLAG:** an actual lease still needs `COMPUTE_RESELL_ENABLED=true` + operator keys + a funded wallet + a Modal account to verify end-to-end.~~ **Discharged 2026-08-01.** Exercised end to end + against a deployed backend on **both** Vast and RunPod: `POST /leases` → the real background + reaper promoted to `ready` with NATed SSH coordinates within one sweep → SSH into a real GPU → + `POST /release` → instance verified gone at the provider. Not Modal — Modal is the CPU sandbox + path, and conflating it with GPU leases is the same error as the `exec:start` bullet above. + +**Fixes:** (1) ~~bump the pin~~ — **publish an atlas release containing the `compute:` commands**; +the pin is already `^0.13.2` and the code is already in the repo, so this is a release action with +no code change. Until then, keep prompts free of `atlas compute:*` (already true — `12a43695`). +(2) ~~enable resale~~ — **done in production**; what remains is wiring `billing.compute` to reality +(mirror `billing.llm`). (3) reconcile the CLI-leasing vs web-dashboard-reseller UX (owner decision). +(4) reconcile substrate naming. (5) decide the BYOK source of truth. + +> **What is actually missing is a client, not a CLI.** OpenScience cannot launch a managed lease by +> any route: `ComputeTools` is `[ComputeStatusTool]` (`src/tool/compute.ts:138`), and the only +> `/api/compute` call in the product is `mode.ts`'s read-only `/options` probe. `compute_launch`, +> `compute_list` and `compute_release` are designed in `docs/specs/compute-design.md` and **unbuilt**. ## Cross-cutting — overlapping BYOK stores @@ -68,33 +156,34 @@ A user's Modal key can live in **three** places with no reconciliation: the loca ## Consolidated backlog (by effort) -| # | Fix | Path | Effort | -| --- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---- | --------------------- | -| 1 | Bump `@synsci/atlas` pin `^0.5.12`→`^0.13.2` so `atlas compute:up` resolves; add a "compute unavailable → BYOK" prompt guard (`prompt.ts:1329`); reconcile `research.txt:229` | D | S | -| 2 | Azure: add Storage cred or drop "Azure" copy | B | XS | -| 3 | Vast/RunPod: "no skill yet" or remove from catalog | A | XS | -| 4 | `last_used`: populate or remove | A | XS | -| 5 | Fix atlas-bin fallback scope `@openscience→@synsci` | A | XS | -| 6 | Modal de-dup across Compute vs Credentials | A | S | -| 7 | Remove or wire SSH-hosts + model-endpoints | C | S (remove) / L (wire) | -| 8 | Document cloud-storage contract + optional rclone seeding | B | S | -| 9 | Enable resale + wire `billing.compute` to reality (mirror `billing.llm`) so managed leasing works end-to-end | D | M | -| 10 | Reconcile 3-way BYOK store + atlas version pin | A/D | M | +| # | Fix | Path | Effort | +| --- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---- | --------------------- | +| 1 | ~~Bump the `@synsci/atlas` pin~~ — **wrong fix, the pin is already `^0.13.2`.** **Publish** an atlas release that contains the `compute:` commands; the code is in the repo, unreleased. Prompt guard no longer needed (`12a43695` removed `atlas compute:up`) | D | S (release, no code) | +| 2 | Azure: add Storage cred or drop "Azure" copy | B | XS | +| 3 | Vast/RunPod: "no skill yet" or remove from catalog | A | XS | +| 4 | `last_used`: populate or remove — **still open** | A | XS | +| 5 | ~~Fix atlas-bin fallback scope `@openscience→@synsci`~~ — **done** | A | XS | +| 6 | Modal de-dup across Compute vs Credentials | A | S | +| 7 | ~~Remove or wire SSH-hosts~~ — **wired** (`aa9b3142`). Remaining: **model endpoints** (remove or wire), and whether `ComputeJobs` gets an agent tool | C | S (remove) / M (tool) | +| 8 | Document cloud-storage contract + optional rclone seeding | B | S | +| 9 | ~~Enable resale~~ (on in production) + wire `billing.compute` to reality (mirror `billing.llm`) | D | M | +| 10 | Reconcile 3-way BYOK store ~~+ atlas version pin~~ (the pin is correct) | A/D | M | +| 11 | **Added 2026-08-01** — build `compute_launch`/`compute_list`/`compute_release`. Atlas can lease; nothing in OpenScience can ask it to. Designed in `docs/specs/compute-design.md` | D | L | ## Risks / decisions needed from the owner -- **Is managed compute in scope this sprint?** Backend is built but the CLI surface + default-off flag mean it's not shippable today — if out of scope, stop advertising it (`Compute.tsx:159-162`, `Spend.tsx:42`). -- **SSH hosts / endpoints:** in scope (wire, +SSH dep + key security) or remove? -- **Infra to verify:** BYOK round-trips need provider accounts; cloud storage needs real creds+bucket+CLIs; managed leases need operator keys + `COMPUTE_RESELL_ENABLED=true` + funded wallet + Modal. **Do not mark any path "works" without exercising it.** +- **Is managed compute in scope this sprint?** ~~Backend is built but the CLI surface + default-off flag mean it's not shippable today~~ — **restated 2026-08-01:** the backend is built _and demonstrated end to end_, and resale is on in production. What blocks it is (a) an unpublished CLI and (b) **no client**: OpenScience has no launch tool. If it stays out of scope, stop advertising it (`Compute.tsx`, the Spend panel). +- **SSH hosts / endpoints:** ~~in scope (wire, +SSH dep + key security) or remove?~~ **SSH hosts: answered — wired in `aa9b3142`.** Model endpoints: still remove-or-wire. +- **Infra to verify:** BYOK round-trips need provider accounts; cloud storage needs real creds+bucket+CLIs. ~~managed leases need operator keys + `COMPUTE_RESELL_ENABLED=true` + funded wallet + Modal~~ — **done 2026-08-01, Vast and RunPod, on a deployed backend.** **Do not mark any path "works" without exercising it.** - **BYOK source-of-truth** is a cross-repo decision. ## Acceptance criteria -- No prompt instructs a command absent from the bundled CLI (grep prompts for every `atlas …` verb; assert each resolves in `atlas --help`). +- No prompt instructs a command absent from the bundled CLI (grep prompts for every `atlas …` verb; assert each resolves in `atlas --help`). **Met for compute** — 0 hits for `compute:up` in `backend/cli/src`. - Connecting Vast/RunPod either drives a real run or the UI no longer implies it will. - Storage lists only credential-backed backends (Azure fixed or removed); a documented smoke test (with creds) round-trips an object on S3 + GCS. -- SSH/endpoint panels are gone, or adding a host + "run nvidia-smi on " executes over SSH. -- With managed enabled + infra: `Compute spend = Managed` starts a real lease, wallet debits per the 60 s tick, auto-releases — demonstrated once. +- ~~SSH/endpoint panels are gone, or adding a host + "run nvidia-smi on " executes over SSH.~~ **Met for SSH** (`aa9b3142`); **not met for model endpoints.** +- With managed enabled + infra: `Compute spend = Managed` starts a real lease, wallet debits per the 60 s tick, auto-releases — demonstrated once. **Demonstrated at the API, not from OpenScience** — `POST /api/compute/leases` → billing tick decrementing the wallet in lockstep → release, on a deployed backend. The `Compute spend = Managed` _path_ still has no launch mechanism to trigger. - One substrate name + one canonical atlas version documented; `billing.compute` changes behavior or is labeled advisory; `settings-compute` tests stay green. **Key files:** `components/settings/{Compute,Storage,Spend}.tsx`, `server/routes/settings/{compute,storage,credentials,billing}.ts`, `openscience/index.ts`, `session/{prompt,billing-gate}.ts`, `agent/prompt/research.txt`, `config/config.ts`. Atlas: `routes/compute.py`, `services/{execution,compute_billing,compute_keys}_service.py`, `compute/{lease_manager,modal_provider}.py`, `config.py`. From 2fa74c446ea0952e7a000195bf4a1473df25bdbd Mon Sep 17 00:00:00 2001 From: KB Date: Sat, 1 Aug 2026 23:16:56 +0530 Subject: [PATCH 47/56] plan(compute): mark the three lease plans executed, and fix one wrong premise All three ran to completion on atlas feat/compute-lease-prerequisites, which is still an unmerged draft PR. Their bodies are kept as written -- they are the record of what was planned -- and each gains a banner stating where it ran, what came out, and what execution proved the plan wrong about. The banners carry what the plans could not know. Promotion needed a reachability gate, justified by RunPod returning desiredStatus=RUNNING with no address, and the first version of that gate broke Modal. A destroyed Vast instance read as provisioning forever. A review finding demanding a 404 was overturned by a live probe: Vast never 404s. Deploying exposed a migration race across all 22 ADD COLUMN sites. A catalog-stability heuristic with perfect retrospective separation failed its first prospective test and is retracted. The budget-cap plan also gets an inline correction it needs to stop misleading a future implementer: Task 3 asserts the effective-balance lookup is already in scope in create_lease. It is not. create_lease never reads the balance -- the wallet check is in lease_manager.acquire_lease, and the adjacent compute_estimate is what has it in scope. Clamping required a new conditional lookup. --- .../2026-07-31-compute-lease-prerequisites.md | 34 ++++++++++++++ docs/plans/2026-08-01-compute-budget-cap.md | 39 ++++++++++++++++ .../plans/2026-08-01-compute-lease-defects.md | 45 +++++++++++++++++++ 3 files changed, 118 insertions(+) diff --git a/docs/plans/2026-07-31-compute-lease-prerequisites.md b/docs/plans/2026-07-31-compute-lease-prerequisites.md index 028c02fc..22097b43 100644 --- a/docs/plans/2026-07-31-compute-lease-prerequisites.md +++ b/docs/plans/2026-07-31-compute-lease-prerequisites.md @@ -1,5 +1,39 @@ # Compute lease prerequisites — implementation plan +> ## ✅ EXECUTED — 2026-08-01. Historical record; do not implement from it again. +> +> **Where:** `~/codes/InkVell/atlas`, branch `feat/compute-lease-prerequisites`, base `8aa66d5`. +> **Not merged** — the PR is deliberately draft while the team waits for compute to be complete. +> **Outcome:** all four tasks landed, plus a six-finding fix wave from the whole-branch review. +> Suite **1610 passed / 1 skipped**, 12 commits at plan completion. Ledger: +> `.superpowers/sdd/2026-07-31-compute-lease-prerequisites/progress.md`. +> +> **The deliverable holds, and was later confirmed against a real provider rather than a fake clock:** +> a user lease sat `ready` for 578s → 687s → **788s** un-reaped against the deployed reaper. Before +> this branch it died at 600s. Task 1's deploy gate is discharged by a **cross-login test**: two +> instances on one Vast operator account, each key accepted for its own box and **refused** +> (`Permission denied (publickey)`) against the other's. The cross-tenant hole is closed, measured, +> not inferred. +> +> **What execution proved this plan wrong about — read these before trusting any step below:** +> +> - **Task 3's promotion was under-specified and the first fix for it broke Modal.** The fix wave +> gated promotion on a non-empty `ssh_host`; Modal's `connection()` returns no `ssh_host` at all +> (it is exec-based, no SSH), so Modal CPU sandboxes stopped promoting and were reaped at 600s. +> The gate is now `if not ssh_host and lease.get("ssh_key_name")` — the four SSH providers set +> `ssh_key_name` in `acquire`, Modal does not. The gate itself was **justified by measurement**, +> not taste: RunPod returns `desiredStatus=RUNNING` from pod creation with **no address**, so +> provider status alone is not readiness. This plan's Step 5 does not say any of that. +> - **The status mapping was wrong for a destroyed Vast instance.** `_get_instance` returned `{}` and +> `_map_status` read that as `provisioning` **forever**, so reaper branch 1 could never fire for +> Vast. Fixed in `9bc19a7` (empty payload → `terminated`). Later sharpened again: Vast returns +> HTTP 200 `{"instances": null}` for a destroyed id **and** for one that never existed, and never +> 404s, while RunPod does 404 — both signals are needed. +> - **Task 4's `_PROVISIONING` set omitted Lambda's in-flight strings.** Flagged during the task and +> deliberately left for a human ruling; fixed in the follow-on plan (`8748057`). +> - **Task 1's minor deferral is still open:** `vast_provider`'s docstring API list still names +> `POST /ssh/` — the endpoint the task removed the call to. + > **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development to implement > this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. diff --git a/docs/plans/2026-08-01-compute-budget-cap.md b/docs/plans/2026-08-01-compute-budget-cap.md index 7930ce0d..73652fd0 100644 --- a/docs/plans/2026-08-01-compute-budget-cap.md +++ b/docs/plans/2026-08-01-compute-budget-cap.md @@ -1,5 +1,36 @@ # Managed compute budget cap — implementation plan +> ## ✅ EXECUTED — 2026-08-01. Historical record; do not implement from it again. +> +> **Where:** `~/codes/InkVell/atlas`, branch `feat/compute-lease-prerequisites` (continuing). +> **Not merged** — the PR is deliberately draft while the team waits for compute to be complete. +> **Outcome:** all four tasks landed — `0a9da07` (set-to-total), `a4a79ca` (the cap binds), +> `31fc598` (`budget_cents`), `71dccba` (the spawn kill). Suite **1689 passed / 1 skipped**, branch +> 38 commits. Spec changes 1 and 2 landed together as this plan required. Ledger: +> `.superpowers/sdd/2026-08-01-compute-budget-cap/progress.md`. +> +> **The headline property holds, measured:** a $10 budget at $6.99/h releases at **5160.0s** against a +> theoretical 5150.21s — a 9.79s overshoot that is pure tick quantisation, well inside ±90s. The +> central design decision (a set-to-total, not an increment) was confirmed by mutation: the naive +> `debit_grant(delta)` misses the duration assertion by ~3590s and leaves `grant.spent_cents = 300` +> where wall-clock is 180 — the acquire debit double-counted, exactly the trap this plan predicted. +> Under that mutation `released == 1` and the wallet moved, so **every "a release happened" assertion +> still passed**; only the duration assertion caught it. +> +> **Task 4's hazard was real and is now quantified.** Every spawn grant was `hard_cap_cents = 500` +> flat for **every SKU** — no caller in either repo ever sent anything else. A spawn requests 4 hours: +> $14.60 on an A100-40GB, $18.36 on an A100-80GB, $27.96 on an H100, so only T4 and A10G ever fitted. +> Measured on the real spawn path against the real tick: **a 4-hour A100 spawn was killed at 1.38h** +> and an **H100 was refused outright at acquire**. `budget_cents` is now `int | None`; `None` sizes +> the ceiling to the spawn's own lifetime at the SKU rate, and an explicitly chosen budget still binds. +> +> **What execution proved this plan wrong about:** **Task 3's premise that the effective-balance +> lookup is "already in scope" in `create_lease`.** It is not — see the correction marked inline at +> Task 3, Step 3. Two judgement calls were also taken that this plan does not contain, both recorded +> in the spec: the `402` keeps one body shape but carries **two** `error` values +> (`insufficient_cli_credit` vs a new `budget_below_hourly_rate`), and the budget is deliberately +> **not** clamped to `rate × ttl_hours`. + > **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development to implement > this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. @@ -308,6 +339,14 @@ with a budget-aware version that keeps the wallet as the outer bound. Read the c before editing — `charge_raw`, `ttl_hours` and the effective balance lookup are already in scope, and the existing `402` for insufficient credit is the one to extend rather than duplicate. +> **⚠️ WRONG PREMISE — caught by the implementer, 2026-08-01. Do not follow this sentence.** +> **The effective-balance lookup is NOT in scope in `create_lease`.** `create_lease` never reads the +> balance at all; the wallet check lives in `lease_manager.acquire_lease`, and it is the **adjacent +> `compute_estimate`** that has `effective_balance` in scope — which is presumably what this plan +> misread. Clamping therefore required adding a **new conditional lookup**, taken only when +> `budget_cents` is present, so the no-budget path adds no query. `charge_raw` and `ttl_hours` are in +> scope as stated; only the balance claim is wrong. + Report the effective cap on the response so a caller can tell the user what was actually authorised. - [ ] **Step 4: Run and confirm they pass** diff --git a/docs/plans/2026-08-01-compute-lease-defects.md b/docs/plans/2026-08-01-compute-lease-defects.md index 65e4ca2f..fd6b5604 100644 --- a/docs/plans/2026-08-01-compute-lease-defects.md +++ b/docs/plans/2026-08-01-compute-lease-defects.md @@ -1,5 +1,50 @@ # Compute lease defects — implementation plan +> ## ✅ EXECUTED — 2026-08-01. Historical record; do not implement from it again. +> +> **Where:** `~/codes/InkVell/atlas`, branch `feat/compute-lease-prerequisites` (continuing). +> **Not merged** — the PR is deliberately draft while the team waits for compute to be complete. +> **Outcome:** all four tasks landed (`b8a4467`, `0eb33c2`, `f65dd47`, `8748057`), followed by an +> eight-commit fix wave closing nine review findings. Ledger: +> `.superpowers/sdd/2026-08-01-compute-lease-defects/progress.md`. +> +> **This branch was then deployed and driven end to end**, which is the part that matters: on a +> deployed backend, `POST /api/compute/leases` → the real reaper promoted within one sweep with +> **NATed** SSH coordinates (`ssh3.vast.ai:15650`, `194.68.245.163:22189` — never the placeholder +> `22`) → SSH into a real GPU → release, with the instance verified gone at the provider. Done on +> **both** Vast and RunPod. Prime Intellect no longer leaks a key per lease; provider releases now +> report what actually happened; BYOK leases are polled with the owner's credential and managed +> leases never are. +> +> **What execution proved wrong — including one finding that was wrong in the safe-looking direction:** +> +> - **A review finding was overturned by a live probe.** Finding M9 required an explicit `404` before +> treating a Vast instance as gone. Vast returns HTTP **200 `{"instances": null}`** for a destroyed +> id _and_ for an id that never existed — it never 404s — so M9 would have silently reversed +> `9bc19a7`, itself live-verified. The `410` it cited was on the `/instances/` **listing** endpoint, +> a different call. Corrected by restoring the null-payload verdict and guarding the original +> concern properly: two consecutive terminal observations before reaping. +> - **Deploying found a real bug nothing else would have.** `uvicorn --workers 2` runs the lifespan in +> both workers, both ran `run_migrations` against the same sqlite file, both saw `provider_key_id` +> absent, both `ALTER`ed, one crashed with `duplicate column name` and failed startup. A +> **pre-existing pattern** at all 22 `ADD COLUMN` sites that this branch's new column exposed. Fixed +> in `84bbbb7`. +> - **A tempting heuristic was falsified.** "Offers present in consecutive catalog fetches are +> launchable" had perfect separation at n=8 retrospectively, and **failed its first prospective +> test**. Retracted, along with the claim that spec change 12's cache would give change 3 its +> correctness fix for free. Retry remains unavoidable. +> - **Task 3 carries a caveat that is still open:** Prime's `DELETE /ssh_keys/{id}` is +> documentation-verified only, never exercised against production. It fails safe — the delete is +> swallowed and cannot block the pod teardown — so a wrong endpoint would leak silently. +> - **Measured facts this plan did not anticipate**, now feeding spec changes 3 and 10: Vast offer ids +> churn **~50% between two consecutive catalog fetches** (7 consecutive `Unknown SKU` failures +> across price ranks 0–25); RunPod advertises GPU types with **zero capacity** and its `500` is +> mapped to a `400` — the same status as Vast's stale offer, with a different message, so a +> resolver must discriminate on the message because the two need opposite responses. +> - **Two openscience fixes came out of this wave**, on `feat/compute-guardrails`: `784633e` (stop +> claiming managed compute at a zero balance) and `a732396d` (tmpfs the XDG cache dir so +> `bubblewrap` stops failing tools with "Read-only file system"). + > **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development to implement > this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. From 196f3b2a3e89220c8cce4efec65d90213821e0c4 Mon Sep 17 00:00:00 2001 From: KB Date: Sat, 1 Aug 2026 23:21:11 +0530 Subject: [PATCH 48/56] spec(compute): mark 0, 8 and 9 against what shipped, and fold in what live testing measured Changes 0, 1, 2, 8 and 9 were listed as shipped in the Part B heading, but only 1 and 2 carried a banner -- so sections 0, 8 and 9 still read as unbuilt work with instructions to go and do them. Each now says what shipped and, more usefully, what execution forced that the section never specified: promotion needs a coordinate gate because RunPod reports RUNNING with no address, and that gate needs an ssh_key_name clause because Modal has no SSH at all. Change 8 is deliberately NOT marked shipped. The honesty half is done across all three providers and release_lease now acts on the verdict, but an unconfirmed teardown leaves the row matching both list_active_leases and count_active_managed_gpu_leases -- so the user who asked to release keeps being charged and keeps burning a concurrency slot, which is verbatim the failure the section says to decide against. Criterion 16 is one-third met. Change 9 is the one item here proven adversarially rather than inferred: two boxes on one Vast operator account, each key refused against the other's. Measured facts folded in where they change a decision. Vast offer ids churn ~50% between consecutive catalog fetches, and a stability heuristic with perfect retrospective separation failed prospectively -- retracted, retry is unavoidable. RunPod's no-capacity 500 is mapped to the same 400 as Vast's stale offer, and the two need opposite responses, so the resolver must discriminate on the message. RunPod's own catalog query asks for a price list, not an inventory. The third-party '>6 minute Vast tail' did not reproduce; successful boots ran 34-50s and the real failure mode is 'never boots', which a longer timeout cannot fix. Ranking by reliability2 beat a 0.98 threshold. The bounds table is rewritten: five of seven rows had changed, and change 5 is now the sharpest gap, since every remaining bound is per-lease. --- docs/specs/compute-design.md | 258 +++++++++++++++++++++++++++++++---- 1 file changed, 233 insertions(+), 25 deletions(-) diff --git a/docs/specs/compute-design.md b/docs/specs/compute-design.md index f562e722..d6f25165 100644 --- a/docs/specs/compute-design.md +++ b/docs/specs/compute-design.md @@ -1,10 +1,17 @@ # Compute — design -Status: **Mode detection shipped. Lease prerequisites and the budget cap shipped. Selection and the -agent-facing tools to build.** -Date: 2026-07-31 · single current compute spec · revised after adversarial review +Status: **Mode detection shipped. Lease prerequisites, the SSH key lifecycle and the budget cap shipped +and verified against a live deployment — on an unmerged draft branch. Selection, the quote, the rolling +cap and every agent-facing tool still to build.** +Date: 2026-07-31 · sweep 2026-08-01 · single current compute spec · revised after adversarial review Roadmap: **5**, **51/2**, **55**, **103**; unblocks **56** +> **What a user can do today: nothing.** Everything marked SHIPPED below lives on Atlas +> `feat/compute-lease-prerequisites` and OpenScience `feat/compute-guardrails`, both deliberately draft +> until compute is complete — and OpenScience has no launch tool in any case (`ComputeTools` is +> `[ComputeStatusTool]`). "Shipped" in this document means _built, tested, and in several cases driven +> against a real provider_. It does not mean reachable. + How OpenScience gets a GPU: who provisions it, who pays, and what stops it. Spans two repos — `atlas` (Python/FastAPI) decides and enforces, `openscience` (Bun/TypeScript) relays and obeys. Each side gets its own implementation plan; this document is the contract between them. @@ -115,14 +122,17 @@ and because `byok` wins whenever a credential exists, managed is suppressed at t They have a key, no skills, and no managed path. Part B makes this sharper rather than fixing it: `ComputeTools` is registered unconditionally -(`src/tool/registry.ts:135`), so `compute_launch` stays callable while `compute_status` tells the agent -not to launch managed leases (`src/tool/compute.ts:22`). **The three new tools must state their behaviour -in `byok` and `none`** — refuse with the reason, rather than attempting a managed lease the mode says is -unavailable. No criterion covered this before. +(`src/tool/registry.ts:135`), so `compute_launch` **will be** callable in every mode once it exists, +while `compute_status` tells the agent not to launch managed leases. **The three new tools must state +their behaviour in `byok` and `none`** — refuse with the reason, rather than attempting a managed lease +the mode says is unavailable. No criterion covered this before. + +_Today `ComputeTools` is `[ComputeStatusTool]` (`src/tool/compute.ts:138`), so the sentence above +describes the state Part B creates, not the state that exists._ --- -# Part B — Managed leases (changes 0, 1, 2, 8, 9 shipped; the rest to build) +# Part B — Managed leases (changes 0, 1, 2 and 9 shipped, 8 half; the rest to build) ## The gap Part A exposed @@ -286,7 +296,35 @@ something that ships alongside the resolver. ## Atlas changes -### Change 0 — make a lease reach `ready`, then scope the reaper _(prerequisite, two parts)_ +### Change 0 — make a lease reach `ready`, then scope the reaper _(prerequisite, two parts)_ — **SHIPPED** + +> **Shipped** across `feat/compute-lease-prerequisites` (promotion + coordinates + reaper scoping + +> `lease_state.normalise_state`), and **verified against a real provider rather than an injected clock.** +> +> - **The headline property, on a wall clock:** a user lease with no runner token sat `ready` for +> 578s → 687s → **788s** against the deployed reaper, un-reaped. Before this change it died at 600s. +> - **Promotion and coordinates, live on both providers:** created `provisioning` with no address → +> the real background sweep promoted it within one interval → `ssh_host` / `ssh_port` persisted as +> the **NATed** values (`ssh3.vast.ai:15650`, `194.68.245.163:22189`), never the placeholder `22` → +> `ready_at` stamped → SSH into a real GPU worked. +> - **0(c) live over HTTP:** `GET /connection` returned `state: "ready"`. +> +> **Three things the section below does not say, all of which execution forced:** +> +> - **Provider status alone is not readiness.** RunPod reports `desiredStatus=RUNNING` from pod +> creation with `publicIp=""` and `portMappings=None`; coordinates appeared at t+15s. Promotion +> therefore waits for coordinates as well — `if not ssh_host and lease.get("ssh_key_name")` — and +> the `ssh_key_name` half is load-bearing because **Modal has no SSH at all**: its `connection()` +> returns no `ssh_host`, so an address-only gate stopped Modal CPU sandboxes promoting and had them +> reaped at 600s. That regression was caught before merge. +> - **A destroyed Vast instance read as `provisioning` forever**, so reaper branch 1 could never fire +> for Vast. Fixed by mapping an empty instance payload to terminated. Vast returns HTTP 200 +> `{"instances": null}` for a destroyed id **and** for an id that never existed and **never 404s**; +> RunPod does 404. Both signals are needed, and the terminal verdict now requires two consecutive +> observations. +> - **Deploying it found a migration race** the tests could not: `uvicorn --workers 2` ran +> `run_migrations` in both workers, both `ALTER`ed the same new column, one crashed startup with +> `duplicate column name`. A pre-existing shape at all 22 `ADD COLUMN` sites, exposed by this work. This is the prerequisite everything else waits on, and **the first draft of this spec got it wrong** — it named the heartbeat branch, which is the one branch that cannot be killing these leases. The @@ -345,11 +383,24 @@ put RunPod's median near 59s but Vast's tail past **6 minutes** — so under che most launches to Vast, a slice of legitimate provisions runs close to the limit and some will exceed it. A box reaped mid-provision is a launch the user paid for and never received. +> **Our own measurement does not reproduce that tail, and it changes what this section is for +> (2026-08-01, n=7 real Vast launches, n=2 RunPod).** Every launch that booted at all booted fast — +> **34s, 46s, 50s** on Vast, ~30s on RunPod. The third-party ">6 minute tail" was not observed. What +> _was_ observed is a different failure entirely: **3 of 9 launches never booted at all** (Vast +> `actual_status=offline` for 7 minutes while `cur_state=running`; RunPod returning no capacity). +> +> **So the failure mode is "never boots", not "boots slowly", and a longer timeout does not help it — +> it only delays the refund.** The per-provider timeout is still worth deriving from real data, but it +> drops in priority behind change 10's ranking work and behind falling through on capacity errors +> (change 3), which is what actually addresses the observed loss. n is small; do not over-fit either +> way. + **Make the timeout per-provider and set it from the p99 of measured boot times** (change 10), not from a round number. Until that data exists, raise it for the providers whose observed tail demands it rather than leaving one value covering a 6× spread. -**Ships first, with its own test, before any budget work.** +~~**Ships first, with its own test, before any budget work.**~~ **It did** — change 0 shipped as its own +work before changes 1 and 2, as required. ### Change 1 — make `hard_cap_cents` a real running cap — **SHIPPED** @@ -506,6 +557,43 @@ catalog, bounded to N attempts, then fail with a structured error rather than lo five provider requests, so N retries is N full catalog rebuilds and change 4 adds another per launch. **Change 12 (catalog cache) ships before this**, or the retry path costs more than the lease. +#### Measured 2026-08-01 — the race is worse than assumed, and `400` is overloaded + +Three findings from driving real launches against a deployed backend. All three are requirements on +the resolver, not colour. + +- **The churn is ~50% per fetch, so client-side selection is roughly a coin flip.** Three back-to-back + `/api/compute/options` calls returned 65 / 67 / 67 Vast offers with only **25 stable across all + three**; 31 of the first 65 were gone by the second fetch. Observed **7 consecutive + `HTTP 400 Unknown SKU` failures across price ranks 0–25** — so this is not a cheapest-first + artifact, it is the whole catalog. A lease landed on attempt 3 of a random-pick loop, which is what + ~50% churn predicts. **This is the quantitative case for change 3 existing at all.** +- **A stability heuristic was falsified — do not resurrect it.** "Offers present in consecutive + catalog fetches are launchable" had _perfect_ separation retrospectively (n=8: the one SKU that + launched was in the 3-fetch stable set, all 7 failures were not) and **failed its first prospective + test** — a SKU in the stable set of two fetches still returned `400`. The attractive corollary, + that change 12's cache would hand change 3 its correctness fix for free by intersecting fetches, + is **retracted**. Retry is unavoidable. +- **`400` means two opposite things and the resolver must read the message.** RunPod advertises GPU + types with **zero capacity**; it returns `500` to Atlas and Atlas maps it to `400` — the _same + status_ as Vast's stale offer. The messages differ: + `"Unknown SKU '' for provider 'vast'."` versus + `"create pod: There are no instances currently available"`. **They need opposite responses**: a + stale offer means re-resolve the same requirement against a fresh catalog; no capacity means pick a + **different** SKU. Retrying the same offer on a no-capacity error loops forever. A resolver that + branches on status alone is wrong on one of the two providers. + +**Why RunPod runs out of capacity is a query bug, not a stale catalog.** `list_options` asks for +`gpuTypes{ id, displayName, memoryInGb, lowestPrice{ uninterruptablePrice } }` — a **price list, not an +inventory**. `lowestPrice` is the cheapest anyone ever offered that model and says nothing about current +availability. Contrast Vast, whose `/bundles/?q={"rentable":{"eq":true},…}` catalog **is** +availability-filtered, which is why its options are at least launchable in principle. RunPod does publish +availability and we never request it: `lowestPrice{ stockStatus }` works, and across the 37 priced types +it reads Low 25 / Medium 3 / High 9 — with **all ten cheapest types "Low"**. Cheapest-first on RunPod +therefore selects systematically into the failure region. Cheapest "High" is $0.34/hr against cheapest +"Low" at $0.12, so **hard-filtering to High would roughly triple the price** — request the signal and use +it to order the fallback, do not filter on it. + ### Change 4 — quote a proposal without spending ``` @@ -605,7 +693,29 @@ closed: - **No warning event required.** An 80% notification is worth adding for humans, but it is advice; the cap must never depend on anyone reading it. -### Change 8 — release must not report success it did not achieve +### Change 8 — release must not report success it did not achieve — **HONESTY HALF SHIPPED; THE DECISION BELOW IS NOT** + +> **Shipped** as `0eb33c2` (Vast) and `be6a3d4` (Prime, RunPod, and `release_lease` itself). +> +> - All three providers now distinguish a confirmed teardown from an unconfirmed one instead of +> returning `{"status": "terminated"}` after a bare `try/except: pass`. **404 is deliberately +> excepted** — a pod that is not there is the outcome we wanted, and calling it unconfirmed would +> strand the lease retrying a `DELETE` that can never succeed. +> - `release_confirmed` classifies through `lease_state.normalise_state`, so the reaper, +> `/connection` and release all read one table. +> - `release_lease` now **acts** on the verdict: an unconfirmed teardown logs +> `metric=lease_release_unconfirmed` and **leaves the row unfinished** so the next sweep retries. +> Previously the row was marked released regardless, which is what made even an honest provider +> answer inert. +> +> **⚠️ The section's own decision is NOT implemented, and the gap is the one it predicted.** An +> unconfirmed release leaves the row at its previous status, and both +> `list_active_leases` (`status NOT IN ('released','failed') AND hourly_rate_cents > 0`) and +> `count_active_managed_gpu_leases` (same predicate) still match it. **So the user who asked to +> release keeps being charged and keeps burning one of their two managed slots** — verbatim the +> failure this section says to decide against. `release_pending` as a distinct status, stopping +> billing at the request, freeing the slot, and a retry set that is not `list_unfinished_leases` are +> all still to build. **Acceptance criterion 16 is therefore only one-third met.** `LeaseManager.release_lease` swallows a provider teardown failure into `provider_result = {"warning": …}` (`lease_manager.py:801-802`) and then **unconditionally** marks the row @@ -630,7 +740,32 @@ exposure is an operator problem, not theirs) and **free the concurrency slot**, sweepable for teardown retry. That means the retry set cannot be `list_unfinished_leases` — it needs its own query. -### Change 9 — SSH key lifecycle _(security prerequisite, ships before change 3)_ +### Change 9 — SSH key lifecycle _(security prerequisite, ships before change 3)_ — **SHIPPED** + +> **9(a) shipped** as `7aac22b` and is the one item in this document proven by a **direct adversarial +> test rather than an inference.** The section below could only say "confirm the per-instance attach +> is sufficient, then remove the account registration". Both halves were then measured: +> +> - **The per-instance attach alone works.** A real Vast GPU, reachable over SSH with the returned +> key, with the account key list at `GET /ssh/` returning `[]` before and after — verified as a +> genuine empty list, not an error masked as one. +> - **The cross-tenant property itself, by cross-login.** Two instances on the **same** Vast operator +> account: key A → box A connected, key B → box B connected (the controls), and key A → box B +> **refused, `Permission denied (publickey)`**, as was B → A. Pre-`7aac22b` that would have +> connected. The exposure is closed, and closed for the stated reason. +> +> **9(b) shipped** as `f65dd47` (Prime) — a nullable `provider_key_id` column on `compute_leases` +> (sqlite + Postgres, additive and guarded), the real key **id** returned from `acquire` instead of +> the pod name, forwarded into `release`, and cleanup inside `acquire` when a launch fails after +> registration. **Vast needs no key deletion any more**, because 9(a) removed the thing that was +> leaking. The migration was later verified against a populated pre-existing database (36 lease rows, +> 5 users), not only a fresh temp one. +> +> **Two caveats stay open.** Prime's `DELETE /ssh_keys/{id}` is **documentation-verified only, never +> exercised against production**; it fails safe (swallowed, cannot block the pod delete), so a wrong +> endpoint would leak silently. And the "acquired but the `create_lease` DB write failed" path calls +> `provider.release(lease_id)` with no kwargs, so neither `provider_key_id` nor Lambda's +> `ssh_key_name` reaches release there — pre-existing, not a regression. Two distinct problems. The first is a live cross-tenant exposure; the second is hygiene that becomes unbounded under cheapest-first. @@ -730,6 +865,30 @@ every offer it reads. `inet_down` matters more than it looks: pulling a 200 GB d of 5 Gbit is hours of GPU time billed for waiting. **But measured beats published** — vendor scores describe the host, our telemetry describes what actually happened to our leases. +> **Measured 2026-08-01 (n=7 Vast launches, one session) — a threshold is not enough; rank on the +> score.** +> +> | Selection | Launches | Never booted | +> | ------------------------------------- | -------- | ---------------------------- | +> | cheapest-first, no filter | 4 | 1 | +> | `reliability >= 0.98`, then cheapest | 2 | 1 | +> | **ranked by `reliability2` (0.9993)** | 1 | **0** — up in 50s, first try | +> +> A 0.98 **threshold did not help**: it admitted an offer that never booted. Ranking by the score +> did. n is tiny — treat it as a direction, not a coefficient — but it argues for the score entering +> the **ranking**, not just a floor, which is a stronger claim than this section currently makes. +> +> **This generalises past Vast: every provider publishes a quality signal and we discard all of +> them.** Vast: `reliability2`, `inet_down`, `dlperf_per_dphtotal`. RunPod: `lowestPrice{ stockStatus }` +> (see change 3 — all ten of its cheapest types are "Low"). The cheapest fix available today is to +> **request the signals we already could and use them to order the fallback**, which needs none of +> the boot-telemetry instrumentation below and would have prevented 3 of the 9 observed launch +> failures. That is a smaller, earlier change than the rest of this section and should be split out. +> +> **One open risk, recorded not fixed:** Vast `actual_status=offline` maps to `stopped` → terminal in +> our chain, so a healthy box reporting `offline` transiently would be reaped after two strikes. It +> is unknown whether healthy boxes ever do; it needs a long observation of a running instance. + ### Change 11 — pin the image, per provider **Nothing in this design says what is on the box**, and the answer today is inconsistent in a way that @@ -932,21 +1091,30 @@ are what actually bind. ## The bounds that remain -Verified against the Atlas checkout at HEAD `7b0e9b6`, source-read (not a running deploy). - -| Bound | Owner | Fires when | Today | -| -------------------- | -------------- | --------------------------------- | ------------------------------------------- | -| `hard_cap_cents` | billing tick | approved money is spent | **column only, does not enforce** — ch. 1 | -| Rolling window cap | lease creation | cumulative spend hits the ceiling | **does not exist** — ch. 5 | -| Wallet exhaustion | billing tick | money actually runs out | **works, but races** (`tick_once:152→:172`) | -| Plan TTL (24h) | billing sweep | anything has run absurdly long | **works** (`_release_stale_gpu_leases:303`) | -| Explicit release | agent/user | asked | **works only if teardown succeeds** — ch. 8 | -| Provisioning timeout | lease reaper | a lease never boots | **fires on every user lease** — ch. 0(a) | -| Heartbeat staleness | lease reaper | a booted lease stops reporting | **will fire once 0(a) lands** — ch. 0(b) | +**Updated 2026-08-01.** The July version of this table was source-read against Atlas HEAD `7b0e9b6`, +not a running deploy. Five of its seven rows have since changed, and the ones marked _deployed_ were +exercised against a live backend rather than inferred. + +| Bound | Owner | Fires when | Today | +| -------------------- | -------------- | --------------------------------- | --------------------------------------------------------------------------------------------------------------------------------- | +| `hard_cap_cents` | billing tick | approved money is spent | **enforces** — ch. 1 shipped. $10 @ $6.99/h releases at 5160s vs 5150s theoretical | +| Per-lease budget | lease creation | a caller states one | **accepted** — ch. 2 shipped. `budget_cents`, clamped to the wallet, effective cap reported | +| Rolling window cap | lease creation | cumulative spend hits the ceiling | **does not exist** — ch. 5. Release-and-reacquire is still unbounded | +| Wallet exhaustion | billing tick | money actually runs out | **works** _(deployed: wallet decremented in lockstep with `total_spent_cents`)_, **still races** — ch. 5 | +| Plan TTL (24h) | billing sweep | anything has run absurdly long | **works** (`_release_stale_gpu_leases:303`) | +| Explicit release | agent/user | asked | **honest now, but not acted on fully** — ch. 8 half shipped: an unconfirmed teardown keeps billing and keeps the concurrency slot | +| Provisioning timeout | lease reaper | a lease never boots | **no longer fires on live user leases** — ch. 0(a) shipped _(deployed: `ready` at 788s)_ | +| Heartbeat staleness | lease reaper | a booted lease stops reporting | **scoped to leases holding a runner token** — ch. 0(b) shipped | +| Agent-spawn ceiling | spawn path | a spawn's budget is exhausted | **was a flat $5 kill for every SKU; now sized to the spawn's own lifetime at the SKU rate** — `71dccba` | All server-side; none client-influenceable. Change 0 _removes_ two bounds from user leases, which is safe precisely because the others apply and necessary because they are bounds those leases cannot satisfy. +**The gap that is now the sharpest.** Every bound above is per-lease. With the cap binding and budgets +accepted, **change 5 is the only thing standing between "a run is bounded" and "a user is bounded"** — a +$30 budget honoured twenty times is still $600, and the wallet clamp still double-authorises under +concurrency. It was the least urgent item on this list in July and is the most urgent now. + `COMPUTE_BILLING_TICK_SECONDS` defaults to 60 (`compute_billing_service.py:44`) and `FIRST_BILL_GRACE_SECONDS` is 30 (`:52`), so a budget can overrun by up to ~90s of rate (~$0.17 on an H100). **Approved budgets are ceilings-plus-90-seconds and must never be described as exact.** @@ -1006,9 +1174,23 @@ creation stays in the workspace UI rather than becoming a fourth tool. GPU. `budget_cents` is now `int | None`; `None` sizes the ceiling to the spawn's own lifetime at the SKU rate, with a one-hour floor, and an explicitly chosen budget still binds. Only CPU spawns were ever safe, because they come out `funding=byok` at rate 0 and the tick skips them. -- **Stale skill names in agent prompts.** `research.txt:353` and `ml.txt:192` name skills by directory +- **Stale skill names in agent prompts.** `research.txt:357` and `ml.txt:193` name skills by directory rather than frontmatter `name` (`vllm` → `serving-llms-vllm`), so the agent is told it has skills it - cannot load. `skills/scholar-evaluation/SKILL.md` has no frontmatter at all. + cannot load. `skills/scholar-evaluation/SKILL.md` has no frontmatter at all. _(Still open at + 2026-08-01; the line numbers moved from `:353`/`:192` when this branch edited the prompts — + re-verified, the defect itself is unchanged.)_ +- **`compute_status` no longer names capabilities the client lacks.** Recorded because it is the same + defect class one layer up, found by the same live testing. At a **zero wallet**, + `/api/compute/options` still reports managed providers — availability and affordability are + independent there — so the tool resolved `mode=managed`, told the agent to run GPU work through + managed compute, **and forbade the BYOK fallback**, while every acquire would return `402`. Fixed on + `feat/compute-guardrails` (`784633e`, then `05ef093f`): the guidance now says the client cannot + launch a managed lease at all, and separately that an empty wallet would be refused. `state.mode` is + deliberately unchanged — managed is genuinely configured; missing funds and a missing tool are not a + missing capability. **Known boundary:** a $0.01 wallet still gets the funded wording and would also + `402`. Only `balance === 0` is gated, because acquire needs one hour of the chosen SKU's rate and + rates span cents to dollars in a catalog the tool never sees, so any non-zero cutoff is a guess. + Fixing it properly means threading the cheapest rate through `/api/compute/options` — spec work. ## Out of scope @@ -1086,6 +1268,14 @@ collapsed to one string. ## Acceptance criteria +> **Met as of 2026-08-01:** **0, 1, 2, 3, 4, 5, 9(a), 9(b), 15**, and criterion 25's ordering clause +> (change 0 was its own work; changes 1 and 2 landed together). **Partly met: 16** — a release whose +> teardown fails is no longer reported as clean and the row stays sweepable, but it does **not** stop +> billing and does **not** free the concurrency slot. **Not met:** 6, 7, 8, 10, 11, 12, 13, 14, 17–24 +> — every one of which needs either the resolver, the quote endpoint, or a tool this client does not +> have. Criteria are annotated here rather than deleted; a criterion that has passed is still the +> thing that would catch a regression. + 0. A user lease survives past both `PROVISION_TIMEOUT_SECONDS` and `HEARTBEAT_STALE_SECONDS`; a runner-token lease is still reaped for heartbeat staleness; the reconcile pass persists `ssh_host`/`ssh_port`. @@ -1165,10 +1355,28 @@ Labels in this document: build on. - **Not verified:** `compute:up`'s internal fetch→pick→estimate sequence. The Atlas CLI source is in neither repo; the commit chronology corroborates the shape but the sequence itself is inferred. +- **Verified on a deployed build of `feat/compute-lease-prerequisites` (2026-08-01):** the whole + managed path, end to end, on **both** Vast and RunPod — `POST /leases` → the real background reaper + promoting within one sweep with NATed SSH coordinates → SSH into a real GPU → release, with the + instance confirmed gone at the provider. Plus: `GET /connection` returning the normalised + `state: "ready"`; a user lease `ready` and un-reaped at 788s on a wall clock; a double release + returning `409`; a zero wallet returning the structured `402`; the billing tick decrementing the + wallet in lockstep with `total_spent_cents`; and the change-1 premise (`grant.spent_cents` frozen + at the acquire debit while the lease accrued) observed in production **before** the fix. The + additive migration was applied to a populated pre-existing database, not a fresh one. + **Not covered by this label:** the concurrency cap (`429`) was never reached — the Vast SKU churn + meant a second simultaneous lease always `400`d first — and the two-strike terminal timing could + not be separated from sweep jitter. Detection was confirmed; the strike count was not. A source-read claim is not a deployed-behaviour claim. Confirm the money path against `pytest` before relying on it. +**And a deploy is not a test either — it catches a third class.** The migration race, the `uvicorn` +`PATH` fault, RunPod's `desiredStatus=RUNNING`-with-no-address, and Vast's never-404 were all invisible +to both source-reading and `pytest`, and all four were found by running the thing. Two of the four were +**pre-existing patterns this work merely exposed**, which is the argument for deploying before merging +rather than after. + **The money path is now confirmed**, in both directions. Against a running deployment: the billing tick charges the wallet correctly (34¢/hr decrementing in lockstep with `total_spent_cents`) while `grant.spent_cents` sat frozen at the acquire debit — the defect. Against tests, after the fix: a $10 From 4fa36cf9eb9e0a7384d693c2c93fcbd20e00c978 Mon Sep 17 00:00:00 2001 From: KB Date: Sun, 2 Aug 2026 17:47:11 +0530 Subject: [PATCH 49/56] plan(compute): cache the catalog, then resolve server-side Changes 12 and 3, planned together because of one load-bearing interaction the spec does not cover: a cache cannot make offers fresher. Vast churns ~50% between consecutive fetches, so a cached catalog is half-stale within seconds exactly as an uncached one is by the time a caller acts. The cache is a cost fix; the resolver's retry must bypass it or it re-reads the same dead offers forever. Scoped by what live testing measured rather than what the spec assumed: both failure modes arrive as 400 and need opposite responses, and the stock/reliability signals are deliberately left out because the one heuristic we tried was falsified prospectively. --- docs/plans/2026-08-02-compute-resolver.md | 299 ++++++++++++++++++++++ 1 file changed, 299 insertions(+) create mode 100644 docs/plans/2026-08-02-compute-resolver.md diff --git a/docs/plans/2026-08-02-compute-resolver.md b/docs/plans/2026-08-02-compute-resolver.md new file mode 100644 index 00000000..65968aa8 --- /dev/null +++ b/docs/plans/2026-08-02-compute-resolver.md @@ -0,0 +1,299 @@ +# Compute catalog cache and server-side resolver — implementation plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development to implement +> this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Make a managed lease launch actually land. Today a caller picks a SKU from +`GET /api/compute/options` and posts it — and on Vast that fails roughly half the time, because offer ids +churn faster than a client can act on them. + +**Architecture:** All work is in the **Atlas** repo (`~/codes/InkVell/atlas`), continuing on +`feat/compute-lease-prerequisites`. Spec changes 12 (catalog cache) and 3 (server-side resolver). The +cache comes first because the resolver's retry multiplies catalog fetches. + +**Tech Stack:** Python 3.12, FastAPI, aiosqlite, pytest + pytest-asyncio, respx. + +**Spec:** `~/codes/InkVell/openscience/docs/specs/compute-design.md`, Part B changes 12 and 3. + +## Global Constraints + +- **Repo:** `~/codes/InkVell/atlas`, branch `feat/compute-lease-prerequisites` (already checked out). + Do not create or switch branches. +- **Run tests with:** `cd ~/codes/InkVell/atlas/backend && .venv/bin/python -m pytest -q`. + No activated virtualenv — always invoke `.venv/bin/python` explicitly. +- **Never add a `Co-Authored-By:` trailer or any AI/assistant attribution to commit messages.** +- **No mocks of our own code.** Fake providers at the HTTP boundary with `respx`, or register a + `_FakeProvider` into the provider registry — the pattern in `backend/tests/test_compute_billing.py` + and `backend/tests/test_compute_vast_provider_http.py`. +- **Every new assertion must be shown failing first.** Paste the failure into your report. +- Baseline: full suite is **1689 passed, 1 skipped**. It must be green when each task finishes. +- Ignore `.claude/worktrees/`. + +## Established facts — measured, do not re-derive + +Everything here was measured against production on 2026-08-01, not inferred: + +1. **Vast offer ids churn ~50% between two consecutive catalog fetches.** Three back-to-back + `GET /api/compute/options` calls returned 65 / 67 / 67 Vast offers with only **25 stable across all + three**; 31 of the first 65 were gone by the second fetch. +2. **Client-side selection therefore fails about half the time.** Seven consecutive + `400 Unknown SKU '' for provider 'vast'` across price ranks 0–25 — so it is not a + cheapest-first artefact, it is the whole catalog. A lease finally landed on the third random attempt. +3. **A tempting heuristic was falsified.** "Offers present in consecutive fetches are launchable" had + perfect separation at n=8 retrospectively and **failed its first prospective test**. Do not + reintroduce it. Retry is unavoidable. +4. **RunPod advertises GPU types with zero capacity.** The two cheapest both failed; the third + succeeded. RunPod returns `500 "create pod: There are no instances currently available"`, which Atlas + maps to a **`400`** — the same status as Vast's stale offer, with a different message. +5. **The two failures need opposite responses.** A stale Vast offer means "re-resolve the same + requirement"; RunPod no-capacity means "pick a different SKU". Retrying the same offer on + no-capacity loops forever. +6. **Providers publish quality signals we discard.** Vast returns `reliability2`, `inet_down` and + `dlperf_per_dphtotal` on every offer and `list_options` reads none of them. RunPod exposes + `lowestPrice { stockStatus }` and the query does not ask for it — across 37 priced types the + distribution is Low 25 / High 9 / Medium 3, and **all ten cheapest types are `Low`**. +7. **Ranking must use `price_cents_per_hour_display`, not `price_cents_per_hour`.** The latter is the + raw provider rate on both funding paths; the former is what the user pays and is `0` on BYOK. +8. `_catalog` is called at three sites — `/options`, `/estimate`, and `create_lease` — and each call + costs **five provider HTTP requests** (four wired providers; Vast issues two). +9. `create_lease` re-validates the posted SKU with `_find_option` and raises the `400 Unknown SKU`. That + is the race point. + +## The tension the cache creates, and how to resolve it + +A cache cannot make offers _fresher_. Given fact 1, a cached catalog is ~50% stale for Vast within +seconds, exactly as an uncached one is by the time a caller acts. So the cache is a **cost** fix, not a +correctness fix — and the resolver's retry must **bypass it**, or the retry re-reads the same dead +offers and can never succeed. + +That is the load-bearing interaction between changes 12 and 3, and it is why they are planned together. + +--- + +### Task 1: Cache the options catalog + +**Files:** + +- Modify: `backend/app/routes/compute.py` (`_catalog` and its three call sites) +- Test: `backend/tests/test_compute_catalog_cache.py` (new) + +**Interfaces:** + +- Produces: `_catalog(db, user_id, *, fresh: bool = False)` — same return shape as today + `(options, providers, byok_eligible)`. `fresh=True` bypasses and repopulates the cache. + +- [ ] **Step 1: Decide the cache key, and report before implementing** + +`_provider_catalog` takes `user_id` and `byok_eligible`, and calls `_byok_for(db, user_id, provider)` +— so for a user holding a BYOK key the offers are fetched **with that user's credentials** and the +`funding` annotation differs. A naive global cache would leak one user's catalog to another. + +Establish and report: whether the offer rows for a **managed-only** user (no BYOK key for that provider) +are user-independent. If they are, the common case can share one entry and only BYOK users need +per-user entries. **Report your finding before writing the key** — a wrong key here is a cross-user data +leak, not a performance bug. + +- [ ] **Step 2: Write the failing tests** + +Create `backend/tests/test_compute_catalog_cache.py`. Count provider calls with a registered fake +provider that increments a counter. Cover: + +1. two `_catalog` calls inside the TTL issue **one** round of provider calls +2. a call after the TTL expires re-fetches +3. `fresh=True` bypasses the cache even inside the TTL, and repopulates it +4. **two users with different BYOK eligibility never see each other's rows** — the key correctness test, + shaped by your Step 1 finding +5. a provider that raises does not poison the cache with an empty catalog that then serves for the whole + TTL + +- [ ] **Step 3: Run and confirm they fail** + +Run: `cd ~/codes/InkVell/atlas/backend && .venv/bin/python -m pytest tests/test_compute_catalog_cache.py -q` + +- [ ] **Step 4: Implement** + +An in-process TTL cache is sufficient — the reaper and billing loop already assume a single process per +worker, and a stale entry costs a retry rather than money. Do **not** reach for Redis. + +Pick the TTL deliberately and justify it in a comment against fact 1: a longer TTL saves requests but +cannot reduce staleness below the churn rate, so there is a point past which it only trades correctness +for nothing. State the number you chose and why. + +- [ ] **Step 5: Run, then the full suite** + +- [ ] **Step 6: Commit** + +--- + +### Task 2: A canonical GPU model map + +**Why:** the resolver takes a requirement (`gpu`, `count`) rather than an opaque SKU, so it needs to know +that Vast's `RTX 4090` and RunPod's `NVIDIA GeForce RTX 4090` are the same card — and that `H100 SXM`, +`H100 PCIe` and `H100 NVL` are **not** interchangeable. Interconnect is part of the model identity: they +differ in throughput and price. + +**Files:** + +- Create: `backend/app/compute/gpu_models.py` +- Test: `backend/tests/test_compute_gpu_models.py` (new) + +**Interfaces:** + +- Produces: `canonical(name: str, *, gpu_ram_gb: int | None = None) -> str | None` — maps a provider's + display name to a canonical model id, or `None` when it cannot be mapped confidently. + +- [ ] **Step 1: Write the failing tests** + +The taxonomy to support, which a comparable aggregator settled on independently: + +``` +A10 · A40 · A100-40GB-PCIe · A100-40GB-SXM · A100-80GB-PCIe · A100-80GB-SXM +H100-PCIe · H100-NVL · H100-SXM · H200-NVL · H200-SXM · B200 +L4 · L40 · L40S · RTX-3090 · RTX-4090 · RTX-5090 · RTX-6000-Ada +RTX-A6000 · RTX-PRO-6000 · RTX-PRO-6000-WK +``` + +Cover: + +1. the same card spelled differently across providers maps to one id — use **real strings** taken from + the provider modules, not invented ones +2. `H100-SXM`, `H100-PCIe` and `H100-NVL` are three distinct ids and never satisfy each other +3. an unmappable name returns `None` — **not** a guess. A mis-mapped card is the wrong machine at the + wrong price, silently +4. matching is not substring-based: a name containing `H100` as a substring of something else does not + map to an H100 + +- [ ] **Step 2: Run and confirm they fail** + +- [ ] **Step 3: Implement** + +Read the real name strings each provider emits before writing the map — `vast_provider.list_options` +(`gpu_name`), `runpod_provider.list_options` (`displayName`), and the Lambda and Prime equivalents. +Build from what they actually produce. + +- [ ] **Step 4: Run, then the full suite** + +- [ ] **Step 5: Commit** + +--- + +### Task 3: Resolve a requirement to an offer + +**Files:** + +- Create: `backend/app/compute/resolver.py` +- Test: `backend/tests/test_compute_resolver.py` (new) + +**Interfaces:** + +- Consumes: `canonical()` from Task 2; catalog rows as `_catalog` returns them. +- Produces: `resolve(options, *, gpu, count, max_hourly_cents=None) -> list[dict]` — the matching offers + in preference order, best first. A **list**, not one offer, because the caller retries down it. + +- [ ] **Step 1: Write the failing tests** + +1. picks the **globally cheapest** matching offer across providers — proven with a catalog where the + winner is neither the first provider listed nor the same provider twice. A resolver that always + returns one provider must fail this. +2. ranks on the **funding-adjusted** rate: a cheaper billed offer never beats a dearer BYOK one (fact 7) +3. honours `max_hourly_cents` +4. matches on the canonical model, so `H100-SXM` never returns an `H100-PCIe` +5. exercised at `count > 1`, not only `count = 1` +6. returns an ordered list with the cheapest first, so a caller can walk it +7. an empty result is a distinct, inspectable outcome — not an exception + +- [ ] **Step 2: Run and confirm they fail** + +- [ ] **Step 3: Implement** + +Rank by `price_cents_per_hour_display` ascending among offers whose canonical model and `count` match. +Keep it a pure function over catalog rows — no I/O, no DB. That is what makes it cheap to test +exhaustively. + +**Do not implement a stock or reliability filter in this task.** Facts 6 and 3 make it tempting; the +signals are real but unrequested by the provider modules, and the one heuristic we tried was falsified +prospectively. Ordering by price alone, with retry underneath, is the behaviour we have actually +measured working. Note the opportunity in a comment and leave it. + +- [ ] **Step 4: Run, then the full suite** + +- [ ] **Step 5: Commit** + +--- + +### Task 4: Accept a requirement on lease creation, and retry + +**Files:** + +- Modify: `backend/app/routes/compute.py` (`LeaseRequest`, `create_lease`) +- Test: `backend/tests/test_compute_resell_routes.py` (append) + +**Interfaces:** + +- Consumes: `resolve()` from Task 3, `_catalog(fresh=…)` from Task 1. +- Produces: `POST /api/compute/leases` accepts `{gpu, count, max_hourly_cents?}` in place of + `{provider, sku}`. Explicit `provider`/`sku` continues to work unchanged. + +- [ ] **Step 1: Write the failing tests** + +1. `{gpu, count}` resolves and leases without the caller naming a SKU +2. **explicit `provider`/`sku` still works exactly as today** — the dashboard and `compute:up` depend on + it, and it must not start requiring `gpu` +3. **a stale-offer `400` retries the next candidate and succeeds** — the Vast case, fact 2 +4. **a no-capacity failure moves to a different SKU rather than retrying the same one** — the RunPod + case, facts 4 and 5. Retrying the same offer must be provably not what happens. +5. retries are bounded, and exhausting them returns a structured error naming what was tried +6. the retry re-resolves against a **fresh** catalog, not the cached one — the load-bearing interaction + from the section above. Assert the provider was re-queried. +7. `budget_cents` still applies to a resolved lease exactly as to an explicit one + +- [ ] **Step 2: Run and confirm they fail** + +- [ ] **Step 3: Implement** + +Discriminate the two failures **on the provider's message**, since both arrive as `400`. The exact +strings, captured live: + +- Vast stale offer — `Unknown SKU '' for provider 'vast'.` +- RunPod no capacity — `create pod: There are no instances currently available` + +Match defensively: these are provider prose and can change. An unrecognised `400` should behave like the +safer of the two — advance to the next candidate rather than retrying the same one, since retrying a +genuinely dead SKU cannot succeed while advancing merely costs one attempt. + +- [ ] **Step 4: Run, then the full suite** + +- [ ] **Step 5: Commit** + +--- + +## Whole-branch verification + +- [ ] Full suite green, no network. +- [ ] Confirm by name that the earlier deliverables still pass: + `test_promoted_lease_is_not_reaped_at_the_provisioning_timeout`, + `test_lease_without_a_budget_is_still_sized_to_the_full_plan_ttl`, + `test_out_of_credit_release_of_a_managed_lease_uses_the_operator_key`, + and the budget headline test. +- [ ] Confirm explicit `provider`/`sku` leases are unchanged — the dashboard's path. + +## Acceptance criteria + +1. Two `_catalog` calls inside the TTL cost one round of provider requests; `fresh=True` bypasses. +2. Users with different BYOK eligibility never share a cache entry. +3. `canonical()` maps real provider strings to one id, keeps the three H100 variants distinct, and + returns `None` rather than guessing. +4. `resolve()` returns the globally cheapest match on the funding-adjusted rate, ordered, exercised at + `count > 1`. +5. `{gpu, count}` leases without a caller-supplied SKU; explicit `provider`/`sku` is unchanged. +6. A stale-offer `400` retries the next candidate against a **fresh** catalog and succeeds. +7. A no-capacity failure advances to a different SKU rather than retrying the same one. +8. Retries are bounded and exhaustion returns a structured error. +9. `pytest` passes with no network access. + +## Out of scope + +The quote endpoint (change 4) — it consumes this resolver and is the next plan. Stock and reliability +filtering (change 10) — the signals are unrequested and the one heuristic tried was falsified; it needs +its own evidence. Volumes (6), budget extension (7), the rolling window cap (5), and the three +OpenScience tools. From d34a36fc33c298d8d954a045d1467a16b458d8f7 Mon Sep 17 00:00:00 2001 From: KB Date: Sun, 2 Aug 2026 21:10:16 +0530 Subject: [PATCH 50/56] spec(compute): change 3 ships, and record where the code overrules the plan The resolver reads no provider error message: it excludes already-refused (provider, sku) pairs and re-resolves against a fresh catalog, which gets Vast and RunPod right with one rule and cannot rot when a provider rewrites its prose. The plan prescribed message matching; the code is right. Ranking on the display rate alone was degenerate within BYOK, where every row displays 0 -- the order collapsed to alphabetical and leased a $9.00/h box over a $3.00/h one. Key is now (display, raw, provider, sku). Also records what the canonical GPU map does not cover: 37% of live Vast rows and 44% of RunPod's, missing B300, MI300X, GH200 and the workstation line, with Prime Intellect unmappable only because list_options drops the offer's socket. --- docs/plans/2026-08-02-compute-resolver.md | 26 +++++++++++++ docs/specs/compute-design.md | 47 ++++++++++++++++++++++- 2 files changed, 72 insertions(+), 1 deletion(-) diff --git a/docs/plans/2026-08-02-compute-resolver.md b/docs/plans/2026-08-02-compute-resolver.md index 65968aa8..22938ac2 100644 --- a/docs/plans/2026-08-02-compute-resolver.md +++ b/docs/plans/2026-08-02-compute-resolver.md @@ -1,5 +1,31 @@ # Compute catalog cache and server-side resolver — implementation plan +> **EXECUTED 2026-08-02** on `feat/compute-lease-prerequisites`. All four tasks shipped and reviewed; +> full suite **1908 passed / 1 skipped**, no network. Commits: `8913be3` (cache), `f9c2da9` (GPU map), +> `9e1664d` (resolver), `d888ba5` + `730ee82` (requirement path + retry), `cf9e7de` + `416c064` + +> `bed63dc` + `fa429c6` (final-review fixes). +> +> **Two things this plan got wrong, both caught only by the whole-branch review:** +> +> 1. **Task 3's tie-break became a purchasing decision in Task 4.** Ranking on +> `price_cents_per_hour_display` alone is correct across funding paths but degenerate *within* BYOK, +> where every row displays `0` — the order collapsed to alphabetical by provider and leased a $9.00/h +> box over a $3.00/h one. Neither task was wrong alone. The key is now +> `(display, raw, provider, sku)`. +> 2. **Task 4 charged a retry attempt for a candidate no provider ever saw.** A region-less Lambda offer +> is rejected locally, but it consumed one of three attempts plus a full catalog fan-out — and Lambda +> emits region-less rows precisely for the types it is out of capacity on, which are also its +> cheapest, so they sort first. Three of them ahead of a launchable offer returned 503 `no_capacity` +> while capacity sat in the list. +> +> **And one thing it prescribed that the implementation was right to refuse:** Task 4 Step 3 says to +> discriminate the two `400`s on the provider's message. The implementation excludes already-refused +> `(provider, sku)` pairs instead — one rule that satisfies both providers and cannot rot when a +> provider rewrites its error prose. See `docs/specs/compute-design.md`, change 3. +> +> Deferred Minor findings, triaged by the final review, are in the ledger at +> `.superpowers/sdd/2026-08-02-compute-resolver/progress.md` (gitignored). + > **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development to implement > this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. diff --git a/docs/specs/compute-design.md b/docs/specs/compute-design.md index d6f25165..3c99e199 100644 --- a/docs/specs/compute-design.md +++ b/docs/specs/compute-design.md @@ -516,12 +516,33 @@ commits, which is right for attribution — but shipping change 1 by itself _is_ regression to every existing caller, because the default-grant fix lives here in change 2. **Changes 1 and 2 land together**, in that order, with criterion 4 gating the pair. Change 0 remains its own commit. -### Change 3 — resolve the cheapest SKU from requirements _(mandatory)_ +### Change 3 — resolve the cheapest SKU from requirements _(mandatory)_ — **SHIPPED** Accept `{gpu, count, max_hourly_cents?}` in place of an explicit `sku`; rank **every** operator provider's options by `price_cents_per_hour_display` — the funding-adjusted rate, not the raw one; lease the cheapest match in the same request. Explicit `provider`/`sku` continues to work for the dashboard and the CLI. +> **Shipped 2026-08-02** on `feat/compute-lease-prerequisites`, built as +> `docs/plans/2026-08-02-compute-resolver.md` (four tasks: cache, GPU map, resolver, retry). Suite +> 1908 passed / 1 skipped, no network. Three things below turned out differently in the building, and +> the code is right where it disagrees with the prose: +> +> - **The retry does not read the provider's message**, though the section below says it must. It +> excludes every `(provider, sku)` already refused and re-resolves against a fresh catalog. That +> gets both providers right with one rule: RunPod's stable type id is excluded, so a no-capacity +> type is never retried, and Vast's requirement is genuinely re-resolved against ids that did not +> exist a moment ago. It also cannot rot when a provider rewrites its error prose — which the +> message-matching design would have depended on. **Do not "restore" the message matching.** +> - **Ranking on the display rate alone was wrong for BYOK.** Every BYOK row displays `0`, so the +> ranking collapsed past its first key to alphabetical order by provider — leasing a $9.00/h box +> over a $3.00/h one and reporting nothing, because on BYOK Atlas bills neither. The rank key is +> now `(display, raw, provider, sku)`. Ordering on the managed path is unchanged, since display +> tracks raw there. `max_hourly_cents` still bounds the display price, and therefore **still +> excludes nothing on BYOK** — the user's own provider bill is not a number Atlas can cap. +> - **Ranking is cheapest outright, not cheapest above a reliability floor.** The floor belongs to +> change 10, which has not shipped. The signals are real and were deliberately left out: the one +> stability heuristic this project tried was falsified prospectively (below). + **`count > 1` is thinner than the catalog size suggests.** RunPod hardcodes `"gpu": 1` in its options and `"gpuCount": 1` on acquire (`runpod_provider.py:124`, `:156`), Vast dedups to one row per `(gpu_name, count)` (`:142-156`), and Prime to one per `(gpuType, upstream)` @@ -550,6 +571,26 @@ mis-mapped card is a wrong machine at the wrong price, silently. This taxonomy is the one a comparable aggregator settled on, which is a reasonable signal that it is the right granularity rather than over-specification. +> **Shipped as `backend/app/compute/gpu_models.py`** — `canonical(name, *, gpu_ram_gb=None)`, a table of +> literal strings, never a matcher. Every key was dumped from a live provider API on 2026-08-02, not +> imagined. `RTX 6000 Ada`, `RTX A6000`, `RTX PRO 6000` and `RTX PRO 6000 WK` are four different cards +> whose names contain each other and RunPod sells all four; `GH200 SXM` contains `H200`. Any substring or +> prefix rule ranks a Grace Hopper superchip as an H200. +> +> **The taxonomy is narrower than what our providers actually sell — this is the open decision.** It +> covers **37% of live Vast rows and 44% of RunPod's**. That is mostly the long tail of consumer cards +> nobody would request, and unmapped rows are dropped rather than mis-ranked, so nothing is priced wrong. +> But these are missing and priced today: **B300** (the top of RunPod's catalog), **MI300X**, +> `RTX PRO 6000 MaxQ`, **GH200**, and the Ada/Ampere workstation line. A caller cannot reach any of them +> through `{gpu, count}`; the explicit `provider`/`sku` path still can. +> +> **Prime Intellect is wholly unmappable, for a one-line reason.** Its `gpuType` carries memory but not +> interconnect (`A100_40GB`), so its rows cannot be placed among the four A100 / three H100 ids. The offer +> *does* carry a `socket` field (`PCIe` / `SXM4`) and `acquire` already forwards it — `list_options` just +> never copies it onto the row. Surfacing it unlocks the whole provider. +> +> Widening is additive and needs no resolver change. + **The retry is not optional here.** Vast supplies most of the catalog and its SKUs are ephemeral offer IDs, so the cheapest pick is usually the raciest one. On a provider `400`, re-resolve against a re-fetched catalog, bounded to N attempts, then fail with a structured error rather than looping. @@ -583,6 +624,10 @@ the resolver, not colour. **different** SKU. Retrying the same offer on a no-capacity error loops forever. A resolver that branches on status alone is wrong on one of the two providers. + _As shipped, the resolver reads neither message._ Excluding every already-refused `(provider, sku)` + and re-resolving against a fresh catalog satisfies both requirements at once, without depending on + provider prose. The measurement above stands; only the prescription changed. + **Why RunPod runs out of capacity is a query bug, not a stale catalog.** `list_options` asks for `gpuTypes{ id, displayName, memoryInGb, lowestPrice{ uninterruptablePrice } }` — a **price list, not an inventory**. `lowestPrice` is the cheapest anyone ever offered that model and says nothing about current From 8f27fd77e1c04f59880ffd26e1ca24b1a50353a5 Mon Sep 17 00:00:00 2001 From: KB Date: Sun, 2 Aug 2026 23:06:49 +0530 Subject: [PATCH 51/56] spec(compute): record what the resolver does against real providers Three real leases through the real route. The path works end to end: resolved in 5.1s, ready with SSH coordinates at t+30s, /connection 200, released and confirmed, one grant, nothing left running. Two findings that change what we can claim. Vast caps every query at 64 offers and ignores limit entirely, so 'cheapest' is cheapest of a narrow window rather than of Vast. And the 201 names the offer id but never the GPU, so a caller that asked for an RTX-3090 cannot tell from the response that it got one. Also records a theory that was tested and refuted: the churn is NOT mostly an artifact of our cheapest-per-(gpu_name, count) dedup. 35 of 42 dropped rows were gone from Vast's raw response too. The retry's premise stands. --- docs/specs/compute-design.md | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/docs/specs/compute-design.md b/docs/specs/compute-design.md index 3c99e199..ec7f4008 100644 --- a/docs/specs/compute-design.md +++ b/docs/specs/compute-design.md @@ -628,6 +628,36 @@ the resolver, not colour. and re-resolving against a fresh catalog satisfies both requirements at once, without depending on provider prose. The measurement above stands; only the prescription changed. +#### Verified live 2026-08-02 — the resolver against real Vast and RunPod + +Three real leases through the real route, real provider keys, nothing faked below the HTTP boundary. + +- **The whole path works.** `{gpu: "RTX-3090", count: 1}` → resolved → leased in **5.1s** → `provisioning` + → `ready` with `ssh4.vast.ai` at **t+30s** via the reaper's promotion → `/connection` 200 with real + coordinates → released, `status: terminated` confirmed. Exactly one grant, no orphans, and zero + instances left under the operator key afterwards. +- **The ranking is right on live data.** Across the 24 canonical models present in a live catalog, every + `resolve()` result was correctly ordered, with winners split across both providers. + `max_hourly_cents` is inclusive at the boundary and excludes one cent below it. +- **Live coverage: 37% of Vast rows, 52% of RunPod's** — and the unmapped remainder is exactly the tail + the map is right to drop: `GTX 1060`, `Tesla P4`, `Titan Xp`, `Quadro P4000`, `RTX 3060 laptop`. +- **`limit: 512` is a no-op — Vast caps every query at 64 offers.** Verified at limits 64 through 2000 and + with the key absent: always 64. So the catalog is 64 cheapest + 64 premium-name offers, deduped to ~65 + rows. **"Cheapest" therefore means cheapest of a narrow window, not of Vast**; for a mid-tier card + there may be cheaper instances outside both windows. Pagination is the fix, and it is not built. +- **Churn re-measured on raw offers: 36% survive 40 seconds** (128 → 128 offers, 47 stable), with deduped + rows tracking it at 35%. **Real offer death dominates.** A theory that the ~50% figure was mostly an + artifact of Atlas's cheapest-per-`(gpu_name, count)` dedup was tested and **refuted**: of 42 rows that + left the catalog, 35 were gone from Vast's raw response too. At most ~16% are dedup drops where the + offer is alive and merely out-ranked — an upper bound, since the two fetches were not simultaneous. + **The retry's premise stands.** +- **The retry did not fire in any of the three launches.** First pick succeeded every time, including + against a deliberately staled cache. Not a disproof — it means first-pick success is common — but the + retry path is still unexercised against a real provider refusal. +- **The 201 response never names the GPU.** It carries `provider` and `requested_sku` (an opaque Vast + offer id like `42093969`), but no model. A caller that asked for an `RTX-3090` cannot confirm from the + response that it got one. This blocks the OpenScience `compute_launch` tool from reporting honestly. + **Why RunPod runs out of capacity is a query bug, not a stale catalog.** `list_options` asks for `gpuTypes{ id, displayName, memoryInGb, lowestPrice{ uninterruptablePrice } }` — a **price list, not an inventory**. `lowestPrice` is the cheapest anyone ever offered that model and says nothing about current From c07864713c5072748bdde8f5658166610f4a826d Mon Sep 17 00:00:00 2001 From: KB Date: Mon, 3 Aug 2026 00:00:20 +0530 Subject: [PATCH 52/56] spec(compute): the lease now says what GPU it is, and the retry fired live A lease carries gpu_model (canonical id, NULL rather than a guess), gpu_name (the provider's own string) and gpu_count, on all three launch paths. Verified against real Vast hardware. Also records the retry firing in production conditions for the first time: RunPod's no-capacity refusal, the offer excluded, a fresh re-resolve, and an honest 503 naming what was tried. --- docs/specs/compute-design.md | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/docs/specs/compute-design.md b/docs/specs/compute-design.md index ec7f4008..fab16bd7 100644 --- a/docs/specs/compute-design.md +++ b/docs/specs/compute-design.md @@ -654,9 +654,19 @@ Three real leases through the real route, real provider keys, nothing faked belo - **The retry did not fire in any of the three launches.** First pick succeeded every time, including against a deliberately staled cache. Not a disproof — it means first-pick success is common — but the retry path is still unexercised against a real provider refusal. -- **The 201 response never names the GPU.** It carries `provider` and `requested_sku` (an opaque Vast - offer id like `42093969`), but no model. A caller that asked for an `RTX-3090` cannot confirm from the - response that it got one. This blocks the OpenScience `compute_launch` tool from reporting honestly. +- **The 201 response never named the GPU — since fixed.** It carried `provider` and `requested_sku` (an + opaque Vast offer id like `42093969`) but no model, so a caller that asked for an `RTX-3090` could not + confirm from the response that it got one, and `GET /api/compute/leases` had the same gap for every + lease. `compute_leases` now carries `gpu_model` (the canonical id, **NULL when `canonical()` cannot + place the row — never a guess**), `gpu_name` (the provider's own string, always present, which is what + keeps a NULL model row useful) and `gpu_count`. Populated on the named path, the requirement path and + the agent-spawn path. Verified live: `{gpu: "RTX-3090", count: 1}` returns + `gpu_model='RTX-3090' gpu_name='RTX 3090' gpu_count=1`. +- **The retry fired live**, on a later run: `{gpu: "RTX-4090", count: 1}` drew RunPod's + `create pod: There are no instances currently available` — the verbatim string above. The loop excluded + that `(provider, sku)`, re-resolved against a fresh catalog, found no other `RTX-4090`, and returned a + structured `503` naming what it tried. No grant leaked. **Both halves of fact 4 are now observed in + production conditions, not just in tests.** **Why RunPod runs out of capacity is a query bug, not a stale catalog.** `list_options` asks for `gpuTypes{ id, displayName, memoryInGb, lowestPrice{ uninterruptablePrice } }` — a **price list, not an From 05016e8ebbd98b8fa21d85da0bde75d80b262755 Mon Sep 17 00:00:00 2001 From: KB Date: Mon, 3 Aug 2026 08:56:55 +0530 Subject: [PATCH 53/56] plan(compute): the five things that must be true before OpenScience gets tools Four defects found by live probing plus one decision already taken. The rate limiter meters reads from the launch budget, so an agent rate-limits itself out on a single launch-and-wait. A Vast 429 empties the catalog rather than degrading. Prime Intellect is unreachable for a one-line reason. An unconfirmed teardown keeps billing the user, and both docstrings claim a reaper retry that does not exist. Also records that Vast has no pagination -- offset and from both 400 -- so widening past the 64-offer window is a design change, not a parameter. --- docs/plans/2026-08-03-compute-preflight.md | 319 +++++++++++++++++++++ 1 file changed, 319 insertions(+) create mode 100644 docs/plans/2026-08-03-compute-preflight.md diff --git a/docs/plans/2026-08-03-compute-preflight.md b/docs/plans/2026-08-03-compute-preflight.md new file mode 100644 index 00000000..5354b075 --- /dev/null +++ b/docs/plans/2026-08-03-compute-preflight.md @@ -0,0 +1,319 @@ +# Compute pre-flight: what must be true before OpenScience gets compute tools + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development to implement +> this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Close the five things standing between a working Atlas compute API and three agent-facing +OpenScience tools. Four are defects found by live probing; one is a decision already taken but unbuilt. + +**Architecture:** All work is in **Atlas** (`~/codes/InkVell/atlas`), continuing on +`feat/compute-lease-prerequisites`. No OpenScience changes here — the tools come after. + +**Tech Stack:** Python 3.12, FastAPI, aiosqlite, pytest + pytest-asyncio, respx. + +**Spec:** `~/codes/InkVell/openscience/docs/specs/compute-design.md`. + +## Global Constraints + +- **Repo:** `~/codes/InkVell/atlas`, branch `feat/compute-lease-prerequisites` (already checked out). + Do not create or switch branches. +- **Run tests with:** `cd ~/codes/InkVell/atlas/backend && .venv/bin/python -m pytest -q`. + No activated virtualenv — always invoke `.venv/bin/python` explicitly. +- **Never add a `Co-Authored-By:` trailer or any AI/assistant attribution to commit messages.** +- **No mocks of our own code.** Fake providers at the HTTP boundary with `respx`, or register a fake + into the provider registry — the pattern in `backend/tests/test_compute_resell_routes.py`. +- **Every new assertion must be shown failing first.** Paste the RED output into your report. +- Baseline: full suite is **1913 passed, 1 skipped**. It must be green when each task finishes. +- Ignore `.claude/worktrees/`. + +## Measured facts — established live on 2026-08-02/03, do not re-derive + +1. **Atlas meters reads and launches from one bucket.** `POST /leases`, `GET /leases`, + `GET /leases/{id}/connection` and `POST /leases/{id}/release` all classify as `compute_acquire`, + 20/min. Measured: 429 at request 21. `GET /options` is `default`, 600/min. +2. **The bucket is per bearer token, per process.** Key is `auth:{sha256(token)[:24]}`; the store is + `InMemoryStore` because neither Fly app sets `REDIS_URL`; production runs `--workers 2` + (`backend/fly.toml:69`). So the effective ceiling is 20–40/min depending on which worker serves. +3. **A launch returns `provisioning` with no SSH coordinates.** Measured `ready` at t+20–30s, via the + reaper's promotion. So any client must poll — which is why fact 1 bites. +4. **Vast rate-limits `/bundles/` at ~1 request/second**, advertising `x-ratelimit-limit: 1.0`, + `x-ratelimit-remaining`, `x-ratelimit-reset` ≈1s out; recovers after 1s idle. Per Vast's docs the + identity is bearer token + session user + api_key param + **client IP**, enforced as a minimum + interval **per endpoint** — so this is shared by every managed user of a deployment and does not + improve when we add machines. +5. **Vast returns exactly 64 offers per query and ignores `limit`** (tested 64…2000 and with the key + absent). **There is no pagination**: `offset` and `from` both return `400`. More inventory is only + reachable through more *filtered* queries, each costing one request against fact 4. +6. **RunPod has no observable limit** — 60 requests at 2.2/s, no 429, no rate-limit headers. +7. **One `list_options` is 2 concurrent GETs** and completes healthily in ~2.1s; three back-to-back + fan-outs (the retry pattern) also all healthy. Today's pattern fits inside fact 4; concurrent + *retrying* launches are what would not. + +--- + +### Task 1: Reads must not spend the launch budget + +**Files:** +- Modify: `backend/app/middleware/rate_limit.py` +- Test: `backend/tests/test_rate_limit_compute.py` (new) + +**Interfaces:** +- Produces: safe methods (`GET`/`HEAD`/`OPTIONS`) on `compute_acquire` paths classify as `default`. + `POST`/`DELETE` on those paths keep `compute_acquire`. + +Two defects, one file. + +**(a) The read/write split.** `_classify` already lets safe methods fall through to the generous +`default` class, but only for classes whose *name contains the substring* `"mutation"` +(`rate_limit.py:203`). That is why `compute_acquire` was missed. Replace the name test with an explicit +field on `_BucketClass` — a name is not a policy — and set it on `compute_acquire`, `mutations` and +`atlas_graph_mutations` so behaviour for the existing two is unchanged. + +Do **not** raise `compute_acquire`'s 20/min. The launch side should stay guarded; it is the reads that +were never meant to be in there. + +**(b) The store is captured too early.** `RateLimitMiddleware.__init__` does +`self._store = get_kv_store()` (`rate_limit.py:225`). `reset_kv_store()` swaps the module-level store, +so the middleware keeps a stale reference and these buckets never reset between tests. This has already +produced a **false pass** in a RED run — a test asserting a 429 received the limiter's 429. Resolve the +store per request (or per call) so a test reset is honoured. Keep it cheap: `get_kv_store()` is memoized. + +- [ ] **Step 1: Write the failing tests** + +1. `GET /api/compute/leases` 30 times on one token stays 200 — it must not consume launch budget. + Fails today at request 21. +2. `POST /api/compute/leases` is still limited at 20/min on one token. +3. Reads and writes do not share a bucket: exhaust the write bucket, then a read still succeeds. +4. `mutations` and `atlas_graph_mutations` keep their existing safe-method fall-through — a regression + guard on the behaviour being generalised. +5. After `reset_kv_store()`, a previously exhausted bucket is clean. **This is (b), and it must fail + first** — if it passes before the change, the test is not testing the defect. + +- [ ] **Step 2: Run and confirm they fail** +- [ ] **Step 3: Implement** +- [ ] **Step 4: Run, then the full suite** +- [ ] **Step 5: Commit** + +--- + +### Task 2: A rate-limited Vast must degrade, not vanish + +**Files:** +- Modify: `backend/app/compute/vast_provider.py`, `backend/app/routes/compute.py` +- Test: `backend/tests/test_compute_vast_provider_http.py` (append), + `backend/tests/test_compute_catalog_cache.py` (append) + +**Interfaces:** +- Produces: a Vast `429` is distinguishable from "Vast has no offers", and does not silently empty the + catalog. + +`list_options` calls `cheap_resp.raise_for_status()`. On a `429` that raises, `_provider_catalog` catches +`Exception` and returns `base` — a block with `options: []`. So a rate-limited fetch is indistinguishable +from Vast having nothing, and Vast is most of the cheap inventory. The caller is told "no listed offer +matches" when the truth is "ask again in one second". + +Given fact 4 this is not hypothetical under concurrency: `fresh=True` retries bypass the cache, and two +concurrent blocked launches exceed Vast's budget. + +Required behaviour: + +- A `429` from the cheap query must be reported as a rate-limit condition, not an empty catalog. +- `_provider_catalog` must **not** replace a usable cached entry with an empty one because of a + transient `429`. Serving slightly stale rows beats serving none — and note the existing comment at + `routes/compute.py:222` already argues the empty result must not be *cached*; this extends that + reasoning to not discarding what we already have. +- The `no_matching_offer` / `no_capacity` error a caller finally sees must not claim nothing matched + when a provider was rate-limited. Say which provider was unavailable. +- Respect `x-ratelimit-reset` where present rather than inventing a backoff. + +**Do not add a retry loop inside `list_options`.** The caller already has one, and Vast's budget is +shared deployment-wide — a provider-level retry multiplies load exactly when the system is busiest. + +- [ ] **Step 1: Write the failing tests** (`respx`, no live calls) + +1. A `429` on the cheap query does not produce a silently empty Vast block. +2. A `429` does not evict or overwrite a healthy cached entry. +3. A genuinely empty Vast catalog (200, no offers) is still reported as empty — the two must stay + distinguishable. +4. A `429` on the *premium* query alone still yields the cheap rows (it is already additive). +5. The user-facing error names the rate-limited provider rather than claiming no match. + +- [ ] **Step 2: Run and confirm they fail** +- [ ] **Step 3: Implement** +- [ ] **Step 4: Run, then the full suite** +- [ ] **Step 5: Commit** + +--- + +### Task 3: Unlock Prime Intellect, and widen the canonical map + +**Files:** +- Modify: `backend/app/compute/prime_intellect_provider.py`, `backend/app/compute/gpu_models.py` +- Test: `backend/tests/test_compute_prime_provider_http.py` (append), + `backend/tests/test_compute_gpu_models.py` (append) + +**Interfaces:** +- Consumes: `canonical(name, *, gpu_ram_gb=None)`. +- Produces: Prime rows carry enough to be canonicalised; the map covers the datacenter cards our + providers actually sell. + +**(a) Prime Intellect.** Its `gpuType` carries memory but not interconnect (`A100_40GB`), so its A100 and +H100 rows cannot be placed among the four A100 / three H100 ids and are dropped entirely. The offer +*does* carry a `socket` field (`PCIe` / `SXM4`) and `acquire` already forwards it — `list_options` +simply never copies it onto the row (`prime_intellect_provider.py:174`). Surface it, and make +`canonical()` able to use it. + +**(b) The map is narrower than what is on sale.** Verified against live catalogs: **37% of Vast rows and +52% of RunPod's** are mapped. Most of the remainder is consumer tail that is correctly dropped +(`GTX 1060`, `Tesla P4`, `Titan Xp`, `Quadro P4000`, `RTX 3060 laptop`) — **leave those unmapped.** But +these are priced today and unreachable through `{gpu, count}`: **B300**, **MI300X**, **GH200**, +`RTX PRO 6000 MaxQ`, and the Ada/Ampere workstation line. + +**The exact-match rule is not negotiable.** `RTX 6000 Ada`, `RTX A6000`, `RTX PRO 6000`, +`RTX PRO 6000 WK` and `RTX PRO 6000 MaxQ` are five different cards whose names contain each other, and +`GH200 SXM` contains `H200`. Every added id is a literal-string table entry plus a canonical id. Adding +`GH200` while a substring rule exists anywhere would rank a Grace Hopper superchip as an H200. + +**Do not invent provider strings.** `VAST_API_KEY` and `RUNPOD_API_KEY` are live in `backend/.env`; dump +the real catalogs (read-only `GET`s, the same ones `list_options` issues — **provision nothing**) and +map what you actually observe. Vast rate-limits `/bundles/` at ~1/s (fact 4), so sleep between fetches. +Record in your report which strings came from a live dump and which from a repo fixture. + +- [ ] **Step 1: Write the failing tests** + +1. A Prime offer with `socket: "SXM4"` and `gpuType: "A100_40GB"` canonicalises to `A100-40GB-SXM`; + with `socket: "PCIe"`, to `A100-40GB-PCIe`. +2. A Prime offer with no `socket` still yields `None` — never a guess. +3. Each newly added card maps from its real provider spelling(s). +4. **`GH200 SXM` is not `H200-*`, and `RTX PRO 6000 MaxQ` is not `RTX-PRO-6000` or `-WK`.** Extend the + existing `_UNMAPPABLE`/adversarial cases rather than adding a separate test. +5. Live coverage rises for the datacenter tier; the consumer tail stays unmapped. + +- [ ] **Step 2: Run and confirm they fail** +- [ ] **Step 3: Implement** +- [ ] **Step 4: Run, then the full suite** +- [ ] **Step 5: Commit** + +--- + +### Task 4: A release that did not happen must not bill the user + +**Files:** +- Modify: `backend/app/compute/lease_manager.py`, `backend/app/services/compute_billing_service.py`, + `backend/app/jobs/lease_reaper.py`, `backend/db` migration for the new column +- Test: `backend/tests/test_lease_reaper_seam.py` (extend), `backend/tests/test_compute_billing.py` + +**Interfaces:** +- Produces: an unconfirmed teardown stops billing and frees the concurrency slot, while remaining + visible to a retry. + +**The decision, already taken by the product owner:** stop billing the user, and flag the lease for +reconciliation so the operator's box is chased separately. The user must never pay for our failure to +tear down. + +Today `release_lease` returns early when `release_confirmed()` is false, leaving `status` untouched. Every +billing and concurrency gate is `status NOT IN ('released','failed')`, so the user keeps being billed and +keeps holding a slot. + +**Both docstrings claim the reaper retries this. It does not.** All four reaper branches were walked: +branch 0 needs terminal telemetry a user lease never emits; branch 1 finds the box `ready` precisely +because the teardown failed; branch 2 only fires on `provisioning`; branch 3 is gated on +`runner_api_key_id`, which only the agent-spawn path sets. **Fix the comments as part of this task** — +they are the reason this looked safe. + +Required behaviour: + +- An unconfirmed teardown marks the lease terminal for billing and concurrency purposes. +- It stays discoverable for a retry — a distinct state or flag, not silently released. Historical rows + must be unaffected. +- Something must actually retry it. If that is the reaper, add the branch and prove it fires for a + **user** lease with no `runner_api_key_id`. +- `CredentialUnavailable` (a deleted BYOK key, a rotated operator key) is the case with no recovery + today; it must not be conflated with a provider that answered and refused. + +- [ ] **Step 1: Write the failing tests** + +1. A managed lease whose provider release is unconfirmed stops accruing charges. +2. …and frees the concurrency slot, so a subsequent launch is not 429'd by a box we failed to kill. +3. …and is still visible to whatever retries it. +4. The retry actually fires for a user lease (no `runner_api_key_id`) — extend + `test_lease_reaper_seam.py`, which sweeps every lease class in one pass. +5. A confirmed release is unchanged. +6. A historical row predating the new column serialises and reaps unchanged. + +- [ ] **Step 2: Run and confirm they fail** +- [ ] **Step 3: Implement** +- [ ] **Step 4: Run, then the full suite** +- [ ] **Step 5: Commit** + +--- + +### Task 5: See more than the cheapest 64 offers + +**Files:** +- Modify: `backend/app/compute/vast_provider.py` +- Test: `backend/tests/test_compute_vast_provider_http.py` (append) + +**Interfaces:** +- Produces: a requirement for a specific GPU reaches offers outside the global cheapest-64 window. + +Atlas asks Vast for `limit: 512` and gets 64. **There is no pagination** — `offset` and `from` both +`400` (fact 5). So "Atlas leases the cheapest box" currently means *cheapest of the 64 cheapest +on-demand offers overall, plus 64 more matching the premium name list* — for a mid-tier card there may +be cheaper instances in neither window. + +The only lever is more filtered queries, and each costs one request against a ~1/s **deployment-wide** +budget (fact 4). A per-model sweep of 22 canonical ids would take 22 seconds and is not viable in a +request path. + +**Start by measuring, then choose.** Before implementing, establish with read-only live queries: +- how much a `gpu_name`-filtered query improves coverage for one card versus the global window +- what the cheapest offer for a given card looks like in each + +Then implement the cheapest widening that fits the budget. A targeted query issued only when a caller +names a requirement — one extra request, for exactly the card wanted — is the shape I expect to win, +but **verify before building it**, and if the measurement says the current windows already contain the +cheapest offers for the cards we canonicalise, **say so and build nothing.** That is a legitimate +outcome and better than a speculative fetch on every launch. + +Whatever ships must not increase the request count on the *cached* path, and must not turn one launch +into more than one extra Vast request. + +- [ ] **Step 1: Measure and report the coverage gap** (read-only, provision nothing) +- [ ] **Step 2: Write the failing tests for the chosen design** (`respx`) +- [ ] **Step 3: Run and confirm they fail** +- [ ] **Step 4: Implement** +- [ ] **Step 5: Run, then the full suite** +- [ ] **Step 6: Commit** + +--- + +## Whole-branch verification + +- [ ] Full suite green, no network. +- [ ] These still pass by name: `test_promoted_lease_is_not_reaped_at_the_provisioning_timeout`, + `test_lease_without_a_budget_is_still_sized_to_the_full_plan_ttl`, + `test_out_of_credit_release_of_a_managed_lease_uses_the_operator_key`, + `test_two_users_with_different_byok_eligibility_never_see_each_others_rows`, + `test_gh200_is_not_an_h200`, `test_two_concurrent_migrators_do_not_crash_on_shared_sqlite_file`. +- [ ] Explicit `provider`/`sku` leases unchanged — the dashboard's path. +- [ ] A live re-probe of launch → ready → connection → release still passes. + +## Acceptance criteria + +1. A readiness poll does not consume launch budget; `POST` stays at 20/min. +2. `reset_kv_store()` actually clears the limiter's buckets. +3. A Vast `429` is distinguishable from an empty Vast catalog and never silently empties it. +4. Prime Intellect rows canonicalise when the offer names a socket, and `None` when it does not. +5. B300, MI300X, GH200, `RTX PRO 6000 MaxQ` and the workstation line are reachable via `{gpu, count}`; + the consumer tail stays unmapped; `GH200 SXM` is still not an H200. +6. An unconfirmed teardown stops billing, frees the slot, stays retryable, and something retries it. +7. The docstrings no longer claim a reaper retry that does not exist. +8. Task 5 ships either a measured widening or a written finding that none is warranted. + +## Out of scope + +- Any OpenScience change — the three tools come after this plan. +- Provisioning Redis for a cross-process rate limit (an infra decision, not code). +- Backfilling historical lease rows. +- The ~14 deferred Minor findings in the resolver plan's ledger. From 8da1c103d3fd5134c23835e02094217074aabad7 Mon Sep 17 00:00:00 2001 From: KB Date: Mon, 3 Aug 2026 10:20:32 +0530 Subject: [PATCH 54/56] plan(compute): the wasted Vast query is the query's fault, not the map's Measured: the cheap query is 3% usable, the premium query 98%. The difference is that premium filters by name and cheap does not -- it sorts by price, and Vast's cheapest inventory is mid-tier consumer cards, of which the map holds exactly three. Task 5 now derives the query from the taxonomy instead of maintaining a second list beside it. Task 3 stays datacenter-only. Widening to the mid-tier was reconsidered and rejected: all of them are 16GB or less, the map already covers 3090/4090/5090, and VRAM is the binding constraint for this product's work. --- .../2026-07-30-compute-mode-detection.md | 1746 +++++++++++++++++ docs/plans/2026-08-03-compute-preflight.md | 88 +- 2 files changed, 1803 insertions(+), 31 deletions(-) create mode 100644 docs/plans/2026-07-30-compute-mode-detection.md diff --git a/docs/plans/2026-07-30-compute-mode-detection.md b/docs/plans/2026-07-30-compute-mode-detection.md new file mode 100644 index 00000000..6c4682c4 --- /dev/null +++ b/docs/plans/2026-07-30-compute-mode-detection.md @@ -0,0 +1,1746 @@ +# Compute Mode Detection Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Replace the static `billing.compute` config default with runtime detection that resolves compute funding to `byok`, `managed`, or `none`, exposes it to the agent through a `compute_status` tool, and filters the skill catalog to the providers the user can actually use. + +**Architecture:** One shared resolver (`src/compute/mode.ts`) reads `process.env` for provider credentials and `Skill.all()` for the matching skills; a provider counts only when it has both. Resolution happens on demand at two per-request seams — `SkillTool.init` (which rebuilds the catalog every turn) and the new `compute_status` tool — never at startup, so it cannot observe a half-initialised environment. Managed availability is one authenticated `GET /api/compute/options` call, made only when no usable provider exists, with a hard 3s timeout and a 5s in-process cache. + +**Tech Stack:** Bun, TypeScript, Zod, Hono. Tests are `bun test` with `globalThis.fetch` stubbed at the network boundary — no mocks, no network. + +**Spec:** `docs/specs/compute-mode-detection-design.md` (14 acceptance criteria). Read it before Task 1. + +## Global Constraints + +- **Style (`AGENTS.md`):** prefer `const` over `let`, avoid `else`, single-word variable names, rely on type inference over explicit annotations, **no `any`**, use Bun APIs (`Bun.file()`, `Bun.write()`). +- **No mocks in tests.** Stub `globalThis.fetch` at the network boundary and exercise the real implementation. Restore the real `fetch` in `afterEach`. +- **No network in `bun test`.** `test/preload.ts` already points `OPENSCIENCE_API_BASE` at `http://127.0.0.1:9` (unroutable) and sets `OPENSCIENCE_DISABLE_BUNDLED_SKILLS=true`, so `Skill.all()` in tests sees only skills the test itself writes into the tmpdir project. +- **Never** add `Co-Authored-By:` or any AI attribution to commit messages or PR bodies. Organisation rule. +- Run `bunx prettier --write ` before every commit. CI has a Format job over the whole repo. +- All commands run from `backend/cli`: `bun test`, `bun run typecheck`. +- **Every new assertion must be demonstrated failing against the specific mutation it guards — the _deletion_ of the logic under test, not merely its inversion.** Each test step below names its mutation. On the preceding `science_fetch` branch seven assertion defects were found and all seven were in plan-authored test code; the ones that held up were proven against deletion. +- Do **not** touch `backend/cli/test/provider/synthetic-model.test.ts`, `compaction-divider.png`, `docs/specs/issue-194-katex-latex-leak.md`, or `open-bench/` — untracked user WIP. +- Do **not** implement `docs/specs/compute-guardrails-design.md`. It is parked. + +## Corrections to the spec, already verified + +The spec was written against skill names that do not exist. These are the real names, confirmed against the live 293-skill catalog index (`~/.cache/openscience/skills-index.json`) and the authored sources in `backend/cli/skills/`: + +| Provider | Spec said | Actual skill name(s) | Source dir | +| --------------- | -------------------------------------------------------- | ----------------------------------------------------------------- | ---------------------------------------------------------- | +| Modal | `cloud-compute/modal`, `cloud-compute/modal-ml-training` | `modal-serverless-gpu`, `modal-ml-training`, `modal-research-gpu` | `skills/cloud-compute/modal{,-ml-training,-research-gpu}/` | +| Lambda | `cloud-compute/lambda-labs` | `lambda-labs-gpu-cloud` | `skills/cloud-compute/lambda-labs/` | +| TensorPool | `cloud-compute/tensorpool` | `tensorpool-gpu-cloud` | `skills/cloud-compute/tensorpool/` | +| Prime Intellect | `ml-training/prime-intellect-lab` | `prime-intellect-lab` | `skills/ml-training/prime-intellect-lab/` | +| RunPod | none | none — and none will be written, see Decision 2 | — | +| Vast.ai | none | none — and none will be written, see Decision 2 | — | + +Skill `name` comes from SKILL.md frontmatter, **not** the directory name, and there is no `category/` prefix in the name. A resolver hardcoding the spec's strings would filter nothing. + +Other verified facts the spec did not have: + +- `GET /api/compute/options` already returns `cli_effective_balance_cents`, so `balance_usd` needs **no second call**. Response envelope (`atlas` `origin/main:backend/app/routes/compute.py:213-232`): `{options[], providers[], resell_enabled, byok_eligible, platform_fee_ratio, cli_balance_cents, cli_effective_balance_cents}`, where each `providers[]` entry is `{provider, has_byok, has_operator, funding, count}` and `funding` is one of `byok | managed | unavailable`. +- `computeBillingMode()` in `src/session/billing-gate.ts:34` has exactly **one** consumer, `src/session/prompt.ts:1554`. Task 5 deletes both. +- `backend/cli/skills/` is loaded as a **dev-only** fallback (`Installation.VERSION === "local"`, `src/skill/skill.ts:219`). In a shipped binary skills come from the server catalog index. Authoring in this repo is correct and is what Tasks 6–7 do, but the resolver must never assume a skill is present — it checks `Skill.all()` at resolve time, so a binary whose catalog lacks `runpod-gpu-cloud` correctly reports RunPod as not usable. + +## Decisions taken (answering the spec's open questions) + +1. **Q1 — per-provider resolution:** no. `byok` when _any_ provider is usable; provider choice is left to the agent and the skills. +2. **Q2 — RunPod and Vast — REVISED mid-execution.** The original answer was "write the two skills". The user then ruled: _"the skill is definitely overkill, a capable agent can figure out how to use the cloud provider out of the box."_ + + That reverses more than Tasks 6–7. The spec's **"key AND skill"** rule rested entirely on the claim that a provider with a credential but no skill "gives the agent nothing to act on". If a capable agent can drive a public cloud API from a bare key, that claim is false and the conjunction is wrong — it would report `none` to a user holding a perfectly workable RunPod key. So: + - **A provider is BYOK-usable when it has a credential. Full stop.** The skill conjunction is deleted. + - `runpod` and `vast` carry `skills: []`. No RunPod or Vast skill is written; **Tasks 6 and 7 are deleted.** + - The `unusable` list disappears from `Resolution` and from `compute_status` — it existed only to explain the key-without-skill dead end, which no longer exists. + - **Catalog filtering is unaffected**: it still lists a provider's skills only when that provider is credentialed. A skill is a quality boost where one exists, not a licence to use the provider. + - Side benefit: this deletes the catalog-drift failure mode. Skills reach a shipped binary from the server catalog, so under the old rule a catalog that dropped or renamed `lambda-labs-gpu-cloud` would have silently marked Lambda unusable for a user whose key was fine. + + Tasks 6 and 7 in this plan are **superseded — do not implement them.** + +3. **Q3 — is `none` a hard block:** no, guidance only. `compute_status` tells the agent not to attempt GPU work; nothing gates `bash` or blocks a direct `skill(name=…)` load. Enforcement is a larger change and is out of scope. Task 4 makes this explicit in a code comment rather than leaving it to be discovered. +4. **Q4 — prompt pointer:** **keep one line.** Task 5 deletes the false `atlas compute:up` / `atlas doctor` text and replaces it with a single unconditional, stateless reminder for `COMPUTE_AGENTS`. It carries no mode, so it can never go stale. +5. **Q5 — config description:** yes, updated in Task 5. +6. **Q6 — how long may `compute_status` block:** **3000 ms**, hard. `OpenScience.atlasFetch`'s 60s default is far too long to sit in front of an agent turn. Cache TTL is **5000 ms**. +7. **Filter scope:** only the six mapped providers' skills are subject to mode filtering. `fireworks-ai-inference`, `together-ai-inference`, `tinker-fine-tuning`, `tinker-training-cost` and `skypilot-multi-cloud-orchestration` share the `cloud-compute` category but are inference APIs and orchestrators keyed by their own credentials, not GPU leases this mode governs. They are never hidden. + +## File Structure + +**Created:** + +| Path | Responsibility | +| ---------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------- | +| `backend/cli/src/compute/mode.ts` | The single shared resolver. Provider→env→skill table, `usable()`, `resolve()`, `invalidate()`. The only place that decides what "usable" means. | +| `backend/cli/src/tool/compute.ts` | The `compute_status` tool. Formats a `ComputeMode.Resolution` for the agent; contains no resolution logic of its own. | +| `backend/cli/skills/cloud-compute/runpod/SKILL.md` | RunPod GPU cloud skill (`runpod-gpu-cloud`). | +| `backend/cli/skills/cloud-compute/vast-ai/SKILL.md` | Vast.ai GPU marketplace skill (`vast-ai-gpu-cloud`). | +| `backend/cli/test/compute/mode.test.ts` | Resolver tests (Tasks 1–2). | +| `backend/cli/test/tool/compute-status.test.ts` | Tool tests (Task 3). | +| `backend/cli/test/tool/skill-compute-filter.test.ts` | Catalog filtering tests (Task 4). | +| `backend/cli/test/session/compute-prompt.test.ts` | Prompt-text regression tests (Task 5). | + +**Modified:** + +| Path | Change | +| ----------------------------------------------- | ---------------------------------------------------------------------- | +| `backend/cli/src/tool/registry.ts` | Import and register `ComputeTools`. | +| `backend/cli/src/tool/skill.ts` | Filter the catalog by resolved mode inside `init`. | +| `backend/cli/src/session/prompt.ts:1550-1564` | Replace the mode-carrying injection with a stateless one-line pointer. | +| `backend/cli/src/session/billing-gate.ts:33-36` | Delete the now-dead `computeBillingMode()`. | +| `backend/cli/src/config/config.ts:1058-1064` | Correct the `billing.compute` description. | + +--- + +### Task 1: The resolver's usable-provider rule + +> **Implemented at `dc125b9` under the original "key AND skill" rule, then superseded by Decision 2. Task 1b below revises it to credential-only. Kept here as the historical record — do not re-implement.** + +Pure logic, no network. Establishes the one definition of "usable" that Tasks 2–4 all consume. + +**Files:** + +- Create: `backend/cli/src/compute/mode.ts` +- Test: `backend/cli/test/compute/mode.test.ts` + +**Interfaces:** + +- Consumes: `Skill.all()` from `@/skill` (returns `Skill.Info[]`, each with a `name`). +- Produces, for Tasks 2, 3 and 4: + - `type ComputeMode.Source = "byok" | "managed" | "none"` + - `ComputeMode.PROVIDERS: Record` + - `ComputeMode.SKILLS: Set` — every provider skill name, the exact set Task 4 filters over. + - `ComputeMode.usable(): Promise<{ providers: string[]; unusable: string[] }>` — `providers` are ids with a key _and_ at least one catalogued skill; `unusable` are ids with a key but no catalogued skill. Both sorted, in `PROVIDERS` declaration order. + +- [ ] **Step 1: Write the failing test** + +Create `backend/cli/test/compute/mode.test.ts`: + +```ts +import { test, expect, afterEach, describe } from "bun:test" +import path from "path" +import { ComputeMode } from "../../src/compute/mode" +import { Instance } from "../../src/project/instance" +import { tmpdir } from "../fixture/fixture" + +const ENV = [ + "MODAL_TOKEN_ID", + "MODAL_TOKEN_SECRET", + "LAMBDA_API_KEY", + "LAMBDA_LABS_API_KEY", + "TENSORPOOL_KEY", + "TENSORPOOL_API_KEY", + "PRIME_API_KEY", + "PRIME_INTELLECT_API_KEY", + "RUNPOD_API_KEY", + "VAST_API_KEY", +] + +function clearEnv() { + for (const name of ENV) delete process.env[name] +} + +afterEach(clearEnv) + +/** A tmpdir project seeded with real SKILL.md files, so Skill.all() finds them + * without a network catalog. `OPENSCIENCE_DISABLE_BUNDLED_SKILLS` in preload.ts + * keeps the dev skills/ dir and the server index out, so the test controls the + * catalog exactly. */ +async function withSkills(names: string[], fn: () => Promise): Promise { + await using tmp = await tmpdir({ + git: true, + init: async (dir) => { + for (const name of names) { + await Bun.write( + path.join(dir, ".openscience", "skill", name, "SKILL.md"), + `---\nname: ${name}\ndescription: Test fixture for ${name}.\ncategory: cloud-compute\n---\n\n# ${name}\n`, + ) + } + }, + }) + return Instance.provide({ directory: tmp.path, fn }) +} + +describe("ComputeMode.usable", () => { + test("a provider with a key and a skill is usable", async () => { + clearEnv() + process.env["LAMBDA_API_KEY"] = "secret_abc" + const result = await withSkills(["lambda-labs-gpu-cloud"], () => ComputeMode.usable()) + expect(result.providers).toEqual(["lambda"]) + expect(result.unusable).toEqual([]) + }) + + test("the alternate env spelling also counts", async () => { + clearEnv() + process.env["LAMBDA_LABS_API_KEY"] = "secret_abc" + const result = await withSkills(["lambda-labs-gpu-cloud"], () => ComputeMode.usable()) + expect(result.providers).toEqual(["lambda"]) + }) + + test("a key with NO catalogued skill is not usable", async () => { + clearEnv() + process.env["RUNPOD_API_KEY"] = "rpa_abc" + const result = await withSkills([], () => ComputeMode.usable()) + expect(result.providers).toEqual([]) + expect(result.unusable).toEqual(["runpod"]) + }) + + test("a catalogued skill with NO key is not usable", async () => { + clearEnv() + const result = await withSkills(["lambda-labs-gpu-cloud"], () => ComputeMode.usable()) + expect(result.providers).toEqual([]) + expect(result.unusable).toEqual([]) + }) + + test("modal needs BOTH token vars — id alone is not a key", async () => { + clearEnv() + process.env["MODAL_TOKEN_ID"] = "ak-abc" + const result = await withSkills(["modal-serverless-gpu"], () => ComputeMode.usable()) + expect(result.providers).toEqual([]) + expect(result.unusable).toEqual([]) + }) + + test("modal with both token vars is usable", async () => { + clearEnv() + process.env["MODAL_TOKEN_ID"] = "ak-abc" + process.env["MODAL_TOKEN_SECRET"] = "as-def" + const result = await withSkills(["modal-serverless-gpu"], () => ComputeMode.usable()) + expect(result.providers).toEqual(["modal"]) + }) + + test("an empty-string key does not count as set", async () => { + clearEnv() + process.env["TENSORPOOL_KEY"] = "" + const result = await withSkills(["tensorpool-gpu-cloud"], () => ComputeMode.usable()) + expect(result.providers).toEqual([]) + expect(result.unusable).toEqual([]) + }) + + test("every provider resolves in isolation, given its own skill", async () => { + const cases: Array<[string, Record, string]> = [ + ["modal", { MODAL_TOKEN_ID: "ak-a", MODAL_TOKEN_SECRET: "as-b" }, "modal-serverless-gpu"], + ["lambda", { LAMBDA_API_KEY: "k" }, "lambda-labs-gpu-cloud"], + ["tensorpool", { TENSORPOOL_KEY: "k" }, "tensorpool-gpu-cloud"], + ["prime", { PRIME_API_KEY: "k" }, "prime-intellect-lab"], + ["runpod", { RUNPOD_API_KEY: "k" }, "runpod-gpu-cloud"], + ["vast", { VAST_API_KEY: "k" }, "vast-ai-gpu-cloud"], + ] + for (const [id, env, skill] of cases) { + clearEnv() + Object.assign(process.env, env) + const result = await withSkills([skill], () => ComputeMode.usable()) + expect(result.providers).toEqual([id]) + } + }) + + test("SKILLS covers every name in PROVIDERS and nothing else", async () => { + const declared = Object.values(ComputeMode.PROVIDERS).flatMap((p) => p.skills) + expect([...ComputeMode.SKILLS].sort()).toEqual([...new Set(declared)].sort()) + expect(ComputeMode.SKILLS.size).toBeGreaterThan(0) + }) + + test("a key injected after the first call is seen on the next call", async () => { + clearEnv() + await withSkills(["lambda-labs-gpu-cloud"], async () => { + expect((await ComputeMode.usable()).providers).toEqual([]) + process.env["LAMBDA_API_KEY"] = "secret_late" + expect((await ComputeMode.usable()).providers).toEqual(["lambda"]) + }) + }) +}) +``` + +Mutations these guard: deleting the skill-presence check (test 3 flips to `["runpod"]`); deleting the env check (test 4 flips to `["lambda"]`); deleting the both-vars-required branch for Modal (test 5 flips to `["modal"]`); deleting the non-empty check (test 7 flips to `["tensorpool"]`); deleting any row from `PROVIDERS` (test 8 fails for that row); caching the env read (test 10's second assertion flips back to `[]`). + +- [ ] **Step 2: Run test to verify it fails** + +Run: `cd backend/cli && bun test test/compute/mode.test.ts` +Expected: FAIL — `Cannot find module '../../src/compute/mode'`. + +- [ ] **Step 3: Write the resolver's usable-provider rule** + +Create `backend/cli/src/compute/mode.ts`: + +```ts +import { Skill } from "@/skill" + +/** + * Runtime resolution of how GPU compute is funded. + * + * `billing.compute` used to answer this from config alone, which meant a + * brand-new user with zero provider keys resolved to "byok" — claiming BYOK + * with nothing to BYOK with. This module answers it from the environment + * instead, and can say "none", which is the state we previously handled worst. + * + * Resolution deliberately happens ON DEMAND and never at startup. Provider keys + * reach process.env from three places — the user's shell, the Credentials panel + * (`applyCredentialEnv`, src/index.ts:102) and the Compute panel + * (`applyComputeEnv`, src/index.ts:106) — and the latter two are wrapped in + * `.catch(() => {})`. Detecting at boot would report "none" for a user whose + * keys are configured through the UI. Both call sites (SkillTool.init and the + * compute_status tool) run per request, long after those injections, so the + * ordering constraint cannot be violated and cannot silently regress if someone + * reorders src/index.ts later. + */ +export namespace ComputeMode { + export type Source = "byok" | "managed" | "none" + + /** + * A provider is BYOK-usable only with BOTH a credential and a skill: the agent + * runs GPU work by loading a provider's skill, so a key with no skill gives it + * nothing to act on. + * + * `env` is a list of ALTERNATIVE groups; a group is satisfied when every var in + * it is set and non-empty. Modal is the only pair — its single pasted key + * splits into a token id + secret, and a half-pasted one maps to nothing + * (mirroring `mapProviderEnv`, server/routes/settings/compute.ts:181). + * + * `skills` are frontmatter `name` values, NOT directory names and NOT + * category-prefixed. Only these names are subject to mode filtering; the other + * cloud-compute skills (tinker, skypilot, fireworks, together) are inference + * APIs and orchestrators keyed by their own credentials, not GPU leases this + * mode governs, and are never hidden. + */ + export const PROVIDERS: Record = { + modal: { + env: [["MODAL_TOKEN_ID", "MODAL_TOKEN_SECRET"]], + skills: ["modal-serverless-gpu", "modal-ml-training", "modal-research-gpu"], + }, + lambda: { + env: [["LAMBDA_API_KEY"], ["LAMBDA_LABS_API_KEY"]], + skills: ["lambda-labs-gpu-cloud"], + }, + tensorpool: { + env: [["TENSORPOOL_KEY"], ["TENSORPOOL_API_KEY"]], + skills: ["tensorpool-gpu-cloud"], + }, + prime: { + env: [["PRIME_API_KEY"], ["PRIME_INTELLECT_API_KEY"]], + skills: ["prime-intellect-lab"], + }, + runpod: { + env: [["RUNPOD_API_KEY"]], + skills: ["runpod-gpu-cloud"], + }, + vast: { + env: [["VAST_API_KEY"]], + skills: ["vast-ai-gpu-cloud"], + }, + } + + /** Every provider skill name — the exact set the catalog filter operates on. */ + export const SKILLS = new Set(Object.values(PROVIDERS).flatMap((p) => p.skills)) + + /** Read process.env directly rather than Env.get: applyComputeEnv writes to + * process.env first and mirrors to Env only when instance state exists, so + * process.env is the one source that is always populated. */ + function keyed(groups: string[][]): boolean { + return groups.some((group) => group.every((name) => !!process.env[name])) + } + + /** + * Split configured providers into those the agent can actually act on and + * those with a stored key but no catalogued skill. The second list exists so + * `none` can say *why* — a user who connected a key and is then told no + * compute is available deserves better than silence. + */ + export async function usable() { + const catalog = new Set(await Skill.all().then((all) => all.map((skill) => skill.name))) + const providers: string[] = [] + const unusable: string[] = [] + for (const [id, spec] of Object.entries(PROVIDERS)) { + if (!keyed(spec.env)) continue + if (spec.skills.some((name) => catalog.has(name))) providers.push(id) + else unusable.push(id) + } + return { providers, unusable } + } +} +``` + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `cd backend/cli && bun test test/compute/mode.test.ts` +Expected: PASS, 11 tests. + +- [ ] **Step 5: Prove each assertion against deletion** + +For each mutation named in Step 1, temporarily apply it to `src/compute/mode.ts`, run the suite, confirm the named test fails, then revert. Specifically: + +1. In `usable()`, delete the `if (!keyed(spec.env)) continue` line → "a catalogued skill with NO key" must fail. +2. In `usable()`, replace the `spec.skills.some(...)` branch with an unconditional `providers.push(id)` → "a key with NO catalogued skill" must fail. +3. In `keyed()`, change `group.every(...)` to `group.some(...)` → "modal needs BOTH token vars" must fail. +4. In `keyed()`, change `!!process.env[name]` to `name in process.env` → "an empty-string key does not count" must fail. +5. Hoist the `catalog` set to module scope so it is computed once → "a key injected after the first call" still passes (env is not cached), but note this in the commit body: freshness of the _skill_ list is `Instance.state`'s job, and Task 4 covers per-turn catalog freshness. + +Record the five results in the commit body. + +- [ ] **Step 6: Typecheck, format, commit** + +```bash +cd backend/cli && bun run typecheck +bunx prettier --write src/compute/mode.ts test/compute/mode.test.ts +git add src/compute/mode.ts test/compute/mode.test.ts +git commit -m "feat(compute): resolve usable GPU providers from key and skill" +``` + +--- + +### Task 1b: Revise the rule to credential-only + +Applies Decision 2. A provider is BYOK-usable when it has a credential; the skill conjunction is deleted, and with it the `unusable` concept. + +**Files:** + +- Modify: `backend/cli/src/compute/mode.ts` +- Modify: `backend/cli/test/compute/mode.test.ts` + +**Interfaces:** + +- Consumes: `Skill.all()` is **no longer needed by the resolver** — remove the import if nothing else uses it. +- Produces, replacing Task 1's contract: + - `ComputeMode.PROVIDERS: Record` — unchanged shape; `runpod` and `vast` now carry `skills: []`. + - `ComputeMode.SKILLS: Set` — unchanged meaning, now six names. + - `ComputeMode.usable(): string[]` — **synchronous.** Returns credentialed provider ids in `PROVIDERS` declaration order. No object, no `unusable`, no `Promise`. + +- [ ] **Step 1: Update the tests first** + +In `backend/cli/test/compute/mode.test.ts`: + +- Every `result.providers` becomes `result` (the return is now the array itself). +- Delete the `unusable` assertions and the two tests that exist only to prove the key-without-skill state: "a key with NO catalogued skill is not usable" and the `unusable`-ordering test. +- **Replace** "a key with NO catalogued skill is not usable" with its inverse, which is now the rule: + +```ts +test("a key with NO catalogued skill IS usable — the agent drives the provider API directly", async () => { + clearEnv() + process.env["RUNPOD_API_KEY"] = "rpa_abc" + expect(await withSkills([], () => ComputeMode.usable())).toEqual(["runpod"]) +}) +``` + +- **Delete** "a catalogued skill with NO key is not usable"? No — keep it. A skill without a credential must still not make a provider usable, and it is now the only guard on the env check. Update it to assert `toEqual([])` against the bare array. +- The `SKILLS` literal drops to six names: `modal-serverless-gpu`, `modal-ml-training`, `modal-research-gpu`, `lambda-labs-gpu-cloud`, `tensorpool-gpu-cloud`, `prime-intellect-lab`. +- Keep the Modal-pair tests, the empty-string test, the declaration-order test, and the mid-session-injection test — all still load-bearing. +- Modal's three-skill-names test no longer proves anything about resolution (Modal resolves on its key alone now). **Delete it**, and instead assert the catalog-facing contract it was really protecting: + +```ts +test("PROVIDERS pins the exact skill names the catalog filter matches on", async () => { + expect([...ComputeMode.SKILLS].sort()).toEqual( + [ + "lambda-labs-gpu-cloud", + "modal-ml-training", + "modal-research-gpu", + "modal-serverless-gpu", + "prime-intellect-lab", + "tensorpool-gpu-cloud", + ].sort(), + ) + expect(ComputeMode.PROVIDERS["runpod"].skills).toEqual([]) + expect(ComputeMode.PROVIDERS["vast"].skills).toEqual([]) +}) +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `cd backend/cli && bun test test/compute/mode.test.ts` +Expected: FAIL — `usable()` still returns an object, and RunPod-with-no-skill still resolves to unusable. + +- [ ] **Step 3: Apply the rule change** + +In `backend/cli/src/compute/mode.ts`: + +- Set `runpod` and `vast` to `skills: []`. +- Replace `usable()` with: + +```ts +/** + * The credentialed GPU providers, in declaration order. + * + * A credential is the whole test. An earlier revision also required a + * matching skill, on the theory that a provider with no skill gives the agent + * nothing to act on — but a capable agent drives a documented cloud API from a + * key, so that conjunction only produced a false "no compute available" for + * users holding a perfectly workable key. A skill, where one exists, is a + * quality boost; the catalog filter still offers a provider's skills only when + * that provider is credentialed. + */ +export function usable(): string[] { + return Object.keys(PROVIDERS).filter((id) => keyed(PROVIDERS[id].env)) +} +``` + +Note it is now **synchronous** — it no longer awaits the skill catalog. Keep the call sites `await`-compatible by leaving `resolve()` async (Task 2); do not add a gratuitous `Promise.resolve`. + +- Remove the `Skill` import if nothing else in the file uses it. + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `cd backend/cli && bun test test/compute/mode.test.ts` +Expected: PASS. + +- [ ] **Step 5: Prove the new assertions against deletion** + +1. Delete the `filter((id) => keyed(...))` predicate so every provider is returned → "a catalogued skill with NO key" must fail. +2. Change `group.every` to `group.some` in `keyed()` → the Modal-pair test must fail. +3. Restore the skill conjunction (`&& PROVIDERS[id].skills.some(...)`) → the new RunPod test must fail. This is the specific regression the change exists to prevent. +4. Corrupt one character of any skill string in `PROVIDERS` → the `SKILLS` pin test must fail. + +- [ ] **Step 6: Typecheck, format, commit** + +```bash +cd backend/cli && bun run typecheck && bun test test/compute/ +bunx prettier --write src/compute/mode.ts test/compute/mode.test.ts +git add src/compute/mode.ts test/compute/mode.test.ts +git commit -m "refactor(compute): a credential alone makes a GPU provider usable" +``` + +--- + +### Task 2: Managed availability and full resolution + +Adds the network half and the override rules, completing the resolver. + +**Files:** + +- Modify: `backend/cli/src/compute/mode.ts` +- Test: `backend/cli/test/compute/mode.test.ts` (append a `describe`) + +**Interfaces:** + +- Consumes: `ComputeMode.usable()` **as revised by Task 1b — synchronous, returns `string[]`**; `OpenScience.getSession()` and `OpenScience.API_BASE` from `@/openscience`; `Config.get()` from `@/config/config`. +- Produces, for Tasks 3 and 4: + - `interface ComputeMode.Resolution { mode: Source; providers: string[]; managed: boolean; balance?: number }` — `balance` is USD and present only when `mode === "managed"`. **There is no `unusable` field** (Decision 2). + - `ComputeMode.resolve(): Promise` + - `ComputeMode.invalidate(): void` — drops the availability cache; tests call it in `beforeEach`. + +- [ ] **Step 1: Write the failing test** + +Append to `backend/cli/test/compute/mode.test.ts`: + +```ts +const OPTIONS_URL = "/api/compute/options" +const SESSION = path.join(Global.Path.data, "openscience-session.json") +const realFetch = globalThis.fetch + +/** Record of every URL the resolver fetched, so "the call is skipped" is a + * positive assertion rather than an absence of failure. */ +let calls: string[] = [] + +function stubOptions(body: unknown, status = 200) { + globalThis.fetch = (async (input: string | URL | Request) => { + const url = String(input instanceof Request ? input.url : input) + calls.push(url) + if (!url.includes(OPTIONS_URL)) return realFetch(input as never) + return new Response(JSON.stringify(body), { status, headers: { "content-type": "application/json" } }) + }) as typeof fetch +} + +async function signIn() { + await fs.mkdir(Global.Path.data, { recursive: true }) + await Bun.write(SESSION, JSON.stringify({ api_key: "thk_test.secret", user_id: "u1" })) +} + +const MANAGED_ON = { + options: [], + providers: [ + { provider: "lambda", has_byok: false, has_operator: true, funding: "managed", count: 3 }, + { provider: "vast", has_byok: false, has_operator: false, funding: "unavailable", count: 0 }, + ], + resell_enabled: true, + cli_effective_balance_cents: 1234, +} + +const MANAGED_OFF = { + options: [], + providers: [{ provider: "lambda", has_byok: false, has_operator: false, funding: "unavailable", count: 0 }], + resell_enabled: false, + cli_effective_balance_cents: 1234, +} + +describe("ComputeMode.resolve", () => { + beforeEach(() => { + clearEnv() + calls = [] + ComputeMode.invalidate() + }) + + afterEach(async () => { + globalThis.fetch = realFetch + await fs.rm(SESSION, { force: true }).catch(() => {}) + }) + + test("a usable provider resolves to byok WITHOUT calling the availability endpoint", async () => { + await signIn() + stubOptions(MANAGED_ON) + process.env["LAMBDA_API_KEY"] = "k" + const result = await withSkills(["lambda-labs-gpu-cloud"], () => ComputeMode.resolve()) + expect(result.mode).toBe("byok") + expect(result.providers).toEqual(["lambda"]) + expect(result.balance).toBeUndefined() + expect(calls.filter((url) => url.includes(OPTIONS_URL))).toEqual([]) + }) + + test("no keys plus managed available resolves to managed, with the balance", async () => { + await signIn() + stubOptions(MANAGED_ON) + const result = await withSkills(["lambda-labs-gpu-cloud"], () => ComputeMode.resolve()) + expect(result.mode).toBe("managed") + expect(result.managed).toBe(true) + expect(result.balance).toBe(12.34) + }) + + test("no keys plus managed unavailable resolves to none", async () => { + await signIn() + stubOptions(MANAGED_OFF) + const result = await withSkills(["lambda-labs-gpu-cloud"], () => ComputeMode.resolve()) + expect(result.mode).toBe("none") + expect(result.managed).toBe(false) + expect(result.balance).toBeUndefined() + }) + + test("a failing availability call resolves to none, not managed", async () => { + await signIn() + globalThis.fetch = (async () => { + throw new Error("network down") + }) as typeof fetch + const result = await withSkills([], () => ComputeMode.resolve()) + expect(result.mode).toBe("none") + expect(result.managed).toBe(false) + }) + + test("a non-ok availability response resolves to none", async () => { + await signIn() + stubOptions({ detail: "unauthorized" }, 401) + const result = await withSkills([], () => ComputeMode.resolve()) + expect(result.mode).toBe("none") + }) + + test("no session means managed is unavailable and no call is made", async () => { + await fs.rm(SESSION, { force: true }).catch(() => {}) + stubOptions(MANAGED_ON) + const result = await withSkills([], () => ComputeMode.resolve()) + expect(result.mode).toBe("none") + expect(calls.filter((url) => url.includes(OPTIONS_URL))).toEqual([]) + }) + + test("a key with no skill still resolves to byok and skips the availability call", async () => { + await signIn() + stubOptions(MANAGED_ON) + process.env["RUNPOD_API_KEY"] = "rpa_x" + const result = await withSkills([], () => ComputeMode.resolve()) + expect(result.mode).toBe("byok") + expect(result.providers).toEqual(["runpod"]) + expect(calls.filter((url) => url.includes(OPTIONS_URL))).toEqual([]) + }) + + test("the availability answer is cached within the TTL", async () => { + await signIn() + stubOptions(MANAGED_ON) + await withSkills([], async () => { + await ComputeMode.resolve() + await ComputeMode.resolve() + }) + expect(calls.filter((url) => url.includes(OPTIONS_URL)).length).toBe(1) + }) + + test("invalidate() drops the cache", async () => { + await signIn() + stubOptions(MANAGED_ON) + await withSkills([], async () => { + await ComputeMode.resolve() + ComputeMode.invalidate() + await ComputeMode.resolve() + }) + expect(calls.filter((url) => url.includes(OPTIONS_URL)).length).toBe(2) + }) +}) + +describe("ComputeMode.resolve override", () => { + beforeEach(() => { + clearEnv() + calls = [] + ComputeMode.invalidate() + }) + afterEach(async () => { + globalThis.fetch = realFetch + await fs.rm(SESSION, { force: true }).catch(() => {}) + }) + + /** Same tmpdir fixture as withSkills, plus an openscience.json setting + * billing.compute. */ + async function withOverride(mode: "byok" | "managed", skills: string[], fn: () => Promise): Promise { + await using tmp = await tmpdir({ + git: true, + init: async (dir) => { + for (const name of skills) { + await Bun.write( + path.join(dir, ".openscience", "skill", name, "SKILL.md"), + `---\nname: ${name}\ndescription: Test fixture for ${name}.\ncategory: cloud-compute\n---\n\n# ${name}\n`, + ) + } + await Bun.write(path.join(dir, "openscience.json"), JSON.stringify({ billing: { compute: mode } })) + }, + }) + return Instance.provide({ directory: tmp.path, fn }) + } + + test("override byok with a usable provider stays byok", async () => { + await signIn() + stubOptions(MANAGED_ON) + process.env["LAMBDA_API_KEY"] = "k" + const result = await withOverride("byok", ["lambda-labs-gpu-cloud"], () => ComputeMode.resolve()) + expect(result.mode).toBe("byok") + }) + + test("override byok with NO usable provider narrows to none, never managed", async () => { + await signIn() + stubOptions(MANAGED_ON) + const result = await withOverride("byok", ["lambda-labs-gpu-cloud"], () => ComputeMode.resolve()) + expect(result.mode).toBe("none") + expect(calls.filter((url) => url.includes(OPTIONS_URL))).toEqual([]) + }) + + test("override managed with managed unavailable narrows to none", async () => { + await signIn() + stubOptions(MANAGED_OFF) + const result = await withOverride("managed", [], () => ComputeMode.resolve()) + expect(result.mode).toBe("none") + }) + + test("override managed beats a usable provider when managed IS available", async () => { + await signIn() + stubOptions(MANAGED_ON) + process.env["LAMBDA_API_KEY"] = "k" + const result = await withOverride("managed", ["lambda-labs-gpu-cloud"], () => ComputeMode.resolve()) + expect(result.mode).toBe("managed") + }) +}) +``` + +Add these imports at the top of the file, alongside the existing ones: + +```ts +import { beforeEach } from "bun:test" +import fs from "fs/promises" +import { Global } from "../../src/global" +``` + +Mutations these guard: deleting the `providers.length` short-circuit before the network call (the two "no call is made" assertions fail); flipping the catch/`!res.ok` fallbacks from `false` to `true` (the two failure tests resolve to `managed`); deleting the cache (the TTL test sees 2 calls); deleting `invalidate()`'s body (that test sees 1 call); deleting the override branch entirely (the two narrow-to-none tests resolve to `byok`/`managed`); deleting only the `"byok"` override arm (narrow-to-none returns `managed`). + +- [ ] **Step 2: Run test to verify it fails** + +Run: `cd backend/cli && bun test test/compute/mode.test.ts` +Expected: FAIL — `ComputeMode.resolve is not a function`. + +- [ ] **Step 3: Implement resolution** + +Append to `backend/cli/src/compute/mode.ts`, inside the `ComputeMode` namespace, and add the imports `import { Config } from "@/config/config"` and `import { OpenScience } from "@/openscience"` at the top: + +```ts +export interface Resolution { + mode: Source + /** Credentialed BYOK providers, in PROVIDERS declaration order. */ + providers: string[] + managed: boolean + /** Wallet balance in USD. Present only when mode === "managed". */ + balance?: number +} + +/** Hard ceiling on how long resolution may block an agent turn. Atlas's own + * 60s default is far too long to sit in front of a tool call; a slow or + * hanging backend must degrade to "none", not stall the turn. */ +const TIMEOUT = 3_000 + +/** Short in-process TTL, enough to stop a chatty agent hammering the endpoint + * inside one turn and no longer. The whole reason this is a tool rather than + * a prompt injection is that the answer changes mid-session, so a long cache + * would reintroduce exactly the staleness the tool exists to avoid. */ +const TTL = 5_000 + +let cache: { at: number; value: { managed: boolean; balance?: number } } | undefined + +/** Drop the availability cache. Called by tests; also safe after a connect. */ +export function invalidate() { + cache = undefined +} + +/** + * One authenticated call to /api/compute/options, which already annotates each + * provider with `funding` — "managed" when reselling is on and an operator key + * exists, else "unavailable". A failed, unauthenticated or timed-out call is + * treated as UNAVAILABLE: failing toward "none" produces an honest "connect a + * key" message, whereas failing toward "managed" would reproduce the bug this + * design exists to fix, promising a capability we never confirmed. + */ +async function available() { + if (cache && Date.now() - cache.at < TTL) return cache.value + const value = await probe() + cache = { at: Date.now(), value } + return value +} + +async function probe(): Promise<{ managed: boolean; balance?: number }> { + const session = await OpenScience.getSession().catch(() => null) + if (!session) return { managed: false } + try { + const res = await fetch(`${OpenScience.API_BASE}/api/compute/options`, { + headers: { Authorization: `Bearer ${session.api_key}` }, + signal: AbortSignal.timeout(TIMEOUT), + }) + if (!res.ok) return { managed: false } + const data = await res.json() + const providers = Array.isArray(data?.providers) ? data.providers : [] + const managed = providers.some((entry: { funding?: string }) => entry?.funding === "managed") + if (!managed) return { managed: false } + const cents = data?.cli_effective_balance_cents + return { managed: true, balance: typeof cents === "number" ? cents / 100 : undefined } + } catch { + return { managed: false } + } +} + +/** + * The single shared entry point. `billing.compute` is an OVERRIDE, not the + * source of truth: it may narrow the outcome to "none", but it may never + * manufacture a capability that isn't there. + */ +export async function resolve(): Promise { + const providers = usable() + const override = (await Config.get()).billing?.compute + + if (override === "byok") { + return { mode: providers.length ? "byok" : "none", providers, managed: false } + } + + if (override === "managed") { + const managed = await available() + return { + mode: managed.managed ? "managed" : "none", + providers, + managed: managed.managed, + balance: managed.managed ? managed.balance : undefined, + } + } + + // BYOK wins when a credentialed provider is present: it is free to the user, + // it works today, and it needs nothing from Atlas. This is also why a BYOK + // user never pays for the availability call. + if (providers.length) return { mode: "byok", providers, managed: false } + + const managed = await available() + return { + mode: managed.managed ? "managed" : "none", + providers, + managed: managed.managed, + balance: managed.managed ? managed.balance : undefined, + } +} +``` + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `cd backend/cli && bun test test/compute/mode.test.ts` +Expected: PASS, 24 tests. + +- [ ] **Step 5: Prove each assertion against deletion** + +Apply each mutation named in Step 1, confirm the named test fails, revert. Additionally verify the timeout is real: temporarily replace the stub with one that never resolves, assert `resolve()` returns `none` in under 4s, then revert. + +- [ ] **Step 6: Typecheck, format, commit** + +```bash +cd backend/cli && bun run typecheck && bun test test/compute/ +bunx prettier --write src/compute/mode.ts test/compute/mode.test.ts +git add src/compute/mode.ts test/compute/mode.test.ts +git commit -m "feat(compute): resolve byok/managed/none at runtime, config becomes an override" +``` + +--- + +### Task 3: The `compute_status` tool + +**Files:** + +- Create: `backend/cli/src/tool/compute.ts` +- Modify: `backend/cli/src/tool/registry.ts` +- Test: `backend/cli/test/tool/compute-status.test.ts` + +**Interfaces:** + +- Consumes: `ComputeMode.resolve()` and `ComputeMode.Resolution` from Task 2; `Tool.define` from `./tool`. +- Produces: `ComputeStatusTool` (id `compute_status`) and `export const ComputeTools = [ComputeStatusTool]`, registered in `ToolRegistry`. + +- [ ] **Step 1: Write the failing test** + +Create `backend/cli/test/tool/compute-status.test.ts`: + +```ts +import { test, expect, describe, beforeEach, afterEach } from "bun:test" +import path from "path" +import fs from "fs/promises" +import { ComputeStatusTool } from "../../src/tool/compute" +import { ComputeMode } from "../../src/compute/mode" +import { ToolRegistry } from "../../src/tool/registry" +import { Instance } from "../../src/project/instance" +import { Global } from "../../src/global" +import { tmpdir } from "../fixture/fixture" + +const ENV = ["LAMBDA_API_KEY", "RUNPOD_API_KEY", "MODAL_TOKEN_ID", "MODAL_TOKEN_SECRET"] +const SESSION = path.join(Global.Path.data, "openscience-session.json") +const realFetch = globalThis.fetch + +const CTX = { + sessionID: "ses_test", + messageID: "msg_test", + agent: "research", + abort: new AbortController().signal, + messages: [], + metadata: () => {}, + ask: async () => {}, +} + +function stub(body: unknown, status = 200) { + globalThis.fetch = (async (input: string | URL | Request) => { + const url = String(input instanceof Request ? input.url : input) + if (!url.includes("/api/compute/options")) return realFetch(input as never) + return new Response(JSON.stringify(body), { status, headers: { "content-type": "application/json" } }) + }) as typeof fetch +} + +const MANAGED_ON = { + providers: [{ provider: "lambda", funding: "managed", has_byok: false, has_operator: true, count: 2 }], + resell_enabled: true, + cli_effective_balance_cents: 4200, +} +const MANAGED_OFF = { providers: [], resell_enabled: false, cli_effective_balance_cents: 0 } + +async function run(skills: string[], fn?: () => Promise) { + await using tmp = await tmpdir({ + git: true, + init: async (dir) => { + for (const name of skills) { + await Bun.write( + path.join(dir, ".openscience", "skill", name, "SKILL.md"), + `---\nname: ${name}\ndescription: Fixture ${name}.\ncategory: cloud-compute\n---\n\n# ${name}\n`, + ) + } + }, + }) + return Instance.provide({ + directory: tmp.path, + fn: async () => { + await fn?.() + const tool = await ComputeStatusTool.init({}) + return tool.execute({}, CTX as never) + }, + }) +} + +describe("compute_status", () => { + beforeEach(async () => { + for (const name of ENV) delete process.env[name] + ComputeMode.invalidate() + await fs.mkdir(Global.Path.data, { recursive: true }) + await Bun.write(SESSION, JSON.stringify({ api_key: "thk_t.s", user_id: "u1" })) + }) + afterEach(async () => { + globalThis.fetch = realFetch + for (const name of ENV) delete process.env[name] + await fs.rm(SESSION, { force: true }).catch(() => {}) + }) + + test("byok reports the mode, the usable providers, and byok guidance", async () => { + stub(MANAGED_ON) + process.env["LAMBDA_API_KEY"] = "k" + const result = await run(["lambda-labs-gpu-cloud"]) + expect(result.metadata.mode).toBe("byok") + expect(result.metadata.providers).toEqual(["lambda"]) + expect(result.output).toContain("lambda") + expect(result.output.toLowerCase()).toContain("do not launch managed") + expect(result.metadata.balance_usd).toBeUndefined() + }) + + test("managed reports the balance and managed guidance", async () => { + stub(MANAGED_ON) + const result = await run([]) + expect(result.metadata.mode).toBe("managed") + expect(result.metadata.balance_usd).toBe(42) + expect(result.output).toContain("42") + expect(result.output.toLowerCase()).toContain("credits") + }) + + test("none tells the agent not to attempt GPU work and how to enable it", async () => { + stub(MANAGED_OFF) + const result = await run([]) + expect(result.metadata.mode).toBe("none") + expect(result.output.toLowerCase()).toContain("do not attempt gpu work") + expect(result.output).toContain("Settings") + expect(result.metadata.balance_usd).toBeUndefined() + }) + + test("a provider with a key but no skill is still reported as usable byok", async () => { + stub(MANAGED_OFF) + process.env["RUNPOD_API_KEY"] = "rpa_x" + const result = await run([]) + expect(result.metadata.mode).toBe("byok") + expect(result.metadata.providers).toEqual(["runpod"]) + expect(result.output).toContain("runpod") + }) + + test("the three modes produce three DIFFERENT guidance strings", async () => { + stub(MANAGED_ON) + process.env["LAMBDA_API_KEY"] = "k" + const byok = await run(["lambda-labs-gpu-cloud"]) + delete process.env["LAMBDA_API_KEY"] + ComputeMode.invalidate() + const managed = await run([]) + stub(MANAGED_OFF) + ComputeMode.invalidate() + const none = await run([]) + const texts = [byok.output, managed.output, none.output] + expect(new Set(texts).size).toBe(3) + }) + + test("a credential connected between two calls changes the answer, no restart", async () => { + stub(MANAGED_OFF) + const before = await run([]) + expect(before.metadata.mode).toBe("none") + process.env["LAMBDA_API_KEY"] = "connected-mid-session" + const after = await run(["lambda-labs-gpu-cloud"]) + expect(after.metadata.mode).toBe("byok") + }) + + test("the description instructs the agent to check before running GPU work", async () => { + const tool = await ComputeStatusTool.init({}) + expect(tool.description.toLowerCase()).toContain("before") + expect(tool.description.toLowerCase()).toContain("gpu") + expect(tool.description).toContain("byok") + expect(tool.description).toContain("managed") + expect(tool.description).toContain("none") + }) + + test("the tool is registered", async () => { + await using tmp = await tmpdir({ git: true }) + await Instance.provide({ + directory: tmp.path, + fn: async () => { + expect(await ToolRegistry.ids()).toContain("compute_status") + }, + }) + }) +}) +``` + +Mutations these guard: deleting the `providers` field from the output (test 1); deleting the balance line (test 2); returning one shared guidance string for every mode (test 5); deleting the `unusable` reporting (test 4); deleting the description's "before" instruction (test 7); removing the registry entry (test 8); caching the resolution at module load (test 6). + +- [ ] **Step 2: Run test to verify it fails** + +Run: `cd backend/cli && bun test test/tool/compute-status.test.ts` +Expected: FAIL — `Cannot find module '../../src/tool/compute'`. + +- [ ] **Step 3: Write the tool** + +Create `backend/cli/src/tool/compute.ts`: + +```ts +import z from "zod" +import { Tool } from "./tool" +import { ComputeMode } from "@/compute/mode" + +/** + * The agent PULLS its compute mode from here; nothing is injected per turn. + * + * An earlier design injected mode guidance into every turn. That was wrong for a + * reason that matters more than token cost: the mode changes mid-session. A user + * connects a Modal key in Settings ▸ Compute while a session is running, and a + * reminder injected at turn 3 is false by turn 12. A tool returns the state at + * the moment it is asked. + * + * The DESCRIPTION carries the constraint — it reaches the agent before it starts + * down a path, which is the one thing an injection did well, and tool definitions + * are in every request regardless, so it costs nothing extra. The RESULT carries + * the specifics. Adding rates or a balance to an every-turn injection would be + * expensive; adding them here is free. + */ + +const GUIDANCE: Record = { + byok: "Run GPU work on the user's connected providers via the cloud-compute skills. Do not launch managed leases — they bill Credits and are not the funded path here.", + managed: + "Run GPU work through managed compute, billed to Credits. Do not use the user's own provider keys — they are not funded here.", + none: "No compute is available. Do not attempt GPU work. Tell the user to connect a provider key in Settings ▸ Compute, or to top up for managed compute.", +} + +export const ComputeStatusTool = Tool.define("compute_status", { + description: [ + "Check how GPU compute is funded before running any GPU, training, or cluster work.", + "Returns one of byok, managed, or none, the providers available, and the rule that applies.", + "Call this first — the answer can change mid-session as the user connects or removes keys.", + ].join(" "), + parameters: z.object({}), + async execute(_params, _ctx) { + const state = await ComputeMode.resolve() + const lines = [ + `**mode**: ${state.mode}`, + `**providers**: ${state.providers.length ? state.providers.join(", ") : "none configured"}`, + `**managed available**: ${state.managed ? "yes" : "no"}`, + ] + if (state.balance !== undefined) lines.push(`**balance**: $${state.balance.toFixed(2)}`) + lines.push("", GUIDANCE[state.mode]) + + return { + title: `Compute: ${state.mode}`, + output: lines.join("\n"), + metadata: { + mode: state.mode, + providers: state.providers, + managed_available: state.managed, + balance_usd: state.balance, + }, + } + }, +}) + +export const ComputeTools = [ComputeStatusTool] +``` + +- [ ] **Step 4: Register the tool** + +In `backend/cli/src/tool/registry.ts`, add the import next to the other tool imports: + +```ts +import { ComputeTools } from "./compute" +``` + +and add `...ComputeTools,` to the array returned by `all()`, immediately after `...ProvenanceTools,`. + +- [ ] **Step 5: Run tests to verify they pass** + +Run: `cd backend/cli && bun test test/tool/compute-status.test.ts` +Expected: PASS, 8 tests. + +- [ ] **Step 6: Prove each assertion against deletion** + +Apply each mutation named in Step 1, confirm the named test fails, revert. In particular, collapse `GUIDANCE` to a single shared string and confirm "the three modes produce three DIFFERENT guidance strings" fails — an inversion would not catch this. + +- [ ] **Step 7: Typecheck, format, commit** + +```bash +cd backend/cli && bun run typecheck && bun test test/tool/ +bunx prettier --write src/tool/compute.ts src/tool/registry.ts test/tool/compute-status.test.ts +git add src/tool/compute.ts src/tool/registry.ts test/tool/compute-status.test.ts +git commit -m "feat(tool): add compute_status so the agent pulls its compute mode" +``` + +--- + +### Task 4: Filter the skill catalog by resolved mode + +**Files:** + +- Modify: `backend/cli/src/tool/skill.ts:32-58` +- Test: `backend/cli/test/tool/skill-compute-filter.test.ts` + +**Interfaces:** + +- Consumes: `ComputeMode.resolve()`, `ComputeMode.SKILLS`, `ComputeMode.PROVIDERS` from Tasks 1–2. +- Produces: nothing new. `SkillTool.init` keeps its existing shape. + +`SkillTool.init` is the right seam for two reasons. `registry.ts:187` calls `await t.init({ agent })` inside `tools()`, so **it runs per request** — a credential connected mid-session appears on the next turn with no cache to invalidate. And by the time a turn is served, every env injection in `src/index.ts` has long since run, so detection cannot observe a half-initialised environment. + +**Filter the catalog; do not auto-load the markdown.** Only usable providers' skills are listed, so the agent picks the right one because it is the only one offered. Auto-injecting a provider's markdown would fight the mechanism `tool/skill.ts` exists to provide, and these files run 500+ lines — unprompted injection is expensive on turns that have nothing to do with compute. + +- [ ] **Step 1: Write the failing test** + +Create `backend/cli/test/tool/skill-compute-filter.test.ts`: + +```ts +import { test, expect, describe, beforeEach, afterEach } from "bun:test" +import path from "path" +import fs from "fs/promises" +import { SkillTool } from "../../src/tool/skill" +import { ComputeMode } from "../../src/compute/mode" +import { Instance } from "../../src/project/instance" +import { Global } from "../../src/global" +import { tmpdir } from "../fixture/fixture" + +const ENV = ["LAMBDA_API_KEY", "RUNPOD_API_KEY", "MODAL_TOKEN_ID", "MODAL_TOKEN_SECRET", "TENSORPOOL_KEY"] +const SESSION = path.join(Global.Path.data, "openscience-session.json") +const realFetch = globalThis.fetch + +// Every provider skill, plus two skills that must never be filtered: a +// non-compute one and a cloud-compute skill that maps to no panel provider. +const ALL = [ + ["modal-serverless-gpu", "cloud-compute"], + ["lambda-labs-gpu-cloud", "cloud-compute"], + ["tensorpool-gpu-cloud", "cloud-compute"], + ["prime-intellect-lab", "ml-training"], + ["tinker-fine-tuning", "cloud-compute"], + ["rdkit", "chemistry"], +] as const + +function stub(managed: boolean) { + globalThis.fetch = (async (input: string | URL | Request) => { + const url = String(input instanceof Request ? input.url : input) + if (!url.includes("/api/compute/options")) return realFetch(input as never) + return new Response( + JSON.stringify({ + providers: managed ? [{ provider: "lambda", funding: "managed" }] : [], + resell_enabled: managed, + cli_effective_balance_cents: 500, + }), + { status: 200, headers: { "content-type": "application/json" } }, + ) + }) as typeof fetch +} + +async function project(fn: (dir: string) => Promise) { + return tmpdir({ + git: true, + init: async (dir) => { + for (const [name, category] of ALL) { + await Bun.write( + path.join(dir, ".openscience", "skill", name, "SKILL.md"), + `---\nname: ${name}\ndescription: Fixture ${name}.\ncategory: ${category}\n---\n\n# ${name}\n`, + ) + } + await fn(dir) + }, + }) +} + +/** Which of the six provider skills does the tool offer? Read from the tool's + * own category listing, which is what the model sees. */ +async function offered(): Promise { + const tool = await SkillTool.init({}) + const found: string[] = [] + for (const category of ["cloud-compute", "ml-training"]) { + const result = await tool + .execute({ category }, { + sessionID: "s", + messageID: "m", + agent: "research", + abort: new AbortController().signal, + messages: [], + metadata: () => {}, + ask: async () => {}, + } as never) + .catch(() => undefined) + if (result) found.push(result.output) + } + const text = found.join("\n") + return [...ComputeMode.SKILLS].filter((name) => text.includes(`**${name}**`)).sort() +} + +async function nonComputeVisible(): Promise { + const tool = await SkillTool.init({}) + const result = await tool.execute({ category: "chemistry" }, { + sessionID: "s", + messageID: "m", + agent: "research", + abort: new AbortController().signal, + messages: [], + metadata: () => {}, + ask: async () => {}, + } as never) + return result.output.includes("**rdkit**") +} + +async function tinkerVisible(): Promise { + const tool = await SkillTool.init({}) + const result = await tool.execute({ category: "cloud-compute" }, { + sessionID: "s", + messageID: "m", + agent: "research", + abort: new AbortController().signal, + messages: [], + metadata: () => {}, + ask: async () => {}, + } as never) + return result.output.includes("**tinker-fine-tuning**") +} + +describe("skill catalog filtering by compute mode", () => { + beforeEach(async () => { + for (const name of ENV) delete process.env[name] + ComputeMode.invalidate() + await fs.mkdir(Global.Path.data, { recursive: true }) + await Bun.write(SESSION, JSON.stringify({ api_key: "thk_t.s", user_id: "u1" })) + }) + afterEach(async () => { + globalThis.fetch = realFetch + for (const name of ENV) delete process.env[name] + await fs.rm(SESSION, { force: true }).catch(() => {}) + }) + + test("with only a Modal credential, only Modal's skills are offered", async () => { + stub(false) + process.env["MODAL_TOKEN_ID"] = "ak-a" + process.env["MODAL_TOKEN_SECRET"] = "as-b" + await using tmp = await project(async () => {}) + const names = await Instance.provide({ directory: tmp.path, fn: offered }) + expect(names).toEqual(["modal-serverless-gpu"]) + }) + + test("a RunPod credential is byok but contributes no skills — nobody else's are offered either", async () => { + stub(false) + process.env["RUNPOD_API_KEY"] = "rpa_x" + await using tmp = await project(async () => {}) + await Instance.provide({ + directory: tmp.path, + fn: async () => { + // RunPod carries skills: [] (Decision 2), so being credentialed makes the + // user byok without unlocking any other provider's skills. + expect((await ComputeMode.resolve()).mode).toBe("byok") + expect(await offered()).toEqual([]) + }, + }) + }) + + test("in managed, no BYOK provider skill is offered", async () => { + stub(true) + await using tmp = await project(async () => {}) + const names = await Instance.provide({ directory: tmp.path, fn: offered }) + expect(names).toEqual([]) + }) + + test("in none, no BYOK provider skill is offered", async () => { + stub(false) + await using tmp = await project(async () => {}) + const names = await Instance.provide({ directory: tmp.path, fn: offered }) + expect(names).toEqual([]) + }) + + test("non-compute skills are unaffected in every mode", async () => { + for (const managed of [true, false]) { + stub(managed) + ComputeMode.invalidate() + await using tmp = await project(async () => {}) + expect(await Instance.provide({ directory: tmp.path, fn: nonComputeVisible })).toBe(true) + } + }) + + test("cloud-compute skills that map to no panel provider are never hidden", async () => { + stub(false) + await using tmp = await project(async () => {}) + expect(await Instance.provide({ directory: tmp.path, fn: tinkerVisible })).toBe(true) + }) + + test("a credential added between two init() calls changes the catalog on the second", async () => { + stub(false) + await using tmp = await project(async () => {}) + await Instance.provide({ + directory: tmp.path, + fn: async () => { + expect(await offered()).toEqual([]) + process.env["TENSORPOOL_KEY"] = "tp-late" + expect(await offered()).toEqual(["tensorpool-gpu-cloud"]) + }, + }) + }) + + test("SkillTool.init and compute_status never disagree about usable providers", async () => { + stub(false) + process.env["LAMBDA_API_KEY"] = "k" + await using tmp = await project(async () => {}) + await Instance.provide({ + directory: tmp.path, + fn: async () => { + const state = await ComputeMode.resolve() + const names = await offered() + const expected = state.providers.flatMap((id) => ComputeMode.PROVIDERS[id].skills) + expect(names.sort()).toEqual([...new Set(expected)].sort()) + }, + }) + }) +}) +``` + +Mutations these guard: deleting the filter entirely (tests 1–4 and 8 fail); widening the filter to the whole `cloud-compute` category (test 6 fails); widening it to every skill (test 5 fails); resolving the mode once at module load instead of inside `init` (test 7's second assertion fails); filtering by provider id instead of skill name (test 1 fails). + +- [ ] **Step 2: Run test to verify it fails** + +Run: `cd backend/cli && bun test test/tool/skill-compute-filter.test.ts` +Expected: FAIL — every provider skill is offered in all modes. + +- [ ] **Step 3: Add the filter** + +In `backend/cli/src/tool/skill.ts`, add the import: + +```ts +import { ComputeMode } from "@/compute/mode" +``` + +and replace the `accessibleSkills` block (currently lines 36-42) with: + +```ts +// Filter skills by agent permissions if agent provided +const agent = ctx?.agent +const permitted = agent + ? skills.filter((skill) => { + const rule = PermissionNext.evaluate("skill", skill.name, agent.permission) + return rule.action !== "deny" + }) + : skills + +// Filter the GPU provider skills by the resolved compute mode, so the agent +// picks the right provider because it is the only one offered. This init runs +// per request (registry.ts calls it inside tools()), which buys two things for +// free: a credential connected mid-session shows up on the next turn with no +// cache to invalidate, and resolution always happens after src/index.ts's env +// injections rather than racing them. +// +// This is a LISTING filter, not a gate. `none` is guidance, not enforcement — +// a hidden skill can still be loaded by exact name, and the agent still has +// bash. Gating the load path is a larger change and is deliberately out of +// scope; see docs/specs/compute-mode-detection-design.md open question 3. +const compute = await ComputeMode.resolve() +const offered = new Set(compute.providers.flatMap((id) => ComputeMode.PROVIDERS[id].skills)) +const accessibleSkills = permitted.filter((skill) => !ComputeMode.SKILLS.has(skill.name) || offered.has(skill.name)) +``` + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `cd backend/cli && bun test test/tool/skill-compute-filter.test.ts` +Expected: PASS, 8 tests. + +- [ ] **Step 5: Run the full suite for regressions** + +Run: `cd backend/cli && bun test` +Expected: PASS. `test/skill/` and `test/session/system-skills.test.ts` exercise the same catalog — if any of them go red, the filter is too wide. + +- [ ] **Step 6: Prove each assertion against deletion** + +Apply each mutation named in Step 1, confirm the named test fails, revert. + +- [ ] **Step 7: Typecheck, format, commit** + +```bash +cd backend/cli && bun run typecheck +bunx prettier --write src/tool/skill.ts test/tool/skill-compute-filter.test.ts +git add src/tool/skill.ts test/tool/skill-compute-filter.test.ts +git commit -m "feat(skill): offer GPU provider skills only for usable providers" +``` + +--- + +### Task 5: Retire the false prompt injection, config text, and dead code + +Three things in the current injected text do not hold: `atlas compute:up` is not in the published `@synsci/atlas@0.13.2`; `atlas doctor` reports no compute field at all, so the stated condition is unobservable; and managed compute is off by default server-side (`COMPUTE_RESELL_ENABLED` defaults to `false`). An agent in managed mode runs an unknown command, cannot check the sanctioned signal, and is pointed at "the user's own GPU providers" as the remedy — which in managed mode is precisely the set of keys that does not exist. + +**Files:** + +- Modify: `backend/cli/src/session/prompt.ts:1546-1564` +- Modify: `backend/cli/src/session/billing-gate.ts:33-36` +- Modify: `backend/cli/src/config/config.ts:1058-1064` +- Test: `backend/cli/test/session/compute-prompt.test.ts` + +**Interfaces:** + +- Consumes: nothing new. +- Produces: `computeBillingMode` no longer exists. `BillingMode`, `llmBillingMode`, `resolveCredentialSource`, `requiresWalletBalance` and `shouldReportUsage` are unchanged. + +- [ ] **Step 1: Write the failing test** + +Create `backend/cli/test/session/compute-prompt.test.ts`: + +```ts +import { test, expect, describe } from "bun:test" +import path from "path" + +const root = path.join(import.meta.dir, "..", "..", "src") + +async function sources() { + const files = await Array.fromAsync( + new Bun.Glob("session/**/*.{ts,txt}").scan({ cwd: root, absolute: true, onlyFiles: true }), + ) + return Promise.all(files.map(async (file) => [file, await Bun.file(file).text()] as const)) +} + +describe("compute prompt text", () => { + test("no prompt or session source references atlas compute:up", async () => { + const hits = (await sources()).filter(([, text]) => text.includes("compute:up")) + expect(hits.map(([file]) => path.relative(root, file))).toEqual([]) + }) + + test("no prompt or session source uses atlas doctor as the compute availability signal", async () => { + const hits = (await sources()).filter(([, text]) => /atlas doctor/i.test(text)) + expect(hits.map(([file]) => path.relative(root, file))).toEqual([]) + }) + + test("the compute reminder points at compute_status and carries no mode", async () => { + const text = await Bun.file(path.join(root, "session", "prompt.ts")).text() + expect(text).toContain("compute_status") + // The reminder must be stateless — a mode baked into an injected string is + // false the moment the user connects a key mid-session. + expect(text).not.toContain("Compute spend is set to") + }) + + test("computeBillingMode is gone and nothing imports it", async () => { + const gate = await Bun.file(path.join(root, "session", "billing-gate.ts")).text() + expect(gate).not.toContain("computeBillingMode") + const files = await Array.fromAsync(new Bun.Glob("**/*.ts").scan({ cwd: root, absolute: true, onlyFiles: true })) + const importers = ( + await Promise.all( + files.map(async (file) => ((await Bun.file(file).text()).includes("computeBillingMode") ? file : undefined)), + ) + ).filter(Boolean) + expect(importers).toEqual([]) + }) + + test("the billing.compute config description no longer claims 'Unset = byok'", async () => { + const text = await Bun.file(path.join(root, "config", "config.ts")).text() + expect(text).not.toContain("Unset = byok") + expect(text).toContain("auto-detect") + }) +}) +``` + +Mutations these guard: leaving either false claim in any session prompt (tests 1–2); re-introducing a mode-carrying injection (test 3); leaving `computeBillingMode` behind as dead code (test 4); forgetting the config description (test 5). + +- [ ] **Step 2: Run test to verify it fails** + +Run: `cd backend/cli && bun test test/session/compute-prompt.test.ts` +Expected: FAIL on tests 1, 2, 3, 4 and 5 — all five conditions currently hold in the wrong direction. + +- [ ] **Step 3: Replace the injection** + +In `backend/cli/src/session/prompt.ts`, replace the whole block currently at lines 1550-1564 (from the `// Compute spend preference` comment through the closing `}` of the `if`) with: + +```ts +// Compute funding is PULLED from the `compute_status` tool, not injected — +// the mode changes mid-session (a key connected in Settings ▸ Compute at +// turn 3 makes a reminder injected then false by turn 12). This line is a +// stateless pointer: it carries no mode, so it can never go stale, and it +// closes the gap where an agent reaches for bash without ever looking. +if (COMPUTE_AGENTS.has(input.agent.name)) { + userMessage.parts.push({ + id: Identifier.ascending("part"), + messageID: userMessage.info.id, + sessionID: userMessage.info.sessionID, + type: "text", + text: "Call `compute_status` before running GPU, training, or cluster work. It reports how compute is funded and which providers are usable right now.", + synthetic: true, + }) +} +``` + +Then delete the now-unused import on line 46: `import { computeBillingMode } from "./billing-gate"`. + +- [ ] **Step 4: Delete the dead function** + +In `backend/cli/src/session/billing-gate.ts`, delete lines 33-36: + +```ts +/** The user-facing compute spend toggle. Defaults to "byok" (own GPU providers). */ +export async function computeBillingMode(): Promise { + return (await Config.get()).billing?.compute ?? "byok" +} +``` + +`prompt.ts:1554` was its only consumer. `Config` is still imported for `llmBillingMode`, so leave the import. + +- [ ] **Step 5: Correct the config description** + +In `backend/cli/src/config/config.ts`, replace the `compute` field's `.describe(...)` string (line 1062) with: + +```ts + "How GPU/compute is paid for. 'managed' runs on Atlas-provisioned compute billed to your wallet; 'byok' uses your own connected GPU providers (Modal, Lambda Labs, TensorPool, Prime Intellect, RunPod, Vast.ai). Unset = auto-detect from your connected providers. Setting this can only narrow the result — if the mode you pick isn't actually available, compute resolves to none rather than pretending.", +``` + +- [ ] **Step 6: Run tests to verify they pass** + +Run: `cd backend/cli && bun test test/session/compute-prompt.test.ts` +Expected: PASS, 5 tests. + +- [ ] **Step 7: Run the full suite** + +Run: `cd backend/cli && bun test && bun run typecheck` +Expected: PASS. Any test asserting the old injected text must be updated to the new reminder, not deleted. + +- [ ] **Step 8: Prove each assertion against deletion** + +Apply each mutation named in Step 1, confirm the named test fails, revert. + +- [ ] **Step 9: Format and commit** + +```bash +cd backend/cli +bunx prettier --write src/session/prompt.ts src/session/billing-gate.ts src/config/config.ts test/session/compute-prompt.test.ts +git add src/session/prompt.ts src/session/billing-gate.ts src/config/config.ts test/session/compute-prompt.test.ts +git commit -m "fix(prompt): drop the false atlas compute:up guidance for a compute_status pointer" +``` + +--- + +### Task 5b: Finish acceptance criterion 12 in the agent prompts + +Task 5 discharged criterion 12 for `src/session/**` only. Its tests glob `session/**/*.{ts,txt}`, which excludes `src/agent/prompt/*.txt` — and the primary `research` agent's prompt still gates managed compute on `atlas doctor`, the same false signal Task 5 just removed from `prompt.ts`. Both the Task 5 implementer and its reviewer surfaced this independently. + +**Files:** + +- Modify: `backend/cli/src/agent/prompt/research.txt` (Stage 5: COMPUTE, lines 252-261) +- Modify: `backend/cli/test/session/compute-prompt.test.ts` + +**Interfaces:** none — prompt text and test scope only. + +**What is wrong, precisely.** `research.txt:253-257` currently reads: + +``` +- Managed compute (Daytona-backed) runs through the bundled `atlas` CLI when your Atlas + session is active. Run `atlas doctor --format=json` first; if it reports the CLI is + unavailable/unauthenticated, print a one-line note and fall back to the BYOK cloud-compute + skills below (Modal, Tinker, TensorPool, Prime Intellect, HF Jobs) — never block on it. +``` + +`atlas doctor` reports `config_path`, `profile`, `base_url`, `auth`, `backend`, `package.skills`, `integrations`, `spool`, `warnings`, `ok` — **nothing about compute**. Using CLI auth as a proxy for managed-compute availability is the wrong signal in the worst direction: managed compute is off behind `COMPUTE_RESELL_ENABLED=false` regardless of authentication, so an authenticated user is told managed compute works when it does not. + +**Out of scope, deliberately.** `research.txt:81-84` also runs `atlas doctor`, to check whether the `atlas` CLI is present and authenticated before loading graph state. That is a valid use of a signal the command genuinely reports — leave it alone. The absence tests must not become so broad that they forbid it. + +- [ ] **Step 1: Widen the test scope to prove the gap exists** + +In `backend/cli/test/session/compute-prompt.test.ts`, change `sources()` to scan agent prompts as well as session ones: + +```ts +async function sources() { + const globs = ["session/**/*.{ts,txt}", "agent/prompt/*.txt"] + const files = ( + await Promise.all( + globs.map((pattern) => + Array.fromAsync(new Bun.Glob(pattern).scan({ cwd: root, absolute: true, onlyFiles: true })), + ), + ) + ).flat() + return Promise.all(files.map(async (file) => [file, await Bun.file(file).text()] as const)) +} +``` + +Then replace the blanket `atlas doctor` test with one that forbids it **as a compute signal** while still permitting the CLI-availability check: + +```ts +test("the file set is non-empty and covers both prompt trees", async () => { + const files = (await sources()).map(([file]) => path.relative(root, file)) + expect(files.length).toBeGreaterThan(20) + expect(files).toContain("session/prompt.ts") + expect(files).toContain("agent/prompt/research.txt") +}) + +test("no prompt uses atlas doctor as the compute availability signal", async () => { + // `atlas doctor` legitimately reports whether the atlas CLI is present and + // authenticated (research.txt uses it that way before loading graph state). + // What it does NOT report is anything about compute — so any paragraph that + // mentions both compute and `atlas doctor` is reading a signal that isn't there. + const hits = (await sources()).filter(([, text]) => + text.split(/\n\s*\n/).some((para) => /atlas doctor/i.test(para) && /\bcompute\b/i.test(para)), + ) + expect(hits.map(([file]) => path.relative(root, file))).toEqual([]) +}) + +test("agent prompts point at compute_status for GPU funding", async () => { + const text = await Bun.file(path.join(root, "agent", "prompt", "research.txt")).text() + expect(text).toContain("compute_status") +}) + +test("prompts name skills that exist in the provider map", async () => { + // `modal` is not a skill name — the real ones are modal-serverless-gpu, + // modal-ml-training, modal-research-gpu. A prompt naming a skill the catalog + // does not have sends the agent to load something that cannot resolve. + const text = await Bun.file(path.join(root, "agent", "prompt", "research.txt")).text() + expect(text).not.toMatch(/`modal`/) +}) +``` + +Keep the existing `compute:up`, `compute_status`-in-`prompt.ts`, `computeBillingMode`, and config-description tests unchanged. + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `cd backend/cli && bun test test/session/compute-prompt.test.ts` +Expected: FAIL on "no prompt uses atlas doctor as the compute availability signal" (naming `agent/prompt/research.txt`), on "agent prompts point at compute_status", and on "prompts name skills that exist in the provider map". + +- [ ] **Step 3: Rewrite the COMPUTE stage guidance** + +In `backend/cli/src/agent/prompt/research.txt`, replace lines 254-257 (the four-line `Managed compute (Daytona-backed) …` bullet) with: + +``` +- Call the `compute_status` tool before launching any GPU work. It reports how compute is + funded right now — `byok`, `managed`, or `none` — which providers are usable, and the rule + that applies. Do not infer this from `atlas doctor`; it reports nothing about compute. +- If it returns `byok`, load the cloud-compute skill for one of the providers it lists. + If `managed`, run the work through managed compute. If `none`, do not launch GPU work — + tell the user to connect a provider key in Settings ▸ Compute. +``` + +Then correct the stale skill name on the following line — `modal` is not a skill; the catalog has `modal-serverless-gpu`: + +``` +- Load: `modal-serverless-gpu` for general serverless GPU (inference, serving) +``` + +Leave lines 81-84 untouched. + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `cd backend/cli && bun test test/session/compute-prompt.test.ts` +Expected: PASS. + +- [ ] **Step 5: Prove each new assertion against deletion** + +1. Re-add the phrase `Run \`atlas doctor --format=json\` first`into the COMPUTE bullet → "no prompt uses atlas doctor as the compute availability signal" must fail, naming`agent/prompt/research.txt`. +2. Confirm the _inverse_: the untouched graph-state use at lines 81-84 must **not** trip that test. Verify the paragraph containing it has no `compute` mention, so the test permits a legitimate `atlas doctor` call. If it does trip, the test is too broad — fix it. +3. Remove `compute_status` from `research.txt` → "agent prompts point at compute_status" must fail. +4. Restore `` `modal` `` in the load line → "prompts name skills that exist in the provider map" must fail. +5. Point `sources()` at a non-existent directory → "the file set is non-empty" must fail. This guards the whole absence-test family against passing on an empty glob. + +- [ ] **Step 6: Full suite, format, commit** + +```bash +cd backend/cli && bun test && bun run typecheck +bunx prettier --write test/session/compute-prompt.test.ts +git add src/agent/prompt/research.txt test/session/compute-prompt.test.ts +git commit -m "fix(prompt): research agent checks compute_status, not atlas doctor" +``` + +Note: `research.txt` is a `.txt` prompt file — do not run prettier on it. + +--- + +### Tasks 6 and 7: RunPod and Vast.ai skills — DELETED + +Superseded by Decision 2. The user ruled that a provider skill is overkill — +a capable agent drives a documented cloud API from a bare key. With the skill +conjunction gone from the resolver, a RunPod or Vast credential already makes +a user BYOK-usable, so there is nothing left for these tasks to fix. + +`PROVIDERS.runpod.skills` and `PROVIDERS.vast.skills` are `[]`, which is the +honest statement of the situation: those providers have credentials and no +catalogued skill, and that is fine. + +--- + +## Final verification + +Run before declaring the branch complete. Evidence before assertions — paste the actual output, do not summarise it. + +- [ ] `cd backend/cli && bun test` — full suite green, no network. +- [ ] `cd backend/cli && bun run typecheck` — clean. If it fails only inside the untracked `test/provider/synthetic-model.test.ts`, that is the user's WIP and is expected; note it rather than fixing it. +- [ ] `bunx prettier --check .` from the repo root — CI's Format job runs over the whole repo. +- [ ] Walk the spec's 14 acceptance criteria and name the test that proves each. + +**Pushing is blocked twice.** The husky pre-push hook pins a bun version from `package.json` `packageManager` and runs `bun typecheck`, which fails on the untracked `synthetic-model.test.ts`. The prior session's workaround was `git stash push -u` on that one path, push, then `git stash pop`. **Restore it immediately; do not leave it stashed, and do not use `--no-verify`** — the user declined that explicitly. + +## Acceptance criteria → task map + +| # | Criterion | Proven by | +| --- | ------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------- | +| 1 | Runtime resolver, single shared implementation | Task 1 + Task 2; Task 4 "SkillTool.init and compute_status never disagree" | +| 2 | **REVISED by Decision 2** — a credential alone makes a provider usable; a half-configured Modal credential still does not count | Task 1b: "a key with NO catalogued skill IS usable", "a catalogued skill with NO key", the Modal-pair test, the empty-string test | +| 3 | Catalog lists usable providers only; none in managed/none; non-compute unaffected | Task 4 tests 1–6 | +| 4 | Markdown never auto-injected | Task 4 — the filter touches the listing only; no auto-load path is added anywhere | +| 5 | No keys + managed unavailable (incl. a failed check) → `none` | Task 2 tests 3, 4, 5, 6 | +| 6 | Override can narrow to `none`, never manufacture | Task 2 override tests 2 and 3 | +| 7 | A key injected by either settings panel at boot is detected | Task 1 test 10 + Task 2's on-demand resolution (never at startup) | +| 8 | Availability call skipped when a usable provider is present | Task 2 test 1 (positive assertion on the recorded call list) | +| 9 | `compute_status` returns mode, providers, guidance, resolved per call | Task 3 tests 1–5 | +| 10 | Mid-session credential reflected without restart | Task 3 test 6; Task 4 test 7 | +| 11 | Nothing mode-carrying injected per turn; description carries the instruction | Task 5 test 3; Task 3 test 7 | +| 12 | No prompt references `atlas compute:up` or `atlas doctor` | Task 5 tests 1, 2 | +| 13 | `none` says don't attempt GPU work and how to enable it | Task 3 tests 3, 4 | +| 14 | `bun test` passes with no network | Final verification | diff --git a/docs/plans/2026-08-03-compute-preflight.md b/docs/plans/2026-08-03-compute-preflight.md index 5354b075..57df6e81 100644 --- a/docs/plans/2026-08-03-compute-preflight.md +++ b/docs/plans/2026-08-03-compute-preflight.md @@ -164,10 +164,15 @@ simply never copies it onto the row (`prime_intellect_provider.py:174`). Surface `canonical()` able to use it. **(b) The map is narrower than what is on sale.** Verified against live catalogs: **37% of Vast rows and -52% of RunPod's** are mapped. Most of the remainder is consumer tail that is correctly dropped -(`GTX 1060`, `Tesla P4`, `Titan Xp`, `Quadro P4000`, `RTX 3060 laptop`) — **leave those unmapped.** But -these are priced today and unreachable through `{gpu, count}`: **B300**, **MI300X**, **GH200**, -`RTX PRO 6000 MaxQ`, and the Ada/Ampere workstation line. +52% of RunPod's** are mapped. These are priced today and unreachable through `{gpu, count}`: **B300**, +**MI300X**, **GH200**, `RTX PRO 6000 MaxQ`, and the Ada/Ampere workstation line. Add those. + +**Leave the mid-tier consumer cards unmapped** — `RTX 3060/3070/3080`, `4060/4070/4080`, `5060/5070`, +and the older `GTX 10xx` / `Tesla P4` / `Quadro` / `Titan Xp` stock. This was reconsidered and the +decision stands: every one of them is ≤16GB, the map already holds the three cards that matter in that +tier (`RTX-3090` and `RTX-4090` at 24GB, `RTX-5090` at 32GB), and for ML research VRAM is the binding +constraint. They remain leasable by explicit `provider`/`sku`. Task 5 fixes the wasted query directly +rather than by widening the taxonomy to absorb it. **The exact-match rule is not negotiable.** `RTX 6000 Ada`, `RTX A6000`, `RTX PRO 6000`, `RTX PRO 6000 WK` and `RTX PRO 6000 MaxQ` are five different cards whose names contain each other, and @@ -248,39 +253,60 @@ Required behaviour: --- -### Task 5: See more than the cheapest 64 offers +### Task 5: Ask Vast for hardware we can actually name **Files:** - Modify: `backend/app/compute/vast_provider.py` - Test: `backend/tests/test_compute_vast_provider_http.py` (append) **Interfaces:** -- Produces: a requirement for a specific GPU reaches offers outside the global cheapest-64 window. - -Atlas asks Vast for `limit: 512` and gets 64. **There is no pagination** — `offset` and `from` both -`400` (fact 5). So "Atlas leases the cheapest box" currently means *cheapest of the 64 cheapest -on-demand offers overall, plus 64 more matching the premium name list* — for a mid-tier card there may -be cheaper instances in neither window. - -The only lever is more filtered queries, and each costs one request against a ~1/s **deployment-wide** -budget (fact 4). A per-model sweep of 22 canonical ids would take 22 seconds and is not viable in a -request path. - -**Start by measuring, then choose.** Before implementing, establish with read-only live queries: -- how much a `gpu_name`-filtered query improves coverage for one card versus the global window -- what the cheapest offer for a given card looks like in each - -Then implement the cheapest widening that fits the budget. A targeted query issued only when a caller -names a requirement — one extra request, for exactly the card wanted — is the shape I expect to win, -but **verify before building it**, and if the measurement says the current windows already contain the -cheapest offers for the cards we canonicalise, **say so and build nothing.** That is a legitimate -outcome and better than a speculative fetch on every launch. - -Whatever ships must not increase the request count on the *cached* path, and must not turn one launch -into more than one extra Vast request. - -- [ ] **Step 1: Measure and report the coverage gap** (read-only, provision nothing) -- [ ] **Step 2: Write the failing tests for the chosen design** (`respx`) +- Consumes: the spellings table in `app/compute/gpu_models.py` (after Task 3). +- Produces: the catalog's Vast rows are overwhelmingly rows the resolver can rank. + +**Measured live 2026-08-03 — this is the defect:** + +``` +cheap query (64 cheapest overall) -> 64 offers, 2 usable ( 3%), 33 distinct cards +premium query (gpu_name in _PREMIUM…) -> 64 offers, 63 usable ( 98%), 13 distinct cards +gpu_name == "RTX 4090" -> 56 offers, 56 usable (100%), 1 card +gpu_name == "A100 SXM4" -> 64 offers, 64 usable (100%), 1 card +gpu_name == "H100 SXM" -> 39 offers, 39 usable (100%), 1 card +``` + +Vast returns exactly 64 per query and has no pagination (fact 5), so each query is a scarce, fixed-size +window against a ~1/s deployment-wide budget (fact 4). The premium query spends its window well because +it **filters by name**. The cheap query filters on nothing and sorts by price ascending — and Vast's +cheapest inventory is mid-tier consumer cards, of which the map contains exactly three +(`RTX-3090`, `RTX-4090`, `RTX-5090`). So it reliably fills its window with rows `canonical()` returns +`None` for, and `resolve()` drops them. + +**The root cause is that the query and the taxonomy were never connected.** Two independently sensible +decisions — "show the cheapest hardware" and "map only cards we can name unambiguously" — combine into a +request that is 97% waste. Note `dph_total <= 0.50` and `reliability2 >= 0.98` were both measured at +3–4% usable: they re-select the same tail, so a price or quality filter does not fix this. A +`gpu_ram >= 40GB` floor reaches 65%, better but still not the point. + +**Derive the query from the taxonomy.** Ask Vast for the cheapest offers *among the card names we can +canonicalise*, rather than the cheapest offers outright. The spellings already exist in +`gpu_models.py`; the fix is to stop maintaining a second, divergent list beside them. `_PREMIUM_GPU_NAMES` +should become a consequence of the map, not an independent constant — if it can be removed entirely in +favour of one derived list, remove it. + +Then, if and only if the measurement in Step 1 shows it earns its request: a **targeted query on the +requirement path** — when a caller names `{gpu, count}`, one query filtered to that card's spellings, +which the numbers above suggest returns 39–64 offers at 100% yield. That is what makes "the cheapest +H100-SXM" true rather than "the cheapest H100-SXM that happened to land in a shared window." + +**Constraints:** do not increase the request count on the cached path; do not turn one launch into more +than one extra Vast request; do not add a retry loop inside `list_options` (Task 2's reasoning applies). +Vast's `gpu_name` values are provider spellings, not canonical ids — `RTX 6000Ada` has no space, RunPod's +do. Filter on what Vast actually emits. + +- [ ] **Step 1: Measure** (read-only live queries, provision nothing, sleep ≥1.3s between them for + fact 4). Confirm the taxonomy-derived filter's yield, and measure whether a targeted per-card + query finds cheaper offers for a named card than the shared windows do. **If it does not, say so + and do not build it** — a speculative extra fetch on every launch is worse than none. +- [ ] **Step 2: Write the failing tests for the chosen design** (`respx`, no live calls) - [ ] **Step 3: Run and confirm they fail** - [ ] **Step 4: Implement** - [ ] **Step 5: Run, then the full suite** From d870e44d23e157b800a7c523722e8de2dce71f8b Mon Sep 17 00:00:00 2001 From: KB Date: Mon, 3 Aug 2026 13:54:01 +0530 Subject: [PATCH 55/56] spec(compute): retract the 84% offer-death figure, and say why That measurement compared two multi-name query windows, which is only sound if a window is the deterministic cheapest-64 of its filter. A single-card query found 45 offers priced below the shared window's own ceiling that the window did not contain -- impossible under that model. But it does not generalise either: a narrow num_gpus in [1] query run twice 1.5s apart returned identical results, 64/64 shared ids. So the window is a sample for some query shapes and not others, and neither 'the window is a sample' nor '36% of offers die per 40s' is established. The retry is unaffected -- a SKU that fails to launch must be replaced whatever the cause. What is not established is WHY a SKU goes stale, which any future staleness or stability heuristic would need measured first. --- docs/specs/compute-design.md | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/docs/specs/compute-design.md b/docs/specs/compute-design.md index fab16bd7..3b58d407 100644 --- a/docs/specs/compute-design.md +++ b/docs/specs/compute-design.md @@ -645,12 +645,28 @@ Three real leases through the real route, real provider keys, nothing faked belo with the key absent: always 64. So the catalog is 64 cheapest + 64 premium-name offers, deduped to ~65 rows. **"Cheapest" therefore means cheapest of a narrow window, not of Vast**; for a mid-tier card there may be cheaper instances outside both windows. Pagination is the fix, and it is not built. -- **Churn re-measured on raw offers: 36% survive 40 seconds** (128 → 128 offers, 47 stable), with deduped - rows tracking it at 35%. **Real offer death dominates.** A theory that the ~50% figure was mostly an - artifact of Atlas's cheapest-per-`(gpu_name, count)` dedup was tested and **refuted**: of 42 rows that - left the catalog, 35 were gone from Vast's raw response too. At most ~16% are dedup drops where the - offer is alive and merely out-ranked — an upper bound, since the two fetches were not simultaneous. - **The retry's premise stands.** +- **Churn measured on raw offers: 36% survive 40 seconds** (128 → 128 offers, 47 stable), with deduped + rows tracking it at 35%. A theory that this was mostly an artifact of Atlas's + cheapest-per-`(gpu_name, count)` dedup was tested and refuted: of 42 rows that left the catalog, 35 + were gone from Vast's next response too. + + > **Correction, 2026-08-03 — do not cite the sentence above as "84% real offer death".** That test + > compared two multi-name query windows, which is only valid if a window is the deterministic + > cheapest-64 of its filter. It may not be. A `gpu_name == "A100 SXM4"` query returned **45 offers + > priced below the shared window's own ceiling that the shared window did not contain** — impossible + > under a deterministic cheapest-64 — and two multi-name queries matching an identical live name set, + > issued 1.6s apart over the identical price range, shared only 27 of 64 ids. + > + > **But the effect is query-shape dependent, so neither claim generalises.** A narrow + > `num_gpus in [1]` query run twice 1.5s apart returned **identical** results: 64/64 shared ids, same + > price range, zero drops. So "the window is a sample" is not true of every query, and "36% of offers + > die per 40s" is not established either — the earlier figure cannot distinguish a dead offer from one + > the window simply did not return. + > + > What is unaffected: the retry is still justified, because a SKU that fails to launch must be + > replaced whatever the cause. What is *not* established is **why** a SKU goes stale, and any future + > design that depends on the answer — a staleness heuristic, a stability filter, a cache-freshness + > rule — needs this measured properly first, per-query-shape. - **The retry did not fire in any of the three launches.** First pick succeeded every time, including against a deliberately staled cache. Not a disproof — it means first-pick success is common — but the retry path is still unexercised against a real provider refusal. From 39c63695d87b2975900505c5c74f541151ed3055 Mon Sep 17 00:00:00 2001 From: KB Date: Mon, 3 Aug 2026 13:56:18 +0530 Subject: [PATCH 56/56] plan(compute): pre-flight executed, and what review caught that it did not anticipate --- docs/plans/2026-08-03-compute-preflight.md | 33 ++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/docs/plans/2026-08-03-compute-preflight.md b/docs/plans/2026-08-03-compute-preflight.md index 57df6e81..f6b510b0 100644 --- a/docs/plans/2026-08-03-compute-preflight.md +++ b/docs/plans/2026-08-03-compute-preflight.md @@ -1,5 +1,38 @@ # Compute pre-flight: what must be true before OpenScience gets compute tools +> **EXECUTED 2026-08-03** on `feat/compute-lease-prerequisites`. All five tasks shipped and reviewed. +> Suite **2004 passed / 1 skipped** (from 1913), no network. Live re-probe after the last task: resolve +> → lease in 5.0s → `gpu_model='RTX-3090'` matching the request → `ready` → released, zero instances +> left running. +> +> **Three things review caught that the plan did not anticipate:** +> +> 1. **Task 2's first age-ceiling fix bounded the wrong thing.** It capped how *long* a stale block +> could starve a launch, not whether it did: under the ceiling, three dead Vast rows still consumed +> every attempt while a live RunPod row went untried. Fixed by making freshness the first rank key. +> 2. **Task 4's retry could never drain.** `lambda_provider.py` had a bare `raise_for_status()` with no +> 404 exemption — RunPod and Prime both have one — so an already-gone box produced 1,440 +> operator-authenticated terminate calls per day forever, from an uncapped serial pass running ahead +> of the promotion sweep. A billing fix was degrading the launch path. +> 3. **Task 3's `RTX-A2000` mapped two different cards to one id.** 6GB and 12GB, and since the resolver +> ranks on price the caller always got the 6GB one. The class of bug matters more than the card: +> `canonical()` never consults VRAM on an exact hit, so no per-name test can fail on it — the old +> tests asserted both values for one id and passed. Only measured inventory catches it. +> +> **One measurement retracted.** The spec's "84% genuine offer death" figure compared two multi-name +> query windows, which is only sound if a window is the deterministic cheapest-64 of its filter. It may +> not be — but a narrow query proved perfectly stable, so the effect is query-shape dependent and +> neither claim generalises. See `docs/specs/compute-design.md`. +> +> **One thing measured, earned, and deliberately not built:** the targeted per-card Vast query. 12 of 22 +> `(card, count)` pairs are absent or up to 50% dearer in the shared windows. It cannot live in +> `vast_provider.py` — `list_options` never receives `{gpu, count}`, and `_provider_catalog`'s cache key +> names no requirement, so a requirement-filtered fetch would poison the entry every caller shares. It +> needs `routes/compute.py` and its own task. +> +> Deferred Minor findings are in the ledger at +> `.superpowers/sdd/2026-08-03-compute-preflight/progress.md` (gitignored). + > **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development to implement > this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.