diff --git a/backend/cli/src/agent/prompt/biology.txt b/backend/cli/src/agent/prompt/biology.txt index 2bb8848b..34aab5f8 100644 --- a/backend/cli/src/agent/prompt/biology.txt +++ b/backend/cli/src/agent/prompt/biology.txt @@ -56,7 +56,8 @@ Before using any external service, verify credentials: [ -n "$VAR_NAME" ] && echo "set" || echo "not set" If not connected, tell the user: - "Connect [service] at https://app.syntheticsciences.ai -> Services, then restart openscience." + "Connect [service] at https://app.syntheticsciences.ai -> Services (GPU provider keys: + Settings ▸ Compute). A newly connected key is picked up on the next message — no restart." ## Native Database & Analysis Tools diff --git a/backend/cli/src/agent/prompt/ml.txt b/backend/cli/src/agent/prompt/ml.txt index f6fcf30a..4ff811ca 100644 --- a/backend/cli/src/agent/prompt/ml.txt +++ b/backend/cli/src/agent/prompt/ml.txt @@ -57,7 +57,8 @@ Before using any external service, verify credentials are set: ``` Common: `TINKER_API_KEY`, `MODAL_TOKEN_ID`+`MODAL_TOKEN_SECRET`, `HF_TOKEN`, `WANDB_API_KEY`, `PRIME_API_KEY`, `TENSORPOOL_KEY`. If a service is not connected, tell the user to connect it -in Settings → Credentials and restart, then continue with what you can do. +in Settings → Credentials (GPU provider keys: Settings → Compute); it is picked up on the +next message, no restart. Continue with what you can do meanwhile. ## CRITICAL: Environment Setup Before running ANY Python, set up an isolated environment in the working directory: diff --git a/backend/cli/src/agent/prompt/research.txt b/backend/cli/src/agent/prompt/research.txt index 2a9269c5..0b3ea22c 100644 --- a/backend/cli/src/agent/prompt/research.txt +++ b/backend/cli/src/agent/prompt/research.txt @@ -42,7 +42,8 @@ Before using any external service, verify credentials: [ -n "$VAR_NAME" ] && echo "set" || echo "not set" If not connected, tell the user: - "Connect [service] at https://app.syntheticsciences.ai -> Services, then restart openscience." + "Connect [service] at https://app.syntheticsciences.ai -> Services (GPU provider keys: + Settings ▸ Compute). A newly connected key is picked up on the next message — no restart." ## CRITICAL: Convergence & Anti-Loop Your job is to finish the task IN THIS SESSION, not to hand it off. @@ -251,12 +252,17 @@ implementation in up-to-date usage rather than potentially stale training data. ### Stage 5: COMPUTE Execute computational work. If `methodology.md` exists, follow the pipeline defined there. -- Managed compute (Daytona-backed) runs through the bundled `atlas` CLI when your Atlas - session is active. Run `atlas doctor --format=json` first; if it reports the CLI is - unavailable/unauthenticated, print a one-line note and fall back to the BYOK cloud-compute - skills below (Modal, Tinker, TensorPool, Prime Intellect, HF Jobs) — never block on it. -- Load: `modal-research-gpu` for GPU-accelerated scientific computing -- Load: `modal` for general serverless GPU (inference, serving) +- Call the `compute_status` tool before launching any GPU work. It reports how compute is + funded right now — `byok`, `managed`, or `none` — which providers are usable, and the rule + that applies. This is the only compute-availability signal — CLI auth status is not one. +- If it returns `byok`, load the cloud-compute skill for one of the providers it lists — + when Modal is one of them, `modal-research-gpu` for GPU-accelerated scientific computing + or `modal-serverless-gpu` for general serverless GPU (inference, serving). Never load a + compute skill for a provider the tool did not list. +- If it returns `managed`, do not launch GPU work either — OpenScience cannot start a + managed lease yet. Follow the rule the tool returns and tell the user what it says. +- If it returns `none`, do not launch GPU work — tell the user to connect a provider key + in Settings ▸ Compute. - Load: domain libraries as needed (see Scientific Computing skills) - Present cost estimate and get approval before launching jobs - Run computations, monitor progress, collect outputs @@ -350,7 +356,7 @@ Tracking: weights-and-biases, mlflow, tensorboard, langsmith, phoenix ### Inference & Deployment — serving, quantization, benchmarking Serving: vllm, sglang, tensorrt-llm, llama-cpp, outlines Quantization: bitsandbytes, awq, gptq, hqq, gguf -Platforms: modal, lambda-labs, tensorpool, skypilot, fireworks-ai, groq, together-ai +Platforms: modal-serverless-gpu, lambda-labs, tensorpool, skypilot, fireworks-ai, groq, together-ai Eval: lm-evaluation-harness, bigcode-evaluation-harness, llm-as-judge-evaluation, hugging-face-evaluation ### Data & Embeddings — DataFrames, datasets, tokenizers, vector stores @@ -384,7 +390,7 @@ Writing: scientific-writing, ml-paper-writing, research-grants, venue-templates, Docs: scientific-slides, paper-2-web, latex-posters, pptx-posters, markitdown, market-research-reports ### Cloud Compute — GPU provisioning, serverless, distributed -GPU: modal-research-gpu, modal, lambda-labs, tensorpool, prime-intellect-lab, skypilot +GPU: modal-research-gpu, modal-serverless-gpu, lambda-labs, tensorpool, prime-intellect-lab, skypilot Distributed: ray-train, ray-data Managed: tinker, hugging-face-jobs Utility: get-available-resources diff --git a/backend/cli/src/compute/mode.ts b/backend/cli/src/compute/mode.ts new file mode 100644 index 00000000..c325ccea --- /dev/null +++ b/backend/cli/src/compute/mode.ts @@ -0,0 +1,257 @@ +/** + * Runtime resolution of how GPU compute is funded. + * + * `billing.compute` used to answer this from config alone, which meant a + * brand-new user with zero provider keys resolved to "byok" — claiming BYOK + * with nothing to BYOK with. This module answers it from the environment + * instead, and can say "none", which is the state we previously handled worst. + * + * Resolution deliberately happens ON DEMAND and never at startup. Provider keys + * reach process.env from three places — the user's shell, the Credentials panel + * (`applyCredentialEnv`, src/index.ts:102) and the Compute panel + * (`applyComputeEnv`, src/index.ts:106) — and the latter two are wrapped in + * `.catch(() => {})`. Detecting at boot would report "none" for a user whose + * keys are configured through the UI. Both call sites (SkillTool.init and the + * compute_status tool) run per request, long after those injections, so the + * ordering constraint cannot be violated and cannot silently regress if someone + * reorders src/index.ts later. + */ +import { Config } from "@/config/config" +import { API_BASE, OpenScience } from "@/openscience" + +export namespace ComputeMode { + export type Source = "byok" | "managed" | "none" + + /** + * `env` is a list of ALTERNATIVE groups; a group is satisfied when every var in + * it is set and non-empty. Modal is the only pair — its single pasted key + * splits into a token id + secret, and a half-pasted one maps to nothing + * (mirroring `mapProviderEnv`, server/routes/settings/compute.ts:181). + * + * `skills` are frontmatter `name` values, NOT directory names and NOT + * category-prefixed. Only these names are subject to mode filtering; the other + * cloud-compute skills (tinker, skypilot, fireworks, together) are inference + * APIs and orchestrators keyed by their own credentials, not GPU leases this + * mode governs, and are never hidden. + */ + export const PROVIDERS: Record = { + modal: { + env: [["MODAL_TOKEN_ID", "MODAL_TOKEN_SECRET"]], + skills: ["modal-serverless-gpu", "modal-ml-training", "modal-research-gpu"], + }, + lambda: { + env: [["LAMBDA_API_KEY"], ["LAMBDA_LABS_API_KEY"]], + skills: ["lambda-labs-gpu-cloud"], + }, + tensorpool: { + env: [["TENSORPOOL_KEY"], ["TENSORPOOL_API_KEY"]], + skills: ["tensorpool-gpu-cloud"], + }, + prime: { + env: [["PRIME_API_KEY"], ["PRIME_INTELLECT_API_KEY"]], + skills: ["prime-intellect-lab"], + }, + runpod: { + env: [["RUNPOD_API_KEY"]], + skills: [], + }, + vast: { + env: [["VAST_API_KEY"]], + skills: [], + }, + } + + /** Every provider skill name — the exact set the catalog filter operates on. */ + export const SKILLS = new Set(Object.values(PROVIDERS).flatMap((p) => p.skills)) + + /** Read process.env directly rather than Env.get: applyComputeEnv writes to + * process.env first and mirrors to Env only when instance state exists, so + * process.env is the one source that is always populated. */ + function keyed(groups: string[][]): boolean { + return groups.some((group) => group.every((name) => !!process.env[name])) + } + + /** + * The credentialed GPU providers, in declaration order. + * + * A credential is the whole test. An earlier revision also required a + * matching skill, on the theory that a provider with no skill gives the agent + * nothing to act on — but a capable agent drives a documented cloud API from a + * key, so that conjunction only produced a false "no compute available" for + * users holding a perfectly workable key. A skill, where one exists, is a + * quality boost; the catalog filter still offers a provider's skills only when + * that provider is credentialed. + */ + export function usable(): string[] { + return Object.keys(PROVIDERS).filter((id) => keyed(PROVIDERS[id].env)) + } + + /** + * WHY `mode` is what it is — the one thing a caller cannot reconstruct from + * `mode` alone, and the thing that decides what advice is even actionable. + * "managed" reached from the environment means the user holds no provider + * credential, so connecting one flips the next call to byok. "managed" + * reached from `billing.compute` means the setting pins it: connecting a + * credential changes neither `resolve()` (`funded()` never consults + * `providers`) nor `offered()` (empty under a managed override), so telling + * the user to connect one is advice the setting itself defeats. The override + * value is carried, not just a forced/not-forced bit, because "none" is + * reachable from BOTH overrides and they need opposite advice. + */ + export type Origin = "environment" | "config:byok" | "config:managed" + + export interface Resolution { + mode: Source + /** Credentialed BYOK providers, in PROVIDERS declaration order. */ + providers: string[] + /** + * Whether managed compute is available — `undefined` when nothing measured + * it. The byok arms return before `available()` runs (that skip is the + * point: a byok user never pays for the round trip), so they have no + * verdict to report. Absence means "not checked", never "no": a hardcoded + * `false` there would be a fact the code never established, which is the + * exact defect this module exists to remove. + */ + managed?: boolean + /** Wallet balance in USD. Present only when mode === "managed". */ + balance?: number + origin: Origin + } + + /** Hard ceiling on how long resolution may block an agent turn. Atlas's own + * 60s default is far too long to sit in front of a tool call; a slow or + * hanging backend must degrade to "none", not stall the turn. */ + const TIMEOUT = 3_000 + + /** Short in-process TTL, enough to stop a chatty agent hammering the endpoint + * inside one turn and no longer. The whole reason this is a tool rather than + * a prompt injection is that the answer changes mid-session, so a long cache + * would reintroduce exactly the staleness the tool exists to avoid. */ + const TTL = 5_000 + + let cache: { at: number; value: { managed: boolean; balance?: number } } | undefined + + /** Drop the availability cache. Called by tests; also safe after a connect. */ + export function invalidate() { + cache = undefined + } + + /** + * One authenticated call to /api/compute/options, which already annotates each + * provider with `funding` — "managed" when reselling is on and an operator key + * exists, else "unavailable". A failed, unauthenticated or timed-out call is + * treated as UNAVAILABLE: failing toward "none" produces an honest "connect a + * key" message, whereas failing toward "managed" would reproduce the bug this + * design exists to fix, promising a capability we never confirmed. + */ + async function available() { + if (cache && Date.now() - cache.at < TTL) return cache.value + const value = await probe() + cache = { at: Date.now(), value } + return value + } + + async function probe(): Promise<{ managed: boolean; balance?: number }> { + const session = await OpenScience.getSession().catch(() => null) + if (!session) return { managed: false } + try { + const res = await fetch(`${API_BASE}/api/compute/options`, { + headers: { Authorization: `Bearer ${session.api_key}` }, + signal: AbortSignal.timeout(TIMEOUT), + }) + if (!res.ok) return { managed: false } + const data = await res.json() + const providers = Array.isArray(data?.providers) ? data.providers : [] + const managed = providers.some((entry: { funding?: string }) => entry?.funding === "managed") + if (!managed) return { managed: false } + const cents = data?.cli_effective_balance_cents + return { managed: true, balance: typeof cents === "number" ? cents / 100 : undefined } + } catch { + return { managed: false } + } + } + + /** Build a Resolution purely from an availability probe's verdict — never + * from `providers`, which is passed through only for display. Shared by the + * managed-override arm and the no-override/no-provider fallback: both trust + * `available()` completely and must never fall back to "byok" just because + * a credential happens to be present (that would silently defeat the + * managed override — see the "managed with a usable provider" test). */ + function funded(providers: string[], state: { managed: boolean; balance?: number }, origin: Origin): Resolution { + return { + mode: state.managed ? "managed" : "none", + providers, + managed: state.managed, + balance: state.managed ? state.balance : undefined, + origin, + } + } + + /** + * The single shared entry point. `billing.compute` is an OVERRIDE, not the + * source of truth: it may narrow the outcome to "none", but it may never + * manufacture a capability that isn't there. + */ + export async function resolve(): Promise { + const providers = usable() + const override = (await Config.get()).billing?.compute + + if (override === "byok") { + return { mode: providers.length ? "byok" : "none", providers, origin: "config:byok" } + } + + if (override === "managed") { + return funded(providers, await available(), "config:managed") + } + + // BYOK wins when a credentialed provider is present: it is free to the user, + // it works today, and it needs nothing from Atlas. This is also why a BYOK + // user never pays for the availability call — and why `managed` is left + // unset here rather than false (see Resolution.managed). + if (providers.length) return { mode: "byok", providers, origin: "environment" } + + return funded(providers, await available(), "environment") + } + + /** + * The skill names the catalog filter should offer. The invariant is two + * one-way implications, not a single "iff" on non-emptiness: + * - `offered()` non-empty IMPLIES `resolve()`'s mode is "byok" — only the + * funded path ever offers anything. + * - mode "byok" IMPLIES `offered()` equals the credentialed providers' + * skills exactly, which is the EMPTY set when those providers carry no + * catalogued skill (`runpod`, `vast`: `skills: []`). A byok user with + * only a RunPod key correctly sees nothing offered here — RunPod has no + * skill to offer, though the agent can still drive its API directly. + * Empty in every other mode too, including "managed" with a real credential + * sitting unused (see `Resolution.providers`'s doc comment) — offering it + * there would dangle the user's own uncapped provider account in front of + * an agent that has just been told not to touch it. + * + * This mirrors `resolve()`'s byok arms exactly but never reaches the + * availability probe, because it doesn't need to: whether the mode is + * "byok" is fully decided by `usable()` (env-only) and the override alone. + * + * - override "byok" -> byok iff a credential exists; no network either way. + * - override "managed" -> mode is "managed" or "none"; either way offered is empty. + * - unset -> byok iff a credential exists; the network arm only + * runs when there are no credentials, and then offered + * is empty regardless of what it returns. + * + * `available()`/`probe()` — the per-LLM-step Atlas round trip this function + * exists to eliminate — is never reached here. The only I/O is + * `Config.get()`, which `Instance.state` memoizes per project instance + * after its first read within that instance; that first read can itself + * issue a fetch when a `wellknown` auth entry is configured + * (config.ts:82-105), but that cost belongs to `Config.get()` and is paid + * at most once per instance, not once per step — it is not a cost this + * function adds. + */ + export async function offered(): Promise> { + const providers = usable() + const override = (await Config.get()).billing?.compute + if (override === "managed") return new Set() + if (!providers.length) return new Set() + return new Set(providers.flatMap((id) => PROVIDERS[id].skills)) + } +} diff --git a/backend/cli/src/config/config.ts b/backend/cli/src/config/config.ts index 5660a1f8..e99a1730 100644 --- a/backend/cli/src/config/config.ts +++ b/backend/cli/src/config/config.ts @@ -1057,9 +1057,10 @@ export namespace Config { ), compute: z .enum(["managed", "byok"]) + .nullable() .optional() .describe( - "How GPU/compute is paid for. 'managed' runs on Atlas-provisioned compute billed to your wallet (via the bundled atlas CLI); 'byok' uses your own connected GPU providers (Modal, Tinker, TensorPool, …). Unset = byok.", + "How GPU/compute is paid for. 'managed' runs on Atlas-provisioned compute billed to your wallet; 'byok' uses your own connected GPU providers (Modal, Lambda Labs, TensorPool, Prime Intellect, RunPod, Vast.ai). Unset or null = auto-detect from your connected providers. Setting this can only narrow the result — if the mode you pick isn't actually available, compute resolves to none rather than pretending.", ), }) .optional() diff --git a/backend/cli/src/sandbox/sandbox.ts b/backend/cli/src/sandbox/sandbox.ts index 2f261ffc..23883d47 100644 --- a/backend/cli/src/sandbox/sandbox.ts +++ b/backend/cli/src/sandbox/sandbox.ts @@ -259,6 +259,16 @@ export namespace Sandbox { // Whole fs read-only, a fresh /dev and /proc, and a throwaway writable /tmp; // then re-mount the bits that must be writable on top. const args = ["--ro-bind", "/", "/", "--dev", "/dev", "--proc", "/proc", "--tmpfs", "/tmp"] + // $HOME is deliberately refused as a writable root (tooBroadToConfine), which + // leaves the whole-fs --ro-bind covering the XDG cache dir too. Startup tools + // that write there unconditionally (zsh's compdump/history lock, pip/npm/uv + // caches, ...) then fail with "Read-only file system". A tmpfs — not a bind — + // is the fix: writes succeed so those tools stop erroring, but nothing here + // persists to the real home, so the containment tooBroadToConfine enforces + // stays intact. Mounted before the writable binds below, so an explicitly + // writable path under the cache dir still wins. + const cache = process.env.XDG_CACHE_HOME || path.join(os.homedir(), ".cache") + args.push("--tmpfs", cache) for (const p of dedupe(policy.writable)) { // Skip only the /tmp mount root itself — it is provided as a fresh tmpfs and // re-binding host /tmp would defeat it. A workspace that lives *under* /tmp diff --git a/backend/cli/src/server/routes/settings/billing.ts b/backend/cli/src/server/routes/settings/billing.ts index b551c5b2..b8fecf96 100644 --- a/backend/cli/src/server/routes/settings/billing.ts +++ b/backend/cli/src/server/routes/settings/billing.ts @@ -10,11 +10,13 @@ const log = Log.create({ service: "settings-billing" }) // The two independent spend toggles (Settings → Spend), backed by the strict // config (`billing.llm` / `billing.compute`). "managed" runs on Credits; -// "byok" runs on the user's own keys/OAuth and is never billed. LLM is nullable -// (unset = auto-detect from the resolved credential); compute defaults to byok. +// "byok" runs on the user's own keys/OAuth and is never billed. Both are +// nullable: unset/null means auto-detect — llm from the resolved credential, +// compute from ComputeMode.resolve() (connected providers, then managed +// availability). export const BillingState = z.object({ llm: z.enum(["managed", "byok"]).nullable(), - compute: z.enum(["managed", "byok"]), + compute: z.enum(["managed", "byok"]).nullable(), wallet: z.object({ signedIn: z.boolean().describe("Whether an Atlas session (thk_ key) is available"), balanceUsd: z.number().describe("Credit balance in USD; -1 when signed out or unavailable"), @@ -22,11 +24,11 @@ export const BillingState = z.object({ }) export type BillingState = z.infer -// `llm: null` sets the toggle back to auto (auto-detect from the resolved -// credential); omitting a field leaves it untouched. +// `llm: null` / `compute: null` sets the toggle back to auto (auto-detect); +// omitting a field leaves it untouched. const BillingPatch = z.object({ llm: z.enum(["managed", "byok"]).nullable().optional(), - compute: z.enum(["managed", "byok"]).optional(), + compute: z.enum(["managed", "byok"]).nullable().optional(), }) async function readState(): Promise { @@ -35,7 +37,7 @@ async function readState(): Promise { const balanceUsd = (session ? await OpenScience.getBalance().catch(() => null) : null) ?? -1 return { llm: cfg.billing?.llm ?? null, - compute: cfg.billing?.compute ?? "byok", + compute: cfg.billing?.compute ?? null, wallet: { signedIn: !!session, balanceUsd }, } } diff --git a/backend/cli/src/session/billing-gate.ts b/backend/cli/src/session/billing-gate.ts index cc77ec2b..a77650d9 100644 --- a/backend/cli/src/session/billing-gate.ts +++ b/backend/cli/src/session/billing-gate.ts @@ -30,11 +30,6 @@ export async function llmBillingMode(): Promise { return (await Config.get()).billing?.llm ?? undefined } -/** The user-facing compute spend toggle. Defaults to "byok" (own GPU providers). */ -export async function computeBillingMode(): Promise { - return (await Config.get()).billing?.compute ?? "byok" -} - /** First-party providers whose OAuth path runs on the user's own subscription * and never debits Credits. */ const OAUTH_FREE_PROVIDERS = new Set([ diff --git a/backend/cli/src/session/prompt.ts b/backend/cli/src/session/prompt.ts index 7c5ae284..74b6cff6 100644 --- a/backend/cli/src/session/prompt.ts +++ b/backend/cli/src/session/prompt.ts @@ -43,7 +43,6 @@ import { Command } from "../command" import { $, fileURLToPath } from "bun" import { ConfigMarkdown } from "../config/markdown" import { Config } from "../config/config" -import { computeBillingMode } from "./billing-gate" import { SessionSummary } from "./summary" import { NamedError } from "@synsci/util/error" import { fn } from "@/util/fn" @@ -1547,19 +1546,18 @@ export namespace SessionPrompt { const userMessage = input.messages.findLast((msg) => msg.info.role === "user") if (!userMessage) return input.messages - // Compute spend preference — make the user's explicit managed/BYOK choice - // authoritative for GPU work. Only injected when the toggle is explicitly set - // (unset = the agent's own atlas-doctor-driven default, unchanged). - if (COMPUTE_AGENTS.has(input.agent.name) && (await Config.get()).billing?.compute) { - const managed = (await computeBillingMode()) === "managed" + // Compute funding is PULLED from the `compute_status` tool, not injected — + // the mode changes mid-session (a key connected in Settings ▸ Compute at + // turn 3 makes a reminder injected then false by turn 12). This line is a + // stateless pointer: it carries no mode, so it can never go stale, and it + // closes the gap where an agent reaches for bash without ever looking. + if (COMPUTE_AGENTS.has(input.agent.name)) { userMessage.parts.push({ id: Identifier.ascending("part"), messageID: userMessage.info.id, sessionID: userMessage.info.sessionID, type: "text", - text: managed - ? "Compute spend is set to MANAGED. Run GPU/training work through the bundled `atlas compute` CLI (e.g. `atlas compute:up`), which bills Credits. Do not fall back to the user's own GPU providers unless `atlas doctor` reports managed compute unavailable." - : "Compute spend is set to BYOK. Run GPU/training work on the user's own connected providers (Modal, Tinker, TensorPool, …) via the cloud-compute skills — do not launch managed `atlas compute` leases that bill Credits.", + text: "Call `compute_status` before running GPU, training, or cluster work. It reports how compute is funded and which providers are usable right now.", synthetic: true, }) } diff --git a/backend/cli/src/tool/compute.ts b/backend/cli/src/tool/compute.ts new file mode 100644 index 00000000..bbe133d4 --- /dev/null +++ b/backend/cli/src/tool/compute.ts @@ -0,0 +1,138 @@ +import z from "zod" +import { Tool } from "./tool" +import { ComputeMode } from "@/compute/mode" + +/** + * The agent PULLS its compute mode from here; nothing is injected per turn. + * + * An earlier design injected mode guidance into every turn. That was wrong for a + * reason that matters more than token cost: the mode changes mid-session. A user + * connects a Modal key in Settings ▸ Compute while a session is running, and a + * reminder injected at turn 3 is false by turn 12. A tool returns the state at + * the moment it is asked. + * + * The DESCRIPTION carries the constraint — it reaches the agent before it starts + * down a path, which is the one thing an injection did well, and tool definitions + * are in every request regardless, so it costs nothing extra. The RESULT carries + * the specifics. Adding rates or a balance to an every-turn injection would be + * expensive; adding them here is free. + * + * Every string below is bound by one rule: never name a capability that does not + * exist. That is what mode resolution enforces for `mode` itself, and the + * guidance text sits one layer above it, where the same defect keeps reappearing + * in a different shape. + */ + +/** + * The two ways out of a mode that cannot run GPU work, and they are mutually + * exclusive. `origin` decides which one is true (see ComputeMode.Origin): a + * config-pinned "managed" makes CONNECT a lie, because `funded()` never reads + * `providers` and `offered()` returns empty under that override — the key would + * be connected, ignored, and its skills still hidden. + */ +const CONNECT = + "Tell the user to connect a provider key in Settings ▸ Compute — a key connected there is picked up on the next call, no restart, and BYOK is the only path that runs GPU work today." +const PINNED = + '`billing.compute` is pinned to "managed" in the config, so connecting a provider key will not switch this session to byok — the setting has to be changed or removed first.' + +function escape(origin: ComputeMode.Origin): string { + return origin === "config:managed" ? PINNED : CONNECT +} + +const BYOK = + "Run GPU work on the user's connected providers via the cloud-compute skills. Do not launch managed leases — they bill Credits and are not the funded path here." + +/** + * `managed` guidance is a function of balance AND origin, not a fixed string. + * + * Balance, because `GET /api/compute/options` reports a provider as managed + * whenever reselling is on and an operator key exists — availability and + * affordability are independent there, and live testing against a deployed + * backend with a zero wallet confirmed it (every lease acquire returns HTTP 402 + * insufficient_cli_credit). Only `balance === 0` counts as unaffordable: + * acquiring a lease requires an hour of the chosen SKU's rate up front, and + * rates span cents to dollars an hour depending on a catalog this tool never + * sees, so any non-zero cutoff would be a guess. `balance === undefined` (the + * probe succeeded but carried no balance field) is missing information, not an + * empty wallet, and is left alone. + * + * Origin, because the way out differs — see `escape`. + * + * Neither branch tells the agent to run the work. THERE IS NO MANAGED LAUNCH + * MECHANISM IN THIS CLIENT: `ComputeTools` is `[ComputeStatusTool]`, the only + * `/api/compute` call in the product is mode.ts's read-only `/options` probe, + * and `compute_launch`/`list`/`release` are Part B, unbuilt (docs/specs/ + * compute-design.md, "The gap Part A exposed"). A funded, keyless user — the + * default for anyone signed in — was previously told to do the one thing that + * cannot be done and forbidden the one thing that can. + * + * This does NOT change `state.mode`. Managed is genuinely configured; an empty + * wallet is missing funds and an unbuilt launch path is a missing tool, and + * neither is a missing capability at the resolution layer. Collapsing them + * would destroy distinctions the override rule in `ComputeMode.resolve` + * (narrow, never manufacture) exists to preserve. + */ +function managedGuidance(balance: number | undefined, origin: ComputeMode.Origin): string { + if (balance === 0) + return `Managed compute is configured, but the wallet is empty — every lease attempt would be refused (HTTP 402), and OpenScience cannot launch it in any case: this client has no managed-lease command yet, only this status tool. Do not launch GPU work through managed compute. Topping up in Settings ▸ Compute fixes the wallet but not the missing launch path. ${escape(origin)}` + return `Managed compute is funded, but OpenScience cannot launch it — this client has no managed-lease command yet, only this status tool. Do not launch GPU work through managed compute. ${escape(origin)}` +} + +/** + * No path to `none` is balance-related — `probe()` returns `managed: false` + * only for no session, a non-2xx response, a network/parse failure, or no + * provider with `funding: "managed"`, and Atlas reports managed regardless of + * balance. So "top up for managed compute" was a remedy that could never move a + * user out of this mode. + */ +function noneGuidance(origin: ComputeMode.Origin): string { + return `No compute is available. Do not attempt GPU work. ${escape(origin)}` +} + +/** + * Tri-state on purpose. Both byok arms return before the availability probe + * runs — that skip is the performance win, so there is no verdict to print and + * "no" would be an unmeasured claim. + */ +function availability(managed: boolean | undefined): string { + if (managed === undefined) return "not checked (byok takes precedence, so availability was never probed)" + return managed ? "yes" : "no" +} + +function guidance(state: ComputeMode.Resolution): string { + if (state.mode === "byok") return BYOK + if (state.mode === "managed") return managedGuidance(state.balance, state.origin) + return noneGuidance(state.origin) +} + +export const ComputeStatusTool = Tool.define("compute_status", { + description: [ + "Check how GPU compute is funded before running any GPU, training, or cluster work.", + "Returns one of byok, managed, or none, the providers available, and the rule that applies.", + "Call this first — the answer can change mid-session as the user connects or removes keys.", + ].join(" "), + parameters: z.object({}), + async execute(_params, _ctx) { + const state = await ComputeMode.resolve() + const lines = [ + `**mode**: ${state.mode}`, + `**providers**: ${state.providers.length ? state.providers.join(", ") : "none configured"}`, + `**managed available**: ${availability(state.managed)}`, + ] + if (state.balance !== undefined) lines.push(`**balance**: $${state.balance.toFixed(2)}`) + lines.push("", guidance(state)) + + return { + title: `Compute: ${state.mode}`, + output: lines.join("\n"), + metadata: { + mode: state.mode, + providers: state.providers, + managed_available: state.managed, + balance_usd: state.balance, + }, + } + }, +}) + +export const ComputeTools = [ComputeStatusTool] diff --git a/backend/cli/src/tool/registry.ts b/backend/cli/src/tool/registry.ts index 3f5096af..34e794cd 100644 --- a/backend/cli/src/tool/registry.ts +++ b/backend/cli/src/tool/registry.ts @@ -33,6 +33,7 @@ import { ArtifactTool } from "./artifact" import { LearnTool } from "./learn" import { ScienceTools } from "./science" import { ProvenanceTools } from "./provenance" +import { ComputeTools } from "./compute" import { NotebookTool } from "./notebook" import { RKernelTool } from "./rkernel" @@ -131,6 +132,7 @@ export namespace ToolRegistry { ...BiologyTools, ...ScienceTools, ...ProvenanceTools, + ...ComputeTools, NotebookTool, RKernelTool, ArtifactTool, diff --git a/backend/cli/src/tool/skill.ts b/backend/cli/src/tool/skill.ts index 48d1a5e9..bd2a3b34 100644 --- a/backend/cli/src/tool/skill.ts +++ b/backend/cli/src/tool/skill.ts @@ -8,6 +8,7 @@ import { PermissionNext } from "../permission/next" import { OpenScience } from "@/openscience" import { RSILifecycle } from "@/session/rsi/lifecycle" import { Global } from "@/global" +import { ComputeMode } from "@/compute/mode" // Lightweight fuzzy score: rewards substring containment + shared bigrams. // Returns 0..1. No external deps needed for a "did you mean?" hint. @@ -34,13 +35,34 @@ export const SkillTool = Tool.define("skill", async (ctx) => { // Filter skills by agent permissions if agent provided const agent = ctx?.agent - const accessibleSkills = agent + const permitted = agent ? skills.filter((skill) => { const rule = PermissionNext.evaluate("skill", skill.name, agent.permission) return rule.action !== "deny" }) : skills + // Filter the GPU provider skills by the resolved compute mode, so the agent + // picks the right provider because it is the only one offered. This init runs + // per request (registry.ts calls it inside tools()), which buys two things for + // free: a credential connected mid-session shows up on the next turn with no + // cache to invalidate, and resolution always happens after src/index.ts's env + // injections rather than racing them. + // + // ComputeMode.offered(), not resolve(): it answers the one question this + // filter needs (which skills are the FUNDED path) without the availability + // probe resolve() sometimes needs for the *mode label* — see its doc + // comment. That keeps this init synchronous-except-for-Config, so a + // signed-in user with no GPU keys never pays a per-step network round trip + // for a value this filter doesn't read. + // + // This is a LISTING filter, not a gate. `none` is guidance, not enforcement — + // a hidden skill can still be loaded by exact name, and the agent still has + // bash. Gating the load path is a larger change and is deliberately out of + // scope; see docs/specs/compute-design.md, Part A. + const offered = await ComputeMode.offered() + const accessibleSkills = permitted.filter((skill) => !ComputeMode.SKILLS.has(skill.name) || offered.has(skill.name)) + // Group skills by category for the description const categories: Record = {} const uncategorized: Skill.Info[] = [] diff --git a/backend/cli/test/compute/mode.test.ts b/backend/cli/test/compute/mode.test.ts new file mode 100644 index 00000000..4646a3fe --- /dev/null +++ b/backend/cli/test/compute/mode.test.ts @@ -0,0 +1,523 @@ +import { test, expect, afterEach, beforeEach, describe } from "bun:test" +import path from "path" +import fs from "fs/promises" +import { ComputeMode } from "../../src/compute/mode" +import { Instance } from "../../src/project/instance" +import { Global } from "../../src/global" +import { tmpdir } from "../fixture/fixture" + +const ENV = [ + "MODAL_TOKEN_ID", + "MODAL_TOKEN_SECRET", + "LAMBDA_API_KEY", + "LAMBDA_LABS_API_KEY", + "TENSORPOOL_KEY", + "TENSORPOOL_API_KEY", + "PRIME_API_KEY", + "PRIME_INTELLECT_API_KEY", + "RUNPOD_API_KEY", + "VAST_API_KEY", +] + +function clearEnv() { + for (const name of ENV) delete process.env[name] +} + +afterEach(clearEnv) + +/** A tmpdir project seeded with real SKILL.md files, so Skill.all() finds them + * without a network catalog. `OPENSCIENCE_DISABLE_BUNDLED_SKILLS` in preload.ts + * keeps the dev skills/ dir and the server index out, so the test controls the + * catalog exactly. */ +async function withSkills(names: string[], fn: () => T): Promise { + await using tmp = await tmpdir({ + git: true, + init: async (dir) => { + for (const name of names) { + await Bun.write( + path.join(dir, ".openscience", "skill", name, "SKILL.md"), + `---\nname: ${name}\ndescription: Test fixture for ${name}.\ncategory: cloud-compute\n---\n\n# ${name}\n`, + ) + } + }, + }) + return Instance.provide({ directory: tmp.path, fn }) +} + +describe("ComputeMode.usable", () => { + test("a provider with a key and a skill is usable", async () => { + clearEnv() + process.env["LAMBDA_API_KEY"] = "secret_abc" + const result = await withSkills(["lambda-labs-gpu-cloud"], () => ComputeMode.usable()) + expect(result).toEqual(["lambda"]) + }) + + test("the alternate env spelling also counts", async () => { + clearEnv() + process.env["LAMBDA_LABS_API_KEY"] = "secret_abc" + const result = await withSkills(["lambda-labs-gpu-cloud"], () => ComputeMode.usable()) + expect(result).toEqual(["lambda"]) + }) + + test("a key with NO catalogued skill IS usable — the agent drives the provider API directly", async () => { + clearEnv() + process.env["RUNPOD_API_KEY"] = "rpa_abc" + expect(await withSkills([], () => ComputeMode.usable())).toEqual(["runpod"]) + }) + + test("a catalogued skill with NO key is not usable", async () => { + clearEnv() + const result = await withSkills(["lambda-labs-gpu-cloud"], () => ComputeMode.usable()) + expect(result).toEqual([]) + }) + + test("modal needs BOTH token vars — id alone is not a key", async () => { + clearEnv() + process.env["MODAL_TOKEN_ID"] = "ak-abc" + const result = await withSkills(["modal-serverless-gpu"], () => ComputeMode.usable()) + expect(result).toEqual([]) + }) + + test("modal with both token vars is usable", async () => { + clearEnv() + process.env["MODAL_TOKEN_ID"] = "ak-abc" + process.env["MODAL_TOKEN_SECRET"] = "as-def" + const result = await withSkills(["modal-serverless-gpu"], () => ComputeMode.usable()) + expect(result).toEqual(["modal"]) + }) + + test("PROVIDERS pins the exact skill names the catalog filter matches on", async () => { + expect([...ComputeMode.SKILLS].sort()).toEqual( + [ + "lambda-labs-gpu-cloud", + "modal-ml-training", + "modal-research-gpu", + "modal-serverless-gpu", + "prime-intellect-lab", + "tensorpool-gpu-cloud", + ].sort(), + ) + expect(ComputeMode.PROVIDERS["runpod"].skills).toEqual([]) + expect(ComputeMode.PROVIDERS["vast"].skills).toEqual([]) + }) + + test("an empty-string key does not count as set", async () => { + clearEnv() + process.env["TENSORPOOL_KEY"] = "" + const result = await withSkills(["tensorpool-gpu-cloud"], () => ComputeMode.usable()) + expect(result).toEqual([]) + }) + + test("every provider resolves in isolation, given its own skill", async () => { + const cases: Array<[string, Record, string]> = [ + ["modal", { MODAL_TOKEN_ID: "ak-a", MODAL_TOKEN_SECRET: "as-b" }, "modal-serverless-gpu"], + ["lambda", { LAMBDA_API_KEY: "k" }, "lambda-labs-gpu-cloud"], + ["tensorpool", { TENSORPOOL_KEY: "k" }, "tensorpool-gpu-cloud"], + ["prime", { PRIME_API_KEY: "k" }, "prime-intellect-lab"], + ["runpod", { RUNPOD_API_KEY: "k" }, "runpod-gpu-cloud"], + ["vast", { VAST_API_KEY: "k" }, "vast-ai-gpu-cloud"], + ] + for (const [id, env, skill] of cases) { + clearEnv() + Object.assign(process.env, env) + const result = await withSkills([skill], () => ComputeMode.usable()) + expect(result).toEqual([id]) + } + }) + + test("SKILLS covers every name in PROVIDERS and nothing else", () => { + const required = [ + "modal-serverless-gpu", + "modal-ml-training", + "modal-research-gpu", + "lambda-labs-gpu-cloud", + "tensorpool-gpu-cloud", + "prime-intellect-lab", + ] + expect([...ComputeMode.SKILLS].sort()).toEqual([...required].sort()) + expect(ComputeMode.SKILLS.size).toBe(required.length) + }) + + test("a key injected after the first call is seen on the next call", async () => { + clearEnv() + await withSkills(["lambda-labs-gpu-cloud"], async () => { + expect(await ComputeMode.usable()).toEqual([]) + process.env["LAMBDA_API_KEY"] = "secret_late" + expect(await ComputeMode.usable()).toEqual(["lambda"]) + }) + }) + + test("providers keyed together return in PROVIDERS declaration order, not set order", async () => { + clearEnv() + process.env["VAST_API_KEY"] = "v" + process.env["MODAL_TOKEN_ID"] = "ak-a" + process.env["MODAL_TOKEN_SECRET"] = "as-b" + process.env["LAMBDA_API_KEY"] = "l" + const result = await withSkills(["vast-ai-gpu-cloud", "modal-serverless-gpu", "lambda-labs-gpu-cloud"], () => + ComputeMode.usable(), + ) + expect(result).toEqual(["modal", "lambda", "vast"]) + }) +}) + +const OPTIONS_URL = "/api/compute/options" +const SESSION = path.join(Global.Path.data, "openscience-session.json") +const realFetch = globalThis.fetch +const realNow = Date.now + +/** Record of every URL the resolver fetched, so "the call is skipped" is a + * positive assertion rather than an absence of failure. */ +let calls: string[] = [] + +function stubOptions(body: unknown, status = 200) { + globalThis.fetch = (async (input: string | URL | Request) => { + const url = String(input instanceof Request ? input.url : input) + calls.push(url) + if (!url.includes(OPTIONS_URL)) return realFetch(input as never) + return new Response(JSON.stringify(body), { status, headers: { "content-type": "application/json" } }) + }) as typeof fetch +} + +async function signIn() { + await fs.mkdir(Global.Path.data, { recursive: true }) + await Bun.write(SESSION, JSON.stringify({ api_key: "thk_test.secret", user_id: "u1" })) +} + +const MANAGED_ON = { + options: [], + providers: [ + { provider: "lambda", has_byok: false, has_operator: true, funding: "managed", count: 3 }, + { provider: "vast", has_byok: false, has_operator: false, funding: "unavailable", count: 0 }, + ], + resell_enabled: true, + cli_effective_balance_cents: 1234, +} + +const MANAGED_OFF = { + options: [], + providers: [{ provider: "lambda", has_byok: false, has_operator: false, funding: "unavailable", count: 0 }], + resell_enabled: false, + cli_effective_balance_cents: 1234, +} + +describe("ComputeMode.resolve", () => { + beforeEach(() => { + clearEnv() + calls = [] + ComputeMode.invalidate() + }) + + afterEach(async () => { + globalThis.fetch = realFetch + Date.now = realNow + await fs.rm(SESSION, { force: true }).catch(() => {}) + }) + + test("a usable provider resolves to byok WITHOUT calling the availability endpoint", async () => { + await signIn() + stubOptions(MANAGED_ON) + process.env["LAMBDA_API_KEY"] = "k" + const result = await withSkills(["lambda-labs-gpu-cloud"], () => ComputeMode.resolve()) + expect(result.mode).toBe("byok") + expect(result.providers).toEqual(["lambda"]) + expect(result.balance).toBeUndefined() + expect(calls.filter((url) => url.includes(OPTIONS_URL))).toEqual([]) + }) + + test("no keys plus managed available resolves to managed, with the balance", async () => { + await signIn() + stubOptions(MANAGED_ON) + const result = await withSkills(["lambda-labs-gpu-cloud"], () => ComputeMode.resolve()) + expect(result.mode).toBe("managed") + expect(result.managed).toBe(true) + expect(result.balance).toBe(12.34) + }) + + test("no keys plus managed unavailable resolves to none", async () => { + await signIn() + stubOptions(MANAGED_OFF) + const result = await withSkills(["lambda-labs-gpu-cloud"], () => ComputeMode.resolve()) + expect(result.mode).toBe("none") + expect(result.managed).toBe(false) + expect(result.balance).toBeUndefined() + }) + + test("a failing availability call resolves to none, not managed", async () => { + await signIn() + globalThis.fetch = (async (input: string | URL | Request): Promise => { + throw new Error("network down") + }) as typeof fetch + const result = await withSkills([], () => ComputeMode.resolve()) + expect(result.mode).toBe("none") + expect(result.managed).toBe(false) + }) + + test("a non-ok availability response resolves to none", async () => { + await signIn() + stubOptions({ detail: "unauthorized" }, 401) + const result = await withSkills([], () => ComputeMode.resolve()) + expect(result.mode).toBe("none") + }) + + test("no session means managed is unavailable and no call is made", async () => { + await fs.rm(SESSION, { force: true }).catch(() => {}) + stubOptions(MANAGED_ON) + const result = await withSkills([], () => ComputeMode.resolve()) + expect(result.mode).toBe("none") + expect(calls.filter((url) => url.includes(OPTIONS_URL))).toEqual([]) + }) + + test("a key with no skill still resolves to byok and skips the availability call", async () => { + await signIn() + stubOptions(MANAGED_ON) + process.env["RUNPOD_API_KEY"] = "rpa_x" + const result = await withSkills([], () => ComputeMode.resolve()) + expect(result.mode).toBe("byok") + expect(result.providers).toEqual(["runpod"]) + expect(calls.filter((url) => url.includes(OPTIONS_URL))).toEqual([]) + }) + + test("a byok resolution leaves managed availability UNMEASURED, not false", async () => { + // The byok arms deliberately skip the availability probe — that skip is the + // performance win. `managed: false` there would be an unmeasured claim, and + // the tool prints it as a fact ("managed available: no"). + await signIn() + stubOptions(MANAGED_ON) + process.env["LAMBDA_API_KEY"] = "k" + const result = await withSkills(["lambda-labs-gpu-cloud"], () => ComputeMode.resolve()) + expect(result.mode).toBe("byok") + expect(result.managed).toBeUndefined() + expect(calls.filter((url) => url.includes(OPTIONS_URL))).toEqual([]) + }) + + test("a probed resolution reports availability as a measured boolean", async () => { + await signIn() + stubOptions(MANAGED_ON) + expect((await withSkills([], () => ComputeMode.resolve())).managed).toBe(true) + ComputeMode.invalidate() + stubOptions(MANAGED_OFF) + expect((await withSkills([], () => ComputeMode.resolve())).managed).toBe(false) + }) + + test("with no override, every arm records origin 'environment'", async () => { + await signIn() + stubOptions(MANAGED_ON) + process.env["LAMBDA_API_KEY"] = "k" + expect((await withSkills(["lambda-labs-gpu-cloud"], () => ComputeMode.resolve())).origin).toBe("environment") + clearEnv() + ComputeMode.invalidate() + expect((await withSkills([], () => ComputeMode.resolve())).origin).toBe("environment") + ComputeMode.invalidate() + stubOptions(MANAGED_OFF) + expect((await withSkills([], () => ComputeMode.resolve())).origin).toBe("environment") + }) + + test("the availability answer is cached within the TTL", async () => { + await signIn() + stubOptions(MANAGED_ON) + let now = realNow() + Date.now = () => now + await withSkills([], async () => { + await ComputeMode.resolve() + now += 4_999 // still inside the 5s TTL + await ComputeMode.resolve() + }) + expect(calls.filter((url) => url.includes(OPTIONS_URL)).length).toBe(1) + }) + + test("the cache expires once the TTL elapses, forcing a re-probe", async () => { + await signIn() + stubOptions(MANAGED_ON) + let now = realNow() + Date.now = () => now + await withSkills([], async () => { + const first = await ComputeMode.resolve() + expect(first.mode).toBe("managed") + now += 5_001 // past the 5s TTL — the cached verdict must be treated as stale + stubOptions(MANAGED_OFF) + const second = await ComputeMode.resolve() + expect(second.mode).toBe("none") + }) + expect(calls.filter((url) => url.includes(OPTIONS_URL)).length).toBe(2) + }) + + test("invalidate() drops the cache", async () => { + await signIn() + stubOptions(MANAGED_ON) + await withSkills([], async () => { + await ComputeMode.resolve() + ComputeMode.invalidate() + await ComputeMode.resolve() + }) + expect(calls.filter((url) => url.includes(OPTIONS_URL)).length).toBe(2) + }) +}) + +describe("ComputeMode.resolve override", () => { + beforeEach(() => { + clearEnv() + calls = [] + ComputeMode.invalidate() + }) + afterEach(async () => { + globalThis.fetch = realFetch + await fs.rm(SESSION, { force: true }).catch(() => {}) + }) + + /** Same tmpdir fixture as withSkills, plus an openscience.json setting + * billing.compute. */ + async function withOverride(mode: "byok" | "managed", skills: string[], fn: () => Promise): Promise { + await using tmp = await tmpdir({ + git: true, + init: async (dir) => { + for (const name of skills) { + await Bun.write( + path.join(dir, ".openscience", "skill", name, "SKILL.md"), + `---\nname: ${name}\ndescription: Test fixture for ${name}.\ncategory: cloud-compute\n---\n\n# ${name}\n`, + ) + } + await Bun.write(path.join(dir, "openscience.json"), JSON.stringify({ billing: { compute: mode } })) + }, + }) + return Instance.provide({ directory: tmp.path, fn }) + } + + test("override byok with a usable provider stays byok", async () => { + await signIn() + stubOptions(MANAGED_ON) + process.env["LAMBDA_API_KEY"] = "k" + const result = await withOverride("byok", ["lambda-labs-gpu-cloud"], () => ComputeMode.resolve()) + expect(result.mode).toBe("byok") + }) + + test("override byok with NO usable provider narrows to none, never managed", async () => { + await signIn() + stubOptions(MANAGED_ON) + const result = await withOverride("byok", ["lambda-labs-gpu-cloud"], () => ComputeMode.resolve()) + expect(result.mode).toBe("none") + expect(calls.filter((url) => url.includes(OPTIONS_URL))).toEqual([]) + }) + + test("override managed with managed unavailable narrows to none", async () => { + await signIn() + stubOptions(MANAGED_OFF) + const result = await withOverride("managed", [], () => ComputeMode.resolve()) + expect(result.mode).toBe("none") + }) + + test("override managed with a usable provider still narrows to none when managed is unavailable", async () => { + await signIn() + stubOptions(MANAGED_OFF) + process.env["LAMBDA_API_KEY"] = "k" + const result = await withOverride("managed", ["lambda-labs-gpu-cloud"], () => ComputeMode.resolve()) + expect(result.mode).toBe("none") + // The credential is real — it's just not the funded path under a forced + // managed override, so it must still be reported, not hidden. + expect(result.providers).toEqual(["lambda"]) + }) + + test("an override stamps the origin with the setting that forced the mode", async () => { + // A caller cannot otherwise tell "managed because the environment says so" + // (where connecting a key flips to byok next call) from "managed because + // billing.compute pins it" (where connecting a key changes nothing) — and + // those two states need opposite advice. + await signIn() + stubOptions(MANAGED_ON) + process.env["LAMBDA_API_KEY"] = "k" + expect((await withOverride("byok", ["lambda-labs-gpu-cloud"], () => ComputeMode.resolve())).origin).toBe( + "config:byok", + ) + ComputeMode.invalidate() + expect((await withOverride("managed", ["lambda-labs-gpu-cloud"], () => ComputeMode.resolve())).origin).toBe( + "config:managed", + ) + // Narrowed to "none" the origin still has to name the setting that narrowed it. + ComputeMode.invalidate() + stubOptions(MANAGED_OFF) + const narrowed = await withOverride("managed", ["lambda-labs-gpu-cloud"], () => ComputeMode.resolve()) + expect(narrowed.mode).toBe("none") + expect(narrowed.origin).toBe("config:managed") + }) + + test("override managed beats a usable provider when managed IS available", async () => { + await signIn() + stubOptions(MANAGED_ON) + process.env["LAMBDA_API_KEY"] = "k" + const result = await withOverride("managed", ["lambda-labs-gpu-cloud"], () => ComputeMode.resolve()) + expect(result.mode).toBe("managed") + }) +}) + +describe("ComputeMode.offered", () => { + beforeEach(() => { + clearEnv() + calls = [] + ComputeMode.invalidate() + }) + afterEach(async () => { + globalThis.fetch = realFetch + await fs.rm(SESSION, { force: true }).catch(() => {}) + }) + + /** Prove the equivalence by construction rather than by example: across + * every combination of credential presence, override, and managed + * availability, `offered()` must equal `resolve()`'s providers' skills + * exactly when the resolved mode is "byok" (empty otherwise, including + * when byok's providers carry no catalogued skill), and must never touch + * the network — a positive assertion (an empty recorded call list), not + * merely the absence of a thrown error. */ + test("offered() equals resolve()'s byok providers' skills when mode is byok, empty otherwise, and never calls the availability endpoint", async () => { + const overrides = [undefined, "byok", "managed"] as const + for (const credential of [true, false]) { + for (const override of overrides) { + for (const managedAvailable of [true, false]) { + clearEnv() + calls = [] + ComputeMode.invalidate() + await signIn() + stubOptions(managedAvailable ? MANAGED_ON : MANAGED_OFF) + if (credential) process.env["LAMBDA_API_KEY"] = "k" + + await using tmp = await tmpdir({ + git: true, + init: async (dir) => { + await Bun.write( + path.join(dir, ".openscience", "skill", "lambda-labs-gpu-cloud", "SKILL.md"), + `---\nname: lambda-labs-gpu-cloud\ndescription: Test fixture.\ncategory: cloud-compute\n---\n\n# lambda-labs-gpu-cloud\n`, + ) + if (override) { + await Bun.write(path.join(dir, "openscience.json"), JSON.stringify({ billing: { compute: override } })) + } + }, + }) + + await Instance.provide({ + directory: tmp.path, + fn: async () => { + const state = `credential=${credential} override=${override ?? "unset"} managedAvailable=${managedAvailable}` + const resolved = await ComputeMode.resolve() + calls = [] // isolate the assertion below to offered()'s own network usage + // resolve() may have just warmed the 5s availability cache — without + // dropping it, a probe-hitting offered() would be served from cache + // and never reach fetch, so the "no network call" assertion below + // would pass even for an implementation that calls available(). + ComputeMode.invalidate() + const result = await ComputeMode.offered() + + expect(calls.filter((url) => url.includes(OPTIONS_URL))).toEqual([]) + + if (resolved.mode === "byok") { + expect(result.size, state).toBeGreaterThan(0) + expect([...result].sort(), state).toEqual( + resolved.providers.flatMap((id) => ComputeMode.PROVIDERS[id].skills).sort(), + ) + } else { + expect(result.size, state).toBe(0) + } + }, + }) + } + } + } + }) +}) diff --git a/backend/cli/test/sandbox/sandbox.test.ts b/backend/cli/test/sandbox/sandbox.test.ts index 503f7c6f..9d0e939e 100644 --- a/backend/cli/test/sandbox/sandbox.test.ts +++ b/backend/cli/test/sandbox/sandbox.test.ts @@ -1,5 +1,6 @@ import { describe, expect, test } from "bun:test" import os from "os" +import path from "path" import { Sandbox } from "../../src/sandbox/sandbox" const shell = "/bin/sh" @@ -66,6 +67,23 @@ describe("Sandbox.bubblewrapArgs", () => { test("unshares the PID namespace so /proc escape vectors are closed", () => { expect(Sandbox.bubblewrapArgs({ writable: ["/w"], network: true })).toContain("--unshare-pid") }) + + test("mounts a tmpfs over the XDG cache dir, after the root ro-bind and before writable binds", () => { + // $HOME is deliberately not writable (tooBroadToConfine), so tools that touch + // the XDG cache on startup (zsh compdump/history lock, pip/npm/uv caches, …) + // hit a read-only $HOME and fail — this is the regression this test pins. + const args = Sandbox.bubblewrapArgs({ writable: ["/work/project"], network: true }) + const cache = process.env.XDG_CACHE_HOME ?? path.join(os.homedir(), ".cache") + const cacheIdx = args.indexOf(cache) + expect(cacheIdx).toBeGreaterThan(-1) + expect(args[cacheIdx - 1]).toBe("--tmpfs") + + const roIdx = args.indexOf("--ro-bind") + expect(cacheIdx).toBeGreaterThan(roIdx) // after the whole-fs read-only mount + + const bindIdx = args.indexOf("--bind-try") + expect(cacheIdx).toBeLessThan(bindIdx) // before the explicit writable binds + }) }) describe("Sandbox.backend/describe", () => { diff --git a/backend/cli/test/server/settings-billing.test.ts b/backend/cli/test/server/settings-billing.test.ts index 6a2c5916..a4b764fd 100644 --- a/backend/cli/test/server/settings-billing.test.ts +++ b/backend/cli/test/server/settings-billing.test.ts @@ -1,11 +1,21 @@ -import { test, expect, afterEach } from "bun:test" +import { test, expect, beforeEach, afterEach } from "bun:test" import path from "path" import fs from "fs/promises" import { Global } from "../../src/global" +import { Config } from "../../src/config/config" import { BillingSettingsRoutes } from "../../src/server/routes/settings/billing" const file = path.join(Global.Path.config, "openscience.json") +// Config.global is a lazy, in-process cache invalidated only by +// Config.updateGlobal()/replaceGlobal() calling .reset() — a bare GET never +// resets it. Force a fresh disk read before every test so a read-only test +// can never observe a previous test's in-memory state after that test's +// afterEach has already deleted the file out from under it. +beforeEach(() => { + Config.global.reset() +}) + afterEach(async () => { await fs.rm(file, { force: true }).catch(() => {}) }) @@ -51,3 +61,32 @@ test("PUT llm null sets the toggle back to auto", async () => { const written = JSON.parse(await Bun.file(file).text()) expect(written.billing.llm).toBeNull() }) + +test("GET compute with no config file round-trips as unset (null), not byok", async () => { + // No config file at all — the state readState() must report for a brand + // new user, who has never touched Settings > Spend > Compute. Coercing + // this to "byok" makes the UI show BYOK as active when nothing was chosen, + // and is exactly the bug that let a user narrow themselves into "none" by + // clicking BYOK to "undo" a state they never set. + const res = await BillingSettingsRoutes().request("/") + expect(res.status).toBe(200) + const state = await res.json() + expect(state.compute).toBeNull() +}) + +test("PUT compute null sets the toggle back to auto", async () => { + await fs.mkdir(Global.Path.config, { recursive: true }) + await Bun.write(file, JSON.stringify({ billing: { compute: "byok" } }, null, 2)) + + const res = await BillingSettingsRoutes().request("/", { + method: "PUT", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ compute: null }), + }) + expect(res.status).toBe(200) + const state = await res.json() + expect(state.compute).toBeNull() + + const written = JSON.parse(await Bun.file(file).text()) + expect(written.billing.compute).toBeNull() +}) diff --git a/backend/cli/test/session/compute-prompt.test.ts b/backend/cli/test/session/compute-prompt.test.ts new file mode 100644 index 00000000..2edb2b15 --- /dev/null +++ b/backend/cli/test/session/compute-prompt.test.ts @@ -0,0 +1,184 @@ +import { test, expect, describe } from "bun:test" +import path from "path" +import { ComputeMode } from "../../src/compute/mode" + +const root = path.join(import.meta.dir, "..", "..", "src") + +async function sources() { + const globs = ["session/**/*.{ts,txt}", "agent/prompt/*.txt"] + const files = ( + await Promise.all( + globs.map((pattern) => + Array.fromAsync(new Bun.Glob(pattern).scan({ cwd: root, absolute: true, onlyFiles: true })), + ), + ) + ).flat() + return Promise.all(files.map(async (file) => [file, await Bun.file(file).text()] as const)) +} + +/** Every primary agent that can send a user down a compute path. `sources()` + * globs the whole prompt tree but the assertions below used to hardcode + * research.txt, which made them structurally unable to catch the same defect + * in a sibling prompt. */ +const COMPUTE_AGENTS = ["research", "biology", "physics", "ml"] + +async function agents() { + return Promise.all( + COMPUTE_AGENTS.map(async (name) => { + const file = path.join(root, "agent", "prompt", `${name}.txt`) + return [file, await Bun.file(file).text()] as const + }), + ) +} + +/** Markdown bullets with continuation lines folded in, so a rule about what an + * instruction says sees the whole instruction rather than its first line. */ +function bullets(text: string) { + const out: string[] = [] + let open = false + for (const line of text.split("\n")) { + if (/^\s*[-*] /.test(line)) { + out.push(line.trim()) + open = true + continue + } + if (open && /^\s+\S/.test(line)) { + out[out.length - 1] += " " + line.trim() + continue + } + open = false + } + return out +} + +describe("compute prompt text", () => { + test("every COMPUTE_AGENTS prompt exists and is non-empty", async () => { + // The bans below are only worth as much as the file set they run over — a + // renamed prompt must fail loudly, not silently drop out of coverage. + const loaded = await agents() + expect(loaded.map(([file]) => path.relative(root, file))).toEqual( + COMPUTE_AGENTS.map((name) => path.join("agent", "prompt", `${name}.txt`)), + ) + for (const [file, text] of loaded) expect(text.length, file).toBeGreaterThan(100) + }) + + /** + * Instructions no compute-capable prompt may carry. + * + * The restart pattern is phrase-shaped, not word-shaped, on purpose: "Do not + * restart on a missing tool" is a legitimate instruction and so is stating + * that a newly connected key needs NO restart. What is banned is telling the + * user to restart to pick a credential up — which contradicts a tested design + * claim (compute-status.test.ts, "a credential connected between two calls + * changes the answer, no restart"): the Compute and Credentials panels call + * applyComputeEnv/applyCredentialEnv on save, and a key added in the hosted + * dashboard lands via refreshIfStale's background sync on the next message. + */ + const BANNED: Array<[string, RegExp]> = [ + ["atlas compute:up", /compute:up/], + ["telling the user to restart", /(?:then|and)\s+restart|restart\s+openscience|restart\s+the\s+(?:cli|session)/i], + ] + + test("every compute-capable agent prompt is free of the banned instructions", async () => { + const loaded = await agents() + const hits = BANNED.flatMap(([label, pattern]) => + loaded.filter(([, text]) => pattern.test(text)).map(([file]) => `${path.relative(root, file)}: ${label}`), + ) + expect(hits).toEqual([]) + }) + + test("no agent prompt tells the agent to load a mode-gated compute skill unconditionally", async () => { + // ComputeMode.SKILLS names are hidden from the catalog unless the provider + // is credentialed. An unconditional "Load: `modal-research-gpu`" both + // overrides whatever compute_status just returned and points at a skill the + // filter may have removed, so any bullet naming one has to be gated on byok. + const gated = [...ComputeMode.SKILLS] + const hits = (await agents()).flatMap(([file, text]) => + bullets(text) + .filter((bullet) => /\bload\b/i.test(bullet)) + .filter((bullet) => gated.some((skill) => bullet.includes(skill))) + .filter((bullet) => !bullet.includes("byok")) + .map((bullet) => `${path.relative(root, file)}: ${bullet}`), + ) + expect(hits).toEqual([]) + }) + + test("no prompt or session source references atlas compute:up", async () => { + const hits = (await sources()).filter(([, text]) => text.includes("compute:up")) + expect(hits.map(([file]) => path.relative(root, file))).toEqual([]) + }) + + test("the file set is non-empty and covers both prompt trees", async () => { + const files = (await sources()).map(([file]) => path.relative(root, file)) + expect(files.length).toBeGreaterThan(20) + expect(files).toContain("session/prompt.ts") + expect(files).toContain("agent/prompt/research.txt") + }) + + test("no prompt uses atlas doctor as the compute availability signal", async () => { + // `atlas doctor` legitimately reports whether the atlas CLI is present and + // authenticated (research.txt uses it that way before loading graph state). + // What it does NOT report is anything about compute — so any paragraph that + // mentions both compute and `atlas doctor` is reading a signal that isn't there. + const hits = (await sources()).filter(([, text]) => + text.split(/\n\s*\n/).some((para) => /atlas doctor/i.test(para) && /\bcompute\b/i.test(para)), + ) + expect(hits.map(([file]) => path.relative(root, file))).toEqual([]) + }) + + test("agent prompts point at compute_status for GPU funding", async () => { + const text = await Bun.file(path.join(root, "agent", "prompt", "research.txt")).text() + expect(text).toContain("compute_status") + }) + + test("modal skill mentions in research.txt resolve against ComputeMode.PROVIDERS", async () => { + // `modal` (bare, backticked or not) is a directory name, not a skill name — + // the frontmatter `name` values the skill tool actually resolves on are + // modal-serverless-gpu, modal-ml-training, modal-research-gpu, which is + // exactly ComputeMode.PROVIDERS.modal.skills. Pull every lowercase + // modal*-shaped token out of the prompt (skill tokens are always + // lowercase-hyphenated; "Modal" the company name in prose is capitalized + // and so never matches) and check it against that list — not the other way + // around, since the map trivially agrees with itself. + const text = await Bun.file(path.join(root, "agent", "prompt", "research.txt")).text() + const tokens = [...new Set(text.match(/\bmodal[a-z-]*\b/g) ?? [])] + expect(tokens.length).toBeGreaterThan(0) + const valid = new Set(ComputeMode.PROVIDERS.modal.skills) + expect(tokens.filter((token) => !valid.has(token))).toEqual([]) + }) + + test("the compute reminder points at compute_status and carries no mode", async () => { + const text = await Bun.file(path.join(root, "session", "prompt.ts")).text() + expect(text).toContain("compute_status") + // The reminder must be stateless — a mode baked into an injected string is + // false the moment the user connects a key mid-session. + expect(text).not.toContain("Compute spend is set to") + }) + + test("computeBillingMode is gone and nothing imports it", async () => { + const gate = await Bun.file(path.join(root, "session", "billing-gate.ts")).text() + expect(gate).not.toContain("computeBillingMode") + const files = await Array.fromAsync(new Bun.Glob("**/*.ts").scan({ cwd: root, absolute: true, onlyFiles: true })) + const importers = ( + await Promise.all( + files.map(async (file) => ((await Bun.file(file).text()).includes("computeBillingMode") ? file : undefined)), + ) + ).filter(Boolean) + expect(importers).toEqual([]) + }) + + test("the billing.compute config description no longer claims 'Unset = byok'", async () => { + const text = await Bun.file(path.join(root, "config", "config.ts")).text() + // Scoped to the compute field's own description, not the whole file — + // billing.llm's untouched description already contains "auto-detect", so + // an unscoped scan would pass even if compute's description regressed to + // something like "Defaults to byok when unset". + const start = text.indexOf("compute: z") + expect(start).toBeGreaterThan(-1) + const end = text.indexOf("username: z", start) + expect(end).toBeGreaterThan(start) + const description = text.slice(start, end) + expect(description).not.toContain("Unset = byok") + expect(description).toContain("auto-detect") + }) +}) diff --git a/backend/cli/test/tool/compute-status.test.ts b/backend/cli/test/tool/compute-status.test.ts new file mode 100644 index 00000000..c22a097d --- /dev/null +++ b/backend/cli/test/tool/compute-status.test.ts @@ -0,0 +1,304 @@ +import { test, expect, describe, beforeEach, afterEach } from "bun:test" +import path from "path" +import fs from "fs/promises" +import { ComputeStatusTool } from "../../src/tool/compute" +import { ComputeMode } from "../../src/compute/mode" +import { ToolRegistry } from "../../src/tool/registry" +import { Instance } from "../../src/project/instance" +import { Global } from "../../src/global" +import { tmpdir } from "../fixture/fixture" + +// Every credential variable across ComputeMode.PROVIDERS, derived rather than +// hand-typed so it can never drift out of sync with the sibling scrub list in +// test/tool/skill-compute-filter.test.ts (a hand-typed subset previously let +// an ambient PRIME_API_KEY, TENSORPOOL_KEY, VAST_API_KEY, or +// LAMBDA_LABS_API_KEY leak into these tests and produce false failures). +const ENV = Object.values(ComputeMode.PROVIDERS).flatMap((provider) => provider.env.flat()) +const SESSION = path.join(Global.Path.data, "openscience-session.json") +const realFetch = globalThis.fetch + +const CTX = { + sessionID: "ses_test", + messageID: "msg_test", + agent: "research", + abort: new AbortController().signal, + messages: [], + metadata: () => {}, + ask: async () => {}, +} + +/** Every URL fetched since the last reset, so "only one network call" is a + * positive assertion rather than an absence of failure (mirrors + * test/compute/mode.test.ts's `calls`). */ +let calls: string[] = [] + +function stub(body: unknown, status = 200) { + globalThis.fetch = (async (input: string | URL | Request) => { + const url = String(input instanceof Request ? input.url : input) + calls.push(url) + if (!url.includes("/api/compute/options")) return realFetch(input as never) + return new Response(JSON.stringify(body), { status, headers: { "content-type": "application/json" } }) + }) as typeof fetch +} + +const MANAGED_ON = { + providers: [{ provider: "lambda", funding: "managed", has_byok: false, has_operator: true, count: 2 }], + resell_enabled: true, + cli_effective_balance_cents: 4200, +} +const MANAGED_OFF = { providers: [], resell_enabled: false, cli_effective_balance_cents: 0 } +const MANAGED_ZERO = { + providers: [{ provider: "lambda", funding: "managed", has_byok: false, has_operator: true, count: 2 }], + resell_enabled: true, + cli_effective_balance_cents: 0, +} + +/** `override` writes billing.compute into the project's openscience.json, the + * only way to reach a config-forced origin. */ +async function run(skills: string[], override?: "byok" | "managed") { + await using tmp = await tmpdir({ + git: true, + init: async (dir) => { + for (const name of skills) { + await Bun.write( + path.join(dir, ".openscience", "skill", name, "SKILL.md"), + `---\nname: ${name}\ndescription: Fixture ${name}.\ncategory: cloud-compute\n---\n\n# ${name}\n`, + ) + } + if (override) { + await Bun.write(path.join(dir, "openscience.json"), JSON.stringify({ billing: { compute: override } })) + } + }, + }) + return Instance.provide({ + directory: tmp.path, + fn: async () => { + const tool = await ComputeStatusTool.init({}) + return tool.execute({}, CTX as never) + }, + }) +} + +describe("compute_status", () => { + beforeEach(async () => { + for (const name of ENV) delete process.env[name] + calls = [] + ComputeMode.invalidate() + await fs.mkdir(Global.Path.data, { recursive: true }) + await Bun.write(SESSION, JSON.stringify({ api_key: "thk_t.s", user_id: "u1" })) + }) + afterEach(async () => { + globalThis.fetch = realFetch + for (const name of ENV) delete process.env[name] + await fs.rm(SESSION, { force: true }).catch(() => {}) + }) + + test("byok reports the mode, the usable providers, and byok guidance", async () => { + stub(MANAGED_ON) + process.env["LAMBDA_API_KEY"] = "k" + const result = await run(["lambda-labs-gpu-cloud"]) + expect(result.metadata.mode).toBe("byok") + expect(result.metadata.providers).toEqual(["lambda"]) + expect(result.output).toContain("lambda") + expect(result.output.toLowerCase()).toContain("do not launch managed") + expect(result.metadata.balance_usd).toBeUndefined() + }) + + test("managed reports the balance and managed guidance, from a single network call", async () => { + stub(MANAGED_ON) + const result = await run([]) + expect(result.metadata.mode).toBe("managed") + expect(result.metadata.balance_usd).toBe(42) + expect(result.output).toContain("42") + expect(result.output.toLowerCase()).toContain("managed compute is funded") + // balance_usd must come from the SAME /api/compute/options response that + // decided managed availability, never a second round trip. + expect(calls.filter((url) => url.includes("/api/compute/options")).length).toBe(1) + }) + + test("managed with a zero balance stops telling the agent to spend it", async () => { + // Live testing against a deployed Atlas backend with a zero wallet found + // /api/compute/options reports a "managed" provider regardless of + // balance — availability and affordability are independent. Every lease + // attempt in this state returns HTTP 402 insufficient_cli_credit, so the + // old guidance sent the agent down a path that cannot work while also + // forbidding the only fallback (the user's own keys). + stub(MANAGED_ZERO) + const result = await run([]) + expect(result.output.toLowerCase()).not.toContain("run gpu work through managed compute") + expect(result.output.toLowerCase()).toContain("wallet is empty") + expect(result.output.toLowerCase()).toContain("topping up") + }) + + test("a funded managed wallet does not claim OpenScience can launch managed compute", async () => { + // There is no managed launch mechanism in this client: ComputeTools is + // [ComputeStatusTool] and the only /api/compute call anywhere is mode.ts's + // read-only /options probe. Telling a funded, keyless user (the default + // path for anyone signed in) to "run GPU work through managed compute" + // while forbidding the only fallback left the agent with nothing that works. + stub(MANAGED_ON) + const result = await run([]) + const output = result.output.toLowerCase() + expect(output).not.toContain("run gpu work through managed compute") + expect(output).not.toContain("do not use the user's own provider keys") + expect(output).toContain("cannot launch it") + expect(output).toContain("settings ▸ compute") + }) + + test("a managed override names billing.compute instead of advice that setting blocks", async () => { + // Under an explicit override, connecting a key flips nothing: funded() + // never consults `providers` and offered() returns empty, so the skills + // stay hidden too. "Connect a key to run BYOK instead" is advice the + // setting itself defeats. + stub(MANAGED_ZERO) + const result = await run([], "managed") + expect(result.metadata.mode).toBe("managed") + expect(result.output).toContain("billing.compute") + expect(result.output.toLowerCase()).toContain("will not switch") + }) + + test("managed resolved from the environment still says to connect a key", async () => { + // The mirror of the override case: with no override, mode is managed only + // because the user holds no credential, so connecting one really does flip + // the next call to byok. + stub(MANAGED_ZERO) + const result = await run([]) + expect(result.output).not.toContain("billing.compute") + expect(result.output).toContain("Settings ▸ Compute") + }) + + test("a zero balance narrows guidance only — mode stays managed, balance stays reported", async () => { + stub(MANAGED_ZERO) + const result = await run([]) + expect(result.metadata.mode).toBe("managed") + expect(result.metadata.balance_usd).toBe(0) + }) + + test("none tells the agent not to attempt GPU work and how to enable it", async () => { + stub(MANAGED_OFF) + const result = await run([]) + expect(result.metadata.mode).toBe("none") + expect(result.output.toLowerCase()).toContain("do not attempt gpu work") + expect(result.output).toContain("Settings") + expect(result.metadata.balance_usd).toBeUndefined() + }) + + test("none never offers a top-up, which cannot move a user out of none", async () => { + // No path to `none` is balance-related: probe() returns managed:false only + // for no session, non-2xx, a network/parse failure, or no provider with + // funding "managed" — and Atlas reports managed regardless of balance. + stub(MANAGED_OFF) + const result = await run([]) + expect(result.output.toLowerCase()).not.toContain("top up") + }) + + test("a none narrowed by a managed override names the setting that narrowed it", async () => { + stub(MANAGED_OFF) + process.env["LAMBDA_API_KEY"] = "k" + const result = await run(["lambda-labs-gpu-cloud"], "managed") + expect(result.metadata.mode).toBe("none") + expect(result.output).toContain("billing.compute") + }) + + test("byok reports managed availability as unchecked, never as 'no'", async () => { + // The byok arms skip the availability probe by design, so "managed + // available: no" would be a fact the tool never measured — harmless today, + // load-bearing the moment Part B reads managed_available from metadata. + stub(MANAGED_ON) + process.env["LAMBDA_API_KEY"] = "k" + const result = await run(["lambda-labs-gpu-cloud"]) + expect(result.output).toContain("**managed available**: not checked") + expect(result.metadata.managed_available).toBeUndefined() + expect(calls.filter((url) => url.includes("/api/compute/options"))).toEqual([]) + }) + + test("a probed mode still reports availability as measured yes/no", async () => { + stub(MANAGED_ON) + const managed = await run([]) + expect(managed.output).toContain("**managed available**: yes") + expect(managed.metadata.managed_available).toBe(true) + ComputeMode.invalidate() + stub(MANAGED_OFF) + const none = await run([]) + expect(none.output).toContain("**managed available**: no") + expect(none.metadata.managed_available).toBe(false) + }) + + test("a provider with a key but no skill is still reported as usable byok", async () => { + stub(MANAGED_OFF) + process.env["RUNPOD_API_KEY"] = "rpa_x" + const result = await run([]) + expect(result.metadata.mode).toBe("byok") + expect(result.metadata.providers).toEqual(["runpod"]) + expect(result.output).toContain("runpod") + }) + + test("the three modes produce three DIFFERENT guidance strings", async () => { + stub(MANAGED_ON) + process.env["LAMBDA_API_KEY"] = "k" + const byok = await run(["lambda-labs-gpu-cloud"]) + delete process.env["LAMBDA_API_KEY"] + ComputeMode.invalidate() + const managed = await run([]) + stub(MANAGED_OFF) + ComputeMode.invalidate() + const none = await run([]) + // Comparing whole `output` strings is a false positive: the leading + // `**mode**: byok|managed|none` line always differs by itself, so a + // whole-string comparison would pass even if GUIDANCE collapsed to one + // shared string. A formatting-position trick (e.g. "text after the last + // blank line") is equally fragile — it breaks the moment the separator + // between the report and the guidance changes shape, which is a pure + // formatting edit that should never fail this test. + // + // Assert against the actual contract instead: each mode's GUIDANCE entry + // carries a short, semantically load-bearing phrase that could not + // survive a collapse to one shared string, and that phrase must appear + // in that mode's output and ONLY that mode's output. + const PHRASE = { + byok: "do not launch managed", + managed: "cannot launch it", + none: "do not attempt gpu work", + } + const output = { + byok: byok.output.toLowerCase(), + managed: managed.output.toLowerCase(), + none: none.output.toLowerCase(), + } + for (const mode of Object.keys(PHRASE) as (keyof typeof PHRASE)[]) { + expect(output[mode]).toContain(PHRASE[mode]) + for (const other of Object.keys(PHRASE) as (keyof typeof PHRASE)[]) { + if (other === mode) continue + expect(output[mode]).not.toContain(PHRASE[other]) + } + } + }) + + test("a credential connected between two calls changes the answer, no restart", async () => { + stub(MANAGED_OFF) + const before = await run([]) + expect(before.metadata.mode).toBe("none") + process.env["LAMBDA_API_KEY"] = "connected-mid-session" + const after = await run(["lambda-labs-gpu-cloud"]) + expect(after.metadata.mode).toBe("byok") + }) + + test("the description instructs the agent to check before running GPU work", async () => { + const tool = await ComputeStatusTool.init({}) + expect(tool.description.toLowerCase()).toContain("before") + expect(tool.description.toLowerCase()).toContain("gpu") + expect(tool.description).toContain("byok") + expect(tool.description).toContain("managed") + expect(tool.description).toContain("none") + }) + + test("the tool is registered", async () => { + await using tmp = await tmpdir({ git: true }) + await Instance.provide({ + directory: tmp.path, + fn: async () => { + expect(await ToolRegistry.ids()).toContain("compute_status") + }, + }) + }) +}) diff --git a/backend/cli/test/tool/skill-compute-filter.test.ts b/backend/cli/test/tool/skill-compute-filter.test.ts new file mode 100644 index 00000000..cde2692d --- /dev/null +++ b/backend/cli/test/tool/skill-compute-filter.test.ts @@ -0,0 +1,246 @@ +import { test, expect, describe, beforeEach, afterEach } from "bun:test" +import path from "path" +import fs from "fs/promises" +import { SkillTool } from "../../src/tool/skill" +import { ComputeStatusTool } from "../../src/tool/compute" +import { ComputeMode } from "../../src/compute/mode" +import { Instance } from "../../src/project/instance" +import { Global } from "../../src/global" +import { tmpdir } from "../fixture/fixture" + +// All ten credential variables across ComputeMode.PROVIDERS (modal x2, lambda x2, +// tensorpool x2, prime x2, runpod x1, vast x1) — matches test/compute/mode.test.ts's +// ENV list. A partial list lets a developer's own ambient shell keys (e.g. a real +// PRIME_API_KEY) leak into tests asserting an empty catalog. +const ENV = [ + "MODAL_TOKEN_ID", + "MODAL_TOKEN_SECRET", + "LAMBDA_API_KEY", + "LAMBDA_LABS_API_KEY", + "TENSORPOOL_KEY", + "TENSORPOOL_API_KEY", + "PRIME_API_KEY", + "PRIME_INTELLECT_API_KEY", + "RUNPOD_API_KEY", + "VAST_API_KEY", +] +const SESSION = path.join(Global.Path.data, "openscience-session.json") +const realFetch = globalThis.fetch + +const CTX = { + sessionID: "s", + messageID: "m", + agent: "research", + abort: new AbortController().signal, + messages: [], + metadata: () => {}, + ask: async () => {}, +} + +// Every provider skill, plus two skills that must never be filtered: a +// non-compute one and a cloud-compute skill that maps to no panel provider. +const ALL = [ + ["modal-serverless-gpu", "cloud-compute"], + ["lambda-labs-gpu-cloud", "cloud-compute"], + ["tensorpool-gpu-cloud", "cloud-compute"], + ["prime-intellect-lab", "ml-training"], + ["tinker-fine-tuning", "cloud-compute"], + ["rdkit", "chemistry"], +] as const + +function stub(managed: boolean) { + globalThis.fetch = (async (input: string | URL | Request) => { + const url = String(input instanceof Request ? input.url : input) + if (!url.includes("/api/compute/options")) return realFetch(input as never) + return new Response( + JSON.stringify({ + providers: managed ? [{ provider: "lambda", funding: "managed" }] : [], + resell_enabled: managed, + cli_effective_balance_cents: 500, + }), + { status: 200, headers: { "content-type": "application/json" } }, + ) + }) as typeof fetch +} + +async function project(fn: (dir: string) => Promise) { + return tmpdir({ + git: true, + init: async (dir) => { + for (const [name, category] of ALL) { + await Bun.write( + path.join(dir, ".openscience", "skill", name, "SKILL.md"), + `---\nname: ${name}\ndescription: Fixture ${name}.\ncategory: ${category}\n---\n\n# ${name}\n`, + ) + } + await fn(dir) + }, + }) +} + +/** Which of the six provider skills does the tool offer? Read from the tool's + * own category listing, which is what the model sees. */ +async function offered(): Promise { + const tool = await SkillTool.init({}) + const found: string[] = [] + for (const category of ["cloud-compute", "ml-training"]) { + const result = await tool.execute({ category }, CTX as never).catch(() => undefined) + if (result) found.push(result.output) + } + const text = found.join("\n") + return [...ComputeMode.SKILLS].filter((name) => text.includes(`**${name}**`)).sort() +} + +async function nonComputeVisible(): Promise { + const tool = await SkillTool.init({}) + const result = await tool.execute({ category: "chemistry" }, CTX as never) + return result.output.includes("**rdkit**") +} + +async function tinkerVisible(): Promise { + const tool = await SkillTool.init({}) + const result = await tool.execute({ category: "cloud-compute" }, CTX as never) + return result.output.includes("**tinker-fine-tuning**") +} + +/** The compute_status tool's own verdict — the second surface that must agree + * with SkillTool's catalog filter about which providers are usable. */ +async function computeStatus(): Promise<{ mode: string; providers: string[] }> { + const tool = await ComputeStatusTool.init({}) + const result = await tool.execute({}, CTX as never) + return result.metadata as { mode: string; providers: string[] } +} + +describe("skill catalog filtering by compute mode", () => { + beforeEach(async () => { + for (const name of ENV) delete process.env[name] + ComputeMode.invalidate() + await fs.mkdir(Global.Path.data, { recursive: true }) + await Bun.write(SESSION, JSON.stringify({ api_key: "thk_t.s", user_id: "u1" })) + }) + afterEach(async () => { + globalThis.fetch = realFetch + for (const name of ENV) delete process.env[name] + await fs.rm(SESSION, { force: true }).catch(() => {}) + }) + + test("with only a Modal credential, only Modal's skills are offered", async () => { + stub(false) + process.env["MODAL_TOKEN_ID"] = "ak-a" + process.env["MODAL_TOKEN_SECRET"] = "as-b" + await using tmp = await project(async () => {}) + const names = await Instance.provide({ directory: tmp.path, fn: offered }) + expect(names).toEqual(["modal-serverless-gpu"]) + }) + + test("a RunPod credential is byok but contributes no skills — nobody else's are offered either", async () => { + stub(false) + process.env["RUNPOD_API_KEY"] = "rpa_x" + await using tmp = await project(async () => {}) + await Instance.provide({ + directory: tmp.path, + fn: async () => { + // RunPod carries skills: [] (Decision 2), so being credentialed makes the + // user byok without unlocking any other provider's skills. + expect((await ComputeMode.resolve()).mode).toBe("byok") + expect(await offered()).toEqual([]) + }, + }) + }) + + test("in managed, a credentialed provider's skill is still not offered — mode governs, not providers", async () => { + // Regression guard for the filter keying off `providers` (the credentialed + // set, reported verbatim in every mode) instead of `mode`. Lambda IS + // credentialed here — ComputeMode.resolve().providers is non-empty — but + // billing.compute forces the managed override and the probe confirms + // managed is funded, so mode is "managed", not "byok". A filter that + // trusted `providers` would offer lambda's skill anyway; this must not. + stub(true) + process.env["LAMBDA_API_KEY"] = "k" + await using tmp = await project(async (dir) => { + await Bun.write(path.join(dir, "openscience.json"), JSON.stringify({ billing: { compute: "managed" } })) + }) + await Instance.provide({ + directory: tmp.path, + fn: async () => { + const resolved = await ComputeMode.resolve() + expect(resolved.mode).toBe("managed") + expect(resolved.providers).toEqual(["lambda"]) + expect(await offered()).toEqual([]) + }, + }) + }) + + test("in none, no BYOK provider skill is offered", async () => { + stub(false) + await using tmp = await project(async () => {}) + const names = await Instance.provide({ directory: tmp.path, fn: offered }) + expect(names).toEqual([]) + }) + + test("non-compute skills are unaffected in every mode", async () => { + for (const managed of [true, false]) { + stub(managed) + ComputeMode.invalidate() + await using tmp = await project(async () => {}) + expect(await Instance.provide({ directory: tmp.path, fn: nonComputeVisible })).toBe(true) + } + + // byok: a credentialed provider must not affect a skill outside its scope either. + stub(false) + process.env["LAMBDA_API_KEY"] = "k" + ComputeMode.invalidate() + await using byok = await project(async () => {}) + expect(await Instance.provide({ directory: byok.path, fn: nonComputeVisible })).toBe(true) + }) + + test("cloud-compute skills that map to no panel provider are never hidden", async () => { + stub(false) + await using tmp = await project(async () => {}) + expect(await Instance.provide({ directory: tmp.path, fn: tinkerVisible })).toBe(true) + + // byok: a credentialed provider must not hide a skill outside ComputeMode.SKILLS either. + process.env["LAMBDA_API_KEY"] = "k" + ComputeMode.invalidate() + await using byok = await project(async () => {}) + expect(await Instance.provide({ directory: byok.path, fn: tinkerVisible })).toBe(true) + }) + + test("a credential added between two init() calls changes the catalog on the second", async () => { + stub(false) + await using tmp = await project(async () => {}) + await Instance.provide({ + directory: tmp.path, + fn: async () => { + expect(await offered()).toEqual([]) + process.env["TENSORPOOL_KEY"] = "tp-late" + expect(await offered()).toEqual(["tensorpool-gpu-cloud"]) + }, + }) + }) + + test("credentialed: SkillTool and compute_status agree lambda is the usable provider", async () => { + stub(false) + process.env["LAMBDA_API_KEY"] = "k" + await using tmp = await project(async () => {}) + await Instance.provide({ + directory: tmp.path, + fn: async () => { + expect(await offered()).toEqual(["lambda-labs-gpu-cloud"]) + expect((await computeStatus()).providers).toEqual(["lambda"]) + }, + }) + }) + + test("no credentials: SkillTool and compute_status agree nothing is usable", async () => { + stub(false) + await using tmp = await project(async () => {}) + await Instance.provide({ + directory: tmp.path, + fn: async () => { + expect(await offered()).toEqual([]) + expect((await computeStatus()).mode).toBe("none") + }, + }) + }) +}) diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 72bf433e..b65a6556 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -5,6 +5,19 @@ Ranking is **findings-first**: every item was checked against the tree at `52845c3` before being placed, so "P0" means _users hit this today_, not _it sounded important_. +> **Audit-note currency — 2026-08-01.** Statuses below are unchanged: `✅ DONE` still means _shipped and +> reachable by a user_, and nothing has been re-graded. Some **audit notes** have been corrected in place, +> each marked `[2026-08-01]`, for two reasons: +> +> - **`aa9b3142` ("feat: add managed compute jobs") merged to `main` on 2026-07-29, after the `52845c3` +> audit.** It shipped an SSH/Slurm/PBS job dispatcher, which falsifies several "no SSH client / nothing +> reads this / 0 hits for slurm" notes. The **status marks were deliberately not re-graded** — that +> needs a fresh audit at a new baseline and a recount of the table above, which is its own task. +> - **Compute guardrail work landed on two Atlas/OpenScience branches that are still unmerged draft PRs** +> (deliberately: the team holds compute PRs draft until the feature is complete). It is real, tested and +> in several cases verified against a live deployment — and it is **not reachable by a user**, so it +> cannot be `✅ DONE` under this document's own legend. Notes say where it lives; marks stay put. + ## Status at a glance | Status | Count | Meaning | @@ -46,7 +59,7 @@ built and then never connected to anything a user can reach. Ten independent ins | **`session-review.tsx`** | ~500 lines: diff view, line comments, focus | `grep SessionReview frontend/workspace/src` → **0 hits** | 12, 80 | | **Command palette** | ⌘K, mounted both routes, e2e-tested | Contains exactly 3 commands (open folder, settings, back) — zero scientific actions | 66, 75 | | **OpenTelemetry flag** | `experimental.openTelemetry` in `config.ts:1234` | No `@opentelemetry` dep, no tracer, no exporter anywhere — **the flag emits no span** | 110 | -| **SSH hosts + model endpoints** | Settings panels, persisted to disk | No SSH client in the dep tree; nothing reads either store | 58 | +| **Model endpoints** | Settings panel, persisted to disk | Nothing reads the store — no inference-routing consumer. (**SSH hosts left this list on 2026-08-01**: `aa9b3142` wired them) | 58 | | **`Run.inputs` / `Artifact.contentHash`** | Declared in `provenance/store.ts:38,47` | `ProvenanceRecordTool` exposes no parameter for either — never populated | 10, 18, 19 | | **`open-bench/`** | 2 benches, real run artifacts (`chembench`, `biomnibench`) | Untracked by git; `src/openbench/**` has **only `.pyc`, zero `.py`**; no `pyproject.toml` | 108 | | **Artifact `type` / `summary`** | Accepted by `register()` | Not persisted — `list()` hardcodes `type:"unknown"` (`artifacts.ts:61`) | 13 | @@ -70,11 +83,19 @@ Nothing on this list needs an architecture decision. 1. **`bash` cannot run anything in the background.** `tool/bash.ts` is synchronous with a `timeout` that `killTree`s on expiry. No job registry, no output persistence, no SQLite. A training run cannot survive a single tool call. This is the keystone: items 2, 3, 4, 51, 52, 56, 57, 63 are all downstream of it. + **[2026-08-01] Half of this is now false, and the important half is not.** `aa9b3142` shipped + `compute/jobs.ts`: a job registry with detached execution, persisted metadata and streamed logs. It is + **not reachable from `bash` or from any tool** — the agent still cannot start a job that outlives a tool + call. The keystone stands; what changed is that the substrate exists and needs exposing, not building. 2. **Six GPU providers are advertised; zero have an API client.** `settings/compute.ts` encrypts a key and injects an env var, then hopes a markdown skill shells out to the vendor CLI. **RunPod and Vast have no skill at all** — and `RUNPOD_API_KEY` is named to the model in **all six session prompts**, so the agent is told a capability exists that nothing implements. (Vast isn't even advertised — connecting it is a pure no-op.) Modal is stored in two panels where Credentials silently wins. + **[2026-08-01] Re-verified and still true in this repo.** All six provider prompts still name + `RUNPOD_API_KEY` at `:119`. RunPod and Vast do have real API clients in **Atlas** now, driven end to end + against a deployed backend — but that is the managed path, behind `POST /api/compute/leases` on an + unmerged draft branch, and OpenScience has no tool that calls it. The finding stands as written. 3. **Sharing is hard-disabled at three layers** (`disabled = true` in `share.ts:74` and `share-next.ts:18`, plus `Session.share` returning empty strings) while ~10 orphaned i18n strings per locale still describe the feature. We ship the vocabulary of a feature we don't have. @@ -113,26 +134,26 @@ Fifteen items: **1 done · 6 partial · 1 skill-only · 7 missing.** ## Group A — Broken or misleading in the shipped product -| # | Item | Status | Current state | -| ------- | --------------------------- | ------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| **5** | Fix existing compute gaps | ❌ | RunPod/Vast keys inject but **no consumer exists**, while `RUNPOD_API_KEY` is named to the model in all 6 `session/prompt/*.txt`; Modal double-stored vs `credentials.ts` (Credentials applies first at boot and wins); `last_used` rendered but never assigned | -| **14** | Shared sessions | 🟡 | Code written, then hard-disabled: `share.ts:74` + `share-next.ts:18` both `disabled = true`; `session/index.ts:253` returns empty strings. No permissions model. Orphaned i18n in every locale | -| **13** | Scientific artifact manager | 🟡 | `session/rlm/artifacts.ts` is a blob cache for context relief: IDs are `Date.now()+random`, **no checksums**, `list()` hardcodes `type:"unknown"` and drops your summary | -| **100** | Sandbox docs update | ✅ | **No work needed — the premise was wrong.** `sandbox.mdx` (91 lines) matches `sandbox/sandbox.ts` (509 lines) including a Limitations section; both shipped in `a737ddc`, no drift since | +| # | Item | Status | Current state | +| ------- | --------------------------- | ------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **5** | Fix existing compute gaps | ❌ | RunPod/Vast keys inject but **no consumer exists**, while `RUNPOD_API_KEY` is named to the model in all 6 `session/prompt/*.txt`; Modal double-stored vs `credentials.ts` (Credentials applies first at boot and wins); `last_used` rendered but never assigned. **[2026-08-01] Re-verified after this branch edited the prompts: the `RUNPOD_API_KEY` claim still holds** — all six provider prompts name it at `:119` (`anthropic`, `beast`, `codex_header`, `copilot-gpt-5`, `gemini`, `qwen`); `last_used` is still only defaulted and carried (`routes/settings/compute.ts:252`, `:272`). RunPod and Vast are now leasable as **managed** providers server-side in Atlas, driven end to end on a deployed backend — but behind `POST /api/compute/leases`, on an unmerged draft branch, with no OpenScience tool that calls it, so a key in this panel is still consumed by nothing | +| **14** | Shared sessions | 🟡 | Code written, then hard-disabled: `share.ts:74` + `share-next.ts:18` both `disabled = true`; `session/index.ts:253` returns empty strings. No permissions model. Orphaned i18n in every locale | +| **13** | Scientific artifact manager | 🟡 | `session/rlm/artifacts.ts` is a blob cache for context relief: IDs are `Date.now()+random`, **no checksums**, `list()` hardcodes `type:"unknown"` and drops your summary | +| **100** | Sandbox docs update | ✅ | **No work needed — the premise was wrong.** `sandbox.mdx` (91 lines) matches `sandbox/sandbox.ts` (509 lines) including a Limitations section; both shipped in `a737ddc`, no drift since | ## Group B — The job system (keystone) One workstream. Guardrails ship **with** the runner — a job system that can spend money before it can stop spending money is a liability. -| # | Item | Status | Current state | -| ------- | ------------------------------------ | ------ | ---------------------------------------------------------------------------------------------------------------------------------------------- | -| **51** | Provider-independent job abstraction | ❌ | `util/queue.ts` is an in-memory `AsyncQueue`; `scheduler/index.ts` is a 61-line `setInterval`. Start here | -| **52** | Queue/history database | ❌ | No SQLite/Drizzle/Prisma anywhere; `storage/storage.ts` is JSON files. Use `bun:sqlite` — preserves the single-binary ship | -| **2** | Real compute jobs page | ❌ | No `/jobs` route. `app.tsx` has exactly 3 routes: `/`, `/:dir`, `/:dir/session/:id?` | -| **61** | Per-job secrets, never into logs | 🟡 | Log redaction exists (`OpenScience.redactSecrets` → `bash.ts:207`). **Per-job scoping does not** — injection is global via `applyComputeEnv()` | -| **55** | Budget guardrails + kill switches | ❌ | 0 hits for `costLimit\|spendLimit\|budget` outside a compaction comment. `cli/cmd/stats.ts` reports cost after the fact | -| **103** | Cost approval gates | ❌ | `session/billing-gate.ts` only _classifies_ calls managed/BYOK/free. No pre-flight estimate, cap, or prompt | +| # | Item | Status | Current state | +| ------- | ------------------------------------ | ------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| **51** | Provider-independent job abstraction | ❌ | `util/queue.ts` is an in-memory `AsyncQueue`; `scheduler/index.ts` is a 61-line `setInterval`. Start here. **[2026-08-01] Partly overtaken by `aa9b3142`**: `src/compute/jobs.ts` (`ComputeJobs`) is a real job abstraction over two targets (`local`, `ssh`) and three schedulers (`none`, `slurm`, `pbs`), with a resource request, `apptainer` support, artifact collection, log streaming and cancel. It is **not** provider-independent in this item's sense — no managed-lease or cloud-GPU backend behind it — and **the agent cannot drive it** (`grep ComputeJobs src/tool/` → 0 hits). Read this item as "generalise the abstraction that now exists", not "start from nothing" | +| **52** | Queue/history database | ❌ | No SQLite/Drizzle/Prisma anywhere; `storage/storage.ts` is JSON files. Use `bun:sqlite` — preserves the single-binary ship. **[2026-08-01] Still accurate**, and now load-bearing: `ComputeJobs` persists job metadata as a JSON blob at mode `0600` (`compute/jobs.ts:162`), which is exactly the store this item replaces | +| **2** | Real compute jobs page | ❌ | No `/jobs` route. `app.tsx` has exactly 3 routes: `/`, `/:dir`, `/:dir/session/:id?`. **[2026-08-01] The route claim still holds — the "no page" claim does not.** `aa9b3142` shipped `frontend/workspace/src/atlas/ComputeJobs.tsx`, mounted in `RightPane.tsx:297` (rendered by the session page), backed by `/jobs`, `/jobs/completed`, `/jobs/:id/log`, `/jobs/:id/cancel`. It is a right-pane panel for SSH/local jobs, not a first-class monitoring route, and it shows nothing about managed leases or cost | +| **61** | Per-job secrets, never into logs | 🟡 | Log redaction exists (`OpenScience.redactSecrets` → `bash.ts:207`). **Per-job scoping does not** — injection is global via `applyComputeEnv()` | +| **55** | Budget guardrails + kill switches | ❌ | **In this repo, still nothing**: 0 hits for `costLimit\|spendLimit\|budget` outside a compaction comment; `cli/cmd/stats.ts` reports cost after the fact. **[2026-08-01] The server-side half now exists and binds** — `hard_cap_cents` was decorative (measured in production: a 34¢/hr lease had `hard_cap_cents = 816` with `spent_cents` frozen at 34 while the lease accrued), and the Atlas billing tick now sets the grant to its wall-clock total and releases when the ceiling refuses it. Measured after the fix: a $10 budget at $6.99/h releases at 5160s vs a theoretical 5150s. `POST /api/compute/leases` also accepts `budget_cents`, clamped to the wallet, reporting the effective cap. **Unmerged draft branch, and OpenScience cannot launch a lease to spend against it** — so the CLI-scoped claim above is the one that describes what a user has | +| **103** | Cost approval gates | ❌ | `session/billing-gate.ts` only _classifies_ calls managed/BYOK/free. No pre-flight estimate, cap, or prompt. **[2026-08-01] The cap half is built** (Atlas, unmerged draft): `budget_cents` on `POST /api/compute/leases` is clamped to the wallet and the response reports the **effective** cap, so a caller can state what was actually authorised. **Both approval halves are still missing**: no pre-flight quote (`POST /api/compute/quote` — spec change 4, unbuilt, and `/estimate` needs an explicit `{provider, sku}` a `{gpu, count}` proposal does not have) and no prompt (a `ctx.ask` gate needs a `compute_launch` tool, which does not exist) | ## Group C — Core science UX that is half-built @@ -154,15 +175,15 @@ Twenty-eight items: **0 done · 14 partial · 6 skill-only · 8 missing.** ## Compute, made real -| # | Item | Status | Current state | -| ------ | --------------------------------- | ------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| **4** | Cloud GPU backends (real clients) | 🟡 | Credential plumbing is real, encrypted, and unit-tested — but grep for `api.runpod`/`modal.com`/`lambdalabs` outside skill markdown returns **0 hits**. AWS/GCP/Azure Batch absent entirely | -| **3** | Slurm/HPC integration | ❌ | 0 hits for `sbatch\|squeue\|scancel\|apptainer\|slurm` in any source tree. Prose in skill docs only | -| **57** | Multi-node training | 📄 | Skills exist (`ray-train`, `deepspeed`, `torchtitan`, `megatron-core`). No product code. Needs **51** | -| **58** | Remote kernels over SSH/Jupyter | 🟡 | The SSH-hosts panel persists data **nothing reads**; no SSH client in the dep tree. Make it real or remove the panel | -| **59** | Interactive tunnels | ❌ | JupyterLab, TensorBoard, MLflow, W&B | -| **53** | Artifact upload/download | 📄 | No `@aws-sdk`/`@google-cloud`/`@azure`/`@huggingface` in any `package.json`. Skills shell out to `aws`/`gsutil`/`hf` CLIs | -| **64** | Pre-launch runtime health checks | ❌ | Cheapest possible way to stop burning GPU-hours on a bad environment | +| # | Item | Status | Current state | +| ------ | --------------------------------- | ------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **4** | Cloud GPU backends (real clients) | 🟡 | Credential plumbing is real, encrypted, and unit-tested — but grep for `api.runpod`/`modal.com`/`lambdalabs` outside skill markdown returns **0 hits**. AWS/GCP/Azure Batch absent entirely | +| **3** | Slurm/HPC integration | ❌ | ~~0 hits for `sbatch\|squeue\|scancel\|apptainer\|slurm` in any source tree. Prose in skill docs only~~ — **[2026-08-01] falsified by `aa9b3142`.** `compute/jobs.ts` submits with `sbatch --wait --parsable` (`:331`), cancels with `scancel --name` (`:710`), wraps in `apptainer exec` (`:275`), and probes a host for Slurm/PBS (`:510`). What is genuinely absent: `squeue` polling, job-array support, module systems, and any HPC path the **agent** can reach | +| **57** | Multi-node training | 📄 | Skills exist (`ray-train`, `deepspeed`, `torchtitan`, `megatron-core`). No product code. Needs **51** | +| **58** | Remote kernels over SSH/Jupyter | 🟡 | ~~The SSH-hosts panel persists data **nothing reads**; no SSH client in the dep tree. Make it real or remove the panel~~ — **[2026-08-01] `aa9b3142` made it real.** `ComputeJobs` reads `ssh_hosts` and dispatches over the system `ssh` binary (`compute/jobs.ts:362-366`), with a reachability probe returning latency + Python/GPU/Slurm/PBS (`routes/settings/compute.ts:394`). "No SSH client in the dep tree" is still literally true and no longer evidence — it shells out rather than adding a dep. **Remote _kernels_ are still missing**: this dispatches batch commands, not a Jupyter/IPython kernel the notebook tool can attach to | +| **59** | Interactive tunnels | ❌ | JupyterLab, TensorBoard, MLflow, W&B | +| **53** | Artifact upload/download | 📄 | No `@aws-sdk`/`@google-cloud`/`@azure`/`@huggingface` in any `package.json`. Skills shell out to `aws`/`gsutil`/`hf` CLIs | +| **64** | Pre-launch runtime health checks | ❌ | Cheapest possible way to stop burning GPU-hours on a bad environment | ## Scientific data as a first-class citizen @@ -248,13 +269,13 @@ Every pack below is 📄 or ❌. The parenthetical counts how many named tools i ## Compute depth -| # | Item | Status | Current state | -| ------ | ---------------------------------------------------- | ------ | -------------------------------------------- | -| **54** | GPU availability planner (VRAM, CUDA, region, price) | ❌ | | -| **56** | Spot/preemptible checkpointing | ❌ | Needs **51** | -| **60** | Container builder/cache with CUDA base images | 📄 | CUDA images appear only in Modal skill prose | -| **62** | Dataset locality planner | ❌ | No code, no skill, no doc | -| **63** | Workflow runners: Argo, Cromwell/WDL, Seqera | ❌ | | +| # | Item | Status | Current state | +| ------ | ---------------------------------------------------- | ------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **54** | GPU availability planner (VRAM, CUDA, region, price) | ❌ | | +| **56** | Spot/preemptible checkpointing | ❌ | Needs **51**. **[2026-08-01] Still nothing, and the compute spec sharpens why it matters**: a managed lease is now released the moment its budget is exhausted, so an un-checkpointed run dies with it. `compute-design.md` states the limit plainly — a volume preserves _files_, not _process state_. Note that cheapest-first is **not** spot: `VastProvider.list_options` queries `"type": "on-demand"`, so this is budget/TTL preemption, not marketplace preemption | +| **60** | Container builder/cache with CUDA base images | 📄 | CUDA images appear only in Modal skill prose | +| **62** | Dataset locality planner | ❌ | No code, no skill, no doc | +| **63** | Workflow runners: Argo, Cromwell/WDL, Seqera | ❌ | | ## UX parity diff --git a/docs/plans/06-compute-integrations.md b/docs/plans/06-compute-integrations.md index 2ce09ed2..9d0c5b8b 100644 --- a/docs/plans/06-compute-integrations.md +++ b/docs/plans/06-compute-integrations.md @@ -2,14 +2,27 @@ Workstream: verify each compute path and fix what's broken — BYOK GPU (confirm), cloud storage, SSH, managed compute via the Atlas CLI. Findings-first. Audited against the Atlas backend (cloned) and the installed `atlas` CLI (`0.13.1` = npm `@synsci/atlas`). Citations `file:line`. +> **Currency note — 2026-08-01.** This audit was written in late July and parts of it have been +> overtaken. Three things changed and each is marked inline below, not silently rewritten: +> +> 1. **Path C is no longer a dead end.** `aa9b3142` ("feat: add managed compute jobs", merged to +> `main` on 2026-07-29 — _after_ this audit) shipped a real SSH/Slurm/PBS dispatcher. +> 2. **Path D's "Correction to the initial audit" is false** and is retracted in place. The pin is +> not `^0.5.12` and the published CLI has no `compute:` commands at all. +> 3. **The managed backend behind Path D now works end to end** — lease → promote → SSH → release, +> on Vast and RunPod — but only on an **unmerged draft** Atlas branch, and OpenScience still +> cannot launch a lease. +> +> Everything not marked below is a July fact that has **not** been re-audited. Treat it as such. + ## Status per path -| Path | Verdict | One-line | -| ------------------------------------ | --------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| **A. BYOK GPU providers** | ✅ works (with gaps) | Key encrypt→env-injection is solid + unit-tested for the 4 providers with skills (Modal, Lambda, TensorPool, Prime). **Vast + RunPod keys inject but no skill reads them.** | -| **B. Cloud storage** | ⚠️ creds-only, unverified | No mount/rclone abstraction — AWS/GCP creds → env → whatever CLI a skill invokes. **Azure object storage is advertised but not backed.** Needs real creds+bucket to verify (FLAG). | -| **C. SSH-based compute** | ❌ dead-end | "SSH hosts" + "Model endpoints" panels persist data **nothing ever reads**. No SSH client, no dispatch, no routing. | -| **D. Managed compute via atlas CLI** | ⚠️ real, but version-gapped | The atlas CLI **does** ship a full compute suite (`compute:up`/`catalog`/`list`/`ssh`/`release` → `/api/compute/leases`) in the **published 0.13.2** — but OpenScience pins `@synsci/atlas@^0.5.12`, so the shipped CLI predates it and the prompt's `atlas compute:up` can't resolve. Resale off by default server-side; `billing.compute` prompt-only. | +| Path | Verdict | One-line | +| ------------------------------------ | ------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **A. BYOK GPU providers** | ✅ works (with gaps) | Key encrypt→env-injection is solid + unit-tested for the 4 providers with skills (Modal, Lambda, TensorPool, Prime). **Vast + RunPod keys inject but no skill reads them.** | +| **B. Cloud storage** | ⚠️ creds-only, unverified | No mount/rclone abstraction — AWS/GCP creds → env → whatever CLI a skill invokes. **Azure object storage is advertised but not backed.** Needs real creds+bucket to verify (FLAG). | +| **C. SSH-based compute** | ~~❌ dead-end~~ → **fixed** | ~~"SSH hosts" + "Model endpoints" panels persist data **nothing ever reads**. No SSH client, no dispatch, no routing.~~ **Overtaken by `aa9b3142`:** `src/compute/jobs.ts` dispatches over `ssh`, with Slurm/PBS/none schedulers and a reachability probe. **Model endpoints are still unread.** | +| **D. Managed compute via atlas CLI** | ❌ command surface unpublished | The published `@synsci/atlas@0.13.2` contains **zero** `compute:` commands. The suite exists only in the atlas **repo**, unreleased. The pin is already `^0.13.2` (`backend/cli/package.json:123`) — there is no version to bump to. Atlas's `/api/compute/leases` itself works; nothing shipped can call it. | Ground truth: `atlas doctor` on this machine is seeded (`~/.config/atlas-cli/config.json`), authed, backend reachable — so atlas **auth/config-seeding works**; only the **compute command surface** is broken. @@ -21,10 +34,18 @@ Compute panel (`Compute.tsx`, 6 cards) → `server/routes/settings/compute.ts`: - ❌ **Vast/RunPod inject but no skill reads them** (`VAST_API_KEY`/`RUNPOD_API_KEY` set, no skill) — connecting does nothing. - ⚠️ `last_used` is declared + rendered but **never written** → always "never". - ⚠️ **Modal double-stored** — also in the Credentials panel, which injects the same vars and runs first at boot (`index.ts:107` before `:111`), so a Modal key set in both panels has the Credentials value silently win. -- ⚠️ latent: the atlas-bin fallback resolver walks for `@openscience/atlas` while the dep is `@synsci/atlas` (`index.ts:225`) — dead fallback. +- ⚠️ ~~latent: the atlas-bin fallback resolver walks for `@openscience/atlas` while the dep is `@synsci/atlas` (`index.ts:225`) — dead fallback.~~ **Fixed** — `grep -r '@openscience/atlas' backend/cli/src` → 0 hits (2026-08-01). - **FLAG:** an actual job round-trip (Modal/Lambda/TensorPool/Prime) needs live provider accounts — plumbing verified, round-trip not. -**Fixes:** author Vast/RunPod skills or drop them from the catalog (interim: "key stored — skill coming"); populate or remove `last_used`; pick one home for Modal (recommend removing from Compute, Credentials owns it) or share one precedence; fix the `@openscience→@synsci` scope typo. +**Still true 2026-08-01:** `last_used` is still only defaulted and carried forward, never assigned +(`routes/settings/compute.ts:252`, `:272`). Vast/RunPod still have no catalogued skill. + +**Note on Vast/RunPod:** both are now leasable **server-side**, as managed providers in Atlas, and +were driven end to end on 2026-08-01. That does not close this gap: the capability lives behind +`POST /api/compute/leases`, on an unmerged draft branch, and OpenScience has no tool that calls it. +A user's `VAST_API_KEY`/`RUNPOD_API_KEY` in this panel is still consumed by nothing. + +**Fixes:** author Vast/RunPod skills or drop them from the catalog (interim: "key stored — skill coming"); populate or remove `last_used`; pick one home for Modal (recommend removing from Compute, Credentials owns it) or share one precedence; ~~fix the `@openscience→@synsci` scope typo~~ (done). ## Path B — Cloud storage (⚠️) @@ -37,30 +58,97 @@ Compute panel (`Compute.tsx`, 6 cards) → `server/routes/settings/compute.ts`: **Fixes:** add an Azure Storage cred or drop "Azure" from the copy; document the "creds-only, needs CLIs, no mount" contract; optionally seed an `rclone` remote from stored creds. -## Path C — SSH-based compute (❌) +## Path C — SSH-based compute (~~❌~~ → **half fixed**, 2026-08-01) Compute.tsx promises "SSH hosts" (dispatch runs over SSH) + "Model endpoints" (route inference). `compute.ts` persists `ssh_hosts` + `endpoints`. -- ❌ **store-only dead-ends** — no SSH client dep anywhere, no `ssh` spawn, `ssh_hosts` read only by its own CRUD/SDK-types/UI; `endpoints` has no inference-routing consumer. The agent cannot dispatch to a saved host or route to a saved endpoint. +**The July verdict below was overtaken by `aa9b3142` ("feat: add managed compute jobs", merged to +`main` 2026-07-29 — two days after this audit's baseline `52845c3`). Option (b) was chosen and +built for the SSH half.** What exists now: + +- ✅ **`ssh_hosts` is read and dispatched to.** `src/compute/jobs.ts` (`ComputeJobs`) spawns the + system `ssh` binary (`:362-366`, `BatchMode=yes`, honours a per-host port), supports `local` and + `ssh` targets, `none`/`slurm`/`pbs` schedulers (`sbatch --wait --parsable` at `:331`, `scancel` + at `:710`), optional `apptainer exec` (`:275`), a resource request (cpus/gpus/memory/time/ + partition), artifact collection, and job metadata persisted as JSON at mode `0600` (`:162`). +- ✅ **Reachability is probed, not assumed.** `POST /ssh/:id/test` returns latency plus + Python / NVIDIA GPU / Slurm / PBS capability flags (`routes/settings/compute.ts:394`). +- ✅ **It is user-reachable** — `frontend/workspace/src/atlas/ComputeJobs.tsx`, mounted in + `RightPane.tsx:297`, which the session page renders. Routes: `/jobs`, `/jobs/completed`, + `/jobs/:id/log`, `/jobs/:id/cancel`. +- ❌ **Model endpoints are still a store-only dead end.** `endpoints` still has no + inference-routing consumer; the panel still promises routing that nothing performs. +- ❌ **No agent tool.** `grep ComputeJobs backend/cli/src/tool/` → **0 hits**. The job system is + driven from the UI; the agent cannot start or cancel a job. +- ⚠️ **No SSH client _dependency_ was added** — it shells out to the host's `ssh`. So audits + phrased as "no SSH client in the dep tree" remain literally true and are no longer evidence of + anything. Key handling is still the user's `~/.ssh`; the store still has no key field, which is + why the security sign-off below never had to happen. - ⚠️ nuance: cloud-compute skills SSH into boxes **they provision** (lambda/tensorpool/skypilot) — real SSH, unrelated to the panel. -**Fixes (pick one):** (a) **remove** the SSH-hosts + model-endpoints sections + routes (recommended near-term — stop advertising vaporware); (b) **wire it** — a remote-exec tool reading `ssh_hosts` (needs an SSH client dep + key handling) + treat `endpoints` as selectable OpenAI-compatible targets, injected into compute-agent context. **Decision required + security sign-off** on private-key storage (the store has no key field today). +~~**Fixes (pick one):** (a) **remove** the SSH-hosts + model-endpoints sections + routes (recommended near-term — stop advertising vaporware); (b) **wire it** — a remote-exec tool reading `ssh_hosts` (needs an SSH client dep + key handling) + treat `endpoints` as selectable OpenAI-compatible targets, injected into compute-agent context. **Decision required + security sign-off** on private-key storage (the store has no key field today).~~ + +**Remaining fixes:** (a) **model endpoints** — still the original decision: remove the section or +wire it as selectable OpenAI-compatible targets. (b) Expose `ComputeJobs` to the agent, or state +that it is deliberately UI-only. ## Path D — Managed compute via the Atlas CLI (❌) Config seeding works (`ensureAtlasCliConfig`, verified by `atlas doctor`). Intended UX: "Compute spend = Managed" (`Spend.tsx`) → `billing.compute` → a `` injected by `insertReminders` (`prompt.ts:1321-1333`) telling the agent to run `atlas compute:up`. Atlas has the machinery: `POST /api/compute/leases` provisions Modal sandboxes + reseller GPU VMs, billed to the wallet (`compute.py:305-421`, `compute_billing_service.py`). -**Correction to the initial audit** (which tested the _installed 0.13.1_): the atlas CLI at the **published latest (0.13.2)** ships a real compute suite — `cli/src/atlas-runtime/commands.mjs:915-923` registers `compute:up` (aliases `launch`/`lease` → `POST /compute/leases`, _"zero flags = cheapest GPU; managed bills the wallet per hour; BYOK free"_), `compute:catalog`/`gpus`/`options` (browse GPUs → `/compute/options`), `compute:list`/`leases` (`GET /compute/leases`), `compute:ssh` (`/connection`), `compute:release`/`down` (`/release`). So `atlas compute:up` **is a real command in 0.13.2**, hitting the exact `/api/compute/leases` API — the prompt is _aspirationally correct_, not naming a phantom. +### ~~Correction to the initial audit~~ — the correction was itself wrong (retracted 2026-08-01) + +**Kept in place rather than deleted, because the mistake is the useful part of this section.** It +was a source-read conclusion about a _published artifact_, checked against neither the artifact nor +the manifest it claimed to be quoting. + +~~**Correction to the initial audit** (which tested the _installed 0.13.1_): the atlas CLI at the **published latest (0.13.2)** ships a real compute suite — `cli/src/atlas-runtime/commands.mjs:915-923` registers `compute:up` (aliases `launch`/`lease` → `POST /compute/leases`, _"zero flags = cheapest GPU; managed bills the wallet per hour; BYOK free"_), `compute:catalog`/`gpus`/`options` (browse GPUs → `/compute/options`), `compute:list`/`leases` (`GET /compute/leases`), `compute:ssh` (`/connection`), `compute:release`/`down` (`/release`). So `atlas compute:up` **is a real command in 0.13.2**, hitting the exact `/api/compute/leases` API — the prompt is _aspirationally correct_, not naming a phantom.~~ -- ⚠️ **Version gap is the core defect.** OpenScience pins `@synsci/atlas@^0.5.12` (`backend/cli/package.json`); the installed CLI is 0.13.1 (whose `--help` doesn't surface compute); **npm latest is 0.13.2** (which does). So the MANAGED prompt (`prompt.ts:1329` `atlas compute:up`) names a real command the **shipped/pinned atlas CLI predates** → it doesn't resolve for users today. +**Both of its load-bearing claims are false. Verified 2026-08-01:** + +- **The pin is not `^0.5.12`.** `backend/cli/package.json:123` reads `"@synsci/atlas": "^0.13.2"`. + There is no version gap, and nothing to bump. +- **The published artifact carries no `compute:` commands at all.** `npm pack @synsci/atlas@latest` + resolves to **0.13.2**, and **zero** files in that tarball contain `compute:up` or + `compute:lease`. Not an older surface — no surface. + +The commands _are_ real, but only in the atlas **repo**: `3e1d1ca` removed them, `0.13.1` **and** +`0.13.2` both shipped without them, and `205bbc0` re-added them **with no version bump**. So the +source tree and the published artifact disagree at an identical version number — which is precisely +why reading the repo told the first correction the opposite of what a user installs. + +**This is a release problem, not a code or pinning problem.** Until a version ships with the +commands in it, no prompt, doc or runbook may name `atlas compute:*`. + +- ✅ **The prompt no longer names it.** `12a43695` replaced the `atlas compute:up` guidance with a + `compute_status` pointer; `grep -r 'compute:up' backend/cli/src` → **0 hits**. The defect this + section was written to describe is closed on the client side. - ⚠️ **The surface has churned** — the CLI CHANGELOG shows a `compute:*` set removed then a richer one re-added; and it also describes provisioning as a **web-dashboard "Lambda Labs reseller" Compute tab**. Confirm the intended UX (CLI leasing vs web dashboard, Modal as agent-runtime-internal) is settled before wiring the prompt hard to it. - ⚠️ `exec:start` is a **separate** graph-ledger command (INSERTs a bookkeeping row, no Modal/lease call, `execution_service.py:45-87`) — not the compute path; don't conflate the two. -- ⚠️ server-side managed GPU is **off by default** — `COMPUTE_RESELL_ENABLED="false"` (`config.py:387`). +- ⚠️ ~~server-side managed GPU is **off by default**~~ — the **default** is still `"false"` + (`config.py:383-384`), but **production has it on**: `resell_enabled: true` with lambda / runpod / + vast / prime_intellect operator-funded and 292 launchable options, verified against + `thesis-synsc` on 2026-07-31. Reading the default and concluding "managed is off" was one of the + four source-read errors this workstream has now made about deployed behaviour. - ⚠️ `billing.compute` is **prompt-only** — unlike `billing.llm` (mirrors to server + resyncs), it just persists + injects the reminder. - ⚠️ substrate named 3 ways — "Daytona-backed" (`research.txt:229`) vs "Modal sandbox" (`atlas agent:run --help`) vs "Atlas-provisioned" (`config.ts:984`). -- **FLAG:** an actual lease still needs `COMPUTE_RESELL_ENABLED=true` + operator keys + a funded wallet + a Modal account to verify end-to-end. - -**Fixes:** (1) **primary — bump `@synsci/atlas` `^0.5.12`→`^0.13.2`** (align the pin + the seeded/expected version to the published CLI that has compute) and verify `atlas compute:*` resolves against the installed version; then `prompt.ts:1329`'s `atlas compute:up` is truthful. Add a prompt-time guard: if the installed `atlas` lacks `compute:*`, the MANAGED reminder falls back to BYOK rather than naming an unresolvable command. (2) enable resale (`COMPUTE_RESELL_ENABLED`) + wire `billing.compute` to reality (mirror `billing.llm`) so the managed path actually leases. (3) reconcile the CLI-leasing vs web-dashboard-reseller UX (owner decision). (4) reconcile substrate naming. (5) decide the BYOK source of truth. +- ~~**FLAG:** an actual lease still needs `COMPUTE_RESELL_ENABLED=true` + operator keys + a funded wallet + a Modal account to verify end-to-end.~~ **Discharged 2026-08-01.** Exercised end to end + against a deployed backend on **both** Vast and RunPod: `POST /leases` → the real background + reaper promoted to `ready` with NATed SSH coordinates within one sweep → SSH into a real GPU → + `POST /release` → instance verified gone at the provider. Not Modal — Modal is the CPU sandbox + path, and conflating it with GPU leases is the same error as the `exec:start` bullet above. + +**Fixes:** (1) ~~bump the pin~~ — **publish an atlas release containing the `compute:` commands**; +the pin is already `^0.13.2` and the code is already in the repo, so this is a release action with +no code change. Until then, keep prompts free of `atlas compute:*` (already true — `12a43695`). +(2) ~~enable resale~~ — **done in production**; what remains is wiring `billing.compute` to reality +(mirror `billing.llm`). (3) reconcile the CLI-leasing vs web-dashboard-reseller UX (owner decision). +(4) reconcile substrate naming. (5) decide the BYOK source of truth. + +> **What is actually missing is a client, not a CLI.** OpenScience cannot launch a managed lease by +> any route: `ComputeTools` is `[ComputeStatusTool]` (`src/tool/compute.ts:138`), and the only +> `/api/compute` call in the product is `mode.ts`'s read-only `/options` probe. `compute_launch`, +> `compute_list` and `compute_release` are designed in `docs/specs/compute-design.md` and **unbuilt**. ## Cross-cutting — overlapping BYOK stores @@ -68,33 +156,34 @@ A user's Modal key can live in **three** places with no reconciliation: the loca ## Consolidated backlog (by effort) -| # | Fix | Path | Effort | -| --- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---- | --------------------- | -| 1 | Bump `@synsci/atlas` pin `^0.5.12`→`^0.13.2` so `atlas compute:up` resolves; add a "compute unavailable → BYOK" prompt guard (`prompt.ts:1329`); reconcile `research.txt:229` | D | S | -| 2 | Azure: add Storage cred or drop "Azure" copy | B | XS | -| 3 | Vast/RunPod: "no skill yet" or remove from catalog | A | XS | -| 4 | `last_used`: populate or remove | A | XS | -| 5 | Fix atlas-bin fallback scope `@openscience→@synsci` | A | XS | -| 6 | Modal de-dup across Compute vs Credentials | A | S | -| 7 | Remove or wire SSH-hosts + model-endpoints | C | S (remove) / L (wire) | -| 8 | Document cloud-storage contract + optional rclone seeding | B | S | -| 9 | Enable resale + wire `billing.compute` to reality (mirror `billing.llm`) so managed leasing works end-to-end | D | M | -| 10 | Reconcile 3-way BYOK store + atlas version pin | A/D | M | +| # | Fix | Path | Effort | +| --- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---- | --------------------- | +| 1 | ~~Bump the `@synsci/atlas` pin~~ — **wrong fix, the pin is already `^0.13.2`.** **Publish** an atlas release that contains the `compute:` commands; the code is in the repo, unreleased. Prompt guard no longer needed (`12a43695` removed `atlas compute:up`) | D | S (release, no code) | +| 2 | Azure: add Storage cred or drop "Azure" copy | B | XS | +| 3 | Vast/RunPod: "no skill yet" or remove from catalog | A | XS | +| 4 | `last_used`: populate or remove — **still open** | A | XS | +| 5 | ~~Fix atlas-bin fallback scope `@openscience→@synsci`~~ — **done** | A | XS | +| 6 | Modal de-dup across Compute vs Credentials | A | S | +| 7 | ~~Remove or wire SSH-hosts~~ — **wired** (`aa9b3142`). Remaining: **model endpoints** (remove or wire), and whether `ComputeJobs` gets an agent tool | C | S (remove) / M (tool) | +| 8 | Document cloud-storage contract + optional rclone seeding | B | S | +| 9 | ~~Enable resale~~ (on in production) + wire `billing.compute` to reality (mirror `billing.llm`) | D | M | +| 10 | Reconcile 3-way BYOK store ~~+ atlas version pin~~ (the pin is correct) | A/D | M | +| 11 | **Added 2026-08-01** — build `compute_launch`/`compute_list`/`compute_release`. Atlas can lease; nothing in OpenScience can ask it to. Designed in `docs/specs/compute-design.md` | D | L | ## Risks / decisions needed from the owner -- **Is managed compute in scope this sprint?** Backend is built but the CLI surface + default-off flag mean it's not shippable today — if out of scope, stop advertising it (`Compute.tsx:159-162`, `Spend.tsx:42`). -- **SSH hosts / endpoints:** in scope (wire, +SSH dep + key security) or remove? -- **Infra to verify:** BYOK round-trips need provider accounts; cloud storage needs real creds+bucket+CLIs; managed leases need operator keys + `COMPUTE_RESELL_ENABLED=true` + funded wallet + Modal. **Do not mark any path "works" without exercising it.** +- **Is managed compute in scope this sprint?** ~~Backend is built but the CLI surface + default-off flag mean it's not shippable today~~ — **restated 2026-08-01:** the backend is built _and demonstrated end to end_, and resale is on in production. What blocks it is (a) an unpublished CLI and (b) **no client**: OpenScience has no launch tool. If it stays out of scope, stop advertising it (`Compute.tsx`, the Spend panel). +- **SSH hosts / endpoints:** ~~in scope (wire, +SSH dep + key security) or remove?~~ **SSH hosts: answered — wired in `aa9b3142`.** Model endpoints: still remove-or-wire. +- **Infra to verify:** BYOK round-trips need provider accounts; cloud storage needs real creds+bucket+CLIs. ~~managed leases need operator keys + `COMPUTE_RESELL_ENABLED=true` + funded wallet + Modal~~ — **done 2026-08-01, Vast and RunPod, on a deployed backend.** **Do not mark any path "works" without exercising it.** - **BYOK source-of-truth** is a cross-repo decision. ## Acceptance criteria -- No prompt instructs a command absent from the bundled CLI (grep prompts for every `atlas …` verb; assert each resolves in `atlas --help`). +- No prompt instructs a command absent from the bundled CLI (grep prompts for every `atlas …` verb; assert each resolves in `atlas --help`). **Met for compute** — 0 hits for `compute:up` in `backend/cli/src`. - Connecting Vast/RunPod either drives a real run or the UI no longer implies it will. - Storage lists only credential-backed backends (Azure fixed or removed); a documented smoke test (with creds) round-trips an object on S3 + GCS. -- SSH/endpoint panels are gone, or adding a host + "run nvidia-smi on " executes over SSH. -- With managed enabled + infra: `Compute spend = Managed` starts a real lease, wallet debits per the 60 s tick, auto-releases — demonstrated once. +- ~~SSH/endpoint panels are gone, or adding a host + "run nvidia-smi on " executes over SSH.~~ **Met for SSH** (`aa9b3142`); **not met for model endpoints.** +- With managed enabled + infra: `Compute spend = Managed` starts a real lease, wallet debits per the 60 s tick, auto-releases — demonstrated once. **Demonstrated at the API, not from OpenScience** — `POST /api/compute/leases` → billing tick decrementing the wallet in lockstep → release, on a deployed backend. The `Compute spend = Managed` _path_ still has no launch mechanism to trigger. - One substrate name + one canonical atlas version documented; `billing.compute` changes behavior or is labeled advisory; `settings-compute` tests stay green. **Key files:** `components/settings/{Compute,Storage,Spend}.tsx`, `server/routes/settings/{compute,storage,credentials,billing}.ts`, `openscience/index.ts`, `session/{prompt,billing-gate}.ts`, `agent/prompt/research.txt`, `config/config.ts`. Atlas: `routes/compute.py`, `services/{execution,compute_billing,compute_keys}_service.py`, `compute/{lease_manager,modal_provider}.py`, `config.py`. diff --git a/docs/plans/2026-07-30-compute-mode-detection.md b/docs/plans/2026-07-30-compute-mode-detection.md new file mode 100644 index 00000000..6c4682c4 --- /dev/null +++ b/docs/plans/2026-07-30-compute-mode-detection.md @@ -0,0 +1,1746 @@ +# Compute Mode Detection Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Replace the static `billing.compute` config default with runtime detection that resolves compute funding to `byok`, `managed`, or `none`, exposes it to the agent through a `compute_status` tool, and filters the skill catalog to the providers the user can actually use. + +**Architecture:** One shared resolver (`src/compute/mode.ts`) reads `process.env` for provider credentials and `Skill.all()` for the matching skills; a provider counts only when it has both. Resolution happens on demand at two per-request seams — `SkillTool.init` (which rebuilds the catalog every turn) and the new `compute_status` tool — never at startup, so it cannot observe a half-initialised environment. Managed availability is one authenticated `GET /api/compute/options` call, made only when no usable provider exists, with a hard 3s timeout and a 5s in-process cache. + +**Tech Stack:** Bun, TypeScript, Zod, Hono. Tests are `bun test` with `globalThis.fetch` stubbed at the network boundary — no mocks, no network. + +**Spec:** `docs/specs/compute-mode-detection-design.md` (14 acceptance criteria). Read it before Task 1. + +## Global Constraints + +- **Style (`AGENTS.md`):** prefer `const` over `let`, avoid `else`, single-word variable names, rely on type inference over explicit annotations, **no `any`**, use Bun APIs (`Bun.file()`, `Bun.write()`). +- **No mocks in tests.** Stub `globalThis.fetch` at the network boundary and exercise the real implementation. Restore the real `fetch` in `afterEach`. +- **No network in `bun test`.** `test/preload.ts` already points `OPENSCIENCE_API_BASE` at `http://127.0.0.1:9` (unroutable) and sets `OPENSCIENCE_DISABLE_BUNDLED_SKILLS=true`, so `Skill.all()` in tests sees only skills the test itself writes into the tmpdir project. +- **Never** add `Co-Authored-By:` or any AI attribution to commit messages or PR bodies. Organisation rule. +- Run `bunx prettier --write ` before every commit. CI has a Format job over the whole repo. +- All commands run from `backend/cli`: `bun test`, `bun run typecheck`. +- **Every new assertion must be demonstrated failing against the specific mutation it guards — the _deletion_ of the logic under test, not merely its inversion.** Each test step below names its mutation. On the preceding `science_fetch` branch seven assertion defects were found and all seven were in plan-authored test code; the ones that held up were proven against deletion. +- Do **not** touch `backend/cli/test/provider/synthetic-model.test.ts`, `compaction-divider.png`, `docs/specs/issue-194-katex-latex-leak.md`, or `open-bench/` — untracked user WIP. +- Do **not** implement `docs/specs/compute-guardrails-design.md`. It is parked. + +## Corrections to the spec, already verified + +The spec was written against skill names that do not exist. These are the real names, confirmed against the live 293-skill catalog index (`~/.cache/openscience/skills-index.json`) and the authored sources in `backend/cli/skills/`: + +| Provider | Spec said | Actual skill name(s) | Source dir | +| --------------- | -------------------------------------------------------- | ----------------------------------------------------------------- | ---------------------------------------------------------- | +| Modal | `cloud-compute/modal`, `cloud-compute/modal-ml-training` | `modal-serverless-gpu`, `modal-ml-training`, `modal-research-gpu` | `skills/cloud-compute/modal{,-ml-training,-research-gpu}/` | +| Lambda | `cloud-compute/lambda-labs` | `lambda-labs-gpu-cloud` | `skills/cloud-compute/lambda-labs/` | +| TensorPool | `cloud-compute/tensorpool` | `tensorpool-gpu-cloud` | `skills/cloud-compute/tensorpool/` | +| Prime Intellect | `ml-training/prime-intellect-lab` | `prime-intellect-lab` | `skills/ml-training/prime-intellect-lab/` | +| RunPod | none | none — and none will be written, see Decision 2 | — | +| Vast.ai | none | none — and none will be written, see Decision 2 | — | + +Skill `name` comes from SKILL.md frontmatter, **not** the directory name, and there is no `category/` prefix in the name. A resolver hardcoding the spec's strings would filter nothing. + +Other verified facts the spec did not have: + +- `GET /api/compute/options` already returns `cli_effective_balance_cents`, so `balance_usd` needs **no second call**. Response envelope (`atlas` `origin/main:backend/app/routes/compute.py:213-232`): `{options[], providers[], resell_enabled, byok_eligible, platform_fee_ratio, cli_balance_cents, cli_effective_balance_cents}`, where each `providers[]` entry is `{provider, has_byok, has_operator, funding, count}` and `funding` is one of `byok | managed | unavailable`. +- `computeBillingMode()` in `src/session/billing-gate.ts:34` has exactly **one** consumer, `src/session/prompt.ts:1554`. Task 5 deletes both. +- `backend/cli/skills/` is loaded as a **dev-only** fallback (`Installation.VERSION === "local"`, `src/skill/skill.ts:219`). In a shipped binary skills come from the server catalog index. Authoring in this repo is correct and is what Tasks 6–7 do, but the resolver must never assume a skill is present — it checks `Skill.all()` at resolve time, so a binary whose catalog lacks `runpod-gpu-cloud` correctly reports RunPod as not usable. + +## Decisions taken (answering the spec's open questions) + +1. **Q1 — per-provider resolution:** no. `byok` when _any_ provider is usable; provider choice is left to the agent and the skills. +2. **Q2 — RunPod and Vast — REVISED mid-execution.** The original answer was "write the two skills". The user then ruled: _"the skill is definitely overkill, a capable agent can figure out how to use the cloud provider out of the box."_ + + That reverses more than Tasks 6–7. The spec's **"key AND skill"** rule rested entirely on the claim that a provider with a credential but no skill "gives the agent nothing to act on". If a capable agent can drive a public cloud API from a bare key, that claim is false and the conjunction is wrong — it would report `none` to a user holding a perfectly workable RunPod key. So: + - **A provider is BYOK-usable when it has a credential. Full stop.** The skill conjunction is deleted. + - `runpod` and `vast` carry `skills: []`. No RunPod or Vast skill is written; **Tasks 6 and 7 are deleted.** + - The `unusable` list disappears from `Resolution` and from `compute_status` — it existed only to explain the key-without-skill dead end, which no longer exists. + - **Catalog filtering is unaffected**: it still lists a provider's skills only when that provider is credentialed. A skill is a quality boost where one exists, not a licence to use the provider. + - Side benefit: this deletes the catalog-drift failure mode. Skills reach a shipped binary from the server catalog, so under the old rule a catalog that dropped or renamed `lambda-labs-gpu-cloud` would have silently marked Lambda unusable for a user whose key was fine. + + Tasks 6 and 7 in this plan are **superseded — do not implement them.** + +3. **Q3 — is `none` a hard block:** no, guidance only. `compute_status` tells the agent not to attempt GPU work; nothing gates `bash` or blocks a direct `skill(name=…)` load. Enforcement is a larger change and is out of scope. Task 4 makes this explicit in a code comment rather than leaving it to be discovered. +4. **Q4 — prompt pointer:** **keep one line.** Task 5 deletes the false `atlas compute:up` / `atlas doctor` text and replaces it with a single unconditional, stateless reminder for `COMPUTE_AGENTS`. It carries no mode, so it can never go stale. +5. **Q5 — config description:** yes, updated in Task 5. +6. **Q6 — how long may `compute_status` block:** **3000 ms**, hard. `OpenScience.atlasFetch`'s 60s default is far too long to sit in front of an agent turn. Cache TTL is **5000 ms**. +7. **Filter scope:** only the six mapped providers' skills are subject to mode filtering. `fireworks-ai-inference`, `together-ai-inference`, `tinker-fine-tuning`, `tinker-training-cost` and `skypilot-multi-cloud-orchestration` share the `cloud-compute` category but are inference APIs and orchestrators keyed by their own credentials, not GPU leases this mode governs. They are never hidden. + +## File Structure + +**Created:** + +| Path | Responsibility | +| ---------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------- | +| `backend/cli/src/compute/mode.ts` | The single shared resolver. Provider→env→skill table, `usable()`, `resolve()`, `invalidate()`. The only place that decides what "usable" means. | +| `backend/cli/src/tool/compute.ts` | The `compute_status` tool. Formats a `ComputeMode.Resolution` for the agent; contains no resolution logic of its own. | +| `backend/cli/skills/cloud-compute/runpod/SKILL.md` | RunPod GPU cloud skill (`runpod-gpu-cloud`). | +| `backend/cli/skills/cloud-compute/vast-ai/SKILL.md` | Vast.ai GPU marketplace skill (`vast-ai-gpu-cloud`). | +| `backend/cli/test/compute/mode.test.ts` | Resolver tests (Tasks 1–2). | +| `backend/cli/test/tool/compute-status.test.ts` | Tool tests (Task 3). | +| `backend/cli/test/tool/skill-compute-filter.test.ts` | Catalog filtering tests (Task 4). | +| `backend/cli/test/session/compute-prompt.test.ts` | Prompt-text regression tests (Task 5). | + +**Modified:** + +| Path | Change | +| ----------------------------------------------- | ---------------------------------------------------------------------- | +| `backend/cli/src/tool/registry.ts` | Import and register `ComputeTools`. | +| `backend/cli/src/tool/skill.ts` | Filter the catalog by resolved mode inside `init`. | +| `backend/cli/src/session/prompt.ts:1550-1564` | Replace the mode-carrying injection with a stateless one-line pointer. | +| `backend/cli/src/session/billing-gate.ts:33-36` | Delete the now-dead `computeBillingMode()`. | +| `backend/cli/src/config/config.ts:1058-1064` | Correct the `billing.compute` description. | + +--- + +### Task 1: The resolver's usable-provider rule + +> **Implemented at `dc125b9` under the original "key AND skill" rule, then superseded by Decision 2. Task 1b below revises it to credential-only. Kept here as the historical record — do not re-implement.** + +Pure logic, no network. Establishes the one definition of "usable" that Tasks 2–4 all consume. + +**Files:** + +- Create: `backend/cli/src/compute/mode.ts` +- Test: `backend/cli/test/compute/mode.test.ts` + +**Interfaces:** + +- Consumes: `Skill.all()` from `@/skill` (returns `Skill.Info[]`, each with a `name`). +- Produces, for Tasks 2, 3 and 4: + - `type ComputeMode.Source = "byok" | "managed" | "none"` + - `ComputeMode.PROVIDERS: Record` + - `ComputeMode.SKILLS: Set` — every provider skill name, the exact set Task 4 filters over. + - `ComputeMode.usable(): Promise<{ providers: string[]; unusable: string[] }>` — `providers` are ids with a key _and_ at least one catalogued skill; `unusable` are ids with a key but no catalogued skill. Both sorted, in `PROVIDERS` declaration order. + +- [ ] **Step 1: Write the failing test** + +Create `backend/cli/test/compute/mode.test.ts`: + +```ts +import { test, expect, afterEach, describe } from "bun:test" +import path from "path" +import { ComputeMode } from "../../src/compute/mode" +import { Instance } from "../../src/project/instance" +import { tmpdir } from "../fixture/fixture" + +const ENV = [ + "MODAL_TOKEN_ID", + "MODAL_TOKEN_SECRET", + "LAMBDA_API_KEY", + "LAMBDA_LABS_API_KEY", + "TENSORPOOL_KEY", + "TENSORPOOL_API_KEY", + "PRIME_API_KEY", + "PRIME_INTELLECT_API_KEY", + "RUNPOD_API_KEY", + "VAST_API_KEY", +] + +function clearEnv() { + for (const name of ENV) delete process.env[name] +} + +afterEach(clearEnv) + +/** A tmpdir project seeded with real SKILL.md files, so Skill.all() finds them + * without a network catalog. `OPENSCIENCE_DISABLE_BUNDLED_SKILLS` in preload.ts + * keeps the dev skills/ dir and the server index out, so the test controls the + * catalog exactly. */ +async function withSkills(names: string[], fn: () => Promise): Promise { + await using tmp = await tmpdir({ + git: true, + init: async (dir) => { + for (const name of names) { + await Bun.write( + path.join(dir, ".openscience", "skill", name, "SKILL.md"), + `---\nname: ${name}\ndescription: Test fixture for ${name}.\ncategory: cloud-compute\n---\n\n# ${name}\n`, + ) + } + }, + }) + return Instance.provide({ directory: tmp.path, fn }) +} + +describe("ComputeMode.usable", () => { + test("a provider with a key and a skill is usable", async () => { + clearEnv() + process.env["LAMBDA_API_KEY"] = "secret_abc" + const result = await withSkills(["lambda-labs-gpu-cloud"], () => ComputeMode.usable()) + expect(result.providers).toEqual(["lambda"]) + expect(result.unusable).toEqual([]) + }) + + test("the alternate env spelling also counts", async () => { + clearEnv() + process.env["LAMBDA_LABS_API_KEY"] = "secret_abc" + const result = await withSkills(["lambda-labs-gpu-cloud"], () => ComputeMode.usable()) + expect(result.providers).toEqual(["lambda"]) + }) + + test("a key with NO catalogued skill is not usable", async () => { + clearEnv() + process.env["RUNPOD_API_KEY"] = "rpa_abc" + const result = await withSkills([], () => ComputeMode.usable()) + expect(result.providers).toEqual([]) + expect(result.unusable).toEqual(["runpod"]) + }) + + test("a catalogued skill with NO key is not usable", async () => { + clearEnv() + const result = await withSkills(["lambda-labs-gpu-cloud"], () => ComputeMode.usable()) + expect(result.providers).toEqual([]) + expect(result.unusable).toEqual([]) + }) + + test("modal needs BOTH token vars — id alone is not a key", async () => { + clearEnv() + process.env["MODAL_TOKEN_ID"] = "ak-abc" + const result = await withSkills(["modal-serverless-gpu"], () => ComputeMode.usable()) + expect(result.providers).toEqual([]) + expect(result.unusable).toEqual([]) + }) + + test("modal with both token vars is usable", async () => { + clearEnv() + process.env["MODAL_TOKEN_ID"] = "ak-abc" + process.env["MODAL_TOKEN_SECRET"] = "as-def" + const result = await withSkills(["modal-serverless-gpu"], () => ComputeMode.usable()) + expect(result.providers).toEqual(["modal"]) + }) + + test("an empty-string key does not count as set", async () => { + clearEnv() + process.env["TENSORPOOL_KEY"] = "" + const result = await withSkills(["tensorpool-gpu-cloud"], () => ComputeMode.usable()) + expect(result.providers).toEqual([]) + expect(result.unusable).toEqual([]) + }) + + test("every provider resolves in isolation, given its own skill", async () => { + const cases: Array<[string, Record, string]> = [ + ["modal", { MODAL_TOKEN_ID: "ak-a", MODAL_TOKEN_SECRET: "as-b" }, "modal-serverless-gpu"], + ["lambda", { LAMBDA_API_KEY: "k" }, "lambda-labs-gpu-cloud"], + ["tensorpool", { TENSORPOOL_KEY: "k" }, "tensorpool-gpu-cloud"], + ["prime", { PRIME_API_KEY: "k" }, "prime-intellect-lab"], + ["runpod", { RUNPOD_API_KEY: "k" }, "runpod-gpu-cloud"], + ["vast", { VAST_API_KEY: "k" }, "vast-ai-gpu-cloud"], + ] + for (const [id, env, skill] of cases) { + clearEnv() + Object.assign(process.env, env) + const result = await withSkills([skill], () => ComputeMode.usable()) + expect(result.providers).toEqual([id]) + } + }) + + test("SKILLS covers every name in PROVIDERS and nothing else", async () => { + const declared = Object.values(ComputeMode.PROVIDERS).flatMap((p) => p.skills) + expect([...ComputeMode.SKILLS].sort()).toEqual([...new Set(declared)].sort()) + expect(ComputeMode.SKILLS.size).toBeGreaterThan(0) + }) + + test("a key injected after the first call is seen on the next call", async () => { + clearEnv() + await withSkills(["lambda-labs-gpu-cloud"], async () => { + expect((await ComputeMode.usable()).providers).toEqual([]) + process.env["LAMBDA_API_KEY"] = "secret_late" + expect((await ComputeMode.usable()).providers).toEqual(["lambda"]) + }) + }) +}) +``` + +Mutations these guard: deleting the skill-presence check (test 3 flips to `["runpod"]`); deleting the env check (test 4 flips to `["lambda"]`); deleting the both-vars-required branch for Modal (test 5 flips to `["modal"]`); deleting the non-empty check (test 7 flips to `["tensorpool"]`); deleting any row from `PROVIDERS` (test 8 fails for that row); caching the env read (test 10's second assertion flips back to `[]`). + +- [ ] **Step 2: Run test to verify it fails** + +Run: `cd backend/cli && bun test test/compute/mode.test.ts` +Expected: FAIL — `Cannot find module '../../src/compute/mode'`. + +- [ ] **Step 3: Write the resolver's usable-provider rule** + +Create `backend/cli/src/compute/mode.ts`: + +```ts +import { Skill } from "@/skill" + +/** + * Runtime resolution of how GPU compute is funded. + * + * `billing.compute` used to answer this from config alone, which meant a + * brand-new user with zero provider keys resolved to "byok" — claiming BYOK + * with nothing to BYOK with. This module answers it from the environment + * instead, and can say "none", which is the state we previously handled worst. + * + * Resolution deliberately happens ON DEMAND and never at startup. Provider keys + * reach process.env from three places — the user's shell, the Credentials panel + * (`applyCredentialEnv`, src/index.ts:102) and the Compute panel + * (`applyComputeEnv`, src/index.ts:106) — and the latter two are wrapped in + * `.catch(() => {})`. Detecting at boot would report "none" for a user whose + * keys are configured through the UI. Both call sites (SkillTool.init and the + * compute_status tool) run per request, long after those injections, so the + * ordering constraint cannot be violated and cannot silently regress if someone + * reorders src/index.ts later. + */ +export namespace ComputeMode { + export type Source = "byok" | "managed" | "none" + + /** + * A provider is BYOK-usable only with BOTH a credential and a skill: the agent + * runs GPU work by loading a provider's skill, so a key with no skill gives it + * nothing to act on. + * + * `env` is a list of ALTERNATIVE groups; a group is satisfied when every var in + * it is set and non-empty. Modal is the only pair — its single pasted key + * splits into a token id + secret, and a half-pasted one maps to nothing + * (mirroring `mapProviderEnv`, server/routes/settings/compute.ts:181). + * + * `skills` are frontmatter `name` values, NOT directory names and NOT + * category-prefixed. Only these names are subject to mode filtering; the other + * cloud-compute skills (tinker, skypilot, fireworks, together) are inference + * APIs and orchestrators keyed by their own credentials, not GPU leases this + * mode governs, and are never hidden. + */ + export const PROVIDERS: Record = { + modal: { + env: [["MODAL_TOKEN_ID", "MODAL_TOKEN_SECRET"]], + skills: ["modal-serverless-gpu", "modal-ml-training", "modal-research-gpu"], + }, + lambda: { + env: [["LAMBDA_API_KEY"], ["LAMBDA_LABS_API_KEY"]], + skills: ["lambda-labs-gpu-cloud"], + }, + tensorpool: { + env: [["TENSORPOOL_KEY"], ["TENSORPOOL_API_KEY"]], + skills: ["tensorpool-gpu-cloud"], + }, + prime: { + env: [["PRIME_API_KEY"], ["PRIME_INTELLECT_API_KEY"]], + skills: ["prime-intellect-lab"], + }, + runpod: { + env: [["RUNPOD_API_KEY"]], + skills: ["runpod-gpu-cloud"], + }, + vast: { + env: [["VAST_API_KEY"]], + skills: ["vast-ai-gpu-cloud"], + }, + } + + /** Every provider skill name — the exact set the catalog filter operates on. */ + export const SKILLS = new Set(Object.values(PROVIDERS).flatMap((p) => p.skills)) + + /** Read process.env directly rather than Env.get: applyComputeEnv writes to + * process.env first and mirrors to Env only when instance state exists, so + * process.env is the one source that is always populated. */ + function keyed(groups: string[][]): boolean { + return groups.some((group) => group.every((name) => !!process.env[name])) + } + + /** + * Split configured providers into those the agent can actually act on and + * those with a stored key but no catalogued skill. The second list exists so + * `none` can say *why* — a user who connected a key and is then told no + * compute is available deserves better than silence. + */ + export async function usable() { + const catalog = new Set(await Skill.all().then((all) => all.map((skill) => skill.name))) + const providers: string[] = [] + const unusable: string[] = [] + for (const [id, spec] of Object.entries(PROVIDERS)) { + if (!keyed(spec.env)) continue + if (spec.skills.some((name) => catalog.has(name))) providers.push(id) + else unusable.push(id) + } + return { providers, unusable } + } +} +``` + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `cd backend/cli && bun test test/compute/mode.test.ts` +Expected: PASS, 11 tests. + +- [ ] **Step 5: Prove each assertion against deletion** + +For each mutation named in Step 1, temporarily apply it to `src/compute/mode.ts`, run the suite, confirm the named test fails, then revert. Specifically: + +1. In `usable()`, delete the `if (!keyed(spec.env)) continue` line → "a catalogued skill with NO key" must fail. +2. In `usable()`, replace the `spec.skills.some(...)` branch with an unconditional `providers.push(id)` → "a key with NO catalogued skill" must fail. +3. In `keyed()`, change `group.every(...)` to `group.some(...)` → "modal needs BOTH token vars" must fail. +4. In `keyed()`, change `!!process.env[name]` to `name in process.env` → "an empty-string key does not count" must fail. +5. Hoist the `catalog` set to module scope so it is computed once → "a key injected after the first call" still passes (env is not cached), but note this in the commit body: freshness of the _skill_ list is `Instance.state`'s job, and Task 4 covers per-turn catalog freshness. + +Record the five results in the commit body. + +- [ ] **Step 6: Typecheck, format, commit** + +```bash +cd backend/cli && bun run typecheck +bunx prettier --write src/compute/mode.ts test/compute/mode.test.ts +git add src/compute/mode.ts test/compute/mode.test.ts +git commit -m "feat(compute): resolve usable GPU providers from key and skill" +``` + +--- + +### Task 1b: Revise the rule to credential-only + +Applies Decision 2. A provider is BYOK-usable when it has a credential; the skill conjunction is deleted, and with it the `unusable` concept. + +**Files:** + +- Modify: `backend/cli/src/compute/mode.ts` +- Modify: `backend/cli/test/compute/mode.test.ts` + +**Interfaces:** + +- Consumes: `Skill.all()` is **no longer needed by the resolver** — remove the import if nothing else uses it. +- Produces, replacing Task 1's contract: + - `ComputeMode.PROVIDERS: Record` — unchanged shape; `runpod` and `vast` now carry `skills: []`. + - `ComputeMode.SKILLS: Set` — unchanged meaning, now six names. + - `ComputeMode.usable(): string[]` — **synchronous.** Returns credentialed provider ids in `PROVIDERS` declaration order. No object, no `unusable`, no `Promise`. + +- [ ] **Step 1: Update the tests first** + +In `backend/cli/test/compute/mode.test.ts`: + +- Every `result.providers` becomes `result` (the return is now the array itself). +- Delete the `unusable` assertions and the two tests that exist only to prove the key-without-skill state: "a key with NO catalogued skill is not usable" and the `unusable`-ordering test. +- **Replace** "a key with NO catalogued skill is not usable" with its inverse, which is now the rule: + +```ts +test("a key with NO catalogued skill IS usable — the agent drives the provider API directly", async () => { + clearEnv() + process.env["RUNPOD_API_KEY"] = "rpa_abc" + expect(await withSkills([], () => ComputeMode.usable())).toEqual(["runpod"]) +}) +``` + +- **Delete** "a catalogued skill with NO key is not usable"? No — keep it. A skill without a credential must still not make a provider usable, and it is now the only guard on the env check. Update it to assert `toEqual([])` against the bare array. +- The `SKILLS` literal drops to six names: `modal-serverless-gpu`, `modal-ml-training`, `modal-research-gpu`, `lambda-labs-gpu-cloud`, `tensorpool-gpu-cloud`, `prime-intellect-lab`. +- Keep the Modal-pair tests, the empty-string test, the declaration-order test, and the mid-session-injection test — all still load-bearing. +- Modal's three-skill-names test no longer proves anything about resolution (Modal resolves on its key alone now). **Delete it**, and instead assert the catalog-facing contract it was really protecting: + +```ts +test("PROVIDERS pins the exact skill names the catalog filter matches on", async () => { + expect([...ComputeMode.SKILLS].sort()).toEqual( + [ + "lambda-labs-gpu-cloud", + "modal-ml-training", + "modal-research-gpu", + "modal-serverless-gpu", + "prime-intellect-lab", + "tensorpool-gpu-cloud", + ].sort(), + ) + expect(ComputeMode.PROVIDERS["runpod"].skills).toEqual([]) + expect(ComputeMode.PROVIDERS["vast"].skills).toEqual([]) +}) +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `cd backend/cli && bun test test/compute/mode.test.ts` +Expected: FAIL — `usable()` still returns an object, and RunPod-with-no-skill still resolves to unusable. + +- [ ] **Step 3: Apply the rule change** + +In `backend/cli/src/compute/mode.ts`: + +- Set `runpod` and `vast` to `skills: []`. +- Replace `usable()` with: + +```ts +/** + * The credentialed GPU providers, in declaration order. + * + * A credential is the whole test. An earlier revision also required a + * matching skill, on the theory that a provider with no skill gives the agent + * nothing to act on — but a capable agent drives a documented cloud API from a + * key, so that conjunction only produced a false "no compute available" for + * users holding a perfectly workable key. A skill, where one exists, is a + * quality boost; the catalog filter still offers a provider's skills only when + * that provider is credentialed. + */ +export function usable(): string[] { + return Object.keys(PROVIDERS).filter((id) => keyed(PROVIDERS[id].env)) +} +``` + +Note it is now **synchronous** — it no longer awaits the skill catalog. Keep the call sites `await`-compatible by leaving `resolve()` async (Task 2); do not add a gratuitous `Promise.resolve`. + +- Remove the `Skill` import if nothing else in the file uses it. + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `cd backend/cli && bun test test/compute/mode.test.ts` +Expected: PASS. + +- [ ] **Step 5: Prove the new assertions against deletion** + +1. Delete the `filter((id) => keyed(...))` predicate so every provider is returned → "a catalogued skill with NO key" must fail. +2. Change `group.every` to `group.some` in `keyed()` → the Modal-pair test must fail. +3. Restore the skill conjunction (`&& PROVIDERS[id].skills.some(...)`) → the new RunPod test must fail. This is the specific regression the change exists to prevent. +4. Corrupt one character of any skill string in `PROVIDERS` → the `SKILLS` pin test must fail. + +- [ ] **Step 6: Typecheck, format, commit** + +```bash +cd backend/cli && bun run typecheck && bun test test/compute/ +bunx prettier --write src/compute/mode.ts test/compute/mode.test.ts +git add src/compute/mode.ts test/compute/mode.test.ts +git commit -m "refactor(compute): a credential alone makes a GPU provider usable" +``` + +--- + +### Task 2: Managed availability and full resolution + +Adds the network half and the override rules, completing the resolver. + +**Files:** + +- Modify: `backend/cli/src/compute/mode.ts` +- Test: `backend/cli/test/compute/mode.test.ts` (append a `describe`) + +**Interfaces:** + +- Consumes: `ComputeMode.usable()` **as revised by Task 1b — synchronous, returns `string[]`**; `OpenScience.getSession()` and `OpenScience.API_BASE` from `@/openscience`; `Config.get()` from `@/config/config`. +- Produces, for Tasks 3 and 4: + - `interface ComputeMode.Resolution { mode: Source; providers: string[]; managed: boolean; balance?: number }` — `balance` is USD and present only when `mode === "managed"`. **There is no `unusable` field** (Decision 2). + - `ComputeMode.resolve(): Promise` + - `ComputeMode.invalidate(): void` — drops the availability cache; tests call it in `beforeEach`. + +- [ ] **Step 1: Write the failing test** + +Append to `backend/cli/test/compute/mode.test.ts`: + +```ts +const OPTIONS_URL = "/api/compute/options" +const SESSION = path.join(Global.Path.data, "openscience-session.json") +const realFetch = globalThis.fetch + +/** Record of every URL the resolver fetched, so "the call is skipped" is a + * positive assertion rather than an absence of failure. */ +let calls: string[] = [] + +function stubOptions(body: unknown, status = 200) { + globalThis.fetch = (async (input: string | URL | Request) => { + const url = String(input instanceof Request ? input.url : input) + calls.push(url) + if (!url.includes(OPTIONS_URL)) return realFetch(input as never) + return new Response(JSON.stringify(body), { status, headers: { "content-type": "application/json" } }) + }) as typeof fetch +} + +async function signIn() { + await fs.mkdir(Global.Path.data, { recursive: true }) + await Bun.write(SESSION, JSON.stringify({ api_key: "thk_test.secret", user_id: "u1" })) +} + +const MANAGED_ON = { + options: [], + providers: [ + { provider: "lambda", has_byok: false, has_operator: true, funding: "managed", count: 3 }, + { provider: "vast", has_byok: false, has_operator: false, funding: "unavailable", count: 0 }, + ], + resell_enabled: true, + cli_effective_balance_cents: 1234, +} + +const MANAGED_OFF = { + options: [], + providers: [{ provider: "lambda", has_byok: false, has_operator: false, funding: "unavailable", count: 0 }], + resell_enabled: false, + cli_effective_balance_cents: 1234, +} + +describe("ComputeMode.resolve", () => { + beforeEach(() => { + clearEnv() + calls = [] + ComputeMode.invalidate() + }) + + afterEach(async () => { + globalThis.fetch = realFetch + await fs.rm(SESSION, { force: true }).catch(() => {}) + }) + + test("a usable provider resolves to byok WITHOUT calling the availability endpoint", async () => { + await signIn() + stubOptions(MANAGED_ON) + process.env["LAMBDA_API_KEY"] = "k" + const result = await withSkills(["lambda-labs-gpu-cloud"], () => ComputeMode.resolve()) + expect(result.mode).toBe("byok") + expect(result.providers).toEqual(["lambda"]) + expect(result.balance).toBeUndefined() + expect(calls.filter((url) => url.includes(OPTIONS_URL))).toEqual([]) + }) + + test("no keys plus managed available resolves to managed, with the balance", async () => { + await signIn() + stubOptions(MANAGED_ON) + const result = await withSkills(["lambda-labs-gpu-cloud"], () => ComputeMode.resolve()) + expect(result.mode).toBe("managed") + expect(result.managed).toBe(true) + expect(result.balance).toBe(12.34) + }) + + test("no keys plus managed unavailable resolves to none", async () => { + await signIn() + stubOptions(MANAGED_OFF) + const result = await withSkills(["lambda-labs-gpu-cloud"], () => ComputeMode.resolve()) + expect(result.mode).toBe("none") + expect(result.managed).toBe(false) + expect(result.balance).toBeUndefined() + }) + + test("a failing availability call resolves to none, not managed", async () => { + await signIn() + globalThis.fetch = (async () => { + throw new Error("network down") + }) as typeof fetch + const result = await withSkills([], () => ComputeMode.resolve()) + expect(result.mode).toBe("none") + expect(result.managed).toBe(false) + }) + + test("a non-ok availability response resolves to none", async () => { + await signIn() + stubOptions({ detail: "unauthorized" }, 401) + const result = await withSkills([], () => ComputeMode.resolve()) + expect(result.mode).toBe("none") + }) + + test("no session means managed is unavailable and no call is made", async () => { + await fs.rm(SESSION, { force: true }).catch(() => {}) + stubOptions(MANAGED_ON) + const result = await withSkills([], () => ComputeMode.resolve()) + expect(result.mode).toBe("none") + expect(calls.filter((url) => url.includes(OPTIONS_URL))).toEqual([]) + }) + + test("a key with no skill still resolves to byok and skips the availability call", async () => { + await signIn() + stubOptions(MANAGED_ON) + process.env["RUNPOD_API_KEY"] = "rpa_x" + const result = await withSkills([], () => ComputeMode.resolve()) + expect(result.mode).toBe("byok") + expect(result.providers).toEqual(["runpod"]) + expect(calls.filter((url) => url.includes(OPTIONS_URL))).toEqual([]) + }) + + test("the availability answer is cached within the TTL", async () => { + await signIn() + stubOptions(MANAGED_ON) + await withSkills([], async () => { + await ComputeMode.resolve() + await ComputeMode.resolve() + }) + expect(calls.filter((url) => url.includes(OPTIONS_URL)).length).toBe(1) + }) + + test("invalidate() drops the cache", async () => { + await signIn() + stubOptions(MANAGED_ON) + await withSkills([], async () => { + await ComputeMode.resolve() + ComputeMode.invalidate() + await ComputeMode.resolve() + }) + expect(calls.filter((url) => url.includes(OPTIONS_URL)).length).toBe(2) + }) +}) + +describe("ComputeMode.resolve override", () => { + beforeEach(() => { + clearEnv() + calls = [] + ComputeMode.invalidate() + }) + afterEach(async () => { + globalThis.fetch = realFetch + await fs.rm(SESSION, { force: true }).catch(() => {}) + }) + + /** Same tmpdir fixture as withSkills, plus an openscience.json setting + * billing.compute. */ + async function withOverride(mode: "byok" | "managed", skills: string[], fn: () => Promise): Promise { + await using tmp = await tmpdir({ + git: true, + init: async (dir) => { + for (const name of skills) { + await Bun.write( + path.join(dir, ".openscience", "skill", name, "SKILL.md"), + `---\nname: ${name}\ndescription: Test fixture for ${name}.\ncategory: cloud-compute\n---\n\n# ${name}\n`, + ) + } + await Bun.write(path.join(dir, "openscience.json"), JSON.stringify({ billing: { compute: mode } })) + }, + }) + return Instance.provide({ directory: tmp.path, fn }) + } + + test("override byok with a usable provider stays byok", async () => { + await signIn() + stubOptions(MANAGED_ON) + process.env["LAMBDA_API_KEY"] = "k" + const result = await withOverride("byok", ["lambda-labs-gpu-cloud"], () => ComputeMode.resolve()) + expect(result.mode).toBe("byok") + }) + + test("override byok with NO usable provider narrows to none, never managed", async () => { + await signIn() + stubOptions(MANAGED_ON) + const result = await withOverride("byok", ["lambda-labs-gpu-cloud"], () => ComputeMode.resolve()) + expect(result.mode).toBe("none") + expect(calls.filter((url) => url.includes(OPTIONS_URL))).toEqual([]) + }) + + test("override managed with managed unavailable narrows to none", async () => { + await signIn() + stubOptions(MANAGED_OFF) + const result = await withOverride("managed", [], () => ComputeMode.resolve()) + expect(result.mode).toBe("none") + }) + + test("override managed beats a usable provider when managed IS available", async () => { + await signIn() + stubOptions(MANAGED_ON) + process.env["LAMBDA_API_KEY"] = "k" + const result = await withOverride("managed", ["lambda-labs-gpu-cloud"], () => ComputeMode.resolve()) + expect(result.mode).toBe("managed") + }) +}) +``` + +Add these imports at the top of the file, alongside the existing ones: + +```ts +import { beforeEach } from "bun:test" +import fs from "fs/promises" +import { Global } from "../../src/global" +``` + +Mutations these guard: deleting the `providers.length` short-circuit before the network call (the two "no call is made" assertions fail); flipping the catch/`!res.ok` fallbacks from `false` to `true` (the two failure tests resolve to `managed`); deleting the cache (the TTL test sees 2 calls); deleting `invalidate()`'s body (that test sees 1 call); deleting the override branch entirely (the two narrow-to-none tests resolve to `byok`/`managed`); deleting only the `"byok"` override arm (narrow-to-none returns `managed`). + +- [ ] **Step 2: Run test to verify it fails** + +Run: `cd backend/cli && bun test test/compute/mode.test.ts` +Expected: FAIL — `ComputeMode.resolve is not a function`. + +- [ ] **Step 3: Implement resolution** + +Append to `backend/cli/src/compute/mode.ts`, inside the `ComputeMode` namespace, and add the imports `import { Config } from "@/config/config"` and `import { OpenScience } from "@/openscience"` at the top: + +```ts +export interface Resolution { + mode: Source + /** Credentialed BYOK providers, in PROVIDERS declaration order. */ + providers: string[] + managed: boolean + /** Wallet balance in USD. Present only when mode === "managed". */ + balance?: number +} + +/** Hard ceiling on how long resolution may block an agent turn. Atlas's own + * 60s default is far too long to sit in front of a tool call; a slow or + * hanging backend must degrade to "none", not stall the turn. */ +const TIMEOUT = 3_000 + +/** Short in-process TTL, enough to stop a chatty agent hammering the endpoint + * inside one turn and no longer. The whole reason this is a tool rather than + * a prompt injection is that the answer changes mid-session, so a long cache + * would reintroduce exactly the staleness the tool exists to avoid. */ +const TTL = 5_000 + +let cache: { at: number; value: { managed: boolean; balance?: number } } | undefined + +/** Drop the availability cache. Called by tests; also safe after a connect. */ +export function invalidate() { + cache = undefined +} + +/** + * One authenticated call to /api/compute/options, which already annotates each + * provider with `funding` — "managed" when reselling is on and an operator key + * exists, else "unavailable". A failed, unauthenticated or timed-out call is + * treated as UNAVAILABLE: failing toward "none" produces an honest "connect a + * key" message, whereas failing toward "managed" would reproduce the bug this + * design exists to fix, promising a capability we never confirmed. + */ +async function available() { + if (cache && Date.now() - cache.at < TTL) return cache.value + const value = await probe() + cache = { at: Date.now(), value } + return value +} + +async function probe(): Promise<{ managed: boolean; balance?: number }> { + const session = await OpenScience.getSession().catch(() => null) + if (!session) return { managed: false } + try { + const res = await fetch(`${OpenScience.API_BASE}/api/compute/options`, { + headers: { Authorization: `Bearer ${session.api_key}` }, + signal: AbortSignal.timeout(TIMEOUT), + }) + if (!res.ok) return { managed: false } + const data = await res.json() + const providers = Array.isArray(data?.providers) ? data.providers : [] + const managed = providers.some((entry: { funding?: string }) => entry?.funding === "managed") + if (!managed) return { managed: false } + const cents = data?.cli_effective_balance_cents + return { managed: true, balance: typeof cents === "number" ? cents / 100 : undefined } + } catch { + return { managed: false } + } +} + +/** + * The single shared entry point. `billing.compute` is an OVERRIDE, not the + * source of truth: it may narrow the outcome to "none", but it may never + * manufacture a capability that isn't there. + */ +export async function resolve(): Promise { + const providers = usable() + const override = (await Config.get()).billing?.compute + + if (override === "byok") { + return { mode: providers.length ? "byok" : "none", providers, managed: false } + } + + if (override === "managed") { + const managed = await available() + return { + mode: managed.managed ? "managed" : "none", + providers, + managed: managed.managed, + balance: managed.managed ? managed.balance : undefined, + } + } + + // BYOK wins when a credentialed provider is present: it is free to the user, + // it works today, and it needs nothing from Atlas. This is also why a BYOK + // user never pays for the availability call. + if (providers.length) return { mode: "byok", providers, managed: false } + + const managed = await available() + return { + mode: managed.managed ? "managed" : "none", + providers, + managed: managed.managed, + balance: managed.managed ? managed.balance : undefined, + } +} +``` + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `cd backend/cli && bun test test/compute/mode.test.ts` +Expected: PASS, 24 tests. + +- [ ] **Step 5: Prove each assertion against deletion** + +Apply each mutation named in Step 1, confirm the named test fails, revert. Additionally verify the timeout is real: temporarily replace the stub with one that never resolves, assert `resolve()` returns `none` in under 4s, then revert. + +- [ ] **Step 6: Typecheck, format, commit** + +```bash +cd backend/cli && bun run typecheck && bun test test/compute/ +bunx prettier --write src/compute/mode.ts test/compute/mode.test.ts +git add src/compute/mode.ts test/compute/mode.test.ts +git commit -m "feat(compute): resolve byok/managed/none at runtime, config becomes an override" +``` + +--- + +### Task 3: The `compute_status` tool + +**Files:** + +- Create: `backend/cli/src/tool/compute.ts` +- Modify: `backend/cli/src/tool/registry.ts` +- Test: `backend/cli/test/tool/compute-status.test.ts` + +**Interfaces:** + +- Consumes: `ComputeMode.resolve()` and `ComputeMode.Resolution` from Task 2; `Tool.define` from `./tool`. +- Produces: `ComputeStatusTool` (id `compute_status`) and `export const ComputeTools = [ComputeStatusTool]`, registered in `ToolRegistry`. + +- [ ] **Step 1: Write the failing test** + +Create `backend/cli/test/tool/compute-status.test.ts`: + +```ts +import { test, expect, describe, beforeEach, afterEach } from "bun:test" +import path from "path" +import fs from "fs/promises" +import { ComputeStatusTool } from "../../src/tool/compute" +import { ComputeMode } from "../../src/compute/mode" +import { ToolRegistry } from "../../src/tool/registry" +import { Instance } from "../../src/project/instance" +import { Global } from "../../src/global" +import { tmpdir } from "../fixture/fixture" + +const ENV = ["LAMBDA_API_KEY", "RUNPOD_API_KEY", "MODAL_TOKEN_ID", "MODAL_TOKEN_SECRET"] +const SESSION = path.join(Global.Path.data, "openscience-session.json") +const realFetch = globalThis.fetch + +const CTX = { + sessionID: "ses_test", + messageID: "msg_test", + agent: "research", + abort: new AbortController().signal, + messages: [], + metadata: () => {}, + ask: async () => {}, +} + +function stub(body: unknown, status = 200) { + globalThis.fetch = (async (input: string | URL | Request) => { + const url = String(input instanceof Request ? input.url : input) + if (!url.includes("/api/compute/options")) return realFetch(input as never) + return new Response(JSON.stringify(body), { status, headers: { "content-type": "application/json" } }) + }) as typeof fetch +} + +const MANAGED_ON = { + providers: [{ provider: "lambda", funding: "managed", has_byok: false, has_operator: true, count: 2 }], + resell_enabled: true, + cli_effective_balance_cents: 4200, +} +const MANAGED_OFF = { providers: [], resell_enabled: false, cli_effective_balance_cents: 0 } + +async function run(skills: string[], fn?: () => Promise) { + await using tmp = await tmpdir({ + git: true, + init: async (dir) => { + for (const name of skills) { + await Bun.write( + path.join(dir, ".openscience", "skill", name, "SKILL.md"), + `---\nname: ${name}\ndescription: Fixture ${name}.\ncategory: cloud-compute\n---\n\n# ${name}\n`, + ) + } + }, + }) + return Instance.provide({ + directory: tmp.path, + fn: async () => { + await fn?.() + const tool = await ComputeStatusTool.init({}) + return tool.execute({}, CTX as never) + }, + }) +} + +describe("compute_status", () => { + beforeEach(async () => { + for (const name of ENV) delete process.env[name] + ComputeMode.invalidate() + await fs.mkdir(Global.Path.data, { recursive: true }) + await Bun.write(SESSION, JSON.stringify({ api_key: "thk_t.s", user_id: "u1" })) + }) + afterEach(async () => { + globalThis.fetch = realFetch + for (const name of ENV) delete process.env[name] + await fs.rm(SESSION, { force: true }).catch(() => {}) + }) + + test("byok reports the mode, the usable providers, and byok guidance", async () => { + stub(MANAGED_ON) + process.env["LAMBDA_API_KEY"] = "k" + const result = await run(["lambda-labs-gpu-cloud"]) + expect(result.metadata.mode).toBe("byok") + expect(result.metadata.providers).toEqual(["lambda"]) + expect(result.output).toContain("lambda") + expect(result.output.toLowerCase()).toContain("do not launch managed") + expect(result.metadata.balance_usd).toBeUndefined() + }) + + test("managed reports the balance and managed guidance", async () => { + stub(MANAGED_ON) + const result = await run([]) + expect(result.metadata.mode).toBe("managed") + expect(result.metadata.balance_usd).toBe(42) + expect(result.output).toContain("42") + expect(result.output.toLowerCase()).toContain("credits") + }) + + test("none tells the agent not to attempt GPU work and how to enable it", async () => { + stub(MANAGED_OFF) + const result = await run([]) + expect(result.metadata.mode).toBe("none") + expect(result.output.toLowerCase()).toContain("do not attempt gpu work") + expect(result.output).toContain("Settings") + expect(result.metadata.balance_usd).toBeUndefined() + }) + + test("a provider with a key but no skill is still reported as usable byok", async () => { + stub(MANAGED_OFF) + process.env["RUNPOD_API_KEY"] = "rpa_x" + const result = await run([]) + expect(result.metadata.mode).toBe("byok") + expect(result.metadata.providers).toEqual(["runpod"]) + expect(result.output).toContain("runpod") + }) + + test("the three modes produce three DIFFERENT guidance strings", async () => { + stub(MANAGED_ON) + process.env["LAMBDA_API_KEY"] = "k" + const byok = await run(["lambda-labs-gpu-cloud"]) + delete process.env["LAMBDA_API_KEY"] + ComputeMode.invalidate() + const managed = await run([]) + stub(MANAGED_OFF) + ComputeMode.invalidate() + const none = await run([]) + const texts = [byok.output, managed.output, none.output] + expect(new Set(texts).size).toBe(3) + }) + + test("a credential connected between two calls changes the answer, no restart", async () => { + stub(MANAGED_OFF) + const before = await run([]) + expect(before.metadata.mode).toBe("none") + process.env["LAMBDA_API_KEY"] = "connected-mid-session" + const after = await run(["lambda-labs-gpu-cloud"]) + expect(after.metadata.mode).toBe("byok") + }) + + test("the description instructs the agent to check before running GPU work", async () => { + const tool = await ComputeStatusTool.init({}) + expect(tool.description.toLowerCase()).toContain("before") + expect(tool.description.toLowerCase()).toContain("gpu") + expect(tool.description).toContain("byok") + expect(tool.description).toContain("managed") + expect(tool.description).toContain("none") + }) + + test("the tool is registered", async () => { + await using tmp = await tmpdir({ git: true }) + await Instance.provide({ + directory: tmp.path, + fn: async () => { + expect(await ToolRegistry.ids()).toContain("compute_status") + }, + }) + }) +}) +``` + +Mutations these guard: deleting the `providers` field from the output (test 1); deleting the balance line (test 2); returning one shared guidance string for every mode (test 5); deleting the `unusable` reporting (test 4); deleting the description's "before" instruction (test 7); removing the registry entry (test 8); caching the resolution at module load (test 6). + +- [ ] **Step 2: Run test to verify it fails** + +Run: `cd backend/cli && bun test test/tool/compute-status.test.ts` +Expected: FAIL — `Cannot find module '../../src/tool/compute'`. + +- [ ] **Step 3: Write the tool** + +Create `backend/cli/src/tool/compute.ts`: + +```ts +import z from "zod" +import { Tool } from "./tool" +import { ComputeMode } from "@/compute/mode" + +/** + * The agent PULLS its compute mode from here; nothing is injected per turn. + * + * An earlier design injected mode guidance into every turn. That was wrong for a + * reason that matters more than token cost: the mode changes mid-session. A user + * connects a Modal key in Settings ▸ Compute while a session is running, and a + * reminder injected at turn 3 is false by turn 12. A tool returns the state at + * the moment it is asked. + * + * The DESCRIPTION carries the constraint — it reaches the agent before it starts + * down a path, which is the one thing an injection did well, and tool definitions + * are in every request regardless, so it costs nothing extra. The RESULT carries + * the specifics. Adding rates or a balance to an every-turn injection would be + * expensive; adding them here is free. + */ + +const GUIDANCE: Record = { + byok: "Run GPU work on the user's connected providers via the cloud-compute skills. Do not launch managed leases — they bill Credits and are not the funded path here.", + managed: + "Run GPU work through managed compute, billed to Credits. Do not use the user's own provider keys — they are not funded here.", + none: "No compute is available. Do not attempt GPU work. Tell the user to connect a provider key in Settings ▸ Compute, or to top up for managed compute.", +} + +export const ComputeStatusTool = Tool.define("compute_status", { + description: [ + "Check how GPU compute is funded before running any GPU, training, or cluster work.", + "Returns one of byok, managed, or none, the providers available, and the rule that applies.", + "Call this first — the answer can change mid-session as the user connects or removes keys.", + ].join(" "), + parameters: z.object({}), + async execute(_params, _ctx) { + const state = await ComputeMode.resolve() + const lines = [ + `**mode**: ${state.mode}`, + `**providers**: ${state.providers.length ? state.providers.join(", ") : "none configured"}`, + `**managed available**: ${state.managed ? "yes" : "no"}`, + ] + if (state.balance !== undefined) lines.push(`**balance**: $${state.balance.toFixed(2)}`) + lines.push("", GUIDANCE[state.mode]) + + return { + title: `Compute: ${state.mode}`, + output: lines.join("\n"), + metadata: { + mode: state.mode, + providers: state.providers, + managed_available: state.managed, + balance_usd: state.balance, + }, + } + }, +}) + +export const ComputeTools = [ComputeStatusTool] +``` + +- [ ] **Step 4: Register the tool** + +In `backend/cli/src/tool/registry.ts`, add the import next to the other tool imports: + +```ts +import { ComputeTools } from "./compute" +``` + +and add `...ComputeTools,` to the array returned by `all()`, immediately after `...ProvenanceTools,`. + +- [ ] **Step 5: Run tests to verify they pass** + +Run: `cd backend/cli && bun test test/tool/compute-status.test.ts` +Expected: PASS, 8 tests. + +- [ ] **Step 6: Prove each assertion against deletion** + +Apply each mutation named in Step 1, confirm the named test fails, revert. In particular, collapse `GUIDANCE` to a single shared string and confirm "the three modes produce three DIFFERENT guidance strings" fails — an inversion would not catch this. + +- [ ] **Step 7: Typecheck, format, commit** + +```bash +cd backend/cli && bun run typecheck && bun test test/tool/ +bunx prettier --write src/tool/compute.ts src/tool/registry.ts test/tool/compute-status.test.ts +git add src/tool/compute.ts src/tool/registry.ts test/tool/compute-status.test.ts +git commit -m "feat(tool): add compute_status so the agent pulls its compute mode" +``` + +--- + +### Task 4: Filter the skill catalog by resolved mode + +**Files:** + +- Modify: `backend/cli/src/tool/skill.ts:32-58` +- Test: `backend/cli/test/tool/skill-compute-filter.test.ts` + +**Interfaces:** + +- Consumes: `ComputeMode.resolve()`, `ComputeMode.SKILLS`, `ComputeMode.PROVIDERS` from Tasks 1–2. +- Produces: nothing new. `SkillTool.init` keeps its existing shape. + +`SkillTool.init` is the right seam for two reasons. `registry.ts:187` calls `await t.init({ agent })` inside `tools()`, so **it runs per request** — a credential connected mid-session appears on the next turn with no cache to invalidate. And by the time a turn is served, every env injection in `src/index.ts` has long since run, so detection cannot observe a half-initialised environment. + +**Filter the catalog; do not auto-load the markdown.** Only usable providers' skills are listed, so the agent picks the right one because it is the only one offered. Auto-injecting a provider's markdown would fight the mechanism `tool/skill.ts` exists to provide, and these files run 500+ lines — unprompted injection is expensive on turns that have nothing to do with compute. + +- [ ] **Step 1: Write the failing test** + +Create `backend/cli/test/tool/skill-compute-filter.test.ts`: + +```ts +import { test, expect, describe, beforeEach, afterEach } from "bun:test" +import path from "path" +import fs from "fs/promises" +import { SkillTool } from "../../src/tool/skill" +import { ComputeMode } from "../../src/compute/mode" +import { Instance } from "../../src/project/instance" +import { Global } from "../../src/global" +import { tmpdir } from "../fixture/fixture" + +const ENV = ["LAMBDA_API_KEY", "RUNPOD_API_KEY", "MODAL_TOKEN_ID", "MODAL_TOKEN_SECRET", "TENSORPOOL_KEY"] +const SESSION = path.join(Global.Path.data, "openscience-session.json") +const realFetch = globalThis.fetch + +// Every provider skill, plus two skills that must never be filtered: a +// non-compute one and a cloud-compute skill that maps to no panel provider. +const ALL = [ + ["modal-serverless-gpu", "cloud-compute"], + ["lambda-labs-gpu-cloud", "cloud-compute"], + ["tensorpool-gpu-cloud", "cloud-compute"], + ["prime-intellect-lab", "ml-training"], + ["tinker-fine-tuning", "cloud-compute"], + ["rdkit", "chemistry"], +] as const + +function stub(managed: boolean) { + globalThis.fetch = (async (input: string | URL | Request) => { + const url = String(input instanceof Request ? input.url : input) + if (!url.includes("/api/compute/options")) return realFetch(input as never) + return new Response( + JSON.stringify({ + providers: managed ? [{ provider: "lambda", funding: "managed" }] : [], + resell_enabled: managed, + cli_effective_balance_cents: 500, + }), + { status: 200, headers: { "content-type": "application/json" } }, + ) + }) as typeof fetch +} + +async function project(fn: (dir: string) => Promise) { + return tmpdir({ + git: true, + init: async (dir) => { + for (const [name, category] of ALL) { + await Bun.write( + path.join(dir, ".openscience", "skill", name, "SKILL.md"), + `---\nname: ${name}\ndescription: Fixture ${name}.\ncategory: ${category}\n---\n\n# ${name}\n`, + ) + } + await fn(dir) + }, + }) +} + +/** Which of the six provider skills does the tool offer? Read from the tool's + * own category listing, which is what the model sees. */ +async function offered(): Promise { + const tool = await SkillTool.init({}) + const found: string[] = [] + for (const category of ["cloud-compute", "ml-training"]) { + const result = await tool + .execute({ category }, { + sessionID: "s", + messageID: "m", + agent: "research", + abort: new AbortController().signal, + messages: [], + metadata: () => {}, + ask: async () => {}, + } as never) + .catch(() => undefined) + if (result) found.push(result.output) + } + const text = found.join("\n") + return [...ComputeMode.SKILLS].filter((name) => text.includes(`**${name}**`)).sort() +} + +async function nonComputeVisible(): Promise { + const tool = await SkillTool.init({}) + const result = await tool.execute({ category: "chemistry" }, { + sessionID: "s", + messageID: "m", + agent: "research", + abort: new AbortController().signal, + messages: [], + metadata: () => {}, + ask: async () => {}, + } as never) + return result.output.includes("**rdkit**") +} + +async function tinkerVisible(): Promise { + const tool = await SkillTool.init({}) + const result = await tool.execute({ category: "cloud-compute" }, { + sessionID: "s", + messageID: "m", + agent: "research", + abort: new AbortController().signal, + messages: [], + metadata: () => {}, + ask: async () => {}, + } as never) + return result.output.includes("**tinker-fine-tuning**") +} + +describe("skill catalog filtering by compute mode", () => { + beforeEach(async () => { + for (const name of ENV) delete process.env[name] + ComputeMode.invalidate() + await fs.mkdir(Global.Path.data, { recursive: true }) + await Bun.write(SESSION, JSON.stringify({ api_key: "thk_t.s", user_id: "u1" })) + }) + afterEach(async () => { + globalThis.fetch = realFetch + for (const name of ENV) delete process.env[name] + await fs.rm(SESSION, { force: true }).catch(() => {}) + }) + + test("with only a Modal credential, only Modal's skills are offered", async () => { + stub(false) + process.env["MODAL_TOKEN_ID"] = "ak-a" + process.env["MODAL_TOKEN_SECRET"] = "as-b" + await using tmp = await project(async () => {}) + const names = await Instance.provide({ directory: tmp.path, fn: offered }) + expect(names).toEqual(["modal-serverless-gpu"]) + }) + + test("a RunPod credential is byok but contributes no skills — nobody else's are offered either", async () => { + stub(false) + process.env["RUNPOD_API_KEY"] = "rpa_x" + await using tmp = await project(async () => {}) + await Instance.provide({ + directory: tmp.path, + fn: async () => { + // RunPod carries skills: [] (Decision 2), so being credentialed makes the + // user byok without unlocking any other provider's skills. + expect((await ComputeMode.resolve()).mode).toBe("byok") + expect(await offered()).toEqual([]) + }, + }) + }) + + test("in managed, no BYOK provider skill is offered", async () => { + stub(true) + await using tmp = await project(async () => {}) + const names = await Instance.provide({ directory: tmp.path, fn: offered }) + expect(names).toEqual([]) + }) + + test("in none, no BYOK provider skill is offered", async () => { + stub(false) + await using tmp = await project(async () => {}) + const names = await Instance.provide({ directory: tmp.path, fn: offered }) + expect(names).toEqual([]) + }) + + test("non-compute skills are unaffected in every mode", async () => { + for (const managed of [true, false]) { + stub(managed) + ComputeMode.invalidate() + await using tmp = await project(async () => {}) + expect(await Instance.provide({ directory: tmp.path, fn: nonComputeVisible })).toBe(true) + } + }) + + test("cloud-compute skills that map to no panel provider are never hidden", async () => { + stub(false) + await using tmp = await project(async () => {}) + expect(await Instance.provide({ directory: tmp.path, fn: tinkerVisible })).toBe(true) + }) + + test("a credential added between two init() calls changes the catalog on the second", async () => { + stub(false) + await using tmp = await project(async () => {}) + await Instance.provide({ + directory: tmp.path, + fn: async () => { + expect(await offered()).toEqual([]) + process.env["TENSORPOOL_KEY"] = "tp-late" + expect(await offered()).toEqual(["tensorpool-gpu-cloud"]) + }, + }) + }) + + test("SkillTool.init and compute_status never disagree about usable providers", async () => { + stub(false) + process.env["LAMBDA_API_KEY"] = "k" + await using tmp = await project(async () => {}) + await Instance.provide({ + directory: tmp.path, + fn: async () => { + const state = await ComputeMode.resolve() + const names = await offered() + const expected = state.providers.flatMap((id) => ComputeMode.PROVIDERS[id].skills) + expect(names.sort()).toEqual([...new Set(expected)].sort()) + }, + }) + }) +}) +``` + +Mutations these guard: deleting the filter entirely (tests 1–4 and 8 fail); widening the filter to the whole `cloud-compute` category (test 6 fails); widening it to every skill (test 5 fails); resolving the mode once at module load instead of inside `init` (test 7's second assertion fails); filtering by provider id instead of skill name (test 1 fails). + +- [ ] **Step 2: Run test to verify it fails** + +Run: `cd backend/cli && bun test test/tool/skill-compute-filter.test.ts` +Expected: FAIL — every provider skill is offered in all modes. + +- [ ] **Step 3: Add the filter** + +In `backend/cli/src/tool/skill.ts`, add the import: + +```ts +import { ComputeMode } from "@/compute/mode" +``` + +and replace the `accessibleSkills` block (currently lines 36-42) with: + +```ts +// Filter skills by agent permissions if agent provided +const agent = ctx?.agent +const permitted = agent + ? skills.filter((skill) => { + const rule = PermissionNext.evaluate("skill", skill.name, agent.permission) + return rule.action !== "deny" + }) + : skills + +// Filter the GPU provider skills by the resolved compute mode, so the agent +// picks the right provider because it is the only one offered. This init runs +// per request (registry.ts calls it inside tools()), which buys two things for +// free: a credential connected mid-session shows up on the next turn with no +// cache to invalidate, and resolution always happens after src/index.ts's env +// injections rather than racing them. +// +// This is a LISTING filter, not a gate. `none` is guidance, not enforcement — +// a hidden skill can still be loaded by exact name, and the agent still has +// bash. Gating the load path is a larger change and is deliberately out of +// scope; see docs/specs/compute-mode-detection-design.md open question 3. +const compute = await ComputeMode.resolve() +const offered = new Set(compute.providers.flatMap((id) => ComputeMode.PROVIDERS[id].skills)) +const accessibleSkills = permitted.filter((skill) => !ComputeMode.SKILLS.has(skill.name) || offered.has(skill.name)) +``` + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `cd backend/cli && bun test test/tool/skill-compute-filter.test.ts` +Expected: PASS, 8 tests. + +- [ ] **Step 5: Run the full suite for regressions** + +Run: `cd backend/cli && bun test` +Expected: PASS. `test/skill/` and `test/session/system-skills.test.ts` exercise the same catalog — if any of them go red, the filter is too wide. + +- [ ] **Step 6: Prove each assertion against deletion** + +Apply each mutation named in Step 1, confirm the named test fails, revert. + +- [ ] **Step 7: Typecheck, format, commit** + +```bash +cd backend/cli && bun run typecheck +bunx prettier --write src/tool/skill.ts test/tool/skill-compute-filter.test.ts +git add src/tool/skill.ts test/tool/skill-compute-filter.test.ts +git commit -m "feat(skill): offer GPU provider skills only for usable providers" +``` + +--- + +### Task 5: Retire the false prompt injection, config text, and dead code + +Three things in the current injected text do not hold: `atlas compute:up` is not in the published `@synsci/atlas@0.13.2`; `atlas doctor` reports no compute field at all, so the stated condition is unobservable; and managed compute is off by default server-side (`COMPUTE_RESELL_ENABLED` defaults to `false`). An agent in managed mode runs an unknown command, cannot check the sanctioned signal, and is pointed at "the user's own GPU providers" as the remedy — which in managed mode is precisely the set of keys that does not exist. + +**Files:** + +- Modify: `backend/cli/src/session/prompt.ts:1546-1564` +- Modify: `backend/cli/src/session/billing-gate.ts:33-36` +- Modify: `backend/cli/src/config/config.ts:1058-1064` +- Test: `backend/cli/test/session/compute-prompt.test.ts` + +**Interfaces:** + +- Consumes: nothing new. +- Produces: `computeBillingMode` no longer exists. `BillingMode`, `llmBillingMode`, `resolveCredentialSource`, `requiresWalletBalance` and `shouldReportUsage` are unchanged. + +- [ ] **Step 1: Write the failing test** + +Create `backend/cli/test/session/compute-prompt.test.ts`: + +```ts +import { test, expect, describe } from "bun:test" +import path from "path" + +const root = path.join(import.meta.dir, "..", "..", "src") + +async function sources() { + const files = await Array.fromAsync( + new Bun.Glob("session/**/*.{ts,txt}").scan({ cwd: root, absolute: true, onlyFiles: true }), + ) + return Promise.all(files.map(async (file) => [file, await Bun.file(file).text()] as const)) +} + +describe("compute prompt text", () => { + test("no prompt or session source references atlas compute:up", async () => { + const hits = (await sources()).filter(([, text]) => text.includes("compute:up")) + expect(hits.map(([file]) => path.relative(root, file))).toEqual([]) + }) + + test("no prompt or session source uses atlas doctor as the compute availability signal", async () => { + const hits = (await sources()).filter(([, text]) => /atlas doctor/i.test(text)) + expect(hits.map(([file]) => path.relative(root, file))).toEqual([]) + }) + + test("the compute reminder points at compute_status and carries no mode", async () => { + const text = await Bun.file(path.join(root, "session", "prompt.ts")).text() + expect(text).toContain("compute_status") + // The reminder must be stateless — a mode baked into an injected string is + // false the moment the user connects a key mid-session. + expect(text).not.toContain("Compute spend is set to") + }) + + test("computeBillingMode is gone and nothing imports it", async () => { + const gate = await Bun.file(path.join(root, "session", "billing-gate.ts")).text() + expect(gate).not.toContain("computeBillingMode") + const files = await Array.fromAsync(new Bun.Glob("**/*.ts").scan({ cwd: root, absolute: true, onlyFiles: true })) + const importers = ( + await Promise.all( + files.map(async (file) => ((await Bun.file(file).text()).includes("computeBillingMode") ? file : undefined)), + ) + ).filter(Boolean) + expect(importers).toEqual([]) + }) + + test("the billing.compute config description no longer claims 'Unset = byok'", async () => { + const text = await Bun.file(path.join(root, "config", "config.ts")).text() + expect(text).not.toContain("Unset = byok") + expect(text).toContain("auto-detect") + }) +}) +``` + +Mutations these guard: leaving either false claim in any session prompt (tests 1–2); re-introducing a mode-carrying injection (test 3); leaving `computeBillingMode` behind as dead code (test 4); forgetting the config description (test 5). + +- [ ] **Step 2: Run test to verify it fails** + +Run: `cd backend/cli && bun test test/session/compute-prompt.test.ts` +Expected: FAIL on tests 1, 2, 3, 4 and 5 — all five conditions currently hold in the wrong direction. + +- [ ] **Step 3: Replace the injection** + +In `backend/cli/src/session/prompt.ts`, replace the whole block currently at lines 1550-1564 (from the `// Compute spend preference` comment through the closing `}` of the `if`) with: + +```ts +// Compute funding is PULLED from the `compute_status` tool, not injected — +// the mode changes mid-session (a key connected in Settings ▸ Compute at +// turn 3 makes a reminder injected then false by turn 12). This line is a +// stateless pointer: it carries no mode, so it can never go stale, and it +// closes the gap where an agent reaches for bash without ever looking. +if (COMPUTE_AGENTS.has(input.agent.name)) { + userMessage.parts.push({ + id: Identifier.ascending("part"), + messageID: userMessage.info.id, + sessionID: userMessage.info.sessionID, + type: "text", + text: "Call `compute_status` before running GPU, training, or cluster work. It reports how compute is funded and which providers are usable right now.", + synthetic: true, + }) +} +``` + +Then delete the now-unused import on line 46: `import { computeBillingMode } from "./billing-gate"`. + +- [ ] **Step 4: Delete the dead function** + +In `backend/cli/src/session/billing-gate.ts`, delete lines 33-36: + +```ts +/** The user-facing compute spend toggle. Defaults to "byok" (own GPU providers). */ +export async function computeBillingMode(): Promise { + return (await Config.get()).billing?.compute ?? "byok" +} +``` + +`prompt.ts:1554` was its only consumer. `Config` is still imported for `llmBillingMode`, so leave the import. + +- [ ] **Step 5: Correct the config description** + +In `backend/cli/src/config/config.ts`, replace the `compute` field's `.describe(...)` string (line 1062) with: + +```ts + "How GPU/compute is paid for. 'managed' runs on Atlas-provisioned compute billed to your wallet; 'byok' uses your own connected GPU providers (Modal, Lambda Labs, TensorPool, Prime Intellect, RunPod, Vast.ai). Unset = auto-detect from your connected providers. Setting this can only narrow the result — if the mode you pick isn't actually available, compute resolves to none rather than pretending.", +``` + +- [ ] **Step 6: Run tests to verify they pass** + +Run: `cd backend/cli && bun test test/session/compute-prompt.test.ts` +Expected: PASS, 5 tests. + +- [ ] **Step 7: Run the full suite** + +Run: `cd backend/cli && bun test && bun run typecheck` +Expected: PASS. Any test asserting the old injected text must be updated to the new reminder, not deleted. + +- [ ] **Step 8: Prove each assertion against deletion** + +Apply each mutation named in Step 1, confirm the named test fails, revert. + +- [ ] **Step 9: Format and commit** + +```bash +cd backend/cli +bunx prettier --write src/session/prompt.ts src/session/billing-gate.ts src/config/config.ts test/session/compute-prompt.test.ts +git add src/session/prompt.ts src/session/billing-gate.ts src/config/config.ts test/session/compute-prompt.test.ts +git commit -m "fix(prompt): drop the false atlas compute:up guidance for a compute_status pointer" +``` + +--- + +### Task 5b: Finish acceptance criterion 12 in the agent prompts + +Task 5 discharged criterion 12 for `src/session/**` only. Its tests glob `session/**/*.{ts,txt}`, which excludes `src/agent/prompt/*.txt` — and the primary `research` agent's prompt still gates managed compute on `atlas doctor`, the same false signal Task 5 just removed from `prompt.ts`. Both the Task 5 implementer and its reviewer surfaced this independently. + +**Files:** + +- Modify: `backend/cli/src/agent/prompt/research.txt` (Stage 5: COMPUTE, lines 252-261) +- Modify: `backend/cli/test/session/compute-prompt.test.ts` + +**Interfaces:** none — prompt text and test scope only. + +**What is wrong, precisely.** `research.txt:253-257` currently reads: + +``` +- Managed compute (Daytona-backed) runs through the bundled `atlas` CLI when your Atlas + session is active. Run `atlas doctor --format=json` first; if it reports the CLI is + unavailable/unauthenticated, print a one-line note and fall back to the BYOK cloud-compute + skills below (Modal, Tinker, TensorPool, Prime Intellect, HF Jobs) — never block on it. +``` + +`atlas doctor` reports `config_path`, `profile`, `base_url`, `auth`, `backend`, `package.skills`, `integrations`, `spool`, `warnings`, `ok` — **nothing about compute**. Using CLI auth as a proxy for managed-compute availability is the wrong signal in the worst direction: managed compute is off behind `COMPUTE_RESELL_ENABLED=false` regardless of authentication, so an authenticated user is told managed compute works when it does not. + +**Out of scope, deliberately.** `research.txt:81-84` also runs `atlas doctor`, to check whether the `atlas` CLI is present and authenticated before loading graph state. That is a valid use of a signal the command genuinely reports — leave it alone. The absence tests must not become so broad that they forbid it. + +- [ ] **Step 1: Widen the test scope to prove the gap exists** + +In `backend/cli/test/session/compute-prompt.test.ts`, change `sources()` to scan agent prompts as well as session ones: + +```ts +async function sources() { + const globs = ["session/**/*.{ts,txt}", "agent/prompt/*.txt"] + const files = ( + await Promise.all( + globs.map((pattern) => + Array.fromAsync(new Bun.Glob(pattern).scan({ cwd: root, absolute: true, onlyFiles: true })), + ), + ) + ).flat() + return Promise.all(files.map(async (file) => [file, await Bun.file(file).text()] as const)) +} +``` + +Then replace the blanket `atlas doctor` test with one that forbids it **as a compute signal** while still permitting the CLI-availability check: + +```ts +test("the file set is non-empty and covers both prompt trees", async () => { + const files = (await sources()).map(([file]) => path.relative(root, file)) + expect(files.length).toBeGreaterThan(20) + expect(files).toContain("session/prompt.ts") + expect(files).toContain("agent/prompt/research.txt") +}) + +test("no prompt uses atlas doctor as the compute availability signal", async () => { + // `atlas doctor` legitimately reports whether the atlas CLI is present and + // authenticated (research.txt uses it that way before loading graph state). + // What it does NOT report is anything about compute — so any paragraph that + // mentions both compute and `atlas doctor` is reading a signal that isn't there. + const hits = (await sources()).filter(([, text]) => + text.split(/\n\s*\n/).some((para) => /atlas doctor/i.test(para) && /\bcompute\b/i.test(para)), + ) + expect(hits.map(([file]) => path.relative(root, file))).toEqual([]) +}) + +test("agent prompts point at compute_status for GPU funding", async () => { + const text = await Bun.file(path.join(root, "agent", "prompt", "research.txt")).text() + expect(text).toContain("compute_status") +}) + +test("prompts name skills that exist in the provider map", async () => { + // `modal` is not a skill name — the real ones are modal-serverless-gpu, + // modal-ml-training, modal-research-gpu. A prompt naming a skill the catalog + // does not have sends the agent to load something that cannot resolve. + const text = await Bun.file(path.join(root, "agent", "prompt", "research.txt")).text() + expect(text).not.toMatch(/`modal`/) +}) +``` + +Keep the existing `compute:up`, `compute_status`-in-`prompt.ts`, `computeBillingMode`, and config-description tests unchanged. + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `cd backend/cli && bun test test/session/compute-prompt.test.ts` +Expected: FAIL on "no prompt uses atlas doctor as the compute availability signal" (naming `agent/prompt/research.txt`), on "agent prompts point at compute_status", and on "prompts name skills that exist in the provider map". + +- [ ] **Step 3: Rewrite the COMPUTE stage guidance** + +In `backend/cli/src/agent/prompt/research.txt`, replace lines 254-257 (the four-line `Managed compute (Daytona-backed) …` bullet) with: + +``` +- Call the `compute_status` tool before launching any GPU work. It reports how compute is + funded right now — `byok`, `managed`, or `none` — which providers are usable, and the rule + that applies. Do not infer this from `atlas doctor`; it reports nothing about compute. +- If it returns `byok`, load the cloud-compute skill for one of the providers it lists. + If `managed`, run the work through managed compute. If `none`, do not launch GPU work — + tell the user to connect a provider key in Settings ▸ Compute. +``` + +Then correct the stale skill name on the following line — `modal` is not a skill; the catalog has `modal-serverless-gpu`: + +``` +- Load: `modal-serverless-gpu` for general serverless GPU (inference, serving) +``` + +Leave lines 81-84 untouched. + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `cd backend/cli && bun test test/session/compute-prompt.test.ts` +Expected: PASS. + +- [ ] **Step 5: Prove each new assertion against deletion** + +1. Re-add the phrase `Run \`atlas doctor --format=json\` first`into the COMPUTE bullet → "no prompt uses atlas doctor as the compute availability signal" must fail, naming`agent/prompt/research.txt`. +2. Confirm the _inverse_: the untouched graph-state use at lines 81-84 must **not** trip that test. Verify the paragraph containing it has no `compute` mention, so the test permits a legitimate `atlas doctor` call. If it does trip, the test is too broad — fix it. +3. Remove `compute_status` from `research.txt` → "agent prompts point at compute_status" must fail. +4. Restore `` `modal` `` in the load line → "prompts name skills that exist in the provider map" must fail. +5. Point `sources()` at a non-existent directory → "the file set is non-empty" must fail. This guards the whole absence-test family against passing on an empty glob. + +- [ ] **Step 6: Full suite, format, commit** + +```bash +cd backend/cli && bun test && bun run typecheck +bunx prettier --write test/session/compute-prompt.test.ts +git add src/agent/prompt/research.txt test/session/compute-prompt.test.ts +git commit -m "fix(prompt): research agent checks compute_status, not atlas doctor" +``` + +Note: `research.txt` is a `.txt` prompt file — do not run prettier on it. + +--- + +### Tasks 6 and 7: RunPod and Vast.ai skills — DELETED + +Superseded by Decision 2. The user ruled that a provider skill is overkill — +a capable agent drives a documented cloud API from a bare key. With the skill +conjunction gone from the resolver, a RunPod or Vast credential already makes +a user BYOK-usable, so there is nothing left for these tasks to fix. + +`PROVIDERS.runpod.skills` and `PROVIDERS.vast.skills` are `[]`, which is the +honest statement of the situation: those providers have credentials and no +catalogued skill, and that is fine. + +--- + +## Final verification + +Run before declaring the branch complete. Evidence before assertions — paste the actual output, do not summarise it. + +- [ ] `cd backend/cli && bun test` — full suite green, no network. +- [ ] `cd backend/cli && bun run typecheck` — clean. If it fails only inside the untracked `test/provider/synthetic-model.test.ts`, that is the user's WIP and is expected; note it rather than fixing it. +- [ ] `bunx prettier --check .` from the repo root — CI's Format job runs over the whole repo. +- [ ] Walk the spec's 14 acceptance criteria and name the test that proves each. + +**Pushing is blocked twice.** The husky pre-push hook pins a bun version from `package.json` `packageManager` and runs `bun typecheck`, which fails on the untracked `synthetic-model.test.ts`. The prior session's workaround was `git stash push -u` on that one path, push, then `git stash pop`. **Restore it immediately; do not leave it stashed, and do not use `--no-verify`** — the user declined that explicitly. + +## Acceptance criteria → task map + +| # | Criterion | Proven by | +| --- | ------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------- | +| 1 | Runtime resolver, single shared implementation | Task 1 + Task 2; Task 4 "SkillTool.init and compute_status never disagree" | +| 2 | **REVISED by Decision 2** — a credential alone makes a provider usable; a half-configured Modal credential still does not count | Task 1b: "a key with NO catalogued skill IS usable", "a catalogued skill with NO key", the Modal-pair test, the empty-string test | +| 3 | Catalog lists usable providers only; none in managed/none; non-compute unaffected | Task 4 tests 1–6 | +| 4 | Markdown never auto-injected | Task 4 — the filter touches the listing only; no auto-load path is added anywhere | +| 5 | No keys + managed unavailable (incl. a failed check) → `none` | Task 2 tests 3, 4, 5, 6 | +| 6 | Override can narrow to `none`, never manufacture | Task 2 override tests 2 and 3 | +| 7 | A key injected by either settings panel at boot is detected | Task 1 test 10 + Task 2's on-demand resolution (never at startup) | +| 8 | Availability call skipped when a usable provider is present | Task 2 test 1 (positive assertion on the recorded call list) | +| 9 | `compute_status` returns mode, providers, guidance, resolved per call | Task 3 tests 1–5 | +| 10 | Mid-session credential reflected without restart | Task 3 test 6; Task 4 test 7 | +| 11 | Nothing mode-carrying injected per turn; description carries the instruction | Task 5 test 3; Task 3 test 7 | +| 12 | No prompt references `atlas compute:up` or `atlas doctor` | Task 5 tests 1, 2 | +| 13 | `none` says don't attempt GPU work and how to enable it | Task 3 tests 3, 4 | +| 14 | `bun test` passes with no network | Final verification | diff --git a/docs/plans/2026-07-31-compute-lease-prerequisites.md b/docs/plans/2026-07-31-compute-lease-prerequisites.md new file mode 100644 index 00000000..22097b43 --- /dev/null +++ b/docs/plans/2026-07-31-compute-lease-prerequisites.md @@ -0,0 +1,797 @@ +# Compute lease prerequisites — implementation plan + +> ## ✅ EXECUTED — 2026-08-01. Historical record; do not implement from it again. +> +> **Where:** `~/codes/InkVell/atlas`, branch `feat/compute-lease-prerequisites`, base `8aa66d5`. +> **Not merged** — the PR is deliberately draft while the team waits for compute to be complete. +> **Outcome:** all four tasks landed, plus a six-finding fix wave from the whole-branch review. +> Suite **1610 passed / 1 skipped**, 12 commits at plan completion. Ledger: +> `.superpowers/sdd/2026-07-31-compute-lease-prerequisites/progress.md`. +> +> **The deliverable holds, and was later confirmed against a real provider rather than a fake clock:** +> a user lease sat `ready` for 578s → 687s → **788s** un-reaped against the deployed reaper. Before +> this branch it died at 600s. Task 1's deploy gate is discharged by a **cross-login test**: two +> instances on one Vast operator account, each key accepted for its own box and **refused** +> (`Permission denied (publickey)`) against the other's. The cross-tenant hole is closed, measured, +> not inferred. +> +> **What execution proved this plan wrong about — read these before trusting any step below:** +> +> - **Task 3's promotion was under-specified and the first fix for it broke Modal.** The fix wave +> gated promotion on a non-empty `ssh_host`; Modal's `connection()` returns no `ssh_host` at all +> (it is exec-based, no SSH), so Modal CPU sandboxes stopped promoting and were reaped at 600s. +> The gate is now `if not ssh_host and lease.get("ssh_key_name")` — the four SSH providers set +> `ssh_key_name` in `acquire`, Modal does not. The gate itself was **justified by measurement**, +> not taste: RunPod returns `desiredStatus=RUNNING` from pod creation with **no address**, so +> provider status alone is not readiness. This plan's Step 5 does not say any of that. +> - **The status mapping was wrong for a destroyed Vast instance.** `_get_instance` returned `{}` and +> `_map_status` read that as `provisioning` **forever**, so reaper branch 1 could never fire for +> Vast. Fixed in `9bc19a7` (empty payload → `terminated`). Later sharpened again: Vast returns +> HTTP 200 `{"instances": null}` for a destroyed id **and** for one that never existed, and never +> 404s, while RunPod does 404 — both signals are needed. +> - **Task 4's `_PROVISIONING` set omitted Lambda's in-flight strings.** Flagged during the task and +> deliberately left for a human ruling; fixed in the follow-on plan (`8748057`). +> - **Task 1's minor deferral is still open:** `vast_provider`'s docstring API list still names +> `POST /ssh/` — the endpoint the task removed the call to. + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development to implement +> this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Close the live cross-tenant SSH exposure on Vast, and make a user-launched GPU lease survive long +enough to be usable — the two things every other compute change waits on. + +**Architecture:** All four tasks are in the **Atlas** repo (`~/codes/InkVell/atlas`), Python/FastAPI. Task 1 +removes an account-level SSH key registration. Tasks 2–3 fix the lease reaper: today a managed GPU lease +never leaves `provisioning` status, so it is reaped as a provisioning timeout ~10 minutes after creation. +Task 4 normalises the status vocabulary the OpenScience client will later poll on. + +**Tech Stack:** Python 3.12, FastAPI, aiosqlite, pytest + pytest-asyncio, respx (HTTP mocking), httpx. + +**Spec:** `~/codes/InkVell/openscience/docs/specs/compute-design.md` — changes 9(a), 0(a), 0(b), 0(c). + +## Global Constraints + +- **Repo:** `~/codes/InkVell/atlas`. All paths below are relative to that repo root. +- **Run tests with:** `cd ~/codes/InkVell/atlas/backend && .venv/bin/python -m pytest -q` + There is no activated virtualenv; always invoke `.venv/bin/python` explicitly. +- **Branch:** work on `feat/compute-lease-prerequisites`, cut from `main`. The repo is currently on + `feat/managed-catalog-opus5-frontier`, which is unrelated work — **do not commit onto it.** +- **Never add a `Co-Authored-By:` trailer or any AI attribution to commits.** Organisation rule. +- **No mocks of our own code.** Stub HTTP at the transport boundary with `respx`; use the real repo and + provider objects. This matches the existing suite. +- **Every new assertion must be shown failing first.** Run the test before writing the implementation and + paste the failure. A test that has never failed is not evidence. +- **Ignore `.claude/worktrees/`** — other branches, will mislead greps. +- Baseline before starting: `.venv/bin/python -m pytest tests/test_lease_reaper.py -q` → **7 passed**. + +--- + +### Task 0: Branch setup + +- [ ] **Step 1: Cut the branch from main** + +```bash +cd ~/codes/InkVell/atlas +git fetch origin +git checkout -b feat/compute-lease-prerequisites origin/main +``` + +- [ ] **Step 2: Confirm the baseline suite passes** + +Run: `cd ~/codes/InkVell/atlas/backend && .venv/bin/python -m pytest tests/test_lease_reaper.py tests/test_compute_providers.py -q` +Expected: all pass. If anything fails here, stop and report — it is pre-existing and not yours to fix. + +--- + +### Task 1: Vast must not register account-level SSH keys + +**Why:** `VastProvider.acquire` posts every lease's public key to the **shared operator account** +(`POST /ssh/`), and the module docstring says the purpose is "so new instances pick it up" +(`app/compute/vast_provider.py:10-12`). Under managed funding all users share one operator credential +(`:78`), so one user's private key opens another user's instance. The per-instance attach at `:244-248` +already exists and its comment describes it as the path that works "even if the account key wasn't applied +at launch" — i.e. it is the reliable one. + +**Files:** + +- Modify: `backend/app/compute/vast_provider.py:202-213` (remove the account POST), `:10-16` (docstring) +- Create: `backend/tests/test_compute_vast_provider_http.py` + +**Interfaces:** + +- Consumes: nothing from earlier tasks. +- Produces: nothing later tasks depend on. Standalone security fix. + +- [ ] **Step 1: Write the failing test** + +Create `backend/tests/test_compute_vast_provider_http.py`: + +```python +"""HTTP-contract tests for the Vast provider. + +Regression anchor: ``acquire`` must NOT register the per-lease public key on +the Vast *account*. Managed leases all run on one operator credential, so an +account-level key is readable by every subsequently created instance — one +user's private key would open another user's box. The per-instance attach is +the only key path. +""" + +from __future__ import annotations + +import httpx +import respx + +from app.compute.vast_provider import VAST_API, VastProvider + +CREDS = {"secret": "vast-test-key"} + + +@respx.mock +async def test_acquire_registers_no_account_ssh_key(): + account = respx.post(f"{VAST_API}/ssh/").mock( + return_value=httpx.Response(200, json={}) + ) + respx.put(f"{VAST_API}/asks/9999/").mock( + return_value=httpx.Response(200, json={"new_contract": 4242}) + ) + instance = respx.post(f"{VAST_API}/instances/4242/ssh/").mock( + return_value=httpx.Response(200, json={}) + ) + + lease = await VastProvider().acquire("9999", "us", user_credentials=CREDS) + + assert lease["lease_id"] == "4242" + assert instance.called, "the per-instance key attach is the only key path" + assert not account.called, "account-level key registration is cross-tenant readable" + + +@respx.mock +async def test_acquire_still_returns_the_private_key(): + respx.put(f"{VAST_API}/asks/9999/").mock( + return_value=httpx.Response(200, json={"new_contract": 4242}) + ) + respx.post(f"{VAST_API}/instances/4242/ssh/").mock( + return_value=httpx.Response(200, json={}) + ) + + lease = await VastProvider().acquire("9999", "us", user_credentials=CREDS) + + assert lease["ssh_private_key"].startswith("-----BEGIN") + assert lease["ssh_public_key"].startswith("ssh-ed25519") + assert lease["status"] == "provisioning" +``` + +- [ ] **Step 2: Run the test to verify it fails** + +Run: `cd ~/codes/InkVell/atlas/backend && .venv/bin/python -m pytest tests/test_compute_vast_provider_http.py -q` +Expected: `test_acquire_registers_no_account_ssh_key` FAILS on +`assert not account.called`. The second test should already pass — it pins behaviour we must not break. +**Paste the failure output into your report.** + +- [ ] **Step 3: Remove the account-level registration** + +In `backend/app/compute/vast_provider.py`, delete this block from `acquire` (currently `:203-213`): + +```python + # Register the public key on the account so the new instance + # picks it up; best-effort (a content-identical key may already + # exist, which Vast tolerates). + try: + await client.post( + f"{VAST_API}/ssh/", + headers=headers, + json={"ssh_key": public_openssh}, + ) + except Exception: # noqa: BLE001 + pass +``` + +Then update the per-instance attach comment (currently `:240-241`) to say it is the only key path: + +```python + # Attach the per-lease key to the instance. This is the ONLY key + # path: the account-level POST /ssh/ that used to run here made the + # key readable by every instance created afterwards on the same + # (shared, operator) account. +``` + +- [ ] **Step 4: Correct the module docstring** + +Replace lines 10-16 of `backend/app/compute/vast_provider.py`: + +```python +SSH: Atlas generates a fresh keypair per lease and attaches the *public* key +to the instance (``POST /instances/{id}/ssh/``). It is deliberately NOT +registered on the account (``POST /ssh/``): managed leases share one operator +credential, so an account key is picked up by every instance created after it, +which would let one user's private key open another user's box. +``` + +- [ ] **Step 5: Run the tests to verify they pass** + +Run: `cd ~/codes/InkVell/atlas/backend && .venv/bin/python -m pytest tests/test_compute_vast_provider_http.py tests/test_compute_providers.py -q` +Expected: all PASS. + +- [ ] **Step 6: Commit** + +```bash +cd ~/codes/InkVell/atlas +git add backend/app/compute/vast_provider.py backend/tests/test_compute_vast_provider_http.py +git commit -m "fix(vast): stop registering per-lease SSH keys on the shared account + +acquire posted each lease's public key to the operator account so that new +instances would pick it up. Managed leases all run on one operator credential, +so every instance created afterwards accepted that key -- one user's private +key opened another user's box. + +The per-instance attach already present is the only key path needed; its own +comment described it as working even when the account key was not applied." +``` + +- [ ] **Step 7: Flag the deploy gate in your report** + +This cannot be fully verified without a live Vast launch. **State in your report** that before this +deploys, someone must lease one real Vast box and confirm SSH still works with the returned key. The +code comment suggests the per-instance attach is sufficient; that is an inference, not a measurement. + +--- + +### Task 2: Scope heartbeat reaping to leases that can answer it + +**Why:** `sweep_once` branch 3 reaps any non-`provisioning` lease whose telemetry is stale past +`HEARTBEAT_STALE_SECONDS` (600). `create_lease` mints no runner token — `set_runner_api_key` has exactly +one caller, the agent-spawn path — so a user lease can never emit telemetry and can never pass this check. +Task 3 makes leases reach a non-`provisioning` status, which is what would expose this; do it first so the +two land in either order safely. + +The predicate is the **`runner_api_key_id` column** (`backend/app/db/migrations.py:567`). Do not confuse it +with the `thrk_*` runner token in the `runner_tokens` table — different credential, different table. + +**Files:** + +- Modify: `backend/app/jobs/lease_reaper.py:138-144` +- Test: `backend/tests/test_lease_reaper.py` (append) + +**Interfaces:** + +- Consumes: nothing. +- Produces: nothing. Behaviour change only. + +- [ ] **Step 1: Write the failing tests** + +Append to `backend/tests/test_lease_reaper.py`: + +```python +@pytest.mark.asyncio +async def test_user_lease_without_runner_token_is_not_heartbeat_reaped(test_db, monkeypatch): + """A user-launched lease has no runner token, so it can never emit + telemetry. Reaping it for heartbeat staleness destroys a box the user is + paying for, ~10 minutes after it becomes ready.""" + db = test_db["db"]; uid = test_db["user_id"] + await compute_repo.create_lease(db, lease_id="Luser", status="ready", + user_id=uid, provider="runpod", requested_sku="h100", region="us", + hourly_rate_cents=279, category="gpu") + # No telemetry at all, and created long ago -> branch 3 would fire today. + old = _iso(datetime.now(timezone.utc) - timedelta(seconds=9999)) + await db.execute( + "UPDATE compute_leases SET created_at = ?, started_at = ? WHERE lease_id = 'Luser'", + (old, old)) + await db.commit() + + class _P: + async def status(self, _id): return {"status": "running"} + monkeypatch.setattr(reaper, "get_provider", lambda _name: _P()) + async def _noop_release(self, _db, _lease): return {"status": "released"} + monkeypatch.setattr(reaper.LeaseManager, "release_lease", _noop_release) + + n = await reaper.sweep_once(db, now=datetime.now(timezone.utc)) + + assert n == 0 + row = await compute_repo.get_lease(db, "Luser") + assert row["status"] == "ready" + + +@pytest.mark.asyncio +async def test_agent_lease_with_runner_token_is_still_heartbeat_reaped(test_db, monkeypatch): + """The narrowing must not disable the check for leases that CAN report.""" + db = test_db["db"]; uid = test_db["user_id"] + await compute_repo.create_lease(db, lease_id="Lagent", status="ready", + user_id=uid, provider="modal", requested_sku="cpu-small", region="us", + hourly_rate_cents=0, category="cpu", node_id="na") + old = _iso(datetime.now(timezone.utc) - timedelta(seconds=9999)) + await db.execute( + "UPDATE compute_leases SET created_at = ?, started_at = ?, " + "runner_api_key_id = 'thk_test' WHERE lease_id = 'Lagent'", (old, old)) + await db.commit() + + class _P: + async def status(self, _id): return {"status": "running"} + monkeypatch.setattr(reaper, "get_provider", lambda _name: _P()) + reaped = [] + async def _fake_reconcile(_db, *, lease_id, node_id, forced_state=None, reason=None): + reaped.append((lease_id, reason)); return {"outcome": "failed"} + monkeypatch.setattr(reaper.run_reconcile_service, "reconcile_completed_run", _fake_reconcile) + async def _noop_release(self, _db, _lease): return {"status": "released"} + monkeypatch.setattr(reaper.LeaseManager, "release_lease", _noop_release) + + n = await reaper.sweep_once(db, now=datetime.now(timezone.utc)) + + assert n == 1 + assert reaped[0] == ("Lagent", "heartbeat_timeout") +``` + +- [ ] **Step 2: Run to verify the first fails and the second passes** + +Run: `cd ~/codes/InkVell/atlas/backend && .venv/bin/python -m pytest tests/test_lease_reaper.py -q` +Expected: `test_user_lease_without_runner_token_is_not_heartbeat_reaped` FAILS (`assert 1 == 0`). +`test_agent_lease_with_runner_token_is_still_heartbeat_reaped` PASSES already — it pins the behaviour the +narrowing must preserve. **Paste the failure.** + +- [ ] **Step 3: Narrow branch 3** + +In `backend/app/jobs/lease_reaper.py`, replace the branch-3 block: + +```python + # 3. Heartbeat staleness — only for leases that have actually booted + # AND are supposed to report. A still-`provisioning` lease legitimately + # has no telemetry yet (PROVISION_TIMEOUT_SECONDS governs it), and a + # user-launched lease is issued no runner token at all, so it can never + # emit telemetry — reaping it would destroy a box the user is paying + # for. `runner_api_key_id` is set only by the agent-spawn path. + if ( + reason is None + and lease.get("status") != "provisioning" + and lease.get("runner_api_key_id") + ): + latest = _parse_iso(live["latest_at"]) or _lease_started(lease) + if latest and (now - latest).total_seconds() > config.HEARTBEAT_STALE_SECONDS: + reason = "heartbeat_timeout" +``` + +- [ ] **Step 4: Run the tests to verify they pass** + +Run: `cd ~/codes/InkVell/atlas/backend && .venv/bin/python -m pytest tests/test_lease_reaper.py -q` +Expected: 9 passed. If `test_sweep_reaps_heartbeat_stale` (the pre-existing one) now fails, its fixture +lease has no `runner_api_key_id` — **fix the fixture, not the predicate**: add +`runner_api_key_id = 'thk_test'` to that lease, since it models an agent-spawned lease. + +- [ ] **Step 5: Commit** + +```bash +cd ~/codes/InkVell/atlas +git add backend/app/jobs/lease_reaper.py backend/tests/test_lease_reaper.py +git commit -m "fix(reaper): heartbeat staleness only applies to leases with a runner token + +create_lease mints no runner token, so a user-launched lease can never emit +telemetry and can never satisfy the heartbeat check. It stayed bounded by plan +TTL, wallet exhaustion and explicit release; it does not need a liveness probe +it has no way to answer." +``` + +--- + +### Task 3: Promote a provisioning lease to ready and persist its SSH coordinates + +**Why:** two facts combine into the defect. `RunPodProvider.acquire` returns `status: "provisioning"` and +`create_lease` persists it; and the only two writers that flip a lease to `ready` are +`_reconcile_active_cpu_leases` (CPU-only, `lease_manager.py:270`) and `LeaseManager.get_lease_status`, +which has **no production caller** (`:690`; only two test callers). So a managed GPU lease stays +`provisioning` forever and branch 2 reaps it at `PROVISION_TIMEOUT_SECONDS` = 600. + +Separately, `acquire` returns a hardcoded `ssh_port: 22` and no `ssh_host` on every provider — the real +values live only in `provider.connection()`. `update_lease_status` already accepts them +(`compute_repo.py:397-405`) and nothing passes them. + +Do both in the reaper sweep, which already runs every 60s and **already calls `provider.status()` for +every unfinished lease** at branch 1 — so the promotion check costs no extra provider call. The +`connection()` call happens once, on the transition only. + +**Files:** + +- Modify: `backend/app/jobs/lease_reaper.py` (add `_PROVIDER_READY`, add promotion between branches 1 and 2) +- Test: `backend/tests/test_lease_reaper.py` (append) + +**Interfaces:** + +- Consumes: nothing. +- Produces: leases now reach `status = "ready"` with `ssh_host` / `ssh_port` / `ready_at` populated. Task 4 + and all later compute work depend on this. + +- [ ] **Step 1: Write the failing tests** + +Append to `backend/tests/test_lease_reaper.py`: + +```python +@pytest.mark.asyncio +async def test_sweep_promotes_provisioning_lease_and_persists_ssh(test_db, monkeypatch): + """A GPU lease is created 'provisioning' and nothing in production ever + moves it. The sweep must promote it once the provider says it is up, and + record the SSH coordinates, which acquire() cannot know.""" + db = test_db["db"]; uid = test_db["user_id"] + await compute_repo.create_lease(db, lease_id="Lprom", status="provisioning", + user_id=uid, provider="runpod", requested_sku="h100", region="us", + hourly_rate_cents=279, category="gpu") + + class _P: + async def status(self, _id): return {"status": "running"} + async def connection(self, _id, **kw): + return {"status": "running", "ssh_host": "194.68.245.162", + "ssh_port": 22065, "ssh_user": "root"} + monkeypatch.setattr(reaper, "get_provider", lambda _name: _P()) + + n = await reaper.sweep_once(db, now=datetime.now(timezone.utc)) + + assert n == 0 # promotion is not a reap + row = await compute_repo.get_lease(db, "Lprom") + assert row["status"] == "ready" + assert row["ssh_host"] == "194.68.245.162" + assert row["ssh_port"] == 22065 + assert row["ready_at"] is not None + + +@pytest.mark.asyncio +async def test_sweep_does_not_promote_a_lease_still_booting(test_db, monkeypatch): + db = test_db["db"]; uid = test_db["user_id"] + await compute_repo.create_lease(db, lease_id="Lboot", status="provisioning", + user_id=uid, provider="runpod", requested_sku="h100", region="us", + hourly_rate_cents=279, category="gpu") + + class _P: + async def status(self, _id): return {"status": "provisioning"} + async def connection(self, _id, **kw): + raise AssertionError("connection() must not be called before the provider is up") + monkeypatch.setattr(reaper, "get_provider", lambda _name: _P()) + + await reaper.sweep_once(db, now=datetime.now(timezone.utc)) + + row = await compute_repo.get_lease(db, "Lboot") + assert row["status"] == "provisioning" + assert row["ready_at"] is None + + +@pytest.mark.asyncio +async def test_promotion_survives_a_connection_failure(test_db, monkeypatch): + """Provider is up but connection() errors: still promote, so the lease is + not reaped as a provisioning timeout. Coordinates arrive on a later sweep.""" + db = test_db["db"]; uid = test_db["user_id"] + await compute_repo.create_lease(db, lease_id="Lconn", status="provisioning", + user_id=uid, provider="runpod", requested_sku="h100", region="us", + hourly_rate_cents=279, category="gpu") + + class _P: + async def status(self, _id): return {"status": "running"} + async def connection(self, _id, **kw): raise RuntimeError("provider 503") + monkeypatch.setattr(reaper, "get_provider", lambda _name: _P()) + + await reaper.sweep_once(db, now=datetime.now(timezone.utc)) + + row = await compute_repo.get_lease(db, "Lconn") + assert row["status"] == "ready" + assert row["ssh_host"] is None + + +@pytest.mark.asyncio +async def test_promoted_lease_is_not_reaped_at_the_provisioning_timeout(test_db, monkeypatch): + """The headline property: a user lease created 20 minutes ago survives. + Today it dies at PROVISION_TIMEOUT_SECONDS = 600.""" + db = test_db["db"]; uid = test_db["user_id"] + await compute_repo.create_lease(db, lease_id="Llive", status="provisioning", + user_id=uid, provider="runpod", requested_sku="h100", region="us", + hourly_rate_cents=279, category="gpu") + old = _iso(datetime.now(timezone.utc) - timedelta(seconds=1200)) + await db.execute( + "UPDATE compute_leases SET created_at = ?, started_at = ? WHERE lease_id = 'Llive'", + (old, old)) + await db.commit() + + class _P: + async def status(self, _id): return {"status": "running"} + async def connection(self, _id, **kw): + return {"ssh_host": "1.2.3.4", "ssh_port": 20095, "ssh_user": "root"} + monkeypatch.setattr(reaper, "get_provider", lambda _name: _P()) + async def _noop_release(self, _db, _lease): return {"status": "released"} + monkeypatch.setattr(reaper.LeaseManager, "release_lease", _noop_release) + + n = await reaper.sweep_once(db, now=datetime.now(timezone.utc)) + + assert n == 0, "a live 20-minute-old lease must not be reaped" + row = await compute_repo.get_lease(db, "Llive") + assert row["status"] == "ready" +``` + +- [ ] **Step 2: Run to verify all four fail** + +Run: `cd ~/codes/InkVell/atlas/backend && .venv/bin/python -m pytest tests/test_lease_reaper.py -q -k "promot or booting or conn or reaped_at_the"` +Expected: `test_sweep_promotes...` FAILS (`status == 'provisioning'`), +`test_promotion_survives_a_connection_failure` FAILS, `test_promoted_lease_is_not_reaped...` FAILS +(`assert 1 == 0` — it was reaped). `test_sweep_does_not_promote_a_lease_still_booting` passes already. +**Paste the failures.** + +- [ ] **Step 3: Add the ready-state set** + +In `backend/app/jobs/lease_reaper.py`, below `_PROVIDER_TERMINAL` (line 34): + +```python +# Provider vocabularies for "the box is up". RunPod and Vast say "running", +# Prime Intellect says "active", Lambda passes its upstream string through. +_PROVIDER_READY = {"running", "active", "ready"} +``` + +- [ ] **Step 4: Add the promotion helper** + +In `backend/app/jobs/lease_reaper.py`, above `sweep_once`: + +```python +async def _promote_to_ready(db, provider, lease: dict) -> None: + """A provisioning lease whose provider reports it up becomes `ready`, with + the SSH coordinates acquire() could not know. + + Without this a managed GPU lease stays `provisioning` forever — the only + other writers of `ready` are CPU-scoped or have no production caller — and + branch 2 reaps it at PROVISION_TIMEOUT_SECONDS. Coordinates are best-effort: + promotion must happen even if connection() fails, or the lease is reaped + while the box is alive. A later sweep fills them in. + """ + ssh_host = ssh_port = None + try: + conn = (await provider.connection(lease["lease_id"])) or {} + ssh_host = conn.get("ssh_host") or None + ssh_port = conn.get("ssh_port") or None + except Exception: + log.exception("reaper: connection lookup failed for %s", lease["lease_id"]) + await compute_repo.update_lease_status( + db, lease["lease_id"], "ready", ssh_host=ssh_host, ssh_port=ssh_port, + ) + lease["status"] = "ready" +``` + +- [ ] **Step 5: Call it from the sweep** + +In `sweep_once`, insert immediately after branch 1's `if provider is not None:` block and **before** +branch 2 (currently line 132). The provider status was already fetched at branch 1; reuse it: + +```python + # 1b. Promotion. `acquire` returns status='provisioning' and nothing + # else in production advances it, so without this every managed GPU + # lease is reaped by branch 2 at PROVISION_TIMEOUT_SECONDS. + if ( + reason is None + and lease.get("status") == "provisioning" + and provider is not None + and (pstat.get("status") or "").lower() in _PROVIDER_READY + ): + await _promote_to_ready(db, provider, lease) +``` + +`pstat` is bound inside `if provider is not None:` at branch 1. Hoist its initialisation so it is always +defined — change the top of branch 1 from `if provider is not None:` to: + +```python + pstat: dict = {} + provider = get_provider((lease.get("provider") or "").lower()) + if provider is not None: +``` + +and delete the now-redundant `pstat = {}` assignments inside the `try`/`except` — keep +`pstat = (await provider.status(lease_id)) or {}` in the `try` and `pstat = {}` in the `except`. + +- [ ] **Step 6: Run the tests to verify they pass** + +Run: `cd ~/codes/InkVell/atlas/backend && .venv/bin/python -m pytest tests/test_lease_reaper.py -q` +Expected: 13 passed. + +- [ ] **Step 7: Run the wider compute suite for regressions** + +Run: `cd ~/codes/InkVell/atlas/backend && .venv/bin/python -m pytest tests/ -q -k "compute or lease or telemetry"` +Expected: all pass. Report any failure rather than fixing it if it looks unrelated. + +- [ ] **Step 8: Commit** + +```bash +cd ~/codes/InkVell/atlas +git add backend/app/jobs/lease_reaper.py backend/tests/test_lease_reaper.py +git commit -m "fix(reaper): promote provisioning leases to ready and record SSH coordinates + +A managed GPU lease was created 'provisioning' and nothing in production ever +advanced it: the only writers of 'ready' are the CPU-scoped reconcile pass and +get_lease_status, which has no production caller. So every user lease was +reaped as a provisioning timeout ~10 minutes after creation. + +The sweep already polls provider.status for each unfinished lease, so the +promotion check is free. connection() runs once on the transition, and its +failure must not block promotion -- a lease left provisioning gets reaped while +the box is alive. This also populates ssh_host/ssh_port, which acquire() cannot +know: every provider returns a hardcoded port 22 and no host." +``` + +--- + +### Task 4: Normalise the lease status vocabulary + +**Why:** `GET /api/compute/leases/{id}/connection` returns +`conn.get("status") or lease.get("status")` (`backend/app/routes/compute.py:500`) — one field carrying five +vocabularies. RunPod maps `running|stopped|terminated|unknown`, Vast `running|provisioning|stopped|unknown`, +Prime `provisioning|active|stopped|error|terminating|terminated|unknown`, Lambda passes its raw upstream +string through unmapped, and when the provider call throws it falls back to the DB's +`provisioning|ready|released`. A client cannot poll on that. Add a normalised field; **do not change +`status`**, which the dashboard consumes. + +**Files:** + +- Modify: `backend/app/routes/compute.py:450-508` (add `state` to the response) +- Create: `backend/app/compute/lease_state.py` +- Test: `backend/tests/test_compute_lease_state.py` + +**Interfaces:** + +- Consumes: Task 3's `ready` status. +- Produces: `normalise_state(raw: str | None) -> str` in `app/compute/lease_state.py`, returning one of + `"provisioning" | "ready" | "terminated" | "unknown"`. The `/connection` response gains a `state` field + with that value. The OpenScience `compute_launch` readiness poll will consume it. + +- [ ] **Step 1: Write the failing test** + +Create `backend/tests/test_compute_lease_state.py`: + +```python +"""One vocabulary for lease state. + +/connection returns `conn.get("status") or lease.get("status")`, which mixes +four provider vocabularies with the DB's own. A client polling for readiness +cannot be written against that: "running" never appears on the DB path, +"ready" never appears on the provider path, Prime says "active", and Lambda +passes its upstream string through unmapped. +""" + +from __future__ import annotations + +import pytest + +from app.compute.lease_state import normalise_state + + +@pytest.mark.parametrize("raw", ["running", "active", "ready", "RUNNING", "Active"]) +def test_up_states_normalise_to_ready(raw): + assert normalise_state(raw) == "ready" + + +@pytest.mark.parametrize( + "raw", ["terminated", "released", "stopped", "failed", "error", "TERMINATED"] +) +def test_dead_states_normalise_to_terminated(raw): + assert normalise_state(raw) == "terminated" + + +@pytest.mark.parametrize("raw", ["provisioning", "terminating", "PROVISIONING"]) +def test_in_flight_states_normalise_to_provisioning(raw): + assert normalise_state(raw) == "provisioning" + + +@pytest.mark.parametrize("raw", [None, "", "unknown", "some-lambda-string"]) +def test_unrecognised_states_are_unknown_not_ready(raw): + """Fail toward 'unknown'. A client must never read an unmapped Lambda + string as readiness and try to SSH into a box that is still booting.""" + assert normalise_state(raw) == "unknown" +``` + +- [ ] **Step 2: Run to verify it fails** + +Run: `cd ~/codes/InkVell/atlas/backend && .venv/bin/python -m pytest tests/test_compute_lease_state.py -q` +Expected: FAIL — `ModuleNotFoundError: No module named 'app.compute.lease_state'`. **Paste it.** + +- [ ] **Step 3: Write the implementation** + +Create `backend/app/compute/lease_state.py`: + +```python +"""Normalise the five lease-status vocabularies into one. + +`/connection` returns `conn.get("status") or lease.get("status")`, so the same +field carries whichever provider answered plus the DB's own values. Callers +that need to act on readiness use `state`, not `status`. + +`status` is left untouched: the dashboard renders it. +""" + +from __future__ import annotations + +_READY = {"running", "active", "ready"} +_TERMINATED = {"terminated", "released", "stopped", "failed", "error"} +_PROVISIONING = {"provisioning", "terminating", "pending", "starting"} + + +def normalise_state(raw: str | None) -> str: + """Map a provider or DB status onto provisioning | ready | terminated | + unknown. + + Unrecognised input is `unknown`, never `ready` — Lambda passes its upstream + string through unmapped, and a client that read an unknown string as + readiness would SSH into a box that is still booting. + """ + value = (raw or "").strip().lower() + if value in _READY: + return "ready" + if value in _TERMINATED: + return "terminated" + if value in _PROVISIONING: + return "provisioning" + return "unknown" +``` + +- [ ] **Step 4: Run to verify it passes** + +Run: `cd ~/codes/InkVell/atlas/backend && .venv/bin/python -m pytest tests/test_compute_lease_state.py -q` +Expected: 18 passed. + +- [ ] **Step 5: Expose it on the connection endpoint** + +In `backend/app/routes/compute.py`, add the import near the other local imports inside +`lease_connection`, and add one key to the returned dict (after `"status"`): + +```python + from app.compute.lease_state import normalise_state +``` + +```python + "status": conn.get("status") or lease.get("status"), + # Normalised: provisioning | ready | terminated | unknown. Poll on this, + # not on `status`, which carries whichever provider vocabulary answered. + "state": normalise_state(conn.get("status") or lease.get("status")), +``` + +- [ ] **Step 6: Verify the route still passes its suite** + +Run: `cd ~/codes/InkVell/atlas/backend && .venv/bin/python -m pytest tests/ -q -k "compute"` +Expected: all pass. + +- [ ] **Step 7: Commit** + +```bash +cd ~/codes/InkVell/atlas +git add backend/app/compute/lease_state.py backend/tests/test_compute_lease_state.py backend/app/routes/compute.py +git commit -m "feat(compute): normalise lease state into one vocabulary + +/connection's status field carries whichever of four provider vocabularies +answered, or the DB's own when the provider call throws. 'running' never +appears on the DB path, 'ready' never on the provider path, Prime says +'active', and Lambda passes its upstream string through unmapped -- so a +readiness poll cannot be written against it. + +Adds a `state` field: provisioning | ready | terminated | unknown. Unmapped +input is 'unknown', never 'ready'. `status` is unchanged; the dashboard reads +it." +``` + +--- + +## Whole-branch verification + +- [ ] **Step 1: Full compute suite** + +Run: `cd ~/codes/InkVell/atlas/backend && .venv/bin/python -m pytest tests/ -q -k "compute or lease or reaper or telemetry"` +Expected: all pass, no network access required. + +- [ ] **Step 2: Confirm the headline property holds end to end** + +Run: `cd ~/codes/InkVell/atlas/backend && .venv/bin/python -m pytest tests/test_lease_reaper.py -q -v` +Confirm by name that both of these pass: +`test_promoted_lease_is_not_reaped_at_the_provisioning_timeout` and +`test_user_lease_without_runner_token_is_not_heartbeat_reaped`. +**Together they are the deliverable:** a user lease now survives both reaper branches that killed it. + +- [ ] **Step 3: Report the deploy gates** + +Two things this plan cannot verify, both of which must be stated in the final report: + +1. **Task 1 needs a live Vast launch** to confirm SSH works without the account key. +2. **Nothing here has run against a real provider.** These are source-level fixes with unit coverage; the + spec's own record is that four source-read conclusions were wrong about deployed behaviour. + +--- + +## Out of scope for this plan + +Budget caps (spec changes 1–2), the resolver and quote endpoint (3–4), rolling window cap (5), volumes (6), +budget extension (7), release honesty (8), the SSH key-id lifecycle (9b), boot telemetry (10), image +pinning (11), catalog cache (12), orphan reap (13), and all three OpenScience tools. Each needs this plan +landed first — until a lease survives, none of them can be observed to work. diff --git a/docs/plans/2026-08-01-compute-budget-cap.md b/docs/plans/2026-08-01-compute-budget-cap.md new file mode 100644 index 00000000..73652fd0 --- /dev/null +++ b/docs/plans/2026-08-01-compute-budget-cap.md @@ -0,0 +1,422 @@ +# Managed compute budget cap — implementation plan + +> ## ✅ EXECUTED — 2026-08-01. Historical record; do not implement from it again. +> +> **Where:** `~/codes/InkVell/atlas`, branch `feat/compute-lease-prerequisites` (continuing). +> **Not merged** — the PR is deliberately draft while the team waits for compute to be complete. +> **Outcome:** all four tasks landed — `0a9da07` (set-to-total), `a4a79ca` (the cap binds), +> `31fc598` (`budget_cents`), `71dccba` (the spawn kill). Suite **1689 passed / 1 skipped**, branch +> 38 commits. Spec changes 1 and 2 landed together as this plan required. Ledger: +> `.superpowers/sdd/2026-08-01-compute-budget-cap/progress.md`. +> +> **The headline property holds, measured:** a $10 budget at $6.99/h releases at **5160.0s** against a +> theoretical 5150.21s — a 9.79s overshoot that is pure tick quantisation, well inside ±90s. The +> central design decision (a set-to-total, not an increment) was confirmed by mutation: the naive +> `debit_grant(delta)` misses the duration assertion by ~3590s and leaves `grant.spent_cents = 300` +> where wall-clock is 180 — the acquire debit double-counted, exactly the trap this plan predicted. +> Under that mutation `released == 1` and the wallet moved, so **every "a release happened" assertion +> still passed**; only the duration assertion caught it. +> +> **Task 4's hazard was real and is now quantified.** Every spawn grant was `hard_cap_cents = 500` +> flat for **every SKU** — no caller in either repo ever sent anything else. A spawn requests 4 hours: +> $14.60 on an A100-40GB, $18.36 on an A100-80GB, $27.96 on an H100, so only T4 and A10G ever fitted. +> Measured on the real spawn path against the real tick: **a 4-hour A100 spawn was killed at 1.38h** +> and an **H100 was refused outright at acquire**. `budget_cents` is now `int | None`; `None` sizes +> the ceiling to the spawn's own lifetime at the SKU rate, and an explicitly chosen budget still binds. +> +> **What execution proved this plan wrong about:** **Task 3's premise that the effective-balance +> lookup is "already in scope" in `create_lease`.** It is not — see the correction marked inline at +> Task 3, Step 3. Two judgement calls were also taken that this plan does not contain, both recorded +> in the spec: the `402` keeps one body shape but carries **two** `error` values +> (`insufficient_cli_credit` vs a new `budget_below_hourly_rate`), and the budget is deliberately +> **not** clamped to `rate × ttl_hours`. + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development to implement +> this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Make `hard_cap_cents` a ceiling that actually binds, and let a caller propose a budget — so a +managed GPU lease is bounded by the money authorised for it rather than by the user's entire wallet. + +**Architecture:** All work is in the **Atlas** repo (`~/codes/InkVell/atlas`), continuing on +`feat/compute-lease-prerequisites`. Spec changes 1 and 2, which **must land together** — see below. + +**Tech Stack:** Python 3.12, FastAPI, aiosqlite, pytest + pytest-asyncio. + +**Spec:** `~/codes/InkVell/openscience/docs/specs/compute-design.md`, Part B changes 1 and 2. + +## Global Constraints + +- **Repo:** `~/codes/InkVell/atlas`, branch `feat/compute-lease-prerequisites` (already checked out). + Do not create or switch branches. +- **Run tests with:** `cd ~/codes/InkVell/atlas/backend && .venv/bin/python -m pytest -q`. + No activated virtualenv — always invoke `.venv/bin/python` explicitly. +- **Never add a `Co-Authored-By:` trailer or any AI/assistant attribution to commit messages.** +- **No mocks of our own code.** Use the real repo functions, real `tick_once`, `aiosqlite` + + `run_migrations`, and a `_FakeProvider` at the boundary — the pattern in + `backend/tests/test_compute_billing.py`. +- **Every new assertion must be shown failing first.** Paste the failure into your report. +- Baseline: full suite is **1670 passed, 1 skipped**. It must be green when each task finishes. +- Ignore `.claude/worktrees/`. + +## Established facts — do not re-derive these + +Verified in the source, and the last two against production: + +1. **`tick_once`'s managed branch is cumulative and replay-safe** + (`backend/app/services/compute_billing_service.py`, the `if funding == "managed":` branch): + ```python + elapsed_total = (now - started).total_seconds() # started = started_at or created_at + already = int(lease.get("total_spent_cents") or 0) + delta_cents = wall_clock_cents(hourly, elapsed_total) - already + if delta_cents <= 0: skip + ``` + A replayed tick yields `delta <= 0` and skips. **The grant debit must mirror this shape.** +2. `wall_clock_cents(hourly, secs) = round(hourly * secs / 3600)` — `app/compute/provider_registry.py:10`. +3. **BYOK leases are skipped by the tick entirely** (`if funding == "byok": continue`), so they can never + be charged and must never be grant-debited. +4. `compute_repo.debit_grant` is an **increment** with an atomic ceiling: + `SET spent_cents = spent_cents + ? … WHERE grant_id = ? AND status = 'active' AND (spent_cents + ?) <= hard_cap_cents`. +5. **Acquire debits exactly one hour up front** and never refunds it — + `first_hour_cents = int(price_cents_per_hour)` then `debit_grant(db, grant_id, first_hour_cents)` in + `lease_manager.acquire_lease`. +6. The grant is sized at lease creation to `max_spend = max(charge_raw * ttl_hours, charge_raw, 1)` with + `ttl_hours = gpu_sandbox_max_ttl_hours` = **24** on every plan — `app/routes/compute.py`. +7. `LeaseRequest` is `{provider, sku, region?, node_id?}` — no budget field today. +8. **Measured in production (2026-08-01):** a live 34¢/hr lease had `grant.hard_cap_cents = 816` + (= 34 × 24), `grant.spent_cents = 34` **frozen**, while `lease.total_spent_cents` climbed 0 → 2 → 3 → + 5 → 6. The tick charges the wallet and never touches the grant, so `spent_cents` can never approach + `hard_cap_cents`. **The cap is decorative.** That is the bug this plan fixes. + +## The central design decision + +The grant update is a **set-to-total, not an increment**: + +``` +grant.spent_cents := wall_clock_cents(hourly, elapsed_total) +``` + +Three things follow, and they are why an increment is wrong: + +- **Replay safety comes for free.** It mirrors the tick's own cumulative model (fact 1). An increment is + not replay-safe: a crash between the charge and the debit either double-counts on the next tick or + loses the debit. +- **It supersedes the acquire-time one-hour debit instead of double-counting it.** This is the trap the + spec records: a naive re-debit makes a $10 budget at $6.99/h die at 25.8 minutes instead of ~1.4 hours. + A set writes the truth and the pre-tick placeholder simply disappears. +- **It preserves the full plan TTL.** At exactly 24h, `spent = 24 × rate = hard_cap`, and the guard is + `<=`, so the lease survives its TTL and the next tick releases it. Adding the acquire debit on top + (`wall_clock + rate`) would kill a no-budget lease at ~23h — a silent regression for the dashboard and + `compute:up`, which is precisely why changes 1 and 2 must land together. + +**Before the first tick the acquire debit still does its job**, bounding an un-ticked lease at one hour of +grant. Only once the tick runs does wall-clock become the truth. + +--- + +### Task 1: An atomic set-to-total for grant spend + +**Files:** + +- Modify: `backend/app/db/repos/compute_repo.py` (add `set_grant_spend` beside `debit_grant`) +- Test: `backend/tests/test_compute_grant_spend.py` (new) + +**Interfaces:** + +- Produces: `async def set_grant_spend(db, grant_id: str, total_cents: int) -> bool` — sets + `spent_cents` to `total_cents` when the grant is active and `total_cents <= hard_cap_cents`; returns + `False` without writing when it would exceed the cap or the grant is not active. `debit_grant` is left + untouched — other callers still use it. + +- [ ] **Step 1: Write the failing tests** + +Create `backend/tests/test_compute_grant_spend.py`. Follow the fixture style of +`backend/tests/test_compute_billing.py` (`aiosqlite` + `run_migrations` against a tmp DB). Cover: + +1. a set below the cap writes the exact total and returns `True` +2. a set **equal** to the cap succeeds — the guard is `<=`, and this is what preserves the full TTL +3. a set above the cap returns `False` **and leaves `spent_cents` unchanged** (assert the row, not just + the return value) +4. a set on a non-`active` grant returns `False` +5. it is idempotent — calling it twice with the same total leaves the same row + +- [ ] **Step 2: Run and confirm they fail** + +Run: `cd ~/codes/InkVell/atlas/backend && .venv/bin/python -m pytest tests/test_compute_grant_spend.py -q` +Expected: `ImportError` / `AttributeError` — `set_grant_spend` does not exist. Paste it. + +- [ ] **Step 3: Implement** + +In `backend/app/db/repos/compute_repo.py`, beside `debit_grant`: + +```python +async def set_grant_spend(db, grant_id: str, total_cents: int) -> bool: + """Set a grant's cumulative spend, refusing to exceed its ceiling. + + A SET rather than an increment, mirroring the billing tick, which charges + ``wall_clock_cents(rate, elapsed_since_started) - total_spent_cents``. Both + sides then derive from the same wall-clock truth, so a replayed tick is a + no-op instead of a double count, and the one-hour debit ``acquire_lease`` + takes up front is superseded rather than added to. + + Returns False without writing when the total would exceed ``hard_cap_cents`` + or the grant is no longer active — the caller releases the lease. + """ + now = _now() + cursor = await db.execute( + """ + UPDATE compute_grants + SET spent_cents = ?, updated_at = ? + WHERE grant_id = ? + AND status = 'active' + AND ? <= hard_cap_cents + """, + (total_cents, now, grant_id, total_cents), + ) + await db.commit() + return cursor.rowcount > 0 +``` + +- [ ] **Step 4: Run and confirm they pass** + +Run: `cd ~/codes/InkVell/atlas/backend && .venv/bin/python -m pytest tests/test_compute_grant_spend.py -q` +Then the full suite. Both green. + +- [ ] **Step 5: Commit** + +```bash +cd ~/codes/InkVell/atlas +git add backend/app/db/repos/compute_repo.py backend/tests/test_compute_grant_spend.py +git commit -m "feat(compute): an atomic set-to-total for grant spend + +debit_grant increments, which cannot mirror a billing tick that charges +cumulatively from started_at. set_grant_spend writes the total instead, so a +replayed tick is a no-op and the one-hour debit taken at acquire is superseded +rather than added to. Refuses above the ceiling without writing." +``` + +--- + +### Task 2: Make the cap bind + +**Files:** + +- Modify: `backend/app/services/compute_billing_service.py` (the `funding == "managed"` branch of + `tick_once`) +- Test: `backend/tests/test_compute_billing.py` (append) + +**Interfaces:** + +- Consumes: `compute_repo.set_grant_spend` from Task 1. +- Produces: a managed lease is released once its grant ceiling is reached. + +- [ ] **Step 1: Write the failing tests** + +Append to `backend/tests/test_compute_billing.py`. **The headline test is the first one** — the spec +records that both previous attempts at this change omitted it from their tests _and_ their acceptance +criteria, and that a $10 budget dying at 25.8 minutes passes every "a release happened" assertion while +being wrong by 3×. + +1. **A budget of $B at $R/h lasts ≈ B/R hours.** Seed a grant with `hard_cap_cents = B`, a managed lease + at `hourly_rate_cents = R`, drive `tick_once` with an injected `now` advancing in steps, and assert on + the **elapsed billable duration at the moment of release** — not merely that a release occurred. + Allow ±90s of rate: `COMPUTE_BILLING_TICK_SECONDS` (60) + `FIRST_BILL_GRACE_SECONDS` (30). +2. A tick whose new total fits under the cap does **not** release, and `grant.spent_cents` now equals + `wall_clock_cents(rate, elapsed_total)` — proving it tracks rather than freezing at the acquire debit. +3. A tick whose new total would exceed the cap **charges the elapsed time first, then releases.** The + user consumed that time and the operator owes the provider for it; skipping the charge loses real + money. Assert both the wallet movement and the release. +4. A replayed tick (same injected `now`) neither double-charges the wallet nor moves `grant.spent_cents`. +5. **A BYOK lease is never grant-debited** — it is skipped before this code runs. +6. **A no-budget lease still runs its full plan TTL.** With `hard_cap = rate × 24`, the lease survives to + 24h. This is the regression guard for the acquire-debit interaction; if the implementation adds the + acquire debit on top of wall-clock instead of superseding it, this test dies at ~23h. + +- [ ] **Step 2: Run and confirm they fail** + +Run: `cd ~/codes/InkVell/atlas/backend && .venv/bin/python -m pytest tests/test_compute_billing.py -q` +Paste the failures. + +- [ ] **Step 3: Implement** + +In the `funding == "managed"` branch, after the existing `mark_billed` call and before `continue`, set the +grant to the same wall-clock total the charge was derived from, and release when the ceiling refuses it: + +```python + grant_id = lease.get("grant_id") + if grant_id: + # Mirror the charge: the tick bills wall-clock-since-started + # minus what was already billed, so the grant's cumulative + # spend is that same wall-clock total. A set, not an increment — + # see set_grant_spend. The one-hour debit taken at acquire is + # superseded here, which is what keeps a no-budget lease alive + # for its full plan TTL instead of dying an hour early. + within = await compute_repo.set_grant_spend( + db, grant_id, wall_clock_cents(hourly, elapsed_total), + ) + if not within: + # The approved money is spent. The charge above already + # captured the time actually used, which the user consumed + # and the operator owes the provider for. + logger.info( + "compute_billing: grant ceiling reached - releasing lease=%s", + lease_id, + ) + await _safe_release(db, lease) + released += 1 + continue +``` + +Import `wall_clock_cents` if it is not already in scope — it lives in `app.compute.provider_registry`. + +**Order matters and is load-bearing:** charge → `mark_billed` → `set_grant_spend` → release on refusal. +Charging after the ceiling check would drop the final increment; setting the grant before the charge +would authorise money that was never taken. + +- [ ] **Step 4: Run and confirm they pass** + +Run the file, then `cd ~/codes/InkVell/atlas/backend && .venv/bin/python -m pytest tests/ -q`. Both green. + +- [ ] **Step 5: Commit** + +```bash +cd ~/codes/InkVell/atlas +git add backend/app/services/compute_billing_service.py backend/tests/test_compute_billing.py +git commit -m "fix(billing): make hard_cap_cents a ceiling that actually binds + +The tick charged the wallet and never touched the grant, so spent_cents froze +at the one-hour debit taken at acquire and could never approach hard_cap_cents. +Measured in production: a 34c/hr lease with hard_cap 816c sat at spent 34c +while the lease accrued past it. The ceiling was decorative and the wallet was +the only real bound. + +The tick now sets the grant to the same wall-clock total it charges from, and +releases when that total is refused." +``` + +--- + +### Task 3: Accept a budget on lease creation + +**Files:** + +- Modify: `backend/app/routes/compute.py` (`LeaseRequest`, and the grant sizing in `create_lease`) +- Test: `backend/tests/test_compute_resell_routes.py` (append) + +**Interfaces:** + +- Consumes: the binding cap from Task 2. +- Produces: `POST /api/compute/leases` accepts optional `budget_cents`; the response carries the + **effective** cap actually authorised. + +- [ ] **Step 1: Write the failing tests** + +Append to `backend/tests/test_compute_resell_routes.py`, following its existing route-test style: + +1. **Omitting `budget_cents` preserves today's behaviour exactly** — the grant is still sized to + `rate × ttl_hours`. Assert the grant row, since the dashboard and `compute:up` both call this endpoint + without the field. +2. `budget_cents` present sizes the grant to it. +3. **A budget larger than the wallet is clamped, not rejected** — the wallet is always the outer bound — + and the response reports the **effective** cap, not the asked-for one. +4. A budget that cannot fund the first hour is refused with the existing structured `402`, extended with + `affordable_budget_cents`. +5. **BYOK ignores `budget_cents`** and is never grant-debited. + +- [ ] **Step 2: Run and confirm they fail** + +Run: `cd ~/codes/InkVell/atlas/backend && .venv/bin/python -m pytest tests/test_compute_resell_routes.py -q` +Paste the failures. + +- [ ] **Step 3: Implement** + +Add `budget_cents: int | None = None` to `LeaseRequest`. In `create_lease`, replace the fixed sizing: + +```python + max_spend = max(charge_raw * ttl_hours, charge_raw, 1) +``` + +with a budget-aware version that keeps the wallet as the outer bound. Read the current code around it +before editing — `charge_raw`, `ttl_hours` and the effective balance lookup are already in scope, and the +existing `402` for insufficient credit is the one to extend rather than duplicate. + +> **⚠️ WRONG PREMISE — caught by the implementer, 2026-08-01. Do not follow this sentence.** +> **The effective-balance lookup is NOT in scope in `create_lease`.** `create_lease` never reads the +> balance at all; the wallet check lives in `lease_manager.acquire_lease`, and it is the **adjacent +> `compute_estimate`** that has `effective_balance` in scope — which is presumably what this plan +> misread. Clamping therefore required adding a **new conditional lookup**, taken only when +> `budget_cents` is present, so the no-budget path adds no query. `charge_raw` and `ttl_hours` are in +> scope as stated; only the balance claim is wrong. + +Report the effective cap on the response so a caller can tell the user what was actually authorised. + +- [ ] **Step 4: Run and confirm they pass** + +Run the file, then the full suite. Both green. + +- [ ] **Step 5: Commit** (message in the same style; state that absent `budget_cents` is unchanged + behaviour, and that the wallet remains the outer bound) + +--- + +### Task 4: The agent-spawn default becomes a hard kill + +**Why:** `budget_cents` already exists on the agent-spawn path with a default of `500` +(`app/services/agent_tools.py`, plus `models/agent.py` and `spawn_queue_service.py`), where it is +**display-only** today. Task 2 makes caps real, so every already-shipped spawn silently acquires a hard +$5 ceiling. The spec calls this out explicitly: _"Not a no-op, and the previous draft called it one."_ + +**Files:** to be determined by Step 1 — do not guess. + +- [ ] **Step 1: Investigate and report before changing anything** + +Trace how the spawn path's `budget_cents` reaches `create_grant`, what `hard_cap_cents` it produces +today, and what a typical spawn actually costs. **Report:** + +- the value a spawn's grant is currently created with +- whether $5 is above or below a realistic spawn cost, with a number +- whether spawn grants are the same `compute_grants` rows the billing tick now enforces against + +- [ ] **Step 2: Choose and implement, having reported** + +Two options the spec names — pick one on the evidence from Step 1 and say why: + +- **raise the default deliberately** to a value chosen with enforcement in mind, or +- **exempt spawn-path grants** until their budgets are set with enforcement in mind. + +Whichever you choose, add a test proving a spawn is not killed at a budget nobody chose. + +- [ ] **Step 3: Commit** + +--- + +## Whole-branch verification + +- [ ] Full suite: `cd ~/codes/InkVell/atlas/backend && .venv/bin/python -m pytest tests/ -q` — green, no + network. +- [ ] Confirm by name that the prior branch's deliverables still pass: + `test_promoted_lease_is_not_reaped_at_the_provisioning_timeout`, + `test_user_lease_without_runner_token_survives`, + `test_class_4_managed_gpu_with_owner_key_never_uses_it`, + `test_out_of_credit_release_of_a_managed_lease_uses_the_operator_key`. +- [ ] Confirm the headline property test exists and asserts on **elapsed billable duration**, not on the + fact that a release happened. + +## Acceptance criteria + +1. A budget of $B at rate $R/h lasts B/R hours ± 90s of rate, asserted on elapsed billable duration. +2. `grant.spent_cents` tracks wall-clock spend instead of freezing at the acquire-time debit. +3. A tick exceeding the ceiling charges the elapsed time and then releases; one that fits does not. +4. A replayed tick neither double-charges the wallet nor moves `grant.spent_cents`. +5. A lease created without `budget_cents` runs its **full** plan TTL — asserted on runtime, not on the + request being accepted. +6. `budget_cents` sizes the grant; a budget above the wallet is clamped and the response reports the + effective cap; one that cannot fund the first hour is refused with a structured `402`. +7. BYOK leases ignore `budget_cents` and are never grant-debited. +8. An agent spawn is not killed at a budget nobody chose. +9. `pytest` passes with no network access. + +## Out of scope + +Rolling window cap across sequential leases (change 5), the resolver and quote endpoint (changes 3, 4, +12), volumes (6), budget extension (7), and the three OpenScience tools. A budget bounds one lease; it +does not yet bound release-and-reacquire. diff --git a/docs/plans/2026-08-01-compute-lease-defects.md b/docs/plans/2026-08-01-compute-lease-defects.md new file mode 100644 index 00000000..fd6b5604 --- /dev/null +++ b/docs/plans/2026-08-01-compute-lease-defects.md @@ -0,0 +1,528 @@ +# Compute lease defects — implementation plan + +> ## ✅ EXECUTED — 2026-08-01. Historical record; do not implement from it again. +> +> **Where:** `~/codes/InkVell/atlas`, branch `feat/compute-lease-prerequisites` (continuing). +> **Not merged** — the PR is deliberately draft while the team waits for compute to be complete. +> **Outcome:** all four tasks landed (`b8a4467`, `0eb33c2`, `f65dd47`, `8748057`), followed by an +> eight-commit fix wave closing nine review findings. Ledger: +> `.superpowers/sdd/2026-08-01-compute-lease-defects/progress.md`. +> +> **This branch was then deployed and driven end to end**, which is the part that matters: on a +> deployed backend, `POST /api/compute/leases` → the real reaper promoted within one sweep with +> **NATed** SSH coordinates (`ssh3.vast.ai:15650`, `194.68.245.163:22189` — never the placeholder +> `22`) → SSH into a real GPU → release, with the instance verified gone at the provider. Done on +> **both** Vast and RunPod. Prime Intellect no longer leaks a key per lease; provider releases now +> report what actually happened; BYOK leases are polled with the owner's credential and managed +> leases never are. +> +> **What execution proved wrong — including one finding that was wrong in the safe-looking direction:** +> +> - **A review finding was overturned by a live probe.** Finding M9 required an explicit `404` before +> treating a Vast instance as gone. Vast returns HTTP **200 `{"instances": null}`** for a destroyed +> id _and_ for an id that never existed — it never 404s — so M9 would have silently reversed +> `9bc19a7`, itself live-verified. The `410` it cited was on the `/instances/` **listing** endpoint, +> a different call. Corrected by restoring the null-payload verdict and guarding the original +> concern properly: two consecutive terminal observations before reaping. +> - **Deploying found a real bug nothing else would have.** `uvicorn --workers 2` runs the lifespan in +> both workers, both ran `run_migrations` against the same sqlite file, both saw `provider_key_id` +> absent, both `ALTER`ed, one crashed with `duplicate column name` and failed startup. A +> **pre-existing pattern** at all 22 `ADD COLUMN` sites that this branch's new column exposed. Fixed +> in `84bbbb7`. +> - **A tempting heuristic was falsified.** "Offers present in consecutive catalog fetches are +> launchable" had perfect separation at n=8 retrospectively, and **failed its first prospective +> test**. Retracted, along with the claim that spec change 12's cache would give change 3 its +> correctness fix for free. Retry remains unavoidable. +> - **Task 3 carries a caveat that is still open:** Prime's `DELETE /ssh_keys/{id}` is +> documentation-verified only, never exercised against production. It fails safe — the delete is +> swallowed and cannot block the pod teardown — so a wrong endpoint would leak silently. +> - **Measured facts this plan did not anticipate**, now feeding spec changes 3 and 10: Vast offer ids +> churn **~50% between two consecutive catalog fetches** (7 consecutive `Unknown SKU` failures +> across price ranks 0–25); RunPod advertises GPU types with **zero capacity** and its `500` is +> mapped to a `400` — the same status as Vast's stale offer, with a different message, so a +> resolver must discriminate on the message because the two need opposite responses. +> - **Two openscience fixes came out of this wave**, on `feat/compute-guardrails`: `784633e` (stop +> claiming managed compute at a zero balance) and `a732396d` (tmpfs the XDG cache dir so +> `bubblewrap` stops failing tools with "Read-only file system"). + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development to implement +> this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Close the four known-unfixed defects in the managed-lease path that live testing and review +surfaced but the prerequisites branch deliberately left alone. + +**Architecture:** All four tasks are in the **Atlas** repo (`~/codes/InkVell/atlas`), continuing on the +`feat/compute-lease-prerequisites` branch. Two are provider-level correctness (Vast release honesty, Prime +key lifecycle), one restores a whole lease class to the promotion path (BYOK), one is a status-mapping +gap. + +**Tech Stack:** Python 3.12, FastAPI, aiosqlite, pytest + pytest-asyncio, respx, httpx. + +**Spec:** `~/codes/InkVell/openscience/docs/specs/compute-design.md` — change 9(b), plus defects recorded +in `.superpowers/sdd/2026-07-31-compute-lease-prerequisites/progress.md`. + +## Global Constraints + +- **Repo:** `~/codes/InkVell/atlas`, branch `feat/compute-lease-prerequisites` (already checked out). + Do not create or switch branches. +- **Run tests with:** `cd ~/codes/InkVell/atlas/backend && .venv/bin/python -m pytest -q`. + No activated virtualenv — always invoke `.venv/bin/python` explicitly. +- **Never add a `Co-Authored-By:` trailer or any AI/assistant attribution to commit messages.** +- **No mocks of our own code.** Stub HTTP at the transport boundary with `respx`; use real repo functions + and the real `sweep_once`, matching `backend/tests/test_lease_reaper.py` and + `backend/tests/test_compute_vast_provider_http.py`. +- Provider HTTP tests use `@respx.mock` only — **no** `@pytest.mark.asyncio` (pytest-asyncio is in auto + mode). +- **Every new assertion must be shown failing first.** Paste the failure into your report. +- Baseline: full suite is **1610 passed, 1 skipped**. It must be green when each task finishes. +- Ignore `.claude/worktrees/`. + +--- + +### Task 1: BYOK GPU leases can never be promoted, so they still die at 600s + +**Why:** `sweep_once` polls providers with no user credentials — the module's own comment concedes it +(`backend/app/jobs/lease_reaper.py`, branch 1: _"status is polled WITHOUT BYOK user_credentials, so +GPU/BYOK leases can't be classified here"_). For a BYOK lease `provider.status()` either raises or returns +`not_configured`, so `pstat` never matches `_PROVIDER_READY`, promotion never runs, the lease stays +`provisioning`, and branch 2 reaps it at `PROVISION_TIMEOUT_SECONDS`. `release_lease` **does** use BYOK +credentials, so the user's own paid instance is genuinely destroyed. + +`LeaseManager` already has the helper: `_credentials_for_lease(db, lease)` +(`backend/app/compute/lease_manager.py`), which returns BYOK creds for a stored lease or `None`. + +**Files:** + +- Modify: `backend/app/jobs/lease_reaper.py` — branch 1's `status()` call, and the `connection()` call + inside `_promote_to_ready` / `_backfill_coordinates` +- Test: `backend/tests/test_lease_reaper.py` (append) + +**Interfaces:** + +- Consumes: `_credentials_for_lease(db, lease) -> dict | None` from `app.compute.lease_manager`. +- Produces: nothing later tasks depend on. + +- [ ] **Step 1: Read the current call sites** + +Read `sweep_once`'s branch 1 and both coordinate helpers. Note how `_promote_to_ready` and +`_backfill_coordinates` obtain a provider and call `connection()`. All three provider calls need the same +credentials. + +- [ ] **Step 2: Write the failing test** + +Append to `backend/tests/test_lease_reaper.py`: + +```python +@pytest.mark.asyncio +async def test_byok_lease_is_promoted_using_the_users_credentials(test_db, monkeypatch): + """A BYOK lease runs on the user's own provider account. The reaper polls + with no credentials, so it could never see the box as up, never promoted, + and branch 2 destroyed the user's own paid instance at 600s.""" + db = test_db["db"]; uid = test_db["user_id"] + await compute_repo.create_lease(db, lease_id="Lbyok", status="provisioning", + user_id=uid, provider="runpod", requested_sku="h100", region="us", + hourly_rate_cents=279, category="gpu", funding="byok", + ssh_key_name="atlas-byok") + old = _iso(datetime.now(timezone.utc) - timedelta(seconds=1200)) + await db.execute( + "UPDATE compute_leases SET created_at = ?, started_at = ? WHERE lease_id = 'Lbyok'", + (old, old)) + await db.commit() + + seen = {"status": None, "connection": None} + + class _P: + async def status(self, _id, **kw): + seen["status"] = kw.get("user_credentials") + if not kw.get("user_credentials"): + return {"status": "not_configured"} + return {"status": "running"} + + async def connection(self, _id, **kw): + seen["connection"] = kw.get("user_credentials") + if not kw.get("user_credentials"): + return {} + return {"ssh_host": "1.2.3.4", "ssh_port": 20095, "ssh_user": "root"} + + monkeypatch.setattr(reaper, "get_provider", lambda _name: _P()) + + async def _creds(_db, _lease): + return {"secret": "user-byok-key"} + monkeypatch.setattr(reaper, "_credentials_for_lease", _creds, raising=False) + + async def _noop_release(self, _db, _lease): return {"status": "released"} + monkeypatch.setattr(reaper.LeaseManager, "release_lease", _noop_release) + + n = await reaper.sweep_once(db, now=datetime.now(timezone.utc)) + + assert n == 0, "a live BYOK lease must not be reaped" + row = await compute_repo.get_lease(db, "Lbyok") + assert row["status"] == "ready" + assert row["ssh_host"] == "1.2.3.4" + assert seen["status"] == {"secret": "user-byok-key"} + assert seen["connection"] == {"secret": "user-byok-key"} +``` + +- [ ] **Step 3: Run it and confirm it fails** + +Run: `cd ~/codes/InkVell/atlas/backend && .venv/bin/python -m pytest tests/test_lease_reaper.py -q -k byok` +Expected: FAIL — the lease is reaped (`n == 1`) or stays `provisioning`, and `seen["status"]` is `None`. +Paste the failure. + +- [ ] **Step 4: Thread credentials through the three provider calls** + +Import `_credentials_for_lease` into `lease_reaper.py` at module level, alongside the existing +`from app.compute.lease_manager import LeaseManager`: + +```python +from app.compute.lease_manager import LeaseManager, _credentials_for_lease +``` + +In `sweep_once`, resolve credentials once per lease before branch 1 and pass them to `status()`: + +```python + creds = await _credentials_for_lease(db, lease) + kw = {"user_credentials": creds} if creds is not None else {} +``` + +Pass `**kw` to `provider.status(lease_id, **kw)` in branch 1, and thread `creds` into +`_promote_to_ready` and `_backfill_coordinates` so their `connection()` calls use it too. Update those +two helpers' signatures to accept the credentials, and update every call site. + +**Keep the failure handling as it is** — a raising `status()` must still leave `pstat = {}` and fall +through to the timeout branches. This change adds credentials; it must not change what happens when a +provider call fails. + +Correct the now-stale comment on branch 1 that says BYOK leases cannot be classified here. + +- [ ] **Step 5: Run the tests** + +Run: `cd ~/codes/InkVell/atlas/backend && .venv/bin/python -m pytest tests/test_lease_reaper.py -q` +Expected: all pass, including every pre-existing test. Then +`cd ~/codes/InkVell/atlas/backend && .venv/bin/python -m pytest tests/ -q` → 1611 passed / 1 skipped. + +- [ ] **Step 6: Commit** + +```bash +cd ~/codes/InkVell/atlas +git add backend/app/jobs/lease_reaper.py backend/tests/test_lease_reaper.py +git commit -m "fix(reaper): poll BYOK leases with the user's own credentials + +The sweep polled every provider with no credentials, so a BYOK lease could +never be seen as up: status() returned not_configured or raised, promotion +never ran, and branch 2 destroyed the user's own paid instance at the +provisioning timeout. release_lease already used BYOK credentials, so the +teardown worked even though the liveness check could not. + +_credentials_for_lease already existed on LeaseManager; the reaper now uses it +for status() and for the connection() lookups behind promotion and backfill." +``` + +--- + +### Task 2: `VastProvider.release` reports success it never verified + +**Why:** the release wraps its DELETE in `try/except: pass` and then returns +`{"lease_id": lease_id, "status": "terminated"}` **unconditionally**. A 4xx, a network error, or a +half-completed teardown all return the same "terminated". Observed live: the return value is not evidence +of anything. + +This is the provider half of spec change 8. The `LeaseManager.release_lease` half — which marks the row +released regardless — stays out of scope here; this task makes the provider able to tell the truth. + +**Files:** + +- Modify: `backend/app/compute/vast_provider.py` — the `release` method +- Test: `backend/tests/test_compute_vast_provider_http.py` (append) + +**Interfaces:** + +- Produces: `VastProvider.release` returns `status: "terminated"` only on a 2xx; otherwise + `status: "unknown"` with a `warning` key describing what happened. Callers that only read `status` are + unaffected on the success path. + +- [ ] **Step 1: Write the failing tests** + +Append to `backend/tests/test_compute_vast_provider_http.py`: + +```python +@respx.mock +async def test_release_reports_failure_instead_of_claiming_terminated(): + respx.delete(f"{VAST_API}/instances/4242/").mock( + return_value=httpx.Response(403, text="forbidden") + ) + out = await VastProvider().release("4242", user_credentials=CREDS) + assert out["status"] != "terminated", "a refused delete must not report success" + assert "warning" in out + assert "403" in str(out["warning"]) + + +@respx.mock +async def test_release_reports_failure_on_transport_error(): + respx.delete(f"{VAST_API}/instances/4242/").mock( + side_effect=httpx.ConnectError("boom") + ) + out = await VastProvider().release("4242", user_credentials=CREDS) + assert out["status"] != "terminated" + assert "warning" in out + + +@respx.mock +async def test_release_reports_terminated_on_success(): + respx.delete(f"{VAST_API}/instances/4242/").mock( + return_value=httpx.Response(200, json={"success": True}) + ) + out = await VastProvider().release("4242", user_credentials=CREDS) + assert out["status"] == "terminated" + assert "warning" not in out +``` + +- [ ] **Step 2: Run and confirm the first two fail** + +Run: `cd ~/codes/InkVell/atlas/backend && .venv/bin/python -m pytest tests/test_compute_vast_provider_http.py -q -k release` +Expected: the first two FAIL (`status` is `"terminated"` regardless), the third passes. Paste the failure. + +- [ ] **Step 3: Make the release honest** + +Rewrite the body of `release` so it inspects the response and reports what actually happened. Keep it +**non-fatal** — it must not start raising, because callers rely on release being best-effort — and log at +warning level, matching the logging added to the key-attach path in the same module: + +```python + try: + async with httpx.AsyncClient(timeout=30.0) as client: + resp = await client.delete( + f"{VAST_API}/instances/{lease_id}/", headers=headers + ) + if resp.status_code >= 400: + detail = (resp.text or "")[:200] + logger.warning( + "vast.release: HTTP %s destroying instance %s: %s", + resp.status_code, lease_id, detail, + ) + return { + "lease_id": lease_id, + "status": "unknown", + "warning": f"HTTP {resp.status_code}: {detail}", + } + except Exception as exc: # noqa: BLE001 + logger.exception("vast.release: destroying instance %s failed", lease_id) + return {"lease_id": lease_id, "status": "unknown", "warning": str(exc)[:200]} + return {"lease_id": lease_id, "status": "terminated"} +``` + +Update the method's docstring to say the return value now distinguishes a confirmed teardown from an +unconfirmed one. + +- [ ] **Step 4: Run the tests** + +Run: `cd ~/codes/InkVell/atlas/backend && .venv/bin/python -m pytest tests/test_compute_vast_provider_http.py -q` +Expected: all pass. Then the full suite → 1614 passed / 1 skipped. + +- [ ] **Step 5: Check for callers that assumed the old contract** + +Grep for callers of `release(` on providers and confirm none of them break when `status` is `"unknown"`. +`LeaseManager.release_lease` already treats the provider result as advisory. **Report what you find** — +if any caller keys off `status == "terminated"` to decide something important, say so rather than +changing it. + +- [ ] **Step 6: Commit** + +```bash +cd ~/codes/InkVell/atlas +git add backend/app/compute/vast_provider.py backend/tests/test_compute_vast_provider_http.py +git commit -m "fix(vast): release reports what happened instead of always 'terminated' + +The DELETE was wrapped in try/except: pass and the method returned +status='terminated' unconditionally, so a 403, a transport error and a real +teardown were indistinguishable to the caller. Still best-effort and still +non-raising; it just tells the truth now." +``` + +--- + +### Task 3: Prime Intellect leaks an account SSH key per lease, and stores the wrong identifier + +**Why:** `PrimeIntellectProvider.acquire` registers the public key with `_register_ssh_key`, which returns +the provider's key **id** — used as `sshKeyId` on the pod. But `acquire` then returns +`"ssh_key_name": name`, the **pod** name (`atlas-{user_id[:8]}`), which is identical for every lease that +user ever creates. The key id is discarded, `release` deletes nothing, and every lease leaves a key in the +operator account permanently. + +Vast no longer leaks (the account registration was removed there), so Prime is the remaining case. This is +spec change 9(b), scoped to the one provider that still needs it. + +There is **no column** to store a provider-side key id — `compute_leases` carries `ssh_key_name` and +`ssh_public_key` only. Follow the additive-column pattern already used for `runner_api_key_id` in +`backend/app/db/migrations.py` (see the `if "runner_api_key_id" not in existing:` guard) and mirror it in +`backend/app/db/pg_migrations.py`. + +**Files:** + +- Modify: `backend/app/db/migrations.py`, `backend/app/db/pg_migrations.py` (add `provider_key_id TEXT`) +- Modify: `backend/app/db/repos/compute_repo.py` (`create_lease` accepts and persists it) +- Modify: `backend/app/compute/lease_manager.py` (pass it from the acquire result into `create_lease`; + forward it to `release`) +- Modify: `backend/app/compute/prime_intellect_provider.py` (return the key id; delete on release and on + failed launch) +- Test: `backend/tests/test_compute_prime_provider_http.py` (append) + +**Interfaces:** + +- Consumes: nothing from earlier tasks. +- Produces: `compute_leases.provider_key_id`, a nullable TEXT column. `acquire` results may carry + `provider_key_id`; `release` accepts `provider_key_id=` and deletes it when present. + +- [ ] **Step 1: Read the existing shapes first** + +Read `PrimeIntellectProvider._register_ssh_key`, `acquire` and `release`; `LambdaProvider._delete_ssh_key` +and how `release_lease` forwards `ssh_key_name` (`lease_manager.py`); and the `runner_api_key_id` +migration guard. **Report the Prime delete endpoint you find** — if the module does not already know how +to delete a key, find it in the provider's API surface before writing code. + +- [ ] **Step 2: Write the failing tests** + +Append to `backend/tests/test_compute_prime_provider_http.py`, matching that file's existing respx style +(read it first — reuse its base-URL constant and credentials fixture rather than inventing new ones): + +1. `acquire` returns `provider_key_id` equal to the id `POST /ssh_keys/` returned, and **not** the pod + name. +2. `release` called with that `provider_key_id` issues a delete for that key. +3. `release` with no `provider_key_id` does not attempt a key delete and still terminates the pod. +4. A launch that fails **after** key registration deletes the key it registered (no leak on the error + path). + +- [ ] **Step 3: Run and confirm they fail** + +Run: `cd ~/codes/InkVell/atlas/backend && .venv/bin/python -m pytest tests/test_compute_prime_provider_http.py -q` +Paste the failures. + +- [ ] **Step 4: Implement** + +Work in this order so each piece is independently runnable: + +1. Migration: `provider_key_id TEXT` on `compute_leases`, both sqlite and Postgres, additive and guarded + exactly like `runner_api_key_id`. +2. `compute_repo.create_lease`: accept `provider_key_id: str | None = None`, persist it, include it in the + returned dict. +3. `prime_intellect_provider.acquire`: return `"provider_key_id": ssh_key_id` alongside the existing keys. + **Leave `ssh_key_name` as it is** — other code reads it, and this task is not a rename. +4. `prime_intellect_provider.acquire`: wrap the pod-creation call so a failure after key registration + deletes the registered key before re-raising. +5. `prime_intellect_provider.release`: accept `provider_key_id: str | None = None` and delete that key + when present, best-effort and logged, never raising. +6. `lease_manager`: pass the acquire result's `provider_key_id` into `create_lease`, and forward the + stored `provider_key_id` into `provider.release(...)` the same way `ssh_key_name` is already forwarded. + +- [ ] **Step 5: Run the tests** + +Run: `cd ~/codes/InkVell/atlas/backend && .venv/bin/python -m pytest tests/ -q -k "prime or compute or lease"` +then the full suite. Both green. + +- [ ] **Step 6: Commit** + +```bash +cd ~/codes/InkVell/atlas +git add -A backend/app backend/tests +git commit -m "fix(prime): delete the per-lease SSH key instead of leaking it + +acquire registered a key, used its id as sshKeyId, then returned the POD name +in ssh_key_name and discarded the id -- so release had nothing correct to +delete and every lease left a key in the operator account forever. The pod name +is also identical across all of a user's leases, so a name-keyed delete would +have missed or over-deleted. + +Adds a nullable provider_key_id column, returns the real id from acquire, +deletes on release and on a launch that fails after registration." +``` + +--- + +### Task 4: Lambda's in-flight status reads as `unknown` + +**Why:** `LambdaProvider.status` returns Lambda's raw upstream string unmapped +(`backend/app/compute/lambda_provider.py`), and `normalise_state`'s `_PROVISIONING` set contains only +`"provisioning"`. So a booting Lambda box reports `state: "unknown"` for its whole boot window. Safe — +`unknown` never reads as ready — but a client polling for progress cannot distinguish "coming up" from +"something is wrong". + +`booting` is named as an in-flight status by this repo's own frontend +(`frontend/src/components/compute/InstancesTab.tsx:29`), which is the in-repo evidence that was missing +when this was first deferred. + +**Files:** + +- Modify: `backend/app/compute/lease_state.py` +- Test: `backend/tests/test_compute_lease_state.py` + +**Interfaces:** + +- Consumes: `normalise_state` from Task-4-of-the-previous-plan. Unchanged signature. + +- [ ] **Step 1: Add the failing case** + +In `backend/tests/test_compute_lease_state.py`, add `"booting"` to the parametrisation of +`test_in_flight_states_normalise_to_provisioning`, and **remove** it from any unknown-case list if present. + +- [ ] **Step 2: Run and confirm it fails** + +Run: `cd ~/codes/InkVell/atlas/backend && .venv/bin/python -m pytest tests/test_compute_lease_state.py -q` +Expected: FAIL — `normalise_state("booting")` returns `"unknown"`. Paste it. + +- [ ] **Step 3: Map it** + +Add `"booting"` to `_PROVISIONING` in `backend/app/compute/lease_state.py`, with a comment naming Lambda +as the emitter and the frontend file as the evidence. + +**Do not add `unhealthy`.** Lambda's enum is not documented in this repo, and `unhealthy` is genuinely +ambiguous between "degraded but alive" and "dead" — mapping it wrong is worse than leaving it `unknown`. +Say so in the comment. + +- [ ] **Step 4: Run the tests** + +Run: `cd ~/codes/InkVell/atlas/backend && .venv/bin/python -m pytest tests/test_compute_lease_state.py -q` +then the full suite. Both green. + +- [ ] **Step 5: Commit** + +```bash +cd ~/codes/InkVell/atlas +git add backend/app/compute/lease_state.py backend/tests/test_compute_lease_state.py +git commit -m "fix(compute): map Lambda's booting status to provisioning + +Lambda passes its upstream status string through unmapped, so a booting box +reported state='unknown' for its whole boot window -- safe, since unknown never +reads as ready, but a client could not tell 'coming up' from 'broken'. The +repo's own frontend already treats booting as an in-flight status. + +unhealthy is deliberately left unmapped: it is ambiguous between degraded and +dead, and guessing is worse than unknown." +``` + +--- + +## Whole-branch verification + +- [ ] Full suite: `cd ~/codes/InkVell/atlas/backend && .venv/bin/python -m pytest tests/ -q` — green, no + network. +- [ ] Confirm by name that the previous plan's deliverable tests still pass: + `test_promoted_lease_is_not_reaped_at_the_provisioning_timeout`, + `test_user_lease_without_runner_token_is_not_heartbeat_reaped`, + `test_modal_cpu_sandbox_promotes_without_ssh_host`, + `test_status_of_destroyed_instance_is_terminated_not_provisioning`. +- [ ] Report which defects remain open (see below) so nothing is assumed closed. + +## Deliberately out of scope + +Two items from the same defect list are **full spec changes, not defects**, and each needs its own plan: + +- **Volumes provision nothing** (spec change 6). `create_volume` writes a DB row and calls no provider + API; no provider except RunPod has any volume concept. This is "make volumes real, per provider", and + under cheapest-first it also narrows the resolver pool. A design task, not a fix. +- **The options catalog is uncached** (spec change 12). Five provider requests per call behind a 3s client + timeout. It needs a cache key, a TTL, and a decision about how the resolver's retry interacts with it — + all design choices the spec has not made. + +Also still open and recorded, not addressed here: the `LeaseManager.release_lease` half of change 8 (marks +a row released even when provider teardown failed), and `PROVISION_TIMEOUT_SECONDS` being a single global +constant. diff --git a/docs/plans/2026-08-02-compute-resolver.md b/docs/plans/2026-08-02-compute-resolver.md new file mode 100644 index 00000000..22938ac2 --- /dev/null +++ b/docs/plans/2026-08-02-compute-resolver.md @@ -0,0 +1,325 @@ +# Compute catalog cache and server-side resolver — implementation plan + +> **EXECUTED 2026-08-02** on `feat/compute-lease-prerequisites`. All four tasks shipped and reviewed; +> full suite **1908 passed / 1 skipped**, no network. Commits: `8913be3` (cache), `f9c2da9` (GPU map), +> `9e1664d` (resolver), `d888ba5` + `730ee82` (requirement path + retry), `cf9e7de` + `416c064` + +> `bed63dc` + `fa429c6` (final-review fixes). +> +> **Two things this plan got wrong, both caught only by the whole-branch review:** +> +> 1. **Task 3's tie-break became a purchasing decision in Task 4.** Ranking on +> `price_cents_per_hour_display` alone is correct across funding paths but degenerate *within* BYOK, +> where every row displays `0` — the order collapsed to alphabetical by provider and leased a $9.00/h +> box over a $3.00/h one. Neither task was wrong alone. The key is now +> `(display, raw, provider, sku)`. +> 2. **Task 4 charged a retry attempt for a candidate no provider ever saw.** A region-less Lambda offer +> is rejected locally, but it consumed one of three attempts plus a full catalog fan-out — and Lambda +> emits region-less rows precisely for the types it is out of capacity on, which are also its +> cheapest, so they sort first. Three of them ahead of a launchable offer returned 503 `no_capacity` +> while capacity sat in the list. +> +> **And one thing it prescribed that the implementation was right to refuse:** Task 4 Step 3 says to +> discriminate the two `400`s on the provider's message. The implementation excludes already-refused +> `(provider, sku)` pairs instead — one rule that satisfies both providers and cannot rot when a +> provider rewrites its error prose. See `docs/specs/compute-design.md`, change 3. +> +> Deferred Minor findings, triaged by the final review, are in the ledger at +> `.superpowers/sdd/2026-08-02-compute-resolver/progress.md` (gitignored). + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development to implement +> this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Make a managed lease launch actually land. Today a caller picks a SKU from +`GET /api/compute/options` and posts it — and on Vast that fails roughly half the time, because offer ids +churn faster than a client can act on them. + +**Architecture:** All work is in the **Atlas** repo (`~/codes/InkVell/atlas`), continuing on +`feat/compute-lease-prerequisites`. Spec changes 12 (catalog cache) and 3 (server-side resolver). The +cache comes first because the resolver's retry multiplies catalog fetches. + +**Tech Stack:** Python 3.12, FastAPI, aiosqlite, pytest + pytest-asyncio, respx. + +**Spec:** `~/codes/InkVell/openscience/docs/specs/compute-design.md`, Part B changes 12 and 3. + +## Global Constraints + +- **Repo:** `~/codes/InkVell/atlas`, branch `feat/compute-lease-prerequisites` (already checked out). + Do not create or switch branches. +- **Run tests with:** `cd ~/codes/InkVell/atlas/backend && .venv/bin/python -m pytest -q`. + No activated virtualenv — always invoke `.venv/bin/python` explicitly. +- **Never add a `Co-Authored-By:` trailer or any AI/assistant attribution to commit messages.** +- **No mocks of our own code.** Fake providers at the HTTP boundary with `respx`, or register a + `_FakeProvider` into the provider registry — the pattern in `backend/tests/test_compute_billing.py` + and `backend/tests/test_compute_vast_provider_http.py`. +- **Every new assertion must be shown failing first.** Paste the failure into your report. +- Baseline: full suite is **1689 passed, 1 skipped**. It must be green when each task finishes. +- Ignore `.claude/worktrees/`. + +## Established facts — measured, do not re-derive + +Everything here was measured against production on 2026-08-01, not inferred: + +1. **Vast offer ids churn ~50% between two consecutive catalog fetches.** Three back-to-back + `GET /api/compute/options` calls returned 65 / 67 / 67 Vast offers with only **25 stable across all + three**; 31 of the first 65 were gone by the second fetch. +2. **Client-side selection therefore fails about half the time.** Seven consecutive + `400 Unknown SKU '' for provider 'vast'` across price ranks 0–25 — so it is not a + cheapest-first artefact, it is the whole catalog. A lease finally landed on the third random attempt. +3. **A tempting heuristic was falsified.** "Offers present in consecutive fetches are launchable" had + perfect separation at n=8 retrospectively and **failed its first prospective test**. Do not + reintroduce it. Retry is unavoidable. +4. **RunPod advertises GPU types with zero capacity.** The two cheapest both failed; the third + succeeded. RunPod returns `500 "create pod: There are no instances currently available"`, which Atlas + maps to a **`400`** — the same status as Vast's stale offer, with a different message. +5. **The two failures need opposite responses.** A stale Vast offer means "re-resolve the same + requirement"; RunPod no-capacity means "pick a different SKU". Retrying the same offer on + no-capacity loops forever. +6. **Providers publish quality signals we discard.** Vast returns `reliability2`, `inet_down` and + `dlperf_per_dphtotal` on every offer and `list_options` reads none of them. RunPod exposes + `lowestPrice { stockStatus }` and the query does not ask for it — across 37 priced types the + distribution is Low 25 / High 9 / Medium 3, and **all ten cheapest types are `Low`**. +7. **Ranking must use `price_cents_per_hour_display`, not `price_cents_per_hour`.** The latter is the + raw provider rate on both funding paths; the former is what the user pays and is `0` on BYOK. +8. `_catalog` is called at three sites — `/options`, `/estimate`, and `create_lease` — and each call + costs **five provider HTTP requests** (four wired providers; Vast issues two). +9. `create_lease` re-validates the posted SKU with `_find_option` and raises the `400 Unknown SKU`. That + is the race point. + +## The tension the cache creates, and how to resolve it + +A cache cannot make offers _fresher_. Given fact 1, a cached catalog is ~50% stale for Vast within +seconds, exactly as an uncached one is by the time a caller acts. So the cache is a **cost** fix, not a +correctness fix — and the resolver's retry must **bypass it**, or the retry re-reads the same dead +offers and can never succeed. + +That is the load-bearing interaction between changes 12 and 3, and it is why they are planned together. + +--- + +### Task 1: Cache the options catalog + +**Files:** + +- Modify: `backend/app/routes/compute.py` (`_catalog` and its three call sites) +- Test: `backend/tests/test_compute_catalog_cache.py` (new) + +**Interfaces:** + +- Produces: `_catalog(db, user_id, *, fresh: bool = False)` — same return shape as today + `(options, providers, byok_eligible)`. `fresh=True` bypasses and repopulates the cache. + +- [ ] **Step 1: Decide the cache key, and report before implementing** + +`_provider_catalog` takes `user_id` and `byok_eligible`, and calls `_byok_for(db, user_id, provider)` +— so for a user holding a BYOK key the offers are fetched **with that user's credentials** and the +`funding` annotation differs. A naive global cache would leak one user's catalog to another. + +Establish and report: whether the offer rows for a **managed-only** user (no BYOK key for that provider) +are user-independent. If they are, the common case can share one entry and only BYOK users need +per-user entries. **Report your finding before writing the key** — a wrong key here is a cross-user data +leak, not a performance bug. + +- [ ] **Step 2: Write the failing tests** + +Create `backend/tests/test_compute_catalog_cache.py`. Count provider calls with a registered fake +provider that increments a counter. Cover: + +1. two `_catalog` calls inside the TTL issue **one** round of provider calls +2. a call after the TTL expires re-fetches +3. `fresh=True` bypasses the cache even inside the TTL, and repopulates it +4. **two users with different BYOK eligibility never see each other's rows** — the key correctness test, + shaped by your Step 1 finding +5. a provider that raises does not poison the cache with an empty catalog that then serves for the whole + TTL + +- [ ] **Step 3: Run and confirm they fail** + +Run: `cd ~/codes/InkVell/atlas/backend && .venv/bin/python -m pytest tests/test_compute_catalog_cache.py -q` + +- [ ] **Step 4: Implement** + +An in-process TTL cache is sufficient — the reaper and billing loop already assume a single process per +worker, and a stale entry costs a retry rather than money. Do **not** reach for Redis. + +Pick the TTL deliberately and justify it in a comment against fact 1: a longer TTL saves requests but +cannot reduce staleness below the churn rate, so there is a point past which it only trades correctness +for nothing. State the number you chose and why. + +- [ ] **Step 5: Run, then the full suite** + +- [ ] **Step 6: Commit** + +--- + +### Task 2: A canonical GPU model map + +**Why:** the resolver takes a requirement (`gpu`, `count`) rather than an opaque SKU, so it needs to know +that Vast's `RTX 4090` and RunPod's `NVIDIA GeForce RTX 4090` are the same card — and that `H100 SXM`, +`H100 PCIe` and `H100 NVL` are **not** interchangeable. Interconnect is part of the model identity: they +differ in throughput and price. + +**Files:** + +- Create: `backend/app/compute/gpu_models.py` +- Test: `backend/tests/test_compute_gpu_models.py` (new) + +**Interfaces:** + +- Produces: `canonical(name: str, *, gpu_ram_gb: int | None = None) -> str | None` — maps a provider's + display name to a canonical model id, or `None` when it cannot be mapped confidently. + +- [ ] **Step 1: Write the failing tests** + +The taxonomy to support, which a comparable aggregator settled on independently: + +``` +A10 · A40 · A100-40GB-PCIe · A100-40GB-SXM · A100-80GB-PCIe · A100-80GB-SXM +H100-PCIe · H100-NVL · H100-SXM · H200-NVL · H200-SXM · B200 +L4 · L40 · L40S · RTX-3090 · RTX-4090 · RTX-5090 · RTX-6000-Ada +RTX-A6000 · RTX-PRO-6000 · RTX-PRO-6000-WK +``` + +Cover: + +1. the same card spelled differently across providers maps to one id — use **real strings** taken from + the provider modules, not invented ones +2. `H100-SXM`, `H100-PCIe` and `H100-NVL` are three distinct ids and never satisfy each other +3. an unmappable name returns `None` — **not** a guess. A mis-mapped card is the wrong machine at the + wrong price, silently +4. matching is not substring-based: a name containing `H100` as a substring of something else does not + map to an H100 + +- [ ] **Step 2: Run and confirm they fail** + +- [ ] **Step 3: Implement** + +Read the real name strings each provider emits before writing the map — `vast_provider.list_options` +(`gpu_name`), `runpod_provider.list_options` (`displayName`), and the Lambda and Prime equivalents. +Build from what they actually produce. + +- [ ] **Step 4: Run, then the full suite** + +- [ ] **Step 5: Commit** + +--- + +### Task 3: Resolve a requirement to an offer + +**Files:** + +- Create: `backend/app/compute/resolver.py` +- Test: `backend/tests/test_compute_resolver.py` (new) + +**Interfaces:** + +- Consumes: `canonical()` from Task 2; catalog rows as `_catalog` returns them. +- Produces: `resolve(options, *, gpu, count, max_hourly_cents=None) -> list[dict]` — the matching offers + in preference order, best first. A **list**, not one offer, because the caller retries down it. + +- [ ] **Step 1: Write the failing tests** + +1. picks the **globally cheapest** matching offer across providers — proven with a catalog where the + winner is neither the first provider listed nor the same provider twice. A resolver that always + returns one provider must fail this. +2. ranks on the **funding-adjusted** rate: a cheaper billed offer never beats a dearer BYOK one (fact 7) +3. honours `max_hourly_cents` +4. matches on the canonical model, so `H100-SXM` never returns an `H100-PCIe` +5. exercised at `count > 1`, not only `count = 1` +6. returns an ordered list with the cheapest first, so a caller can walk it +7. an empty result is a distinct, inspectable outcome — not an exception + +- [ ] **Step 2: Run and confirm they fail** + +- [ ] **Step 3: Implement** + +Rank by `price_cents_per_hour_display` ascending among offers whose canonical model and `count` match. +Keep it a pure function over catalog rows — no I/O, no DB. That is what makes it cheap to test +exhaustively. + +**Do not implement a stock or reliability filter in this task.** Facts 6 and 3 make it tempting; the +signals are real but unrequested by the provider modules, and the one heuristic we tried was falsified +prospectively. Ordering by price alone, with retry underneath, is the behaviour we have actually +measured working. Note the opportunity in a comment and leave it. + +- [ ] **Step 4: Run, then the full suite** + +- [ ] **Step 5: Commit** + +--- + +### Task 4: Accept a requirement on lease creation, and retry + +**Files:** + +- Modify: `backend/app/routes/compute.py` (`LeaseRequest`, `create_lease`) +- Test: `backend/tests/test_compute_resell_routes.py` (append) + +**Interfaces:** + +- Consumes: `resolve()` from Task 3, `_catalog(fresh=…)` from Task 1. +- Produces: `POST /api/compute/leases` accepts `{gpu, count, max_hourly_cents?}` in place of + `{provider, sku}`. Explicit `provider`/`sku` continues to work unchanged. + +- [ ] **Step 1: Write the failing tests** + +1. `{gpu, count}` resolves and leases without the caller naming a SKU +2. **explicit `provider`/`sku` still works exactly as today** — the dashboard and `compute:up` depend on + it, and it must not start requiring `gpu` +3. **a stale-offer `400` retries the next candidate and succeeds** — the Vast case, fact 2 +4. **a no-capacity failure moves to a different SKU rather than retrying the same one** — the RunPod + case, facts 4 and 5. Retrying the same offer must be provably not what happens. +5. retries are bounded, and exhausting them returns a structured error naming what was tried +6. the retry re-resolves against a **fresh** catalog, not the cached one — the load-bearing interaction + from the section above. Assert the provider was re-queried. +7. `budget_cents` still applies to a resolved lease exactly as to an explicit one + +- [ ] **Step 2: Run and confirm they fail** + +- [ ] **Step 3: Implement** + +Discriminate the two failures **on the provider's message**, since both arrive as `400`. The exact +strings, captured live: + +- Vast stale offer — `Unknown SKU '' for provider 'vast'.` +- RunPod no capacity — `create pod: There are no instances currently available` + +Match defensively: these are provider prose and can change. An unrecognised `400` should behave like the +safer of the two — advance to the next candidate rather than retrying the same one, since retrying a +genuinely dead SKU cannot succeed while advancing merely costs one attempt. + +- [ ] **Step 4: Run, then the full suite** + +- [ ] **Step 5: Commit** + +--- + +## Whole-branch verification + +- [ ] Full suite green, no network. +- [ ] Confirm by name that the earlier deliverables still pass: + `test_promoted_lease_is_not_reaped_at_the_provisioning_timeout`, + `test_lease_without_a_budget_is_still_sized_to_the_full_plan_ttl`, + `test_out_of_credit_release_of_a_managed_lease_uses_the_operator_key`, + and the budget headline test. +- [ ] Confirm explicit `provider`/`sku` leases are unchanged — the dashboard's path. + +## Acceptance criteria + +1. Two `_catalog` calls inside the TTL cost one round of provider requests; `fresh=True` bypasses. +2. Users with different BYOK eligibility never share a cache entry. +3. `canonical()` maps real provider strings to one id, keeps the three H100 variants distinct, and + returns `None` rather than guessing. +4. `resolve()` returns the globally cheapest match on the funding-adjusted rate, ordered, exercised at + `count > 1`. +5. `{gpu, count}` leases without a caller-supplied SKU; explicit `provider`/`sku` is unchanged. +6. A stale-offer `400` retries the next candidate against a **fresh** catalog and succeeds. +7. A no-capacity failure advances to a different SKU rather than retrying the same one. +8. Retries are bounded and exhaustion returns a structured error. +9. `pytest` passes with no network access. + +## Out of scope + +The quote endpoint (change 4) — it consumes this resolver and is the next plan. Stock and reliability +filtering (change 10) — the signals are unrequested and the one heuristic tried was falsified; it needs +its own evidence. Volumes (6), budget extension (7), the rolling window cap (5), and the three +OpenScience tools. diff --git a/docs/plans/2026-08-03-compute-preflight.md b/docs/plans/2026-08-03-compute-preflight.md new file mode 100644 index 00000000..f6b510b0 --- /dev/null +++ b/docs/plans/2026-08-03-compute-preflight.md @@ -0,0 +1,378 @@ +# Compute pre-flight: what must be true before OpenScience gets compute tools + +> **EXECUTED 2026-08-03** on `feat/compute-lease-prerequisites`. All five tasks shipped and reviewed. +> Suite **2004 passed / 1 skipped** (from 1913), no network. Live re-probe after the last task: resolve +> → lease in 5.0s → `gpu_model='RTX-3090'` matching the request → `ready` → released, zero instances +> left running. +> +> **Three things review caught that the plan did not anticipate:** +> +> 1. **Task 2's first age-ceiling fix bounded the wrong thing.** It capped how *long* a stale block +> could starve a launch, not whether it did: under the ceiling, three dead Vast rows still consumed +> every attempt while a live RunPod row went untried. Fixed by making freshness the first rank key. +> 2. **Task 4's retry could never drain.** `lambda_provider.py` had a bare `raise_for_status()` with no +> 404 exemption — RunPod and Prime both have one — so an already-gone box produced 1,440 +> operator-authenticated terminate calls per day forever, from an uncapped serial pass running ahead +> of the promotion sweep. A billing fix was degrading the launch path. +> 3. **Task 3's `RTX-A2000` mapped two different cards to one id.** 6GB and 12GB, and since the resolver +> ranks on price the caller always got the 6GB one. The class of bug matters more than the card: +> `canonical()` never consults VRAM on an exact hit, so no per-name test can fail on it — the old +> tests asserted both values for one id and passed. Only measured inventory catches it. +> +> **One measurement retracted.** The spec's "84% genuine offer death" figure compared two multi-name +> query windows, which is only sound if a window is the deterministic cheapest-64 of its filter. It may +> not be — but a narrow query proved perfectly stable, so the effect is query-shape dependent and +> neither claim generalises. See `docs/specs/compute-design.md`. +> +> **One thing measured, earned, and deliberately not built:** the targeted per-card Vast query. 12 of 22 +> `(card, count)` pairs are absent or up to 50% dearer in the shared windows. It cannot live in +> `vast_provider.py` — `list_options` never receives `{gpu, count}`, and `_provider_catalog`'s cache key +> names no requirement, so a requirement-filtered fetch would poison the entry every caller shares. It +> needs `routes/compute.py` and its own task. +> +> Deferred Minor findings are in the ledger at +> `.superpowers/sdd/2026-08-03-compute-preflight/progress.md` (gitignored). + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development to implement +> this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Close the five things standing between a working Atlas compute API and three agent-facing +OpenScience tools. Four are defects found by live probing; one is a decision already taken but unbuilt. + +**Architecture:** All work is in **Atlas** (`~/codes/InkVell/atlas`), continuing on +`feat/compute-lease-prerequisites`. No OpenScience changes here — the tools come after. + +**Tech Stack:** Python 3.12, FastAPI, aiosqlite, pytest + pytest-asyncio, respx. + +**Spec:** `~/codes/InkVell/openscience/docs/specs/compute-design.md`. + +## Global Constraints + +- **Repo:** `~/codes/InkVell/atlas`, branch `feat/compute-lease-prerequisites` (already checked out). + Do not create or switch branches. +- **Run tests with:** `cd ~/codes/InkVell/atlas/backend && .venv/bin/python -m pytest -q`. + No activated virtualenv — always invoke `.venv/bin/python` explicitly. +- **Never add a `Co-Authored-By:` trailer or any AI/assistant attribution to commit messages.** +- **No mocks of our own code.** Fake providers at the HTTP boundary with `respx`, or register a fake + into the provider registry — the pattern in `backend/tests/test_compute_resell_routes.py`. +- **Every new assertion must be shown failing first.** Paste the RED output into your report. +- Baseline: full suite is **1913 passed, 1 skipped**. It must be green when each task finishes. +- Ignore `.claude/worktrees/`. + +## Measured facts — established live on 2026-08-02/03, do not re-derive + +1. **Atlas meters reads and launches from one bucket.** `POST /leases`, `GET /leases`, + `GET /leases/{id}/connection` and `POST /leases/{id}/release` all classify as `compute_acquire`, + 20/min. Measured: 429 at request 21. `GET /options` is `default`, 600/min. +2. **The bucket is per bearer token, per process.** Key is `auth:{sha256(token)[:24]}`; the store is + `InMemoryStore` because neither Fly app sets `REDIS_URL`; production runs `--workers 2` + (`backend/fly.toml:69`). So the effective ceiling is 20–40/min depending on which worker serves. +3. **A launch returns `provisioning` with no SSH coordinates.** Measured `ready` at t+20–30s, via the + reaper's promotion. So any client must poll — which is why fact 1 bites. +4. **Vast rate-limits `/bundles/` at ~1 request/second**, advertising `x-ratelimit-limit: 1.0`, + `x-ratelimit-remaining`, `x-ratelimit-reset` ≈1s out; recovers after 1s idle. Per Vast's docs the + identity is bearer token + session user + api_key param + **client IP**, enforced as a minimum + interval **per endpoint** — so this is shared by every managed user of a deployment and does not + improve when we add machines. +5. **Vast returns exactly 64 offers per query and ignores `limit`** (tested 64…2000 and with the key + absent). **There is no pagination**: `offset` and `from` both return `400`. More inventory is only + reachable through more *filtered* queries, each costing one request against fact 4. +6. **RunPod has no observable limit** — 60 requests at 2.2/s, no 429, no rate-limit headers. +7. **One `list_options` is 2 concurrent GETs** and completes healthily in ~2.1s; three back-to-back + fan-outs (the retry pattern) also all healthy. Today's pattern fits inside fact 4; concurrent + *retrying* launches are what would not. + +--- + +### Task 1: Reads must not spend the launch budget + +**Files:** +- Modify: `backend/app/middleware/rate_limit.py` +- Test: `backend/tests/test_rate_limit_compute.py` (new) + +**Interfaces:** +- Produces: safe methods (`GET`/`HEAD`/`OPTIONS`) on `compute_acquire` paths classify as `default`. + `POST`/`DELETE` on those paths keep `compute_acquire`. + +Two defects, one file. + +**(a) The read/write split.** `_classify` already lets safe methods fall through to the generous +`default` class, but only for classes whose *name contains the substring* `"mutation"` +(`rate_limit.py:203`). That is why `compute_acquire` was missed. Replace the name test with an explicit +field on `_BucketClass` — a name is not a policy — and set it on `compute_acquire`, `mutations` and +`atlas_graph_mutations` so behaviour for the existing two is unchanged. + +Do **not** raise `compute_acquire`'s 20/min. The launch side should stay guarded; it is the reads that +were never meant to be in there. + +**(b) The store is captured too early.** `RateLimitMiddleware.__init__` does +`self._store = get_kv_store()` (`rate_limit.py:225`). `reset_kv_store()` swaps the module-level store, +so the middleware keeps a stale reference and these buckets never reset between tests. This has already +produced a **false pass** in a RED run — a test asserting a 429 received the limiter's 429. Resolve the +store per request (or per call) so a test reset is honoured. Keep it cheap: `get_kv_store()` is memoized. + +- [ ] **Step 1: Write the failing tests** + +1. `GET /api/compute/leases` 30 times on one token stays 200 — it must not consume launch budget. + Fails today at request 21. +2. `POST /api/compute/leases` is still limited at 20/min on one token. +3. Reads and writes do not share a bucket: exhaust the write bucket, then a read still succeeds. +4. `mutations` and `atlas_graph_mutations` keep their existing safe-method fall-through — a regression + guard on the behaviour being generalised. +5. After `reset_kv_store()`, a previously exhausted bucket is clean. **This is (b), and it must fail + first** — if it passes before the change, the test is not testing the defect. + +- [ ] **Step 2: Run and confirm they fail** +- [ ] **Step 3: Implement** +- [ ] **Step 4: Run, then the full suite** +- [ ] **Step 5: Commit** + +--- + +### Task 2: A rate-limited Vast must degrade, not vanish + +**Files:** +- Modify: `backend/app/compute/vast_provider.py`, `backend/app/routes/compute.py` +- Test: `backend/tests/test_compute_vast_provider_http.py` (append), + `backend/tests/test_compute_catalog_cache.py` (append) + +**Interfaces:** +- Produces: a Vast `429` is distinguishable from "Vast has no offers", and does not silently empty the + catalog. + +`list_options` calls `cheap_resp.raise_for_status()`. On a `429` that raises, `_provider_catalog` catches +`Exception` and returns `base` — a block with `options: []`. So a rate-limited fetch is indistinguishable +from Vast having nothing, and Vast is most of the cheap inventory. The caller is told "no listed offer +matches" when the truth is "ask again in one second". + +Given fact 4 this is not hypothetical under concurrency: `fresh=True` retries bypass the cache, and two +concurrent blocked launches exceed Vast's budget. + +Required behaviour: + +- A `429` from the cheap query must be reported as a rate-limit condition, not an empty catalog. +- `_provider_catalog` must **not** replace a usable cached entry with an empty one because of a + transient `429`. Serving slightly stale rows beats serving none — and note the existing comment at + `routes/compute.py:222` already argues the empty result must not be *cached*; this extends that + reasoning to not discarding what we already have. +- The `no_matching_offer` / `no_capacity` error a caller finally sees must not claim nothing matched + when a provider was rate-limited. Say which provider was unavailable. +- Respect `x-ratelimit-reset` where present rather than inventing a backoff. + +**Do not add a retry loop inside `list_options`.** The caller already has one, and Vast's budget is +shared deployment-wide — a provider-level retry multiplies load exactly when the system is busiest. + +- [ ] **Step 1: Write the failing tests** (`respx`, no live calls) + +1. A `429` on the cheap query does not produce a silently empty Vast block. +2. A `429` does not evict or overwrite a healthy cached entry. +3. A genuinely empty Vast catalog (200, no offers) is still reported as empty — the two must stay + distinguishable. +4. A `429` on the *premium* query alone still yields the cheap rows (it is already additive). +5. The user-facing error names the rate-limited provider rather than claiming no match. + +- [ ] **Step 2: Run and confirm they fail** +- [ ] **Step 3: Implement** +- [ ] **Step 4: Run, then the full suite** +- [ ] **Step 5: Commit** + +--- + +### Task 3: Unlock Prime Intellect, and widen the canonical map + +**Files:** +- Modify: `backend/app/compute/prime_intellect_provider.py`, `backend/app/compute/gpu_models.py` +- Test: `backend/tests/test_compute_prime_provider_http.py` (append), + `backend/tests/test_compute_gpu_models.py` (append) + +**Interfaces:** +- Consumes: `canonical(name, *, gpu_ram_gb=None)`. +- Produces: Prime rows carry enough to be canonicalised; the map covers the datacenter cards our + providers actually sell. + +**(a) Prime Intellect.** Its `gpuType` carries memory but not interconnect (`A100_40GB`), so its A100 and +H100 rows cannot be placed among the four A100 / three H100 ids and are dropped entirely. The offer +*does* carry a `socket` field (`PCIe` / `SXM4`) and `acquire` already forwards it — `list_options` +simply never copies it onto the row (`prime_intellect_provider.py:174`). Surface it, and make +`canonical()` able to use it. + +**(b) The map is narrower than what is on sale.** Verified against live catalogs: **37% of Vast rows and +52% of RunPod's** are mapped. These are priced today and unreachable through `{gpu, count}`: **B300**, +**MI300X**, **GH200**, `RTX PRO 6000 MaxQ`, and the Ada/Ampere workstation line. Add those. + +**Leave the mid-tier consumer cards unmapped** — `RTX 3060/3070/3080`, `4060/4070/4080`, `5060/5070`, +and the older `GTX 10xx` / `Tesla P4` / `Quadro` / `Titan Xp` stock. This was reconsidered and the +decision stands: every one of them is ≤16GB, the map already holds the three cards that matter in that +tier (`RTX-3090` and `RTX-4090` at 24GB, `RTX-5090` at 32GB), and for ML research VRAM is the binding +constraint. They remain leasable by explicit `provider`/`sku`. Task 5 fixes the wasted query directly +rather than by widening the taxonomy to absorb it. + +**The exact-match rule is not negotiable.** `RTX 6000 Ada`, `RTX A6000`, `RTX PRO 6000`, +`RTX PRO 6000 WK` and `RTX PRO 6000 MaxQ` are five different cards whose names contain each other, and +`GH200 SXM` contains `H200`. Every added id is a literal-string table entry plus a canonical id. Adding +`GH200` while a substring rule exists anywhere would rank a Grace Hopper superchip as an H200. + +**Do not invent provider strings.** `VAST_API_KEY` and `RUNPOD_API_KEY` are live in `backend/.env`; dump +the real catalogs (read-only `GET`s, the same ones `list_options` issues — **provision nothing**) and +map what you actually observe. Vast rate-limits `/bundles/` at ~1/s (fact 4), so sleep between fetches. +Record in your report which strings came from a live dump and which from a repo fixture. + +- [ ] **Step 1: Write the failing tests** + +1. A Prime offer with `socket: "SXM4"` and `gpuType: "A100_40GB"` canonicalises to `A100-40GB-SXM`; + with `socket: "PCIe"`, to `A100-40GB-PCIe`. +2. A Prime offer with no `socket` still yields `None` — never a guess. +3. Each newly added card maps from its real provider spelling(s). +4. **`GH200 SXM` is not `H200-*`, and `RTX PRO 6000 MaxQ` is not `RTX-PRO-6000` or `-WK`.** Extend the + existing `_UNMAPPABLE`/adversarial cases rather than adding a separate test. +5. Live coverage rises for the datacenter tier; the consumer tail stays unmapped. + +- [ ] **Step 2: Run and confirm they fail** +- [ ] **Step 3: Implement** +- [ ] **Step 4: Run, then the full suite** +- [ ] **Step 5: Commit** + +--- + +### Task 4: A release that did not happen must not bill the user + +**Files:** +- Modify: `backend/app/compute/lease_manager.py`, `backend/app/services/compute_billing_service.py`, + `backend/app/jobs/lease_reaper.py`, `backend/db` migration for the new column +- Test: `backend/tests/test_lease_reaper_seam.py` (extend), `backend/tests/test_compute_billing.py` + +**Interfaces:** +- Produces: an unconfirmed teardown stops billing and frees the concurrency slot, while remaining + visible to a retry. + +**The decision, already taken by the product owner:** stop billing the user, and flag the lease for +reconciliation so the operator's box is chased separately. The user must never pay for our failure to +tear down. + +Today `release_lease` returns early when `release_confirmed()` is false, leaving `status` untouched. Every +billing and concurrency gate is `status NOT IN ('released','failed')`, so the user keeps being billed and +keeps holding a slot. + +**Both docstrings claim the reaper retries this. It does not.** All four reaper branches were walked: +branch 0 needs terminal telemetry a user lease never emits; branch 1 finds the box `ready` precisely +because the teardown failed; branch 2 only fires on `provisioning`; branch 3 is gated on +`runner_api_key_id`, which only the agent-spawn path sets. **Fix the comments as part of this task** — +they are the reason this looked safe. + +Required behaviour: + +- An unconfirmed teardown marks the lease terminal for billing and concurrency purposes. +- It stays discoverable for a retry — a distinct state or flag, not silently released. Historical rows + must be unaffected. +- Something must actually retry it. If that is the reaper, add the branch and prove it fires for a + **user** lease with no `runner_api_key_id`. +- `CredentialUnavailable` (a deleted BYOK key, a rotated operator key) is the case with no recovery + today; it must not be conflated with a provider that answered and refused. + +- [ ] **Step 1: Write the failing tests** + +1. A managed lease whose provider release is unconfirmed stops accruing charges. +2. …and frees the concurrency slot, so a subsequent launch is not 429'd by a box we failed to kill. +3. …and is still visible to whatever retries it. +4. The retry actually fires for a user lease (no `runner_api_key_id`) — extend + `test_lease_reaper_seam.py`, which sweeps every lease class in one pass. +5. A confirmed release is unchanged. +6. A historical row predating the new column serialises and reaps unchanged. + +- [ ] **Step 2: Run and confirm they fail** +- [ ] **Step 3: Implement** +- [ ] **Step 4: Run, then the full suite** +- [ ] **Step 5: Commit** + +--- + +### Task 5: Ask Vast for hardware we can actually name + +**Files:** +- Modify: `backend/app/compute/vast_provider.py` +- Test: `backend/tests/test_compute_vast_provider_http.py` (append) + +**Interfaces:** +- Consumes: the spellings table in `app/compute/gpu_models.py` (after Task 3). +- Produces: the catalog's Vast rows are overwhelmingly rows the resolver can rank. + +**Measured live 2026-08-03 — this is the defect:** + +``` +cheap query (64 cheapest overall) -> 64 offers, 2 usable ( 3%), 33 distinct cards +premium query (gpu_name in _PREMIUM…) -> 64 offers, 63 usable ( 98%), 13 distinct cards +gpu_name == "RTX 4090" -> 56 offers, 56 usable (100%), 1 card +gpu_name == "A100 SXM4" -> 64 offers, 64 usable (100%), 1 card +gpu_name == "H100 SXM" -> 39 offers, 39 usable (100%), 1 card +``` + +Vast returns exactly 64 per query and has no pagination (fact 5), so each query is a scarce, fixed-size +window against a ~1/s deployment-wide budget (fact 4). The premium query spends its window well because +it **filters by name**. The cheap query filters on nothing and sorts by price ascending — and Vast's +cheapest inventory is mid-tier consumer cards, of which the map contains exactly three +(`RTX-3090`, `RTX-4090`, `RTX-5090`). So it reliably fills its window with rows `canonical()` returns +`None` for, and `resolve()` drops them. + +**The root cause is that the query and the taxonomy were never connected.** Two independently sensible +decisions — "show the cheapest hardware" and "map only cards we can name unambiguously" — combine into a +request that is 97% waste. Note `dph_total <= 0.50` and `reliability2 >= 0.98` were both measured at +3–4% usable: they re-select the same tail, so a price or quality filter does not fix this. A +`gpu_ram >= 40GB` floor reaches 65%, better but still not the point. + +**Derive the query from the taxonomy.** Ask Vast for the cheapest offers *among the card names we can +canonicalise*, rather than the cheapest offers outright. The spellings already exist in +`gpu_models.py`; the fix is to stop maintaining a second, divergent list beside them. `_PREMIUM_GPU_NAMES` +should become a consequence of the map, not an independent constant — if it can be removed entirely in +favour of one derived list, remove it. + +Then, if and only if the measurement in Step 1 shows it earns its request: a **targeted query on the +requirement path** — when a caller names `{gpu, count}`, one query filtered to that card's spellings, +which the numbers above suggest returns 39–64 offers at 100% yield. That is what makes "the cheapest +H100-SXM" true rather than "the cheapest H100-SXM that happened to land in a shared window." + +**Constraints:** do not increase the request count on the cached path; do not turn one launch into more +than one extra Vast request; do not add a retry loop inside `list_options` (Task 2's reasoning applies). +Vast's `gpu_name` values are provider spellings, not canonical ids — `RTX 6000Ada` has no space, RunPod's +do. Filter on what Vast actually emits. + +- [ ] **Step 1: Measure** (read-only live queries, provision nothing, sleep ≥1.3s between them for + fact 4). Confirm the taxonomy-derived filter's yield, and measure whether a targeted per-card + query finds cheaper offers for a named card than the shared windows do. **If it does not, say so + and do not build it** — a speculative extra fetch on every launch is worse than none. +- [ ] **Step 2: Write the failing tests for the chosen design** (`respx`, no live calls) +- [ ] **Step 3: Run and confirm they fail** +- [ ] **Step 4: Implement** +- [ ] **Step 5: Run, then the full suite** +- [ ] **Step 6: Commit** + +--- + +## Whole-branch verification + +- [ ] Full suite green, no network. +- [ ] These still pass by name: `test_promoted_lease_is_not_reaped_at_the_provisioning_timeout`, + `test_lease_without_a_budget_is_still_sized_to_the_full_plan_ttl`, + `test_out_of_credit_release_of_a_managed_lease_uses_the_operator_key`, + `test_two_users_with_different_byok_eligibility_never_see_each_others_rows`, + `test_gh200_is_not_an_h200`, `test_two_concurrent_migrators_do_not_crash_on_shared_sqlite_file`. +- [ ] Explicit `provider`/`sku` leases unchanged — the dashboard's path. +- [ ] A live re-probe of launch → ready → connection → release still passes. + +## Acceptance criteria + +1. A readiness poll does not consume launch budget; `POST` stays at 20/min. +2. `reset_kv_store()` actually clears the limiter's buckets. +3. A Vast `429` is distinguishable from an empty Vast catalog and never silently empties it. +4. Prime Intellect rows canonicalise when the offer names a socket, and `None` when it does not. +5. B300, MI300X, GH200, `RTX PRO 6000 MaxQ` and the workstation line are reachable via `{gpu, count}`; + the consumer tail stays unmapped; `GH200 SXM` is still not an H200. +6. An unconfirmed teardown stops billing, frees the slot, stays retryable, and something retries it. +7. The docstrings no longer claim a reaper retry that does not exist. +8. Task 5 ships either a measured widening or a written finding that none is warranted. + +## Out of scope + +- Any OpenScience change — the three tools come after this plan. +- Provisioning Redis for a cross-process rate limit (an infra decision, not code). +- Backfilling historical lease rows. +- The ~14 deferred Minor findings in the resolver plan's ledger. diff --git a/docs/specs/compute-design.md b/docs/specs/compute-design.md new file mode 100644 index 00000000..3b58d407 --- /dev/null +++ b/docs/specs/compute-design.md @@ -0,0 +1,1540 @@ +# Compute — design + +Status: **Mode detection shipped. Lease prerequisites, the SSH key lifecycle and the budget cap shipped +and verified against a live deployment — on an unmerged draft branch. Selection, the quote, the rolling +cap and every agent-facing tool still to build.** +Date: 2026-07-31 · sweep 2026-08-01 · single current compute spec · revised after adversarial review +Roadmap: **5**, **51/2**, **55**, **103**; unblocks **56** + +> **What a user can do today: nothing.** Everything marked SHIPPED below lives on Atlas +> `feat/compute-lease-prerequisites` and OpenScience `feat/compute-guardrails`, both deliberately draft +> until compute is complete — and OpenScience has no launch tool in any case (`ComputeTools` is +> `[ComputeStatusTool]`). "Shipped" in this document means _built, tested, and in several cases driven +> against a real provider_. It does not mean reachable. + +How OpenScience gets a GPU: who provisions it, who pays, and what stops it. Spans two repos — +`atlas` (Python/FastAPI) decides and enforces, `openscience` (Bun/TypeScript) relays and obeys. Each +side gets its own implementation plan; this document is the contract between them. + +Replaces `compute-management-design.md`, `compute-guardrails-design.md` and +`compute-mode-detection-design.md`, which disagreed with each other and, in places, with the code. + +--- + +## Two paths + +| Mode | Who provisions | Atlas role | What bounds it | +| --------- | --------------------------------- | ----------------------------------- | --------------------------- | +| `byok` | the agent, direct to the provider | none — never sees the key | the user's own account | +| `managed` | Atlas | provisions, meters, enforces, reaps | budget → wallet → cap → TTL | +| `none` | — | — | — | + +These are genuinely different mechanisms, not one flow with a funding flag. BYOK runs on the user's +provider account, which Atlas neither meters nor bills, so there is nothing for Atlas to decide and no +reason for it to custody a provider credential. **Everything below Part A is `managed` only.** + +**Two different things are called BYOK, and they do not coincide.** OpenScience's `byok` means a provider +credential is in `process.env` (`src/compute/mode.ts:70-72`). Atlas has its own, independent notion: a +provider key stored server-side, gated to paid plans (`lease_manager.py:479-484`, +`routes/compute.py:109-123`). A user with an Atlas-stored key and no local env var resolves to `managed` +client-side while Atlas funds the lease as `byok` — rate 0, never debited, `budget_cents` ignored. + +Consequence for the gate below: **the quote is the authority on price, not the client's mode.** A prompt +that quotes an hourly rate the user is never charged is a lie in the safe direction, but it is still a +lie; the quote endpoint (change 4) returns the funding classification so the prompt can say "billed to +your own provider account" instead of a price. + +--- + +# Part A — Mode detection (shipped) + +Shipped on `feat/compute-guardrails`. Summarised here because Part B depends on it; the detail lives in +the code and its tests. + +`computeBillingMode()` used to return `config.billing?.compute ?? "byok"` without inspecting the +environment, so a user with zero GPU credentials resolved to `byok` — claiming BYOK with nothing to BYOK +with — and "no compute available" had no representation at all. + +```ts +export type ComputeSource = "byok" | "managed" | "none" +``` + +**A credential is the whole test.** An earlier draft required a credential _and_ a matching skill; that +was overruled during implementation, because a capable agent drives a documented cloud API from a bare +key, so the conjunction only produced a false `none` for users holding a workable key. + +| Provider | Env (any group satisfies; all vars within a group required) | +| --------------- | ----------------------------------------------------------- | +| Modal | `MODAL_TOKEN_ID` **+** `MODAL_TOKEN_SECRET` | +| Lambda | `LAMBDA_API_KEY` \| `LAMBDA_LABS_API_KEY` | +| TensorPool | `TENSORPOOL_KEY` \| `TENSORPOOL_API_KEY` | +| Prime Intellect | `PRIME_API_KEY` \| `PRIME_INTELLECT_API_KEY` | +| RunPod | `RUNPOD_API_KEY` | +| Vast | `VAST_API_KEY` | + +BYOK wins whenever a credential is present: it is free to the user and needs nothing from Atlas — which +is also why a BYOK user never pays for the availability network call. + +`billing.compute` is an **override, never the source of truth**: unset → detection, `"byok"` → BYOK if a +credential exists else `none`, `"managed"` → managed if available else `none`. **An override may narrow +to `none`; it may never manufacture a capability.** + +**Only `compute_status` resolves.** `SkillTool.init` calls `ComputeMode.offered()` (`src/tool/skill.ts:63`), +which deliberately never reaches the availability probe — it is answerable from credentials alone, so a +BYOK or keyless user pays no per-turn network cost (`src/compute/mode.ts:205-224`). Resolution is still +per request rather than at startup, because credentials arrive at unpredictable times: the shell, the +Credentials panel, the Compute panel, and — as the developer's own machine proved — dashboard sync → +`synced-env.json` → `preload-env` replay. Startup detection would have reported `none`. + +`session/prompt.ts:1549-1562` injects a stateless per-turn pointer at `compute_status` for +`COMPUTE_AGENTS`. **It must not enumerate tools.** Part B adds three more; a reminder listing them goes +stale the moment the set changes, and the tool descriptions already ride in every request. + +Availability is one authenticated `GET /api/compute/options`, 3s timeout (`mode.ts:101`), 5s TTL cache +(`:107`). A failed, unauthenticated or timed-out call resolves to **unavailable** (`:139-148`) — failing +toward `managed` would reproduce the original bug of promising an unconfirmed capability. + +**That probe is expensive on the Atlas side and is not cached there.** `_catalog` gathers over +`RESELL_PROVIDERS` (`routes/compute.py:34-45`, `:186-189`) and per-provider exceptions are swallowed into +an empty option list (`:129-132`). + +Precisely: **five HTTP requests, not ten.** Six of the ten entries are `ScaffoldProvider`s with no +credential, and `:126-127` returns before any network call — their `list_options` is a pure-Python +sentinel stripped by the `sku` filter at `:133`. Only lambda / runpod / vast / prime_intellect are wired +(`main.py:144-151`), and Vast issues two requests (`vast_provider.py:122-134`). Five requests at 15–20s +provider timeouts still blows the client's 3s budget, so the conclusion holds — but the number matters, +because it is the quantitative argument for change 12. + +A paying managed user therefore resolves to `none` whenever the aggregate exceeds 3s. Fail-closed is still +right, but **the catalog needs a server-side cache** (change 12) — and change 3 makes it load-bearing, +because a transiently-erroring Vast silently removes most of the catalog and changes which provider is +"cheapest" with no signal to anyone. + +`ComputeMode.offered()` filters the skill catalog and is non-empty **only in `byok`**. **This is a listing +filter, not a gate:** a hidden skill remains loadable by exact name and the agent still has `bash`. Gating +the load path was considered and declined. + +### The gap Part A left, which Part B must answer + +**RunPod and Vast have no catalogued skills** (`mode.ts:56`, `:60`). A user whose only credential is +`RUNPOD_API_KEY` therefore resolves to `byok`, is told to use the cloud-compute skills, is offered none — +and because `byok` wins whenever a credential exists, managed is suppressed at the same time (`:186`). +They have a key, no skills, and no managed path. + +Part B makes this sharper rather than fixing it: `ComputeTools` is registered unconditionally +(`src/tool/registry.ts:135`), so `compute_launch` **will be** callable in every mode once it exists, +while `compute_status` tells the agent not to launch managed leases. **The three new tools must state +their behaviour in `byok` and `none`** — refuse with the reason, rather than attempting a managed lease +the mode says is unavailable. No criterion covered this before. + +_Today `ComputeTools` is `[ComputeStatusTool]` (`src/tool/compute.ts:138`), so the sentence above +describes the state Part B creates, not the state that exists._ + +--- + +# Part B — Managed leases (changes 0, 1, 2 and 9 shipped, 8 half; the rest to build) + +## The gap Part A exposed + +`compute_status` tells an agent in `managed` mode to run GPU work billed to Credits — and **no mechanism +exists**. There is no launch tool, and the Atlas CLI's `compute:*` commands are unpublished. Since managed +is live in production, every keyless user lands there. + +## Trust boundary + +**The agent proposes; the server decides.** OpenScience is open source and the agent has `bash`, so any +client-side check is one a fork can delete. Only a decision made over HTTP, behind auth, in a process the +agent does not run, is one it cannot influence. + +Corollary: **OpenScience holds no pricing, balance, selection or approval logic.** It relays a proposal +and obeys a verdict. + +## The flow + +``` +agent compute_status → mode=managed, balance_usd + +agent compute_launch { gpu:"H100-SXM", count:1, budget_cents:3000, max_hourly_cents?, volume_id? } + (gpu is a canonical model id — "h100" is too coarse, interconnect changes price and throughput) + +OS → POST /api/compute/quote { gpu, count, max_hourly_cents, budget_cents } (change 4) + ← { provider, sku, hourly_cents, effective_cap_cents, balance_cents, funding } ADVISORY + └─ permission gate (default ask) — names the provider, because cheapest-first means it varies: + "Vast H100 · $1.94/hr · cap $30.00 · balance $198.83" + +OS → POST /api/compute/leases { gpu, count, max_hourly_cents, budget_cents, volume_id? } +Atlas RE-RESOLVE — the quote is never trusted or reused + reserve against wallet + rolling cap ATOMICALLY · clamp budget to effective balance + mint Ed25519 · size grant to effective cap · launch pod + ← { lease_id, private_key, effective_cap_cents, hourly_cents, provider, sku } + (no ssh_host yet — the pod is still `provisioning`) + +OS → poll GET /api/compute/leases/{id}/connection until status is running AND ssh_host is non-empty + (bounded; on timeout, RELEASE and report — never leave a paid box the agent cannot reach) + write private_key → /compute/.pem (0600) + ← { lease_id, ssh_host, ssh_port, ssh_user, key_path, effective_cap_cents, hourly_cents } + +agent bash: ssh -i -p @ … scp results back + +agent compute_release { lease_id } → Atlas terminates · OS deletes the .pem + (.pem missing? re-fetch from /connection and rewrite it — Atlas holds it encrypted) +``` + +**`compute_launch` is three Atlas calls, not one.** An earlier draft claimed each verb maps 1:1 to an +endpoint; that was wrong in a way that hid two missing pieces — see changes 4 and 0. + +## Why budget, not balance + +The obvious design is to check the wallet each hour and stop the box when it can no longer fund another +one. That bound is **the user's entire balance** — a forgotten H100 against a $500 wallet costs $500, and +sequential re-leasing is unbounded on top of that. + +A per-run budget bounds the _run_. The agent proposes _"this is worth up to $30"_, not _"this needs four +hours"_ — an LLM can judge the first and cannot predict the second, and it is a wrong duration estimate +that forces extension paths, warning windows and re-estimation loops into a design. + +Three consequences, all simplifying: exhaustion is a fact the server observes, so no completion signal is +needed; it is arithmetic, so no extension is needed for correctness; and it is server-side, so no agent +liveness is needed. + +The wallet remains the outer bound. A $1000 budget against a $15 balance is not an error — it buys $15 of +compute — but **the response reports the effective cap** so the agent can tell the user what was actually +authorised rather than what was asked for. + +## Why Atlas resolves the SKU + +The agent states requirements (`gpu`, `count`, optional `max_hourly_cents`); Atlas picks a matching live +offer and leases it in the same request. Three reasons, in order of weight: + +1. **It narrows the offer-ID race.** `compute:up` fetches options, picks, then estimates — and Vast's SKUs + are ephemeral marketplace offer IDs that churn in between, so the default path fails with a raw + `HTTP 400: Unknown SKU`. +2. **292 options never enter the context window.** +3. **It keeps every price decision server-side**, which the trust boundary already required. + +**It does not eliminate the race, and an earlier draft claiming "by construction" was wrong.** +`create_lease` already re-fetches the catalog and re-validates the SKU server-side immediately before +`provider.acquire` (`routes/compute.py:327-333`), so a server-internal window exists today and change 3 +does not close it. No transaction can span a third-party marketplace. **The resolver must therefore +re-resolve and retry on a provider `400`, bounded to N attempts against a re-fetched catalog**, and give +up with a structured error rather than looping. + +### Cheapest wins, across every operator provider + +**The resolver ranks purely by price.** No provider allow-list, no default provider, no preference — the +cheapest live offer matching the requirements is the one leased. That is the product decision, and the +rest of this document conforms to it. + +**Rank on `price_cents_per_hour_display`, not `price_cents_per_hour`.** This distinction is the whole +correctness of the rule. `price_cents_per_hour` is the provider's raw pass-through rate and is set +identically on both funding paths (`routes/compute.py:161`); the column reflecting what the **user** pays +is `price_cents_per_hour_display`, which is `0` when that provider resolves to BYOK (`:162`). Funding is +decided per provider (`:117-123`), so a paid-plan user holding Atlas-stored keys for some providers and +not others has a genuinely mixed catalog — and ranking on the raw rate would prefer a **$1.90/h billed** +offer over a **$2.00/h free** one. Cheapest means cheapest _to the user_. + +Match on GPU model and `count`, honour `max_hourly_cents` if given, then take the minimum display rate. +The one real implementation detail is GPU-name normalisation: options expose `name`, `gpu_ram_gb` and +`upstream`, and providers spell the same card differently, so the resolver needs a canonical model map +rather than a substring match. + +An earlier draft of this section proposed an allow-list restricted to providers whose release cleans up +after itself, which would have excluded Vast — and Vast supplies 204 of 292 live options. **That is +overruled.** Two consequences follow, and both are now in scope rather than deferred: + +- **The key leak stops being rare and becomes the norm** — see below. Change 9 fixes it, and it is no + longer somebody else's ticket. +- **The offer-ID race is live on the common path**, because Vast's SKUs are the ephemeral ones. Changes 3 + and 4 are therefore mandatory, not optional. There is no version of this design where the agent picks a + SKU itself and the default path still works. + +**Vast is not spot, and a predecessor claim that it was is wrong.** `VastProvider.list_options` queries +`"type": "on-demand"` (`vast_provider.py:110-115`), so cheapest-first does not buy preemption risk. It +also runs two queries — cheapest-first alone never surfaces the datacenter cards, so the premium tier is +fetched by name (`:105-121`) — which means an H100 request reaches real H100 offers rather than bottoming +out in consumer GPUs. + +### What cheapest-first obliges us to fix + +Atlas generates a fresh Ed25519 keypair per lease and shows the provider only the public half. An earlier +draft concluded from that "the returned key opens exactly one box everywhere." **That is false on Vast, +and the consequence is cross-tenant.** + +| Provider | Key attachment | Account artifact | Cleaned up on release | +| --------------- | ---------------------------------------------- | ---------------- | ---------------------------------------------- | +| RunPod | `PUBLIC_KEY` env var consumed on boot | **none** | nothing to clean | +| Lambda | account key registry | yes | yes (`lambda_provider.py:248-268`) | +| Vast | account `/ssh/` **and** `/instances/{id}/ssh/` | yes | **no** (`vast_provider.py:307-325`) | +| Prime Intellect | `POST /ssh_keys/` → `sshKeyId` | yes | **no** (`prime_intellect_provider.py:306-325`) | + +RunPod leaves no account-level trace and its creation body takes an arbitrary `env` dict, so boot-time +setup needs no SSH bootstrap. That makes it the **easiest** provider to operate — it is not the default, +and nothing in this design prefers it. + +**Vast's account key registration is a live cross-tenant exposure, today, independent of anything in this +document.** `VastProvider.acquire` posts the per-lease public key to the **account** endpoint +(`vast_provider.py:206-211`), and the module docstring states the purpose plainly: _"registers the public +key on the Vast account (`POST /ssh/`) **so new instances pick it up**"_ (`:10-12`). Under managed funding +every user's lease runs on the single operator credential (`_headers_for` falls back to +`config.VAST_API_KEY`, `:78`). So one user's private key opens another user's box, for as long as the +first key sits on the account when the second instance is created. + +The docstring's justification — the account key is _"keyed by content and harmless to reuse"_ (`:15-16`) — +is true for a single-tenant account and false for a reseller. + +**Delete-on-release cannot fix this.** Concurrently live leases have their keys on the account by +construction, and `MANAGED_GPU_CONCURRENT` defaults to 2 per user (`config.py:117`) with no global bound. +The fix is to stop registering account-level keys at all: `acquire` already attaches the key to the +instance directly (`:244-248`), described in the code as the fallback _"so SSH works even if the account +key wasn't applied at launch"_. **Verify the per-instance attach alone suffices, then drop the account +`POST /ssh/`.** + +This is change 9, and it is a **blocking prerequisite** for making Vast reachable by default — not +something that ships alongside the resolver. + +--- + +## Atlas changes + +### Change 0 — make a lease reach `ready`, then scope the reaper _(prerequisite, two parts)_ — **SHIPPED** + +> **Shipped** across `feat/compute-lease-prerequisites` (promotion + coordinates + reaper scoping + +> `lease_state.normalise_state`), and **verified against a real provider rather than an injected clock.** +> +> - **The headline property, on a wall clock:** a user lease with no runner token sat `ready` for +> 578s → 687s → **788s** against the deployed reaper, un-reaped. Before this change it died at 600s. +> - **Promotion and coordinates, live on both providers:** created `provisioning` with no address → +> the real background sweep promoted it within one interval → `ssh_host` / `ssh_port` persisted as +> the **NATed** values (`ssh3.vast.ai:15650`, `194.68.245.163:22189`), never the placeholder `22` → +> `ready_at` stamped → SSH into a real GPU worked. +> - **0(c) live over HTTP:** `GET /connection` returned `state: "ready"`. +> +> **Three things the section below does not say, all of which execution forced:** +> +> - **Provider status alone is not readiness.** RunPod reports `desiredStatus=RUNNING` from pod +> creation with `publicIp=""` and `portMappings=None`; coordinates appeared at t+15s. Promotion +> therefore waits for coordinates as well — `if not ssh_host and lease.get("ssh_key_name")` — and +> the `ssh_key_name` half is load-bearing because **Modal has no SSH at all**: its `connection()` +> returns no `ssh_host`, so an address-only gate stopped Modal CPU sandboxes promoting and had them +> reaped at 600s. That regression was caught before merge. +> - **A destroyed Vast instance read as `provisioning` forever**, so reaper branch 1 could never fire +> for Vast. Fixed by mapping an empty instance payload to terminated. Vast returns HTTP 200 +> `{"instances": null}` for a destroyed id **and** for an id that never existed and **never 404s**; +> RunPod does 404. Both signals are needed, and the terminal verdict now requires two consecutive +> observations. +> - **Deploying it found a migration race** the tests could not: `uvicorn --workers 2` ran +> `run_migrations` in both workers, both `ALTER`ed the same new column, one crashed startup with +> `duplicate column name`. A pre-existing shape at all 22 `ADD COLUMN` sites, exposed by this work. + +This is the prerequisite everything else waits on, and **the first draft of this spec got it wrong** — +it named the heartbeat branch, which is the one branch that cannot be killing these leases. The +correction matters because the acceptance criterion it implied would have passed green while every user +lease still died at ten minutes. + +**Part (a) — GPU leases never leave `provisioning`.** `RunPodProvider.acquire` returns +`status: "provisioning"` (`runpod_provider.py:183-191`) and `create_lease` persists it. Only two writers +ever flip a lease to `ready`, and neither runs for a managed GPU lease: + +- `lease_manager.py:270`, inside `_reconcile_active_cpu_leases`, which iterates `list_active_cpu_leases` + — **CPU only**. +- `lease_manager.py:690`, inside `LeaseManager.get_lease_status` — which has **no production caller**. + `grep` finds only `backend/tests/test_compute_lease_manager.py:192` and `:603`; there is no status route + in `routes/compute.py`. + +So the lease dies at **branch 2, `provisioning_timeout`** (`lease_reaper.py:133-136`, +`PROVISION_TIMEOUT_SECONDS` = 600, `config.py:70`) — not branch 3. Branch 3 explicitly excludes +provisioning leases and says so in its own comment (`lease_reaper.py:139-141`). + +The fix is a GPU reconcile pass that polls the provider and flips `provisioning` → `ready`, mirroring +what `_reconcile_active_cpu_leases` already does for CPU. `update_lease_status` already accepts +`ssh_host`/`ssh_port` (`compute_repo.py:397-405`) and nothing in production passes them — **the same pass +must persist them**, which is what makes `compute_list` and the `/connection` poll useful. + +**Part (b) — then the heartbeat branch becomes the killer.** Once a lease reaches `ready`, branch 3 +(`lease_reaper.py:141-144`) applies `HEARTBEAT_STALE_SECONDS` = 600 (`config.py:69`) to it, over +`list_unfinished_leases`, whose docstring reads _"category-agnostic"_ (`compute_repo.py:456-462`). +`create_lease` mints no runner token and the telemetry endpoint requires one, so **a user lease cannot +prove liveness**. Scope the heartbeat check to leases holding a runner token. + +**Pin that predicate to a column, because two different credentials are called "the runner token".** The +lease row carries `runner_api_key_id` (`migrations.py:568`), a `thk_*` value set only by the spawn path. +The token the telemetry endpoint authenticates is a separate `thrk_*` value in `runner_tokens` +(`agent_repo.py:371-381`, `routes/agent.py:1829-1840`). The predicate is `runner_api_key_id IS NOT NULL`; +leaving it as prose invites the implementation to guess. + +**Part (c) — normalise the status field.** `routes/compute.py:500` returns +`conn.get("status") or lease.get("status")`, mixing four provider vocabularies with the DB's own. The +client's readiness poll cannot be written against it as-is; see the OpenScience section. Normalising it +belongs here because change 0(a) is what introduces the `ready` transition in the first place. + +Order matters: (b) alone changes nothing, because branch 2 kills the lease first. (a) alone moves the +death from 10 minutes to 10 minutes. **Both, or neither.** + +User leases then stay bounded by plan TTL, wallet exhaustion, explicit release, and the budget cap. The +provider-terminal branch (`:117-131`) continues to apply to everything. + +**Part (a) also produces the boot-time dataset** change 10 ranks on, because flipping to `ready` is what +stamps `ready_at`. + +#### `PROVISION_TIMEOUT_SECONDS` = 600 is a guess, and cheapest-first tests it + +It is a single global number (`config.py:70`) applied to every provider. Measured 7-day boot distributions +put RunPod's median near 59s but Vast's tail past **6 minutes** — so under cheapest-first, which sends +most launches to Vast, a slice of legitimate provisions runs close to the limit and some will exceed it. +A box reaped mid-provision is a launch the user paid for and never received. + +> **Our own measurement does not reproduce that tail, and it changes what this section is for +> (2026-08-01, n=7 real Vast launches, n=2 RunPod).** Every launch that booted at all booted fast — +> **34s, 46s, 50s** on Vast, ~30s on RunPod. The third-party ">6 minute tail" was not observed. What +> _was_ observed is a different failure entirely: **3 of 9 launches never booted at all** (Vast +> `actual_status=offline` for 7 minutes while `cur_state=running`; RunPod returning no capacity). +> +> **So the failure mode is "never boots", not "boots slowly", and a longer timeout does not help it — +> it only delays the refund.** The per-provider timeout is still worth deriving from real data, but it +> drops in priority behind change 10's ranking work and behind falling through on capacity errors +> (change 3), which is what actually addresses the observed loss. n is small; do not over-fit either +> way. + +**Make the timeout per-provider and set it from the p99 of measured boot times** (change 10), not from a +round number. Until that data exists, raise it for the providers whose observed tail demands it rather +than leaving one value covering a 6× spread. + +~~**Ships first, with its own test, before any budget work.**~~ **It did** — change 0 shipped as its own +work before changes 1 and 2, as required. + +### Change 1 — make `hard_cap_cents` a real running cap — **SHIPPED** + +> **Shipped** as `0a9da07` + `a4a79ca`. **The mechanism this section originally prescribed was wrong** and +> is corrected below — read the correction before touching this code. +> +> **The premise was confirmed in production before the fix.** A live 34¢/hr lease had +> `hard_cap_cents = 816` (= 34 × 24) with `spent_cents` **frozen at 34** while `lease.total_spent_cents` +> climbed 0 → 2 → 3 → 5 → 6. The ceiling was decorative and the wallet was the only real bound. This +> section previously carried a "read from source only — confirm against a test" caveat; that is now +> discharged, against both a test and a running deployment. +> +> **The correction: the grant update is a _set-to-total_, not a re-debit.** `debit_grant` increments, and +> an increment cannot mirror a tick that charges cumulatively +> (`wall_clock_cents(rate, now - started_at) - total_spent_cents`). Incrementing double-counts the hour +> `acquire_lease` debits up front — measured by mutation: a $10 budget at $6.99/h died at **26.0 minutes** +> instead of ~1.4 hours, and `grant.spent_cents` read 300 where wall-clock was 180. `set_grant_spend` +> writes the wall-clock total instead, so replay safety is inherited from the tick and the acquire debit +> is superseded rather than added to. +> +> **Measured after the fix:** a $10 budget at $6.99/h releases at 5160s against a theoretical 5150s — a +> 9.8s overshoot that is pure tick quantisation. +> +> A wrong implementation still passes every "a release happened" assertion. Under the mutation above, +> `released == 1`, the status was `released`, and the wallet moved. **Only an assertion on elapsed +> billable duration catches it** — which is why the spec insisted on that property and why both earlier +> attempts, which omitted it, shipped the bug. + +The column exists (`migrations.py:522`, `pg_migrations.py:775`) and an atomic ceiling exists +(`compute_repo.py:196` — `AND (spent_cents + ?) <= hard_cap_cents`). But `debit_grant` is called in +exactly four places, all in `lease_manager.py`: acquire for one hour (`:529`), wallet-insufficient +rollback (`:545`), pre-provisioning failure undo (`:77`), and a settle true-up (`:209`). +**`compute_billing_service.tick_once` never calls it** — only `usage_service.charge` and `mark_billed`. +So `spent_cents` freezes at hour one and the ceiling is never re-evaluated. + +**The billing tick must update the grant** (see the correction above — a set, not a re-debit), releasing +the lease when the new total would exceed the cap — +reusing the path that already fires on wallet exhaustion. + +**This is not a double charge.** The wallet is money; the grant is an authorisation envelope drawn against +it. An implementer who "de-duplicates" these has removed the cap. + +**The acquire-time debit is never refunded on the managed path.** `:209` is unreachable there: it sits in +`reconcile_managed_hold`'s `hold_id`-present branch, and the managed GPU path never sets `hold_id` — it is +initialised `None` at `:456`, never assigned, and the code says so at `:519-525` ("`hold_id` stays None"). +`reconcile_managed_hold` returns from `if not hold_id:` at `:120`. Only `:77` fires, and only when +provisioning fails before the box exists. + +_(A predecessor spec said the debit "is never rolled back". A correction in the first draft of this +document called that wrong and cited `:77` and `:209`. On the happy managed path the predecessor was +closer to right, and the correction was the error — the fourth-order instance of the failure this +document's last section is about.)_ + +**Make the grant debit cumulative, mirroring the tick's own idempotency.** The tick is already replay-safe +because it charges `wall_clock_cents(rate, elapsed) - total_spent_cents` +(`compute_billing_service.py:144-148`) — a replay yields `delta <= 0` and skips. `debit_grant` is an +increment (`compute_repo.py:193`), which is not replay-safe and would double-count hour one. Add a +set-to-total variant — `spent_cents = + `, guarded by the same atomic +`WHERE … <= hard_cap_cents AND status = 'active'` — so a replayed tick is a no-op by construction and the +acquire debit is counted exactly once. + +Two things the implementer must decide explicitly, because both are money: + +- **Write order.** Charge → `mark_billed` → grant debit. A crash between them must not double-charge; the + cumulative form is what guarantees that. +- **The final increment.** When the debit would exceed the cap, **charge the elapsed time, then release.** + The user consumed it and the operator owes the provider for it. Skipping the charge loses real money. + +_Latent, not live:_ `debit_grant`'s predicate includes `status = 'active'` (`compute_repo.py:195`). +`expire_grants_by_session` (`:204`) is defined and never called, so no grant expires today — but under +this change, any future grant expiry silently becomes "release the lease". + +### Change 2 — accept a budget on lease creation — **SHIPPED** + +> **Shipped** as `31fc598`, together with change 1 as this section requires. `budget_cents` is optional +> on `POST /api/compute/leases`, clamped to the effective balance, and the response reports the effective +> cap. Absent, the grant is still sized to `rate × ttl_hours` — pinned by a test that fails if that +> number moves, because it is now a live ceiling on the dashboard and `compute:up`, which both call this +> endpoint without a budget. +> +> Two decisions taken during implementation, neither in this section's original text: +> +> - The `402` keeps one body shape but carries **two `error` values** — `insufficient_cli_credit` when the +> wallet is short, and a new `budget_below_hourly_rate` when the wallet is fine and only the budget is. +> Reusing the credit code would make a client tell a user with a full wallet to top up. +> - The budget is deliberately **not** clamped to `rate × ttl_hours`. The TTL is already a time bound, so +> a larger grant is unreachable spend, and clamping to it would shrink a stated budget for a reason that +> has nothing to do with affordability. + +``` +POST /api/compute/leases +{ provider?, sku?, gpu?, count?, max_hourly_cents?, region?, node_id?, budget_cents?, volume_id? } +``` + +`budget_cents` is optional. Rejection reuses the structured `402`, extended with +`affordable_budget_cents`. **A budget larger than the wallet is clamped, not rejected**, and the response +reports the **effective** cap. **Managed only** — BYOK ignores it and is never debited. + +**"Absent preserves today's behaviour" is false once change 1 lands, and an earlier draft claimed it was +true.** `routes/compute.py:359-362` already sizes every user grant to +`max(charge_raw * ttl_hours, charge_raw, 1)` with `ttl_hours = 24` on every plan tier (`config.py:468`, +`:481`, `:494`). That number is inert today because nothing enforces it. After change 1 it becomes a live +ceiling — and because the acquire debit consumes hour one up front and is never refunded, **a no-budget +lease dies at ~23h instead of the 24h plan TTL**, for every existing caller including the dashboard and +`compute:up`. + +Fix it deliberately: size the default grant to `rate * (ttl + 1)`, or have the cumulative debit account +for the acquire debit so the two do not stack. **Assert unchanged runtime, not merely an accepted +request.** + +**Therefore change 1 must not ship alone.** An earlier draft required changes 0 and 1 to be separate +commits, which is right for attribution — but shipping change 1 by itself _is_ shipping the ~23h +regression to every existing caller, because the default-grant fix lives here in change 2. **Changes 1 +and 2 land together**, in that order, with criterion 4 gating the pair. Change 0 remains its own commit. + +### Change 3 — resolve the cheapest SKU from requirements _(mandatory)_ — **SHIPPED** + +Accept `{gpu, count, max_hourly_cents?}` in place of an explicit `sku`; rank **every** operator provider's +options by `price_cents_per_hour_display` — the funding-adjusted rate, not the raw one; lease the cheapest +match in the same request. Explicit `provider`/`sku` continues to work for the dashboard and the CLI. + +> **Shipped 2026-08-02** on `feat/compute-lease-prerequisites`, built as +> `docs/plans/2026-08-02-compute-resolver.md` (four tasks: cache, GPU map, resolver, retry). Suite +> 1908 passed / 1 skipped, no network. Three things below turned out differently in the building, and +> the code is right where it disagrees with the prose: +> +> - **The retry does not read the provider's message**, though the section below says it must. It +> excludes every `(provider, sku)` already refused and re-resolves against a fresh catalog. That +> gets both providers right with one rule: RunPod's stable type id is excluded, so a no-capacity +> type is never retried, and Vast's requirement is genuinely re-resolved against ids that did not +> exist a moment ago. It also cannot rot when a provider rewrites its error prose — which the +> message-matching design would have depended on. **Do not "restore" the message matching.** +> - **Ranking on the display rate alone was wrong for BYOK.** Every BYOK row displays `0`, so the +> ranking collapsed past its first key to alphabetical order by provider — leasing a $9.00/h box +> over a $3.00/h one and reporting nothing, because on BYOK Atlas bills neither. The rank key is +> now `(display, raw, provider, sku)`. Ordering on the managed path is unchanged, since display +> tracks raw there. `max_hourly_cents` still bounds the display price, and therefore **still +> excludes nothing on BYOK** — the user's own provider bill is not a number Atlas can cap. +> - **Ranking is cheapest outright, not cheapest above a reliability floor.** The floor belongs to +> change 10, which has not shipped. The signals are real and were deliberately left out: the one +> stability heuristic this project tried was falsified prospectively (below). + +**`count > 1` is thinner than the catalog size suggests.** RunPod hardcodes `"gpu": 1` in its options and +`"gpuCount": 1` on acquire (`runpod_provider.py:124`, `:156`), Vast dedups to one row per +`(gpu_name, count)` (`:142-156`), and Prime to one per `(gpuType, upstream)` +(`prime_intellect_provider.py:138-156`). A resolver tested only at `count = 1` will not exercise the path +most multi-GPU training needs. + +Ranking is **cheapest above a reliability floor**, not cheapest outright — see change 10. + +#### The canonical GPU map + +Providers spell the same card differently, and a substring match on `name` will silently mis-rank. +`"h100"` is too coarse to be an input: **interconnect is part of the model identity**, and the three H100 +variants differ in both throughput and price. + +``` +A10 · A40 · A100-40GB-PCIe · A100-40GB-SXM · A100-80GB-PCIe · A100-80GB-SXM +H100-PCIe · H100-NVL · H100-SXM · H200-NVL · H200-SXM · B200 +L4 · L40 · L40S · RTX-3090 · RTX-4090 · RTX-5090 · RTX-6000-Ada +RTX-A6000 · RTX-PRO-6000 · RTX-PRO-6000-WK +``` + +The resolver takes a canonical id from this set plus `count`. Each provider module maps its own naming +into it, and an option that cannot be mapped is **excluded from ranking rather than guessed at** — a +mis-mapped card is a wrong machine at the wrong price, silently. + +This taxonomy is the one a comparable aggregator settled on, which is a reasonable signal that it is the +right granularity rather than over-specification. + +> **Shipped as `backend/app/compute/gpu_models.py`** — `canonical(name, *, gpu_ram_gb=None)`, a table of +> literal strings, never a matcher. Every key was dumped from a live provider API on 2026-08-02, not +> imagined. `RTX 6000 Ada`, `RTX A6000`, `RTX PRO 6000` and `RTX PRO 6000 WK` are four different cards +> whose names contain each other and RunPod sells all four; `GH200 SXM` contains `H200`. Any substring or +> prefix rule ranks a Grace Hopper superchip as an H200. +> +> **The taxonomy is narrower than what our providers actually sell — this is the open decision.** It +> covers **37% of live Vast rows and 44% of RunPod's**. That is mostly the long tail of consumer cards +> nobody would request, and unmapped rows are dropped rather than mis-ranked, so nothing is priced wrong. +> But these are missing and priced today: **B300** (the top of RunPod's catalog), **MI300X**, +> `RTX PRO 6000 MaxQ`, **GH200**, and the Ada/Ampere workstation line. A caller cannot reach any of them +> through `{gpu, count}`; the explicit `provider`/`sku` path still can. +> +> **Prime Intellect is wholly unmappable, for a one-line reason.** Its `gpuType` carries memory but not +> interconnect (`A100_40GB`), so its rows cannot be placed among the four A100 / three H100 ids. The offer +> *does* carry a `socket` field (`PCIe` / `SXM4`) and `acquire` already forwards it — `list_options` just +> never copies it onto the row. Surfacing it unlocks the whole provider. +> +> Widening is additive and needs no resolver change. + +**The retry is not optional here.** Vast supplies most of the catalog and its SKUs are ephemeral offer IDs, +so the cheapest pick is usually the raciest one. On a provider `400`, re-resolve against a re-fetched +catalog, bounded to N attempts, then fail with a structured error rather than looping. +`GET /api/compute/options` (`routes/compute.py:214`) already does the read — but it is uncached and costs +five provider requests, so N retries is N full catalog rebuilds and change 4 adds another per launch. +**Change 12 (catalog cache) ships before this**, or the retry path costs more than the lease. + +#### Measured 2026-08-01 — the race is worse than assumed, and `400` is overloaded + +Three findings from driving real launches against a deployed backend. All three are requirements on +the resolver, not colour. + +- **The churn is ~50% per fetch, so client-side selection is roughly a coin flip.** Three back-to-back + `/api/compute/options` calls returned 65 / 67 / 67 Vast offers with only **25 stable across all + three**; 31 of the first 65 were gone by the second fetch. Observed **7 consecutive + `HTTP 400 Unknown SKU` failures across price ranks 0–25** — so this is not a cheapest-first + artifact, it is the whole catalog. A lease landed on attempt 3 of a random-pick loop, which is what + ~50% churn predicts. **This is the quantitative case for change 3 existing at all.** +- **A stability heuristic was falsified — do not resurrect it.** "Offers present in consecutive + catalog fetches are launchable" had _perfect_ separation retrospectively (n=8: the one SKU that + launched was in the 3-fetch stable set, all 7 failures were not) and **failed its first prospective + test** — a SKU in the stable set of two fetches still returned `400`. The attractive corollary, + that change 12's cache would hand change 3 its correctness fix for free by intersecting fetches, + is **retracted**. Retry is unavoidable. +- **`400` means two opposite things and the resolver must read the message.** RunPod advertises GPU + types with **zero capacity**; it returns `500` to Atlas and Atlas maps it to `400` — the _same + status_ as Vast's stale offer. The messages differ: + `"Unknown SKU '' for provider 'vast'."` versus + `"create pod: There are no instances currently available"`. **They need opposite responses**: a + stale offer means re-resolve the same requirement against a fresh catalog; no capacity means pick a + **different** SKU. Retrying the same offer on a no-capacity error loops forever. A resolver that + branches on status alone is wrong on one of the two providers. + + _As shipped, the resolver reads neither message._ Excluding every already-refused `(provider, sku)` + and re-resolving against a fresh catalog satisfies both requirements at once, without depending on + provider prose. The measurement above stands; only the prescription changed. + +#### Verified live 2026-08-02 — the resolver against real Vast and RunPod + +Three real leases through the real route, real provider keys, nothing faked below the HTTP boundary. + +- **The whole path works.** `{gpu: "RTX-3090", count: 1}` → resolved → leased in **5.1s** → `provisioning` + → `ready` with `ssh4.vast.ai` at **t+30s** via the reaper's promotion → `/connection` 200 with real + coordinates → released, `status: terminated` confirmed. Exactly one grant, no orphans, and zero + instances left under the operator key afterwards. +- **The ranking is right on live data.** Across the 24 canonical models present in a live catalog, every + `resolve()` result was correctly ordered, with winners split across both providers. + `max_hourly_cents` is inclusive at the boundary and excludes one cent below it. +- **Live coverage: 37% of Vast rows, 52% of RunPod's** — and the unmapped remainder is exactly the tail + the map is right to drop: `GTX 1060`, `Tesla P4`, `Titan Xp`, `Quadro P4000`, `RTX 3060 laptop`. +- **`limit: 512` is a no-op — Vast caps every query at 64 offers.** Verified at limits 64 through 2000 and + with the key absent: always 64. So the catalog is 64 cheapest + 64 premium-name offers, deduped to ~65 + rows. **"Cheapest" therefore means cheapest of a narrow window, not of Vast**; for a mid-tier card + there may be cheaper instances outside both windows. Pagination is the fix, and it is not built. +- **Churn measured on raw offers: 36% survive 40 seconds** (128 → 128 offers, 47 stable), with deduped + rows tracking it at 35%. A theory that this was mostly an artifact of Atlas's + cheapest-per-`(gpu_name, count)` dedup was tested and refuted: of 42 rows that left the catalog, 35 + were gone from Vast's next response too. + + > **Correction, 2026-08-03 — do not cite the sentence above as "84% real offer death".** That test + > compared two multi-name query windows, which is only valid if a window is the deterministic + > cheapest-64 of its filter. It may not be. A `gpu_name == "A100 SXM4"` query returned **45 offers + > priced below the shared window's own ceiling that the shared window did not contain** — impossible + > under a deterministic cheapest-64 — and two multi-name queries matching an identical live name set, + > issued 1.6s apart over the identical price range, shared only 27 of 64 ids. + > + > **But the effect is query-shape dependent, so neither claim generalises.** A narrow + > `num_gpus in [1]` query run twice 1.5s apart returned **identical** results: 64/64 shared ids, same + > price range, zero drops. So "the window is a sample" is not true of every query, and "36% of offers + > die per 40s" is not established either — the earlier figure cannot distinguish a dead offer from one + > the window simply did not return. + > + > What is unaffected: the retry is still justified, because a SKU that fails to launch must be + > replaced whatever the cause. What is *not* established is **why** a SKU goes stale, and any future + > design that depends on the answer — a staleness heuristic, a stability filter, a cache-freshness + > rule — needs this measured properly first, per-query-shape. +- **The retry did not fire in any of the three launches.** First pick succeeded every time, including + against a deliberately staled cache. Not a disproof — it means first-pick success is common — but the + retry path is still unexercised against a real provider refusal. +- **The 201 response never named the GPU — since fixed.** It carried `provider` and `requested_sku` (an + opaque Vast offer id like `42093969`) but no model, so a caller that asked for an `RTX-3090` could not + confirm from the response that it got one, and `GET /api/compute/leases` had the same gap for every + lease. `compute_leases` now carries `gpu_model` (the canonical id, **NULL when `canonical()` cannot + place the row — never a guess**), `gpu_name` (the provider's own string, always present, which is what + keeps a NULL model row useful) and `gpu_count`. Populated on the named path, the requirement path and + the agent-spawn path. Verified live: `{gpu: "RTX-3090", count: 1}` returns + `gpu_model='RTX-3090' gpu_name='RTX 3090' gpu_count=1`. +- **The retry fired live**, on a later run: `{gpu: "RTX-4090", count: 1}` drew RunPod's + `create pod: There are no instances currently available` — the verbatim string above. The loop excluded + that `(provider, sku)`, re-resolved against a fresh catalog, found no other `RTX-4090`, and returned a + structured `503` naming what it tried. No grant leaked. **Both halves of fact 4 are now observed in + production conditions, not just in tests.** + +**Why RunPod runs out of capacity is a query bug, not a stale catalog.** `list_options` asks for +`gpuTypes{ id, displayName, memoryInGb, lowestPrice{ uninterruptablePrice } }` — a **price list, not an +inventory**. `lowestPrice` is the cheapest anyone ever offered that model and says nothing about current +availability. Contrast Vast, whose `/bundles/?q={"rentable":{"eq":true},…}` catalog **is** +availability-filtered, which is why its options are at least launchable in principle. RunPod does publish +availability and we never request it: `lowestPrice{ stockStatus }` works, and across the 37 priced types +it reads Low 25 / Medium 3 / High 9 — with **all ten cheapest types "Low"**. Cheapest-first on RunPod +therefore selects systematically into the failure region. Cheapest "High" is $0.34/hr against cheapest +"Low" at $0.12, so **hard-filtering to High would roughly triple the price** — request the signal and use +it to order the fallback, do not filter on it. + +### Change 4 — quote a proposal without spending + +``` +POST /api/compute/quote { gpu, count, max_hourly_cents?, budget_cents } +→ { provider, sku, hourly_cents, effective_cap_cents, balance_cents, funding } +``` + +**Without this the permission gate cannot exist.** The gate must show provider, SKU, rate and effective +cap _before_ money moves, and the client is forbidden from computing any of them. Nothing today can +supply them: `POST /api/compute/estimate` requires an explicit `{provider, sku}` (`EstimateRequest`, +`routes/compute.py:238-241`), which a `{gpu, count}` proposal does not have, and there is no dry-run flag +anywhere in `routes/compute.py`. + +**The quote is advisory and is never reused.** `POST /leases` re-resolves from scratch; a quote token +carried into the lease call would reintroduce exactly the stale-offer race change 3 exists to narrow. The +agent may therefore be shown a rate that differs by cents from the one billed — acceptable, and the launch +response's `effective_cap_cents` is what the user is told they authorised. + +`funding` lets the prompt distinguish an operator-billed lease from an Atlas-BYOK one, which is charged +at rate 0. + +### Change 5 — bound cumulative spend, atomically + +The cap is per-grant and a grant is per-lease, so release-and-reacquire is unbounded. What exists is +`MANAGED_GPU_CONCURRENT` (read at `lease_manager.py:508`, default **2** at `config.py:117`), which bounds +concurrent boxes, not total cost. A $30 budget honoured twenty times is $600. + +Add a **rolling window cap** at lease creation, with window and ceiling as plan config alongside +`gpu_sandbox_max_ttl_hours`, and a distinct `error` code so clients can tell "this box is too expensive" +from "you have spent enough today". + +**It must be atomic, and so must the wallet clamp — but they need different mechanisms, and conflating +them is a vacuous-pass trap.** + +_The window cap_ can follow `debit_grant` (`compute_repo.py:187-201`), the one atomic primitive in the +money path: express it as a single conditional write rather than a `SUM` followed by an `INSERT`. + +_The wallet clamp cannot._ The balance lives behind `usage_service.effective_balance`, read with a bare +read-then-compare at `lease_manager.py:540-543` — and read **after** the grant was already debited at +`:529`. Two concurrent launches create two separate grants (`routes/compute.py:365`), each sized +`rate × 24h`, so `debit_grant` succeeds for both and the entire race sits in the wallet read. With a +default concurrency of 2 (`config.py:117`), two simultaneous launches on a $3 wallet both pass, and change +2's clamp makes it worse because each clamps to the _full_ effective balance and authorises 2× the wallet. + +The one reservation primitive that exists — the pre-auth `hold_id` path (`lease_manager.py:180-216`) — is +deliberately unused on the managed GPU path (`:456`, `:519-525`). So the wallet clamp needs either a real +hold or per-user serialisation, decided explicitly. **A `debit_grant`-shaped window cap satisfies +criterion 6's first half while the wallet still double-authorises**, which is exactly the failure mode the +closing section of this document is about. + +`compute_grants` is indexed on `user_id` and `session_id` only (`migrations.py:586-587`, +`pg_migrations.py:1062-1063`). `(user_id, created_at)` is the migration. + +Design it in now — retrofitting changes the meaning of a number users already trust. + +### Change 6 — attach a persistent volume _(larger than it looks, and cheapest-first makes it harder)_ + +Atlas has `POST /api/compute/volumes` (`routes/compute.py:565`), `list_volumes` and `delete_volume` — but +**they provision nothing.** `create_volume` clamps a size and writes a `compute_volume_repo` row; no +provider API is called anywhere. The only volume in the compute providers is RunPod's `volumeInGb: 20` +(`runpod_provider.py:159`), which is pod-scoped and destroyed with the pod. So this is not "pass an +existing volume through" — it is "make volumes real", per provider. + +**Cheapest-first turns that into a per-provider matrix.** The volume has to exist wherever the resolver +lands, and the four operator providers do not share a network-volume primitive with the same semantics. + +Resolution: **`volume_id` is a requirement, not a preference.** When the request carries one, the resolver +ranks only providers with real network-volume support and takes the cheapest of those. That is still +cheapest-first — a volume is a constraint like `gpu` or `count`, not an override of the pricing rule — and +it degrades honestly: a user who wants durable storage pays whatever the cheapest volume-capable provider +costs, and is told which one. + +Start with RunPod (`networkVolumeId`, mounted at `/workspace`) and add providers as their volume APIs are +wired. **Releasing a lease must not cascade a volume delete.** + +Budget exhaustion then costs the compute, not the work — a network volume is cents per GB-month against +dollars per GPU-hour. + +**Honest limit:** a volume preserves _files_, not _process state_. A run killed mid-epoch still dies +unless it checkpointed to `/workspace`. The volume is the substrate roadmap **56** needs, not a substitute +for it. + +### Change 7 — extend a live budget + +``` +POST /api/compute/leases/{lease_id}/budget { additional_cents } +→ { hard_cap_cents, spent_cents, effective_cap_cents } +``` + +Raises the cap on the existing grant, clamped by wallet and rolling cap. No new state — it edits a number +change 1 already reads every tick. Three constraints keep it from reopening the door earlier drafts +closed: + +- **Pull, never push.** Atlas never auto-extends. A budget that quietly refills is not a budget. +- **Not part of enforcement.** If no extension arrives, exhaustion proceeds unchanged, so a dead agent + costs nothing. +- **No warning event required.** An 80% notification is worth adding for humans, but it is advice; the cap + must never depend on anyone reading it. + +### Change 8 — release must not report success it did not achieve — **HONESTY HALF SHIPPED; THE DECISION BELOW IS NOT** + +> **Shipped** as `0eb33c2` (Vast) and `be6a3d4` (Prime, RunPod, and `release_lease` itself). +> +> - All three providers now distinguish a confirmed teardown from an unconfirmed one instead of +> returning `{"status": "terminated"}` after a bare `try/except: pass`. **404 is deliberately +> excepted** — a pod that is not there is the outcome we wanted, and calling it unconfirmed would +> strand the lease retrying a `DELETE` that can never succeed. +> - `release_confirmed` classifies through `lease_state.normalise_state`, so the reaper, +> `/connection` and release all read one table. +> - `release_lease` now **acts** on the verdict: an unconfirmed teardown logs +> `metric=lease_release_unconfirmed` and **leaves the row unfinished** so the next sweep retries. +> Previously the row was marked released regardless, which is what made even an honest provider +> answer inert. +> +> **⚠️ The section's own decision is NOT implemented, and the gap is the one it predicted.** An +> unconfirmed release leaves the row at its previous status, and both +> `list_active_leases` (`status NOT IN ('released','failed') AND hourly_rate_cents > 0`) and +> `count_active_managed_gpu_leases` (same predicate) still match it. **So the user who asked to +> release keeps being charged and keeps burning one of their two managed slots** — verbatim the +> failure this section says to decide against. `release_pending` as a distinct status, stopping +> billing at the request, freeing the slot, and a retry set that is not `list_unfinished_leases` are +> all still to build. **Acceptance criterion 16 is therefore only one-third met.** + +`LeaseManager.release_lease` swallows a provider teardown failure into +`provider_result = {"warning": …}` (`lease_manager.py:801-802`) and then **unconditionally** marks the row +released (`:808`). A released row leaves `list_active_leases` (`compute_repo.py:470-487`) so billing +stops, and leaves `list_unfinished_leases` (`:463-464`, `status NOT IN ('released','failed')`) so the +reaper never revisits it. **The box runs indefinitely on the operator's account with nothing metering +it**, and the route returns 200. + +Add a distinct terminal-pending status the reaper re-sweeps, or at minimum surface `provider_result` +so the caller knows teardown failed. Until then, "explicit release works" is only true when the provider +call succeeds. + +**A new non-terminal status is not free, and the spec must choose.** Every gate is +`status NOT IN ('released','failed')` — `list_active_leases` (`compute_repo.py:481`, which drives billing +in `tick_once`), `count_active_managed_gpu_leases` (`:347`, which drives `MANAGED_GPU_CONCURRENT`), and +`list_unfinished_leases` (`:464`). So a `release_pending` status lands inside all three: **the user who +asked to release keeps being charged and keeps burning one of their two managed slots**, indefinitely, +because the retry that would clear it is the one that keeps failing. + +Decide both explicitly: **stop billing at the release request** (the user asked; the operator's continued +exposure is an operator problem, not theirs) and **free the concurrency slot**, while keeping the row +sweepable for teardown retry. That means the retry set cannot be `list_unfinished_leases` — it needs its +own query. + +### Change 9 — SSH key lifecycle _(security prerequisite, ships before change 3)_ — **SHIPPED** + +> **9(a) shipped** as `7aac22b` and is the one item in this document proven by a **direct adversarial +> test rather than an inference.** The section below could only say "confirm the per-instance attach +> is sufficient, then remove the account registration". Both halves were then measured: +> +> - **The per-instance attach alone works.** A real Vast GPU, reachable over SSH with the returned +> key, with the account key list at `GET /ssh/` returning `[]` before and after — verified as a +> genuine empty list, not an error masked as one. +> - **The cross-tenant property itself, by cross-login.** Two instances on the **same** Vast operator +> account: key A → box A connected, key B → box B connected (the controls), and key A → box B +> **refused, `Permission denied (publickey)`**, as was B → A. Pre-`7aac22b` that would have +> connected. The exposure is closed, and closed for the stated reason. +> +> **9(b) shipped** as `f65dd47` (Prime) — a nullable `provider_key_id` column on `compute_leases` +> (sqlite + Postgres, additive and guarded), the real key **id** returned from `acquire` instead of +> the pod name, forwarded into `release`, and cleanup inside `acquire` when a launch fails after +> registration. **Vast needs no key deletion any more**, because 9(a) removed the thing that was +> leaking. The migration was later verified against a populated pre-existing database (36 lease rows, +> 5 users), not only a fresh temp one. +> +> **Two caveats stay open.** Prime's `DELETE /ssh_keys/{id}` is **documentation-verified only, never +> exercised against production**; it fails safe (swallowed, cannot block the pod delete), so a wrong +> endpoint would leak silently. And the "acquired but the `create_lease` DB write failed" path calls +> `provider.release(lease_id)` with no kwargs, so neither `provider_key_id` nor Lambda's +> `ssh_key_name` reaches release there — pre-existing, not a regression. + +Two distinct problems. The first is a live cross-tenant exposure; the second is hygiene that becomes +unbounded under cheapest-first. + +**9(a) — stop Vast registering account-level keys.** As above: `POST /ssh/` (`vast_provider.py:206-211`) +puts every lease's public key on the shared operator account so that new instances pick it up. Confirm the +per-instance attach at `:244-248` is sufficient on its own, then remove the account registration. If it is +not sufficient, the account key must be deleted **immediately after instance creation**, not on release — +deleting on release leaves it live for the whole lease, which is exactly the window that matters. + +Until 9(a) ships, managed Vast leases are mutually reachable. This is true of production today. + +**9(b) — delete the registered key on release and on failed launch.** Lambda is the pattern +(`lambda_provider.py:248-268`), but **it cannot simply be copied**, because Lambda works only through +machinery the other two lack: + +- Lambda's `acquire` persists `ssh_key_name = key_name` (`lambda_provider.py:196`), `_delete_ssh_key` + looks it up by that name (`:230-246`), and `release_lease` forwards it (`lease_manager.py:798-799`). +- **Prime Intellect persists the wrong string.** The key is registered under `key_name` + (`prime_intellect_provider.py:220`, `:228-230`) and the API returns an `id` (`:203`) used as `sshKeyId` + (`:245`) — but `acquire` returns `"ssh_key_name": name` (`:274`), the **pod** name, which is + `atlas-{user_id[:8]}` (`routes/compute.py:384`) and therefore identical across every lease that user + ever creates. A name-keyed delete would either miss or delete all of that user's keys. The `sshKeyId` is + discarded. +- **Vast has no identifier at all.** The key is posted as `{"ssh_key": public_openssh}` with no name + (`:207-211`) and the response is swallowed by a bare `except` (`:212-213`). +- **There is nowhere to store one.** `compute_leases` carries `ssh_key_name` and `ssh_public_key` only + (`migrations.py:549-553`, `pg_migrations.py:800-804`). +- **"On failed launch" has no hook.** `lease_manager.py:573-584` re-raises and + `_release_managed_reservation` (`:63-79`) touches only the grant and hold. Cleanup must live inside each + provider's `acquire`, as Lambda's does at `:186`. + +So 9(b) is: fix `prime_intellect_provider.py:274` to return the key id, add a nullable provider-key-id +column, capture Vast's response instead of discarding it, and put failure cleanup inside each `acquire`. +**Test against the provider's key listing, not against the release return value** — a release that returns +success having deleted nothing is the defect. + +### Change 10 — measure boot and availability, and rank on it + +**Cheapest per hour is a proxy that inverts.** Boot time is billed wall-clock: a six-minute provision on a +$2.16/h H100 spends $0.22 before any work starts, and a box that boots six times slower is not the cheap +one. A comparable aggregator publishes exactly this measurement across providers, and its 7-day +distribution is stark — RunPod a ~59s median with a tight spread, Vast a ~1m9s median with a tail running +past **6m** and a long scatter across the whole range. Vast is worse on the median and far worse on the +variance, and cheapest-first sends most launches to Vast. + +**An earlier draft claimed this dataset was free once change 0(a) landed** — `compute_leases.ready_at` +exists (`migrations.py:555`, `pg_migrations.py:806`) and `update_lease_status` stamps it on the transition +to ready (`compute_repo.py:425-427`), so `ready_at - created_at` looked like a boot-time series for +nothing. **Every one of those citations is correct and the conclusion does not follow, three times over.** + +**It is quantised to the sweep interval, and the signal is smaller than the interval.** `ready_at` is +`_now()` evaluated when the reconcile _poll_ observes readiness (`compute_repo.py:408`, `:426-428`), not +when the box became ready. Change 0(a) mirrors `_reconcile_active_cpu_leases`, and every existing sweeper +ticks at 60s (`REAPER_TICK_SECONDS`, `config.py:68`; `COMPUTE_BILLING_TICK_SECONDS`, +`compute_billing_service.py:44`). The discrimination this change exists to make is a ~59s median against a +~1m9s median — **a 10s difference under 60s quantisation, biased upward on both sides.** As specified the +measurement cannot resolve the signal it ranks on. + +**It is censored at the timeout, which makes the timeout criterion circular.** A boot exceeding +`PROVISION_TIMEOUT_SECONDS` is reaped (`lease_reaper.py:133-136`) and never gets a `ready_at`, so the +sample is truncated at exactly the value we wanted to derive from it. The p99 of the surviving sample can +never exceed the timeout that produced the sample. + +**The availability series does not exist.** A reaped lease and a user-released lease write the same row: +`_reap` → `release_lease` → `compute_repo.release_lease` sets `status = 'released'` for both +(`compute_repo.py:549-556`). **Nothing ever writes `'failed'` to `compute_leases`** — the string appears +only inside `NOT IN` filters. The reap reason exists only on an `agent_telemetry` row +(`lease_reaper.py:79-85`), an unmentioned join, and compute-route leases have a `NULL` `node_id` +(`routes/compute.py:303`). + +So change 10 needs real instrumentation, and its cost must be priced rather than assumed away: + +- **Record boot duration explicitly**, not as a difference of two poll-quantised timestamps. Either poll + provisioning leases on a tighter cadence than the 60s sweep — which is a provider API call per + provisioning lease per tick, a cost this document must state — or take the provider's own ready + timestamp where it reports one. +- **Record a reap reason on the lease**, so released-by-user and reaped-for-timeout are distinguishable + without joining telemetry that compute leases do not write. +- **Derive timeouts from a deliberately uncensored window** — a period with the timeout raised well past + the expected tail — rather than from the post-reap sample. + +Then aggregate per `(provider, canonical_gpu)` over a rolling window and use it two ways: + +- **A floor.** Exclude offers from a provider whose recent failure rate or p95 boot time is beyond + threshold, then rank the survivors. This keeps the product rule — cheapest wins — while measuring + "cheapest" correctly. +- **Timeouts** (change 0). + +**Cold start must degrade to today's behaviour.** With no history a provider is not penalised; ranking is +pure price until enough leases exist to say otherwise. A floor that silently excludes every provider on +day one is worse than no floor. + +Vendor-published signals are worth folding in where they exist and cost nothing — +`VastProvider.list_options` currently discards `reliability2`, `dlperf_per_dphtotal` and `inet_down` from +every offer it reads. `inet_down` matters more than it looks: pulling a 200 GB dataset at 50 Mbit instead +of 5 Gbit is hours of GPU time billed for waiting. **But measured beats published** — vendor scores +describe the host, our telemetry describes what actually happened to our leases. + +> **Measured 2026-08-01 (n=7 Vast launches, one session) — a threshold is not enough; rank on the +> score.** +> +> | Selection | Launches | Never booted | +> | ------------------------------------- | -------- | ---------------------------- | +> | cheapest-first, no filter | 4 | 1 | +> | `reliability >= 0.98`, then cheapest | 2 | 1 | +> | **ranked by `reliability2` (0.9993)** | 1 | **0** — up in 50s, first try | +> +> A 0.98 **threshold did not help**: it admitted an offer that never booted. Ranking by the score +> did. n is tiny — treat it as a direction, not a coefficient — but it argues for the score entering +> the **ranking**, not just a floor, which is a stronger claim than this section currently makes. +> +> **This generalises past Vast: every provider publishes a quality signal and we discard all of +> them.** Vast: `reliability2`, `inet_down`, `dlperf_per_dphtotal`. RunPod: `lowestPrice{ stockStatus }` +> (see change 3 — all ten of its cheapest types are "Low"). The cheapest fix available today is to +> **request the signals we already could and use them to order the fallback**, which needs none of +> the boot-telemetry instrumentation below and would have prevented 3 of the 9 observed launch +> failures. That is a smaller, earlier change than the rest of this section and should be split out. +> +> **One open risk, recorded not fixed:** Vast `actual_status=offline` maps to `stopped` → terminal in +> our chain, so a healthy box reporting `offline` transiently would be reaped after two strikes. It +> is unknown whether healthy boxes ever do; it needs a long observation of a running instance. + +### Change 11 — pin the image, per provider + +**Nothing in this design says what is on the box**, and the answer today is inconsistent in a way that +breaks reproducibility: + +| Provider | Image | Pinnable? | +| --------------- | ------------------------------------------------------------------------------- | ----------------------------- | +| RunPod | `runpod/pytorch:2.4.0-py3.11-cuda12.4.1-devel-ubuntu22.04` (pinned, has `nvcc`) | yes — `imageName` in the body | +| Vast | `pytorch/pytorch:latest` (`vast_provider.py:220`) — **a floating tag** | yes — `image` in the body | +| Lambda | none set — whatever the provider defaults to | **no image parameter exists** | +| Prime Intellect | none set | **no image parameter exists** | + +Under cheapest-first the provider varies per launch, so **the environment varies per launch too** — and +`latest` means the same experiment run a month apart gets a different toolchain with no record of it. +For a tool whose purpose is reproducible research, that is a correctness bug, not an ergonomics one. + +Define a **minimum environment contract** the agent may rely on and nothing more — a pinned CUDA runtime, +a pinned Python, and a package manager. Anything beyond it the agent installs itself, and **that install +is billed at GPU rates out of the user's budget**, which is the argument for keeping the contract small +and the image warm rather than bootstrapping from bare Ubuntu on every launch. + +**Only two of the four providers can be pinned at all.** Lambda's launch body is +`{region_name, instance_type_name, name, quantity, ssh_key_names}` (`lambda_provider.py:174-180`) and +Prime's pod body is `{name, cloudId, gpuType, socket, gpuCount, …}` (`prime_intellect_provider.py:231-249`) +— neither takes an image, and Prime fronts heterogeneous upstreams. An earlier draft demanded pinning for +"every provider the resolver can select" while its own table said two were unset; that criterion could not +have passed. + +Resolve it the same way change 6 resolves volumes: **satisfying the environment contract is a +requirement**, so a provider that cannot be pinned is excluded from ranking until its image story is +understood — Lambda ships a fixed OS image that may already satisfy the contract, in which case it is +declared compliant rather than pinned, and that determination is the work. This narrows the pool exactly +like `volume_id` does, and for the same reason: cheapest wins **among boxes that can do the job**. + +**Needs a migration.** `compute_leases` has no image column (`migrations.py:532-584`). Record the resolved +image on the lease so a run can be reproduced later. + +### Change 12 — cache the options catalog _(prerequisite for 3 and 4)_ + +`_catalog` is uncached, and every consumer rebuilds it: the availability probe on every `compute_status`, +`/quote` on every launch, `POST /leases` on every launch, and once more per resolver retry. Five provider +requests each at 15–20s timeouts. + +Two earlier sections say this must ship before change 3, and an earlier draft then filed it under "known +defects, owned elsewhere" — no number, no owner, no criterion. **It is a numbered change because two +others depend on it.** + +Cache per `(user_id, byok_eligible)` with a short TTL, and make the resolver's retry re-fetch +deliberately rather than reusing a cached miss — the retry exists precisely because the cached offer went +stale. + +### Change 13 — reap orphaned user leases + +Change 0 removes both reaper branches that currently apply to user leases, and `create_lease` mints no +runner token — `set_runner_api_key` has exactly one caller, the spawn path (`agent_tools.py:1755`) — so a +user lease can **never** be heartbeat-reaped by design. That is correct: it is a bound those leases cannot +satisfy. + +But it leaves a hole the rest of the design assumes is closed. If the OpenScience process dies between +`POST /leases` and the readiness poll's release — the exact case the release-on-timeout rule exists for — +**nothing releases the box.** `budget_cents` is optional (change 2) and the default grant is `rate × 24h`, +so the worst case is a full-TTL H100 the user never reached and whose key was never written to disk. + +Two options, and the spec picks the first: **make `budget_cents` mandatory on the tool path**, so an +orphan is bounded by the budget rather than the TTL. Second, additionally: reap a lease with no runner +token whose `/connection` has not been fetched in N minutes — liveness a user lease _can_ answer, unlike +telemetry. + +## OpenScience changes + +### Three verbs + +| Tool | Input | Output | +| ----------------- | ----------------------------------------------------------------- | ------------------------------------------------------------------------------------------------- | +| `compute_launch` | `gpu`, `count`, `budget_cents`, `max_hourly_cents?`, `volume_id?` | `lease_id`, `ssh_host`, `ssh_port`, `ssh_user`, `key_path`, `effective_cap_cents`, `hourly_cents` | +| `compute_list` | — | unfinished leases: `lease_id`, `provider`, `sku`, `status`, `ssh_host`, `ssh_port`, rate, cap | +| `compute_release` | `lease_id` | released, plus any provider teardown warning | + +Plus `compute_status` (shipped, unchanged). Separate tools rather than one `action` parameter, so the +permission rule can ask on launch and allow on list. + +The agent does the actual work with plain `ssh`/`scp` from `bash`. **No relay, no exec wrapper, no +file-transfer helper** — that path is a remote-execution platform, and it was already designed once and +rejected. + +Behaviour: + +- On `402`, surface `affordable_budget_cents` and stop. **Never auto-retry at a smaller budget** — a + truncated training run is not a cheaper result, it is a discarded one. +- On `429` (concurrency cap) and `409` (already released, `routes/compute.py:531-535`), surface rather + than retry. +- A non-2xx or malformed launch response **writes no `.pem` and reports no lease**. +- **The readiness poll outlives the server's provisioning timeout, and defers to it.** A client bound set + _shorter_ than the server's is the bug to avoid: measured Vast boots run past six minutes, so a + three-minute client timeout would kill launches that were about to succeed. On timeout with the lease + still live, **release it** — a paid box the agent cannot reach is worse than no box — but a lease the + server has already reaped needs reporting, not releasing. +- **No client-side deadline timer**, price table, or SKU ranking. + +Two prerequisites for that poll, neither of which exists today. + +**The bound is a server constant no endpoint returns.** `PROVISION_TIMEOUT_SECONDS` is server env config +(`config.py:70`) and change 0 makes it per-provider. It appears in no response — not `/options`, not +`/estimate`, not the proposed `/quote`, not the launch payload — and under cheapest-first the client does +not know which provider it got until the launch returns. Telling the client to bound at "that provider's +timeout" while exposing neither the provider nor the timeout means hardcoding a duplicate of a server +constant that silently goes stale. **Change 2's response must carry `provider` and +`provisioning_timeout_seconds`.** + +**There is no status vocabulary to poll on.** `routes/compute.py:500` returns +`conn.get("status") or lease.get("status")` — two disjoint vocabularies in one field. On the provider +path RunPod maps `running|stopped|terminated|unknown` (`runpod_provider.py:39-43`), Vast +`running|provisioning|stopped|unknown` (`:55-65`), Prime +`provisioning|active|stopped|error|terminating|terminated|unknown` (`prime_intellect_provider.py:42-51`), +and **Lambda passes the raw upstream string through unmapped** (`lambda_provider.py:214`, `:224`). When +the provider call throws it falls back to the DB status: `provisioning|ready|released`. + +So `"running"` is unreachable on the DB path, `"ready"` — what change 0(a) writes — is unreachable on the +provider path, Prime says `"active"`, Lambda says whatever Lambda says, and a dead RunPod pod maps to +`"unknown"`. **A client written against either vocabulary breaks on at least two of the four providers.** +Normalise the field as part of change 0 and enumerate the accepted values here; "poll until running" and +"poll until terminal" are both unimplementable as written. + +**Polling is not free either.** `get_lease_connection` calls `provider.connection()` on every poll +(`lease_manager.py:742-753`) — an uncached provider HTTP GET. A six-minute Vast boot polled every 5s is +~70 operator-key provider calls per launch. Specify the interval and a backoff. + +### SSH coordinates do not exist at launch + +Every provider's `acquire()` returns a hardcoded `ssh_port: 22` and **no `ssh_host`** — +`runpod_provider.py:183-191`, `lambda_provider.py:190-198`, `vast_provider.py:257`, +`prime_intellect_provider.py:273`. The real values live only in `provider.connection()` +(`runpod_provider.py:214-232` → `ssh_host: pod.publicIp`, `ssh_port: _ssh_port(pod)`), reachable through +`GET /leases/{id}/connection`. + +So `compute_launch` **must poll `/connection`** before returning. A tool that returns the launch payload +directly hands the agent `ssh_host: null, ssh_port: 22` — and a test asserting "`-p` appears when the port +is not 22" passes vacuously, because the port at launch is always 22. + +**`-p ` is required, not optional.** RunPod NATs SSH to a high public port and Vast routes +through an ssh-proxy port; `ssh -i key user@host` times out on both. Assert it against a real +RunPod-shaped `/connection` payload, never a launch payload. + +### The agent never holds key material + +`compute_launch` writes the private key at `/compute/.pem` +(`src/global/index.ts:82`). **Not a hardcoded `~/.config`**, which is wrong whenever `XDG_CONFIG_HOME` is +set — and **not `:46`**, which an earlier draft cited and which is the _cache_ directory (`:47` is +config). A private key under a cache path is the wrong answer twice over. +and returns only `key_path`. The key stays out of the transcript, out of compaction, and out of session +storage. `compute_release` deletes it. + +Modes are load-bearing in both directions — `ssh` refuses a group-readable key outright, so this is a +functional requirement as much as a security one: + +- `mkdir(dir, { recursive: true, mode: 0o700 })` +- `writeFile(path, key, { mode: 0o600 })` **followed by an explicit `chmod`** — `mode` is ignored when the + file already exists, so a re-fetch over a stale loose-permission `.pem` would stay loose. +- **`Bun.write()` has no `mode` option.** The house style prefers Bun APIs; here it produces `0644` and a + broken feature. Use `node:fs/promises`. + +### Atlas is the truth for everything, including the key + +The private key is **not** one-time and **not** local state. Atlas stores it encrypted on the lease row +(`compute_leases.ssh_key`, via `secret_store`) and `GET /api/compute/leases/{lease_id}/connection` +decrypts it for the authenticated owner (`routes/compute.py:450-508`), added so the Compute tab could +offer a reliable download after a page reload. `GET /api/compute/leases` redacts the blob +(`_redact_lease:424`); the connection endpoint is the one that returns it. + +So the `.pem` on disk is a **cache, not a record**. If it is missing — new machine, cleaned config dir, +another session — re-fetch and rewrite it. OpenScience keeps no durable local state it would have to +reconcile against Atlas. + +**But a cache with no eviction is not a cache.** `compute_release` is the only deletion path, and every +_server-side_ termination bypasses it: wallet exhaustion (`compute_billing_service.py:172`), plan TTL +(`:342`), the reaper (`lease_reaper.py:58`). Each leaves a `0600` private key on disk permanently. Sweep +on `compute_list` — any local `.pem` whose lease Atlas reports as terminal gets deleted — so the eviction +rides on a call the agent already makes. + +`compute_list` calls `GET /api/compute/leases`, which returns `SELECT *` over every lease for the user, +newest first (`compute_repo.py:446-453`). It returns **terminated leases too**, so the tool filters to +non-terminal status rather than presenting the raw list as "what is running". + +### The approval gate + +**There is no generic per-tool permission gate.** `PermissionNext.evaluate`'s `ask` default +(`src/permission/next.ts:237`) is only consulted when a tool explicitly calls `ctx.ask` +(`src/tool/tool.ts:25`; `src/tool/bash.ts:148,157` is the precedent). A `compute_launch` that omits +`ctx.ask` prompts for nothing and never reaches `evaluate`. + +So: `compute_launch` **must call `ctx.ask({ permission: "compute_launch", … })`** with the quote's +provider, SKU, hourly rate, effective cap and balance. `evaluate` then defaults to `ask`, and `Permission` +has a `.catchall` (`src/config/config.ts:642`), so no config schema change is needed. + +Settings ▸ Compute exposes it as `permission.compute_launch`, so a user who wants unattended overnight +runs sets `allow` deliberately. **This is UX, not enforcement** — a fork can delete it. The server caps +are what actually bind. + +--- + +## The bounds that remain + +**Updated 2026-08-01.** The July version of this table was source-read against Atlas HEAD `7b0e9b6`, +not a running deploy. Five of its seven rows have since changed, and the ones marked _deployed_ were +exercised against a live backend rather than inferred. + +| Bound | Owner | Fires when | Today | +| -------------------- | -------------- | --------------------------------- | --------------------------------------------------------------------------------------------------------------------------------- | +| `hard_cap_cents` | billing tick | approved money is spent | **enforces** — ch. 1 shipped. $10 @ $6.99/h releases at 5160s vs 5150s theoretical | +| Per-lease budget | lease creation | a caller states one | **accepted** — ch. 2 shipped. `budget_cents`, clamped to the wallet, effective cap reported | +| Rolling window cap | lease creation | cumulative spend hits the ceiling | **does not exist** — ch. 5. Release-and-reacquire is still unbounded | +| Wallet exhaustion | billing tick | money actually runs out | **works** _(deployed: wallet decremented in lockstep with `total_spent_cents`)_, **still races** — ch. 5 | +| Plan TTL (24h) | billing sweep | anything has run absurdly long | **works** (`_release_stale_gpu_leases:303`) | +| Explicit release | agent/user | asked | **honest now, but not acted on fully** — ch. 8 half shipped: an unconfirmed teardown keeps billing and keeps the concurrency slot | +| Provisioning timeout | lease reaper | a lease never boots | **no longer fires on live user leases** — ch. 0(a) shipped _(deployed: `ready` at 788s)_ | +| Heartbeat staleness | lease reaper | a booted lease stops reporting | **scoped to leases holding a runner token** — ch. 0(b) shipped | +| Agent-spawn ceiling | spawn path | a spawn's budget is exhausted | **was a flat $5 kill for every SKU; now sized to the spawn's own lifetime at the SKU rate** — `71dccba` | + +All server-side; none client-influenceable. Change 0 _removes_ two bounds from user leases, which is safe +precisely because the others apply and necessary because they are bounds those leases cannot satisfy. + +**The gap that is now the sharpest.** Every bound above is per-lease. With the cap binding and budgets +accepted, **change 5 is the only thing standing between "a run is bounded" and "a user is bounded"** — a +$30 budget honoured twenty times is still $600, and the wallet clamp still double-authorises under +concurrency. It was the least urgent item on this list in July and is the most urgent now. + +`COMPUTE_BILLING_TICK_SECONDS` defaults to 60 (`compute_billing_service.py:44`) and +`FIRST_BILL_GRACE_SECONDS` is 30 (`:52`), so a budget can overrun by up to ~90s of rate (~$0.17 on an +H100). **Approved budgets are ceilings-plus-90-seconds and must never be described as exact.** + +--- + +## What we deliberately did not build + +Design exploration reached a persistent runner daemon on the box, a command queue, a result stream, +scoped storage credentials and a PTY relay — a remote-execution platform, designed before a single box had +been leased through the agent. + +The test that settled it: **a productivity feature is admissible only if its absence changes nothing about +enforcement.** Volumes pass (files move off the box before anything fails). Optional extension passes. A +command relay, per-lease `expires_at`, client-side timers and auto-extension all fail — each needs +something alive to act on a signal. + +Also excluded: an ephemeral sandbox/exec path alongside SSH leases. It has no user-facing endpoint today +and cannot run multi-hour training, which is the use case that motivated GPU compute at all. Volume +creation stays in the workspace UI rather than becoming a fourth tool. + +## What you are accepting + +- **A run killed mid-epoch that was not checkpointing is gone.** No server-side mechanism can fix that. + Change 6 converts "you lost the job" into "you lost the GPU"; roadmap **56** is what makes it good. +- **Time is unbounded within a budget.** A cheap CPU lease could run for days inside a small budget. The + 24-hour plan TTL is the only backstop, deliberately. +- **The quoted rate is advisory.** The lease re-resolves, so the billed rate can differ by cents. +- **The provider varies per launch, and so does the box.** Cheapest-first means image, disk, region and + network differ run to run. The prompt names the provider for exactly this reason. A user who needs a + specific one passes `provider`/`sku` explicitly, which still works. +- **Durable storage narrows the pool.** Asking for a volume means paying the cheapest volume-capable + provider, not the cheapest provider. +- **Boot time is billed and varies by minutes across providers.** The floor bounds how bad it gets; it + does not make a marketplace box boot like a datacenter one. +- **The environment is a contract, not a guarantee of parity.** Two providers satisfying the same pinned + CUDA/Python contract are still different machines. +- **`none` is guidance, not enforcement.** The agent still has `bash`. + +## Known defects, owned elsewhere + +- **Atlas CLI unpublished.** `@synsci/atlas@0.13.2` on npm carries 155 command specs and zero `compute:`; + `3e1d1ca` removed them, `205bbc0` re-added them, no version bump followed. A release, not code. +- **CLI usability.** Prints the private key and never saves it, so the `ssh_command` it prints cannot + work as shown. Recoverable — `/leases/{id}/connection` re-serves the key — but the CLI does not call it. + No file transfer, no exec, no compute tests. +- **Six of the ten `RESELL_PROVIDERS` are `ScaffoldProvider` stubs** — registered, `operator: false`, zero + options, no network call. Scaffolding, not integrations. +- ~~**`budget_cents` already exists** on the agent-spawn path defaulting to `500`, display-only. Change 1 + makes caps real, silently giving every shipped spawn a hard $5 kill.~~ **FIXED** (`71dccba`), and it was + worse than this entry claimed. Every spawn grant was `hard_cap_cents = 500` flat for **every SKU** — no + caller in either repo ever sent anything else. A spawn requests 4 hours; that is $14.60 on an + A100-40GB, $18.36 on an A100-80GB, $27.96 on an H100, so only T4 and A10G fitted under $5. Measured on + the real spawn path against the real tick: **a 4-hour A100 spawn was killed at 1.38h**, 35% of its + requested life. An H100 was **refused outright at acquire** ("need 699 cents, have 500") — that half + was pre-existing, dating to `1e49dc9`, which is itself evidence the 500 was never sized against any + GPU. `budget_cents` is now `int | None`; `None` sizes the ceiling to the spawn's own lifetime at the + SKU rate, with a one-hour floor, and an explicitly chosen budget still binds. Only CPU spawns were ever + safe, because they come out `funding=byok` at rate 0 and the tick skips them. +- **Stale skill names in agent prompts.** `research.txt:357` and `ml.txt:193` name skills by directory + rather than frontmatter `name` (`vllm` → `serving-llms-vllm`), so the agent is told it has skills it + cannot load. `skills/scholar-evaluation/SKILL.md` has no frontmatter at all. _(Still open at + 2026-08-01; the line numbers moved from `:353`/`:192` when this branch edited the prompts — + re-verified, the defect itself is unchanged.)_ +- **`compute_status` no longer names capabilities the client lacks.** Recorded because it is the same + defect class one layer up, found by the same live testing. At a **zero wallet**, + `/api/compute/options` still reports managed providers — availability and affordability are + independent there — so the tool resolved `mode=managed`, told the agent to run GPU work through + managed compute, **and forbade the BYOK fallback**, while every acquire would return `402`. Fixed on + `feat/compute-guardrails` (`784633e`, then `05ef093f`): the guidance now says the client cannot + launch a managed lease at all, and separately that an empty wallet would be refused. `state.mode` is + deliberately unchanged — managed is genuinely configured; missing funds and a missing tool are not a + missing capability. **Known boundary:** a $0.01 wallet still gets the funded wording and would also + `402`. Only `balance === 0` is gated, because acquire needs one hour of the chosen SKU's rate and + rates span cents to dollars in a catalog the tool never sees, so any non-zero cutoff is a guess. + Fixing it properly means threading the cheapest rate through `/api/compute/options` — spec work. + +## Out of scope + +Roadmap **61** (per-job secrets), **56** (checkpointing), **4** (BYOK provider API clients), **52** +(`bun:sqlite`). Per-lease `expires_at`, client-side deadline timers, client-side price tables, +auto-extension, sandbox/exec paths. Any change to how `billing.llm` resolves. + +--- + +## Testing + +Atlas follows `backend/tests/test_compute_billing.py` — `_FakeProvider`, `aiosqlite` + `run_migrations`, +plain `pytest` before any deploy. OpenScience stubs `globalThis.fetch` and exercises the real tool; no +mocks, no network. + +- **A budget of $B at $R/h lasts B/R hours ± 90s of rate**, asserted on elapsed billable duration. The + tolerance is not a detail: it is `COMPUTE_BILLING_TICK_SECONDS` + `FIRST_BILL_GRACE_SECONDS`, and + leaving it unstated lets an implementer pick a tolerance that hides the acquire-debit double-count. +- A user lease survives past **both** `PROVISION_TIMEOUT_SECONDS` **and** `HEARTBEAT_STALE_SECONDS`; a + runner-token lease is still reaped for heartbeat staleness. Testing only the second passes green while + the box dies at ten minutes. +- The GPU reconcile pass flips `provisioning` → `ready` **and persists `ssh_host`/`ssh_port`**. +- A replayed tick does not double-charge and does not double-debit the grant. +- **Two concurrent launches against a wallet that funds only one**: exactly one succeeds. Asserted + separately from the rolling cap, because a `debit_grant`-shaped window cap passes the cap half while the + wallet still double-authorises. +- The rolling cap rejects an N+1th lease even when each individual budget is affordable. +- Resolution picks the **globally cheapest** matching offer across all operator providers — asserted with + a fake catalog where the cheapest match is deliberately not the first provider polled, and again where + it is not the provider the previous test picked. A resolver that always returns one provider must fail. +- Ranking is on the **funding-adjusted** rate: a catalog holding a cheaper billed offer and a dearer BYOK + offer resolves to the BYOK one. +- Resolution is exercised at `count > 1`, not only `count = 1`. +- GPU-model matching is canonical, not substring: `H100-SXM`, `H100-PCIe` and `H100-NVL` are three + distinct targets and never satisfy each other; the same card spelled differently across providers maps + to one id; an unmappable option is excluded rather than guessed. +- The reliability floor excludes a provider whose recent record is beyond threshold **and** leaves ranking + on pure price when there is no history — a cold-start floor that excludes everything must fail. +- Boot telemetry has **resolution finer than the difference it must detect** — a recorded duration, not a + difference of 60s-quantised poll timestamps — and a reaped lease is distinguishable from a + released-by-user one without joining `agent_telemetry`. +- A provision that exceeds a short global timeout but fits that provider's measured p99 is **not** reaped. +- The resolved image is recorded on the lease row, and no provider is launched on a floating tag. A + provider that cannot satisfy the environment contract is excluded from ranking, not launched anyway. +- Resolution leases the offer it ranked; a provider `400` triggers at most N re-resolves against a + re-fetched catalog, then a structured error. +- **Vast registers no account-level SSH key** — asserted against the account key listing after a launch, + which is the cross-tenant property, not the hygiene one. Then: a second user's instance created while + the first user's lease is live is **not** reachable with the first user's private key. +- Release deletes the provider-side key for Vast and Prime Intellect, on normal release and on failed + launch, asserted against the provider's key listing. Prime's stored identifier is the key id, not the + pod name — a test that passes with `name` in that field is testing nothing. +- The launch response carries `provider` and `provisioning_timeout_seconds`. +- The `/connection` status field is normalised: every provider's terminal and ready states map to one + documented vocabulary, including Lambda's unmapped upstream strings. +- A terminated lease's `.pem` is deleted on the next `compute_list`, including when the termination was + server-side. +- `compute_launch` in `byok` or `none` refuses with the reason rather than attempting a managed lease. +- The quote spends nothing and creates no lease row. +- `volume_id` mounts, and **release does not delete the volume**. +- Extension raises the cap, is clamped, and **exhaustion proceeds normally when none arrives**. +- A tick exceeding `hard_cap_cents` charges the elapsed time and then releases; one that fits does not. +- A no-budget lease still runs the **full** plan TTL after change 1 — asserted on runtime, not on the + request being accepted. +- A release whose provider teardown fails does not silently present as success. +- OpenScience: the key is written `0600` in a `0700` directory and **never appears in the tool result**; a + re-fetch over an existing loose-mode file tightens it; a malformed launch response writes no `.pem`; a + readiness timeout releases the lease; the connect string carries `-p ` against a real + `/connection` payload; `compute_list` filters terminated leases; `402`/`429`/`409` surface without retry. + +Every new assertion must be shown failing against the specific mutation it guards — ideally the _deletion_ +of the logic, not its inversion. On this branch alone, five plan-authored test defects were caught in +review, including an assertion that compared whole tool outputs and so passed even with all guidance +collapsed to one string. + +## Acceptance criteria + +> **Met as of 2026-08-01:** **0, 1, 2, 3, 4, 5, 9(a), 9(b), 15**, and criterion 25's ordering clause +> (change 0 was its own work; changes 1 and 2 landed together). **Partly met: 16** — a release whose +> teardown fails is no longer reported as clean and the row stays sweepable, but it does **not** stop +> billing and does **not** free the concurrency slot. **Not met:** 6, 7, 8, 10, 11, 12, 13, 14, 17–24 +> — every one of which needs either the resolver, the quote endpoint, or a tool this client does not +> have. Criteria are annotated here rather than deleted; a criterion that has passed is still the +> thing that would catch a regression. + +0. A user lease survives past both `PROVISION_TIMEOUT_SECONDS` and `HEARTBEAT_STALE_SECONDS`; a + runner-token lease is still reaped for heartbeat staleness; the reconcile pass persists + `ssh_host`/`ssh_port`. +1. A budget of $B at rate $R/h lasts B/R hours ± 90s of rate, asserted on elapsed billable duration. +2. A replayed tick neither double-charges the wallet nor double-debits the grant. +3. The billing tick re-debits the grant cumulatively; a tick exceeding the cap charges elapsed time and + then releases. +4. A lease created without `budget_cents` runs the full plan TTL — unchanged **runtime**, not merely an + accepted request. +5. A budget exceeding the wallet is clamped, and the response reports the effective cap. +6. **Separately asserted:** (a) two concurrent launches against a wallet funding one — exactly one + succeeds; (b) the rolling cap rejects an N+1th lease. One test covering both passes while the wallet + still double-authorises. +7. `{gpu, count, max_hourly_cents}` resolves to the **globally cheapest** matching offer across all + operator providers, ranked on the **funding-adjusted** rate — a cheaper billed offer never beats a + dearer BYOK one — proven against a catalog where the winner is neither the first provider polled nor + the same provider twice, exercised at `count > 1`, honouring `max_hourly_cents`, matching GPU models + canonically, retrying a provider `400` against a re-fetched catalog at most N times, then failing with + a structured error. +8. `POST /quote` returns provider, SKU, rate, effective cap, balance and funding, and **spends nothing**. +9. **(a)** A launch registers no account-level SSH key on Vast, and a second user's instance created while + another user's lease is live is not reachable with that user's private key. **(b)** Release deletes the + provider-side key on Vast and Prime Intellect, on normal release and on failed launch, asserted against + the provider's key listing; Prime stores the key id, not the pod name. +10. Boot telemetry resolves finer than the difference it must detect, and a reaped lease is + distinguishable from a released-by-user one without joining `agent_telemetry`. Ranking applies a + reliability floor and falls back to pure price with no history. +11. `PROVISION_TIMEOUT_SECONDS` is per-provider and derived from an **uncensored** measurement window; a + provision that exceeds a short global constant but fits its provider's p99 is not reaped. +12. Every provider the resolver can select satisfies the environment contract with a non-floating image, + the resolved image is recorded on the lease, and a provider that cannot be pinned or declared + compliant is excluded from ranking. +13. `volume_id` attaches a volume that survives lease release, and narrows the resolver to volume-capable + providers rather than failing. +14. Extension raises the cap when affordable, refuses with a structured `402` when not, never fires + automatically. +15. BYOK ignores `budget_cents`. Plan TTL fires independently. +16. A release whose provider teardown fails is not reported as a clean release, **stops billing**, and + **frees the concurrency slot** while remaining sweepable for teardown retry. +17. An orphaned lease — client dead between creation and the first poll — is bounded by its budget, not + only by the 24h TTL. +18. `compute_launch` on a non-2xx or malformed response writes no `.pem` and reports no lease; its + readiness poll is bounded by the `provisioning_timeout_seconds` the launch response carried, against a + normalised status vocabulary; it never returns key material; it holds no pricing or selection logic. +19. The key is written `0600` inside a `0700` directory, under `Global.Path.config` (**not the cache + dir**), and a re-fetch over an existing loose-mode file tightens it. +20. A terminated lease's `.pem` is deleted on the next `compute_list`, including on server-side + termination. +21. `compute_launch` calls `ctx.ask` and prompts by default; `permission.compute_launch: "allow"` silences + it. +22. The connect string carries `-p ` when a real `/connection` payload reports a non-22 port. +23. `compute_list` filters terminated leases and reflects Atlas, not local state. +24. `compute_launch` in `byok` or `none` refuses with the reason rather than attempting a managed lease. +25. `pytest` and `bun test` pass with no network. Change 0 is its own commit; **changes 1 and 2 land + together** — shipping 1 alone ships the ~23h regression. + +--- + +## Verification discipline + +Four conclusions in the predecessor investigation came from reading source and were wrong about deployed +reality: the CLI's contents, whether the prompt was broken, whether managed compute was reachable, and the +claim that reselling was off — **the fourth made by a correction to the third.** + +Labels in this document: + +- **Verified against production `thesis-synsc` (2026-07-31):** `resell_enabled: true`; lambda / runpod / + vast / prime_intellect operator-funded with 292 launchable options; `/compute/estimate` returns + `funding: "managed"` with a real rate and runway; the published npm artifact contains no `compute:` + command; `compute:up`'s default path fails on the Vast SKU race while `--provider lambda|runpod` + succeeds. **The 204-of-292 Vast share is load-bearing — it is why cheapest-first makes Vast the common + path, and therefore why change 9 is in scope — and it comes from this production check, not from + source.** +- **Verified against the Atlas checkout at HEAD `7b0e9b6` (source-read, 2026-07-31):** the `file:line` + citations in Part B, re-checked after two adversarial reviews. The claim "every citation verified" + has now been falsified twice; treat it as "checked, not guaranteed", and check the one you are about to + build on. +- **Not verified:** `compute:up`'s internal fetch→pick→estimate sequence. The Atlas CLI source is in + neither repo; the commit chronology corroborates the shape but the sequence itself is inferred. +- **Verified on a deployed build of `feat/compute-lease-prerequisites` (2026-08-01):** the whole + managed path, end to end, on **both** Vast and RunPod — `POST /leases` → the real background reaper + promoting within one sweep with NATed SSH coordinates → SSH into a real GPU → release, with the + instance confirmed gone at the provider. Plus: `GET /connection` returning the normalised + `state: "ready"`; a user lease `ready` and un-reaped at 788s on a wall clock; a double release + returning `409`; a zero wallet returning the structured `402`; the billing tick decrementing the + wallet in lockstep with `total_spent_cents`; and the change-1 premise (`grant.spent_cents` frozen + at the acquire debit while the lease accrued) observed in production **before** the fix. The + additive migration was applied to a populated pre-existing database, not a fresh one. + **Not covered by this label:** the concurrency cap (`429`) was never reached — the Vast SKU churn + meant a second simultaneous lease always `400`d first — and the two-strike terminal timing could + not be separated from sweep jitter. Detection was confirmed; the strike count was not. + +A source-read claim is not a deployed-behaviour claim. Confirm the money path against `pytest` before +relying on it. + +**And a deploy is not a test either — it catches a third class.** The migration race, the `uvicorn` +`PATH` fault, RunPod's `desiredStatus=RUNNING`-with-no-address, and Vast's never-404 were all invisible +to both source-reading and `pytest`, and all four were found by running the thing. Two of the four were +**pre-existing patterns this work merely exposed**, which is the argument for deploying before merging +rather than after. + +**The money path is now confirmed**, in both directions. Against a running deployment: the billing tick +charges the wallet correctly (34¢/hr decrementing in lockstep with `total_spent_cents`) while +`grant.spent_cents` sat frozen at the acquire debit — the defect. Against tests, after the fix: a $10 +budget at $6.99/h lasts 5160s versus a theoretical 5150s. The caveat that stood over change 1 is +discharged; the ones over the resolver, volumes and the rolling cap are not. + +### What review caught in this document + +Recorded because the pattern is the point. The first draft was written the same way the predecessors +were, and an adversarial pass over the same source found: + +- **Change 0 named the wrong reaper branch.** Branch 3 explicitly skips `provisioning` leases; GPU leases + never leave `provisioning` because both status writers are CPU-only or uncalled. The prerequisite fix + and its acceptance criterion would both have shipped green while every user lease still died at ten + minutes. +- **The launch response cannot carry SSH coordinates.** Providers return `ssh_port: 22` and no host at + acquire. The criterion guarding this passed vacuously, since the port at launch is always 22. +- **The approval gate had no endpoint to source its numbers from.** No dry-run, and `/estimate` needs an + explicit SKU. The gate was specified as the UX centrepiece and could not have been built. +- **"Cheapest offer" contradicted "RunPod is the default"** two sections apart. Settled by product + decision in favour of cheapest, which makes the key-leaking providers the common path — and so promotes + the leak fix from a background ticket into change 9, and makes changes 3 and 4 mandatory rather than + optional. +- **`:209` is unreachable on the managed path**, so a correction this document made to a predecessor was + itself the error — the same failure, one generation on. +- **One plain citation miss** (`lease_manager.py:563` for `:508`), against a section claiming every + citation had been verified. + +Two lessons, both cheap to apply: a citation that is literally correct can still fail to support the claim +built on it, and the acceptance criteria are where a wrong mechanism hides — a criterion that cannot fail +is worse than no criterion. + +**A second adversarial pass, after that rewrite, found more — including one live security defect.** + +- **Vast registers every lease's public key on the shared operator account** so that new instances pick it + up (`vast_provider.py:206-211`, docstring `:10-16`). One user's private key opens another user's box. + The draft called this "leaked into the operator account", i.e. hygiene, and proposed delete-on-release — + which cannot fix it, because concurrently live leases have their keys on the account by construction. + **This is true of production today and is now change 9(a), a blocking prerequisite.** +- **Change 9 was unimplementable as "follow Lambda".** Prime stores the pod name where the key id belongs + (`prime_intellect_provider.py:274`, identical across all of a user's leases), Vast keeps no identifier at + all, there is no column to store one, and "on failed launch" has no hook outside each provider's + `acquire`. +- **"The dataset is free" was wrong three ways.** `ready_at` is stamped at poll time under a 60s sweep, + against a signal that is a 10s difference; the sample is censored at the timeout it was meant to derive; + and no availability series exists, because **nothing ever writes `'failed'`** to `compute_leases`. All + three citations were literally correct — the same failure as the first pass, one layer deeper. +- **Cheapest-first ranked on the wrong column.** `price_cents_per_hour` is the raw provider rate on both + funding paths; what the user pays is `price_cents_per_hour_display`, zero on BYOK. The resolver would + have preferred a billed offer over a free one. +- **Criterion 22 mandated shipping a regression** — change 1 as its own commit _is_ the ~23h regression, + because the fix lives in change 2. +- **Three criteria could pass while broken:** the concurrency one (satisfied by the window cap while the + wallet races), the resolver one (silent on funding), the readiness one (no status vocabulary exists, and + the timeout it bounds on is returned by no endpoint). +- **A citation miss with teeth:** `global/index.ts:46` is the **cache** directory, not config. Following it + writes a private key to a cache path. + +The pattern is now three-for-three: **every round, the errors are in claims built on correct citations, +and in criteria that cannot fail.** Both are cheap to check and neither is caught by re-reading. diff --git a/frontend/workspace/src/components/settings/Billing.tsx b/frontend/workspace/src/components/settings/Billing.tsx index 1b76a2b1..088ef9f2 100644 --- a/frontend/workspace/src/components/settings/Billing.tsx +++ b/frontend/workspace/src/components/settings/Billing.tsx @@ -83,6 +83,11 @@ const COMPUTE_MODES = [ title: "BYOK", body: "Your own connected GPU providers (Settings → Compute). Your provider bills you directly.", }, + { + value: null, + title: "Auto", + body: "Detect from your setup — a connected provider key runs BYOK, otherwise managed compute when available.", + }, ] export default function Billing(): JSX.Element { @@ -117,7 +122,7 @@ export default function Billing(): JSX.Element { if (res.data) return setBilling(res.data) setBillingError("Couldn't load spend settings.") } - const updateBilling = async (patch: { llm?: "managed" | "byok" | null; compute?: "managed" | "byok" }) => { + const updateBilling = async (patch: { llm?: "managed" | "byok" | null; compute?: "managed" | "byok" | null }) => { setBillingBusy(true) setBillingError(undefined) const res = await sdk.client.settings.billing.update(patch) @@ -275,7 +280,7 @@ export default function Billing(): JSX.Element {
Compute -
+
{(m) => ( ( parameters?: { llm?: "managed" | "byok" | null - compute?: "managed" | "byok" + compute?: "managed" | "byok" | null }, options?: Options, ) { diff --git a/tooling/sdk/js/src/v2/gen/types.gen.ts b/tooling/sdk/js/src/v2/gen/types.gen.ts index 710cb283..b5f576af 100644 --- a/tooling/sdk/js/src/v2/gen/types.gen.ts +++ b/tooling/sdk/js/src/v2/gen/types.gen.ts @@ -1710,9 +1710,9 @@ export type Config = { */ llm?: "managed" | "byok" | null /** - * How GPU/compute is paid for. 'managed' runs on Atlas-provisioned compute billed to your wallet (via the bundled atlas CLI); 'byok' uses your own connected GPU providers (Modal, Tinker, TensorPool, …). Unset = byok. + * How GPU/compute is paid for. 'managed' runs on Atlas-provisioned compute billed to your wallet; 'byok' uses your own connected GPU providers (Modal, Lambda Labs, TensorPool, Prime Intellect, RunPod, Vast.ai). Unset or null = auto-detect from your connected providers. Setting this can only narrow the result — if the mode you pick isn't actually available, compute resolves to none rather than pretending. */ - compute?: "managed" | "byok" + compute?: "managed" | "byok" | null } /** * Custom username to display in conversations instead of system username @@ -3761,7 +3761,7 @@ export type SettingsBillingGetResponses = { */ 200: { llm: "managed" | "byok" | null - compute: "managed" | "byok" + compute: "managed" | "byok" | null wallet: { /** * Whether an Atlas session (thk_ key) is available @@ -3780,7 +3780,7 @@ export type SettingsBillingGetResponse = SettingsBillingGetResponses[keyof Setti export type SettingsBillingUpdateData = { body?: { llm?: "managed" | "byok" | null - compute?: "managed" | "byok" + compute?: "managed" | "byok" | null } path?: never query?: never @@ -3793,7 +3793,7 @@ export type SettingsBillingUpdateResponses = { */ 200: { llm: "managed" | "byok" | null - compute: "managed" | "byok" + compute: "managed" | "byok" | null wallet: { /** * Whether an Atlas session (thk_ key) is available diff --git a/tooling/sdk/openapi.json b/tooling/sdk/openapi.json index c9f16bd3..7053439d 100644 --- a/tooling/sdk/openapi.json +++ b/tooling/sdk/openapi.json @@ -4505,10 +4505,17 @@ ] }, "compute": { - "type": "string", - "enum": [ - "managed", - "byok" + "anyOf": [ + { + "type": "string", + "enum": [ + "managed", + "byok" + ] + }, + { + "type": "null" + } ] }, "wallet": { @@ -4572,10 +4579,17 @@ ] }, "compute": { - "type": "string", - "enum": [ - "managed", - "byok" + "anyOf": [ + { + "type": "string", + "enum": [ + "managed", + "byok" + ] + }, + { + "type": "null" + } ] }, "wallet": { @@ -4627,10 +4641,17 @@ ] }, "compute": { - "type": "string", - "enum": [ - "managed", - "byok" + "anyOf": [ + { + "type": "string", + "enum": [ + "managed", + "byok" + ] + }, + { + "type": "null" + } ] } } @@ -19770,11 +19791,18 @@ ] }, "compute": { - "description": "How GPU/compute is paid for. 'managed' runs on Atlas-provisioned compute billed to your wallet (via the bundled atlas CLI); 'byok' uses your own connected GPU providers (Modal, Tinker, TensorPool, …). Unset = byok.", - "type": "string", - "enum": [ - "managed", - "byok" + "description": "How GPU/compute is paid for. 'managed' runs on Atlas-provisioned compute billed to your wallet; 'byok' uses your own connected GPU providers (Modal, Lambda Labs, TensorPool, Prime Intellect, RunPod, Vast.ai). Unset or null = auto-detect from your connected providers. Setting this can only narrow the result — if the mode you pick isn't actually available, compute resolves to none rather than pretending.", + "anyOf": [ + { + "type": "string", + "enum": [ + "managed", + "byok" + ] + }, + { + "type": "null" + } ] } }