diff --git a/.github/trigger-deploy b/.github/trigger-deploy index 16c24eb5..b906fa33 100644 --- a/.github/trigger-deploy +++ b/.github/trigger-deploy @@ -30,3 +30,6 @@ f21-ratelimit e1f992f f13-f21-ci 669033a audit-hardening 94b7fcb guard-falsepos-fix 1f8ea44 +fourth-review 59e2782 +gateway-cache-settle c90565e +stricter-form 6985691 diff --git a/.github/trigger-test b/.github/trigger-test index 35301258..244da615 100644 --- a/.github/trigger-test +++ b/.github/trigger-test @@ -45,3 +45,6 @@ f21-ratelimit e1f992f f13-f21-ci 669033a audit-hardening 94b7fcb guard-falsepos-fix 1f8ea44 +fourth-review 59e2782 +gateway-cache-settle c90565e +stricter-form 6985691 diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 85821650..5503bc77 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -927,8 +927,8 @@ jobs: - name: Bounded rate-limit maps + global waitlist ceiling (F21 — stale one-time-IP keys swept, distributed flood capped — offline) run: node scripts/test-rate-sweep.js - - name: Idempotent render credit (F13 — capacity-refused paid render retried without a second charge; single-use, TTL, eviction — offline) - run: node scripts/test-render-credit.js + - name: Settlement-aware idempotency (FR4-01 — a non-200/unsettled response is NOT cached; @x402/express settles AFTER the handler) + run: node scripts/test-idempotency-settlement.js - name: /health envelope + .flags shape (boolean discipline locks the Yahoo-relay-flap class) run: node scripts/test-health-envelope.js @@ -1133,6 +1133,9 @@ jobs: - name: Tollbooth atomic replay + origin binding (F05/F18 — Durable Object single-use claim + PoW origin/method binding — offline) run: node scripts/test-tollbooth-replay.js + - name: Tollbooth verifier timeout/cancellation (FR4-10 — x402VerifierFromExpress honors the AbortSignal; a late settle can't grant — offline) + run: node scripts/test-tollbooth-verifier.js + - name: Tollbooth dashboard runtime smoke (boots CLI, hits /__tollbooth + /__tollbooth/stats over HTTP) run: node scripts/test-tollbooth-runtime.js diff --git a/CLAUDE.md b/CLAUDE.md index 403b1466..73c09fb0 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -66,8 +66,21 @@ because /v1 settles before the handler and an empty balance = charged-but-failed - Unit/offline: `scripts/test-memory.js`, `test-find.js`, `test-revenue-scan.js`, `test-util-kit.js`, `test-discovery.js`, `tollbooth/test.js`+`edge.test.js`+`features.test.js`. - Raise the MCP free-tier limit for sweeps: `AGENT402_MCP_MAX_PER_MIN=999999 AGENT402_MCP_MAX_PER_HOUR=9999999`. +## x402 settlement ordering (CRITICAL — get this right) +The installed **`@x402/express` v2.16 runs the handler FIRST, then settles**, and +ONLY settles a `<400` response — for any handler `statusCode >= 400` it CANCELS +settlement (`reason: "handler_failed"`) so the buyer is **NOT charged**; if +settlement of a `<400` response fails, it discards the buffered body and returns a +402. So: **a 4xx/5xx (incl. a capacity 503 or an upstream 502) is never charged**, +and a 200 is only charged if settlement then succeeds. Do NOT assume "settles +before the handler" (an earlier, wrong belief that produced the F13 free-render +bypass and the pre-settlement idempotency cache — both since fixed). Anything that +caches, credits, or bills based on handler status BEFORE settlement is unsafe; +key such logic off the FINAL (post-settlement) response, e.g. `res.on("finish")` +with `res.statusCode === 200`. (`node_modules/@x402/express/dist/esm/index.mjs`.) + ## Notable features (current) -- **Idempotency:** opt-in `Idempotency-Key` header; cache key = `sha256(METHOD /path + key + gate-credential)`; replays a paid result without re-charging; no-op without the header. Hooks `res.json` only — streamed responses are never replayable. +- **Idempotency:** opt-in `Idempotency-Key` header; cache key = `sha256(METHOD /path + key + gate-credential)`. **Settlement-aware (FR4-01):** the body is captured at `res.json` but COMMITTED to the cache only on `res.on("finish")` when the FINAL `statusCode === 200` — i.e. after `@x402/express` has settled — so an unsettled 200 (settlement-failure → 402) is never cached/replayed. No-op without the header; streamed responses are never replayable. `scripts/test-idempotency-settlement.js`. - **Tollbooth:** charge modes (`bots`/`all`/`strict`), adaptive PoW, analytics (`gate.stats()` + `/__tollbooth/stats` + `/__tollbooth` dashboard), deploy templates (Cloudflare/Next.js/Docker). Defaults preserve original behavior. - **Buyer SDK (`agent402-client`):** `find()` + `call()` with auto-payment (PoW free / x402 paid), caching, idempotent retries, non-custodial. - **LLM gateway (`src/tools/llm-gateway-kit.js`, OpenAI wire paths):** five tiers — diff --git a/docs/audit-closeout-f01-f25.md b/docs/audit-closeout-f01-f25.md index 98d814bf..c9189d8e 100644 --- a/docs/audit-closeout-f01-f25.md +++ b/docs/audit-closeout-f01-f25.md @@ -5,8 +5,9 @@ is one of: **Fixed** (shipped in code, live), **Accepted** (won't-fix with rationale), or **Owner action** (cannot be closed from this repo — needs a GitHub-admin, DNS/registrar, or product decision; exact steps below). -As of this writing, 22 of 25 are Fixed (F13 flag-gated OFF), 1 is Accepted, and 2 need owner action -(plus the owner half of F01). +As of this writing (post 4th review): 22 of 25 Fixed, F13 REMOVED (its premise was +wrong — see the 4th-review section below), F22 Accepted, and F17/F25 + the owner +half of F01 need owner action. ## Ledger @@ -24,7 +25,7 @@ As of this writing, 22 of 25 are Fixed (F13 flag-gated OFF), 1 is Accepted, and | F10 | Med | Fixed | waitlist drops PII fallback, fails closed | | F11 | Med | Fixed | route-aware Postgres TLS (`src/db-ssl.js`) | | F12 | Med | Fixed | E2B output cap + concurrency ceiling | -| F13 | Med | Fixed (code, flag-gated OFF) | idempotent render credit (`src/render-credit.js`); enable with `RENDER_CREDIT_ENABLED` | +| F13 | Med | **REMOVED (was wrong)** — see §4th-review FR4-02 | the render-credit premise (a charged 503) can't occur under `@x402/express`; feature deleted | | F14 | Med | Fixed | MCP abort + await-before-release | | F15 | Med | Fixed | PostHog proxy timeout + concurrency + byte cap | | F16 | Med | Fixed | `Dockerfile.mcp` digest-pinned + non-root | @@ -42,10 +43,9 @@ As of this writing, 22 of 25 are Fixed (F13 flag-gated OFF), 1 is Accepted, and ## Owner-action items — exact steps -These need an owner action from outside the repo (GitHub-admin, DNS/registrar, or -flipping a reviewed prod flag). Each is copy-paste ready. F13 is fully built and -only needs its flag enabled; F01 (owner half), F17, and F25 are the genuine -external actions. +These need an owner action from outside the repo (GitHub-admin, DNS/registrar). +Each is copy-paste ready. (F13 is no longer here — it was removed; see the +4th-review section.) ### F01 (owner half) — GitHub branch rulesets + protected environment @@ -67,35 +67,6 @@ jobs): add a required reviewer (yourself is fine), set the deployment-branch policy to allow `main` **and** `claude/sweet-brown-i99jl3`, and move the deploy/publish secrets to env scope. -### F13 — render capacity refused after settlement — BUILT (approach B), flag-gated OFF - -Approach **B (idempotent durable credit)** is implemented in `src/render-credit.js` -and wired into the paywall in `src/server.js`, **flagged OFF by default** so -billing is byte-identical until enabled. On a capacity refusal (503) of a -*settled* paid render/screenshot, the server mints a one-time **credit token** (a -256-bit bearer secret) returned only in that buyer's own 503 response -(`X-Render-Credit` header + JSON `renderCredit`), bound to the exact request -(route + body). A retry presenting the token via `X-Render-Credit` skips the gate -(PoW + replay guard + USDC paywall) and is served once; if that retry also fails, -a fresh token is re-issued so the paid credit is never lost. 10-minute TTL, -FIFO-capped. - -Security (adversarially reviewed): the token is unguessable and delivered only to -the paying caller, so knowing a (public) wallet cannot forge or steal it; it is -bound to route+body so it can't be spent on a different or costlier render; -verified live that a forged token still 402s. **Single-use is enforced atomically -at admission** via `claim()` (validate-and-remove in one synchronous step), NOT a -validate-now/consume-on-finish pattern — so a burst of concurrent retries with -the same token can't each be served (the earlier draft had that double-spend; the -test now asserts exactly one of 100 concurrent claims wins). Unit-tested -(`scripts/test-render-credit.js`, 24 cases: atomic single-use / no double-spend, -forgery + cross-request rejection, TTL, re-issue on failed delivery, eviction). - -**To turn it on (owner, after review):** set `RENDER_CREDIT_ENABLED=true` on the -main Railway service. Rollback = unset it (billing reverts, no data). Approach A -(reserve-before-settle) writeup remains in `docs/f13-capacity-options.md` if the -ordering-change route is ever preferred. - ### F17 — publish DMARC (DNS) MX is Zoho; SPF is currently `~all` (soft-fail) with no DMARC. @@ -143,3 +114,29 @@ finding. If we later choose to close it: add a per-request nonce, remove `'unsafe-inline'` from `script-src` first (styles second), and roll out behind `Content-Security-Policy-Report-Only` to catch misses before enforcing. + +--- + +## 4th review (A402-FR4-01..14) — dispositions + +The 4th review's key correction: **`@x402/express` v2.16 settles AFTER the handler +and CANCELS settlement for any `>=400` response** (see CLAUDE.md "x402 settlement +ordering"). Several fixes below follow from that. + +| ID | Sev | Disposition | Where | +|---|---|---|---| +| FR4-01 | High | **Fixed** — idempotency is settlement-aware: cache commits on `finish` only when final status is 200 | `src/server.js` idempotency middleware; `scripts/test-idempotency-settlement.js` | +| FR4-02 | High | **Fixed** — render-credit REMOVED (a 503 is never charged, so a credit was a free-render bypass); `RENDER_CREDIT_ENABLED=false` in prod + code deleted | `src/server.js` (F13 block/import gone), `src/render-credit.js` + test deleted | +| FR4-03 | High | **Fixed** — skill-pack inline `render` now routes through the worker | `src/server.js` `SKILL_INLINE_HANDLERS.render` | +| FR4-04 | High | **Fixed** — Tollbooth enforcing mode FAILS CLOSED without an atomic replay store (unless `TOLLBOOTH_ALLOW_NON_ATOMIC_REPLAY=true`) | `tollbooth/worker.js` | +| FR4-05 | Med | **Fixed** — worker guard also blocks URL/DSN/DB/LEDGER/redis/mongo names (Railway metadata URLs exempt) | `worker/server.js` `forbiddenSecretsIn` | +| FR4-06 | Med | **Fixed** — `workerEnabled()` requires BOTH url+token; `assertWorkerConfig()` fails boot on partial config; bounded worker response read | `src/worker-client.js`, `src/server.js` boot | +| FR4-07 | Med | **Fixed** — `httpHeadersHandler` cancels the body instead of draining it | `src/tools/network-kit2.js` | +| FR4-08 | Med | **Fixed** — x402 quote/audit/RPC reads are byte-capped (`boundedText`) | `src/tools/x402-kit.js` | +| FR4-09 | Med | **Fixed** — code-run error name+value go through the aggregate output budget | `src/tools/code-run-kit.js`; `scripts/test-code-run-caps.js` | +| FR4-10 | Med | **Fixed** — Tollbooth README verifier example honors the `AbortSignal` | `tollbooth/README.md` | +| FR4-11 | Med | **Residual (platform)** — Chromium `--no-sandbox` + no egress firewall after RCE; secretless worker + egress proxy bound impact. Owner/platform work (`docs/worker-isolation-plan.md`). | +| FR4-12 | Low | **Fixed** — worker `isMain` uses resolved-path compare (Windows-safe) | `worker/server.js` | +| FR4-13 | Low | **Accepted** — CSP `unsafe-inline` (see F22). | +| FR4-14 | Low | **Owner action** — GitHub ruleset / env / DNS evidence (see F01/F17/F25). | +| FR4-15 | Low | **Fixed (self-found)** — the LLM-gateway prompt/embeddings cache also wrote pre-settlement (same class as FR4-01); now deferred to a final-200 commit via `req.__deferredCache` | `src/server.js` binder, `src/tools/llm-gateway-kit.js` | diff --git a/scripts/test-code-run-caps.js b/scripts/test-code-run-caps.js index 0ffcc3b3..dfc0d211 100644 --- a/scripts/test-code-run-caps.js +++ b/scripts/test-code-run-caps.js @@ -4,6 +4,9 @@ // before creation. Offline unit test of the cap logic + config. // // node scripts/test-code-run-caps.js +import { readFileSync } from "node:fs"; +import { join, dirname } from "node:path"; +import { fileURLToPath } from "node:url"; import { __test } from "../src/tools/code-run-kit.js"; const { capUtf8, TIERS, E2B_MAX_CONCURRENT } = __test; @@ -42,6 +45,24 @@ const ok = (c, m) => { c ? pass++ : fail++; console.log(`${c ? "ok" : "FAIL"} - ok(budget === 0, "aggregate budget fully consumed, never negative"); } +// FR4-09: the thrown-error name + value (execution.error.value) go through the +// SAME aggregate budget as stdout/stderr/result/traceback — a multi-megabyte +// error message must not bypass the cap. +{ + let budget = 1000; + const take = (v) => { const c = capUtf8(v, Math.max(0, budget)); budget -= c.used; return c; }; + const stdout = take("s".repeat(300)); + const errName = take("Error"); + const errMsg = take("E".repeat(5_000_000)); // huge thrown error value + const traceback = take("t".repeat(5_000_000)); + const total = stdout.used + errName.used + errMsg.used + traceback.used; + ok(errMsg.truncated, "a multi-megabyte error message is truncated"); + ok(total <= 1000, `error name+message+traceback stay inside the aggregate cap (used ${total} <= 1000)`); + // The handler wires these fields through take() (guards against regression). + const src = readFileSync(join(dirname(fileURLToPath(import.meta.url)), "..", "src", "tools", "code-run-kit.js"), "utf8"); + ok(/const errMsg = execution\.error \? take\(execution\.error\.value/.test(src), "handler routes execution.error.value through take() (budgeted)"); +} + // Config sanity. { ok(TIERS["code-run"].maxOutputBytes === 256 * 1024 && TIERS["code-run-pro"].maxOutputBytes === 1024 * 1024, "tiers carry an output byte cap"); diff --git a/scripts/test-idempotency-settlement.js b/scripts/test-idempotency-settlement.js new file mode 100644 index 00000000..6298d36d --- /dev/null +++ b/scripts/test-idempotency-settlement.js @@ -0,0 +1,86 @@ +// FR4-01: idempotency must be SETTLEMENT-AWARE. @x402/express (v2.16) runs the +// handler first and only settles a <400 response; on settlement failure it +// rewrites the buffered 200 into a 402. The old idempotency cache committed at +// res.json() time (handler completion, BEFORE settlement), so a 200 whose +// payment never settled could be replayed. The fix commits only on 'finish' when +// the FINAL status is 200 (post-settlement reality). +// +// This boots the real server (FREE_MODE) and proves the caching now keys on the +// FINAL response status: a 200 is cached + replayed, a NON-200 (stand-in for a +// settlement-failure 402) is NOT cached. Uses X-Pow-Solution as the credential so +// idemHashKey engages without needing a live payment. +// +// node scripts/test-idempotency-settlement.js +import { spawn } from "node:child_process"; +import { readFileSync } from "node:fs"; +import { join, dirname } from "node:path"; +import { fileURLToPath } from "node:url"; + +let pass = 0, fail = 0; +const ok = (c, m) => { c ? pass++ : fail++; console.log(`${c ? "ok" : "FAIL"} - ${m}`); }; +const ROOT = join(dirname(fileURLToPath(import.meta.url)), ".."); + +// Same-class as FR4-01: the LLM-gateway prompt/embeddings cache must ALSO commit +// only after settlement. The handlers now stash on req.__deferredCache and the +// route binder commits on a final 200 (an e2e needs upstream keys; lock the wiring). +{ + const gw = readFileSync(join(ROOT, "src", "tools", "llm-gateway-kit.js"), "utf8"); + const srv = readFileSync(join(ROOT, "src", "server.js"), "utf8"); + ok(/req\.__deferredCache \?\?= \[\]\)\.push\(w\)/.test(gw) && (gw.match(/__deferredCache/g) || []).length >= 2, "gateway chat+embeddings handlers STASH the cache write on req.__deferredCache (not a direct pre-settlement write)"); + ok(/res\.on\("finish", \(\) => \{[\s\S]*?if \(res\.statusCode !== 200\) return;[\s\S]*?req\.__deferredCache/.test(srv), "route binder commits req.__deferredCache only on a final 200 (post-settlement)"); +} + +const PORT = 3959; +const BASE = `http://127.0.0.1:${PORT}`; + +const srv = spawn(process.execPath, ["src/server.js"], { + env: { ...process.env, FREE_MODE: "true", PORT: String(PORT), NODE_ENV: "test" }, + stdio: ["ignore", "ignore", "inherit"], +}); +const until = async (fn, ms = 15000) => { + const t0 = Date.now(); + for (;;) { try { if (await fn()) return true; } catch { /* retry */ } if (Date.now() - t0 > ms) return false; await new Promise((r) => setTimeout(r, 200)); } +}; + +try { + const up = await until(async () => (await fetch(`${BASE}/health`)).ok); + if (!up) { console.error("server did not start"); srv.kill("SIGKILL"); process.exit(1); } + + const call = (path, { key, body, method = "POST" }) => fetch(`${BASE}${path}`, { + method, + headers: { "content-type": "application/json", "idempotency-key": key, "x-pow-solution": "test-credential" }, + body: body === undefined ? undefined : JSON.stringify(body), + }); + + // 1. A 200 IS cached and replayed on retry with the same key+credential. + { + const r1 = await call("/api/hash", { key: "k-ok", body: { text: "idem-settlement-test" } }); + ok(r1.status === 200, "first hash call returns 200"); + const r2 = await call("/api/hash", { key: "k-ok", body: { text: "idem-settlement-test" } }); + ok(r2.status === 200 && r2.headers.get("x-idempotent-replay") === "true", "identical retry is served from cache (X-Idempotent-Replay: true)"); + } + + // 2. A NON-200 final response (stand-in for a settlement-failure 402) is NOT + // cached — the whole point of FR4-01. /api/render with no url returns 400 + // before any Chromium work; the retry must re-run (no replay), not serve a + // cached body. + { + const r1 = await call("/api/render", { key: "k-bad", body: {} }); + ok(r1.status >= 400 && r1.status < 500, `non-200 request returns ${r1.status} (not cached)`); + const r2 = await call("/api/render", { key: "k-bad", body: {} }); + ok(r2.headers.get("x-idempotent-replay") !== "true", "a non-200 response is NOT replayed from cache (settlement-failure bypass closed)"); + ok(r2.status === r1.status, "the retry re-runs the handler and gets the same non-200 status"); + } + + // 3. Sanity: a different body under the same key does not collide. + { + const a = await call("/api/hash", { key: "k-body", body: { text: "aaa" } }); + const b = await call("/api/hash", { key: "k-body", body: { text: "bbb" } }); + ok(a.status === 200 && b.status === 200 && b.headers.get("x-idempotent-replay") !== "true", "different body under the same key is not a cache hit"); + } +} finally { + srv.kill("SIGKILL"); +} + +console.log(`\n${fail ? "FAILED" : "OK"}: ${pass} passed, ${fail} failed`); +process.exit(fail ? 1 : 0); diff --git a/scripts/test-render-credit.js b/scripts/test-render-credit.js deleted file mode 100644 index b9d59423..00000000 --- a/scripts/test-render-credit.js +++ /dev/null @@ -1,89 +0,0 @@ -// F13: unit tests for the idempotent render-credit ledger (offline, no server). -// node scripts/test-render-credit.js -import { createRenderCreditLedger, bodyHashFor, mintCreditToken } from "../src/render-credit.js"; - -let pass = 0, fail = 0; -const ok = (c, m) => { c ? pass++ : fail++; console.log(`${c ? "ok" : "FAIL"} - ${m}`); }; - -const META = { route: "POST /api/render", bodyHash: bodyHashFor({ url: "https://example.com" }) }; -const OTHER_BODY = { route: "POST /api/render", bodyHash: bodyHashFor({ url: "https://evil.com" }) }; -const OTHER_ROUTE = { route: "GET /api/screenshot?url=x", bodyHash: META.bodyHash }; - -// --- token secrecy / shape ------------------------------------------------- -{ - const a = mintCreditToken(), b = mintCreditToken(); - ok(typeof a === "string" && a.length >= 43, "minted token is a long (256-bit) URL-safe string"); - ok(a !== b, "each minted token is unique (unguessable bearer secret)"); - ok(bodyHashFor(undefined) === "-" && bodyHashFor({}) === "-", "empty/absent body → '-' sentinel"); -} - -// --- issue / valid (read-only) --------------------------------------------- -{ - const L = createRenderCreditLedger({ ttlMs: 1000 }); - const tok = L.issue(META, 0); - ok(typeof tok === "string" && tok.length >= 43, "issue returns the bearer token"); - ok(L.valid(tok, META, 0) === true, "the token is valid for the exact request it was issued for"); - ok(L.valid(tok, META, 0) === true, "valid() is read-only — it does NOT consume (so admission must use claim())"); -} - -// --- ATOMIC claim: this is what closes the double-spend -------------------- -{ - const L = createRenderCreditLedger({ ttlMs: 10_000 }); - const tok = L.issue(META, 0); - // Simulate a concurrent burst: every retry validates before any finishes, but - // admission goes through claim(), which removes the token on the FIRST call. - ok(L.claim(tok, META, 0) === true, "first concurrent retry claims the credit"); - ok(L.claim(tok, META, 0) === false, "a second concurrent retry with the same token is refused (no double-spend)"); - ok(L.claim(tok, META, 0) === false, "and a third — one paid credit yields exactly one delivery"); - // 100 racers, one token: - const L2 = createRenderCreditLedger({ ttlMs: 10_000 }); - const t2 = L2.issue(META, 0); - let wins = 0; - for (let i = 0; i < 100; i++) if (L2.claim(t2, META, 0)) wins++; - ok(wins === 1, "exactly ONE of 100 concurrent claims of the same token succeeds"); -} - -// --- claim: forgery / cross-request resistance (and no consume on mismatch) - -{ - const L = createRenderCreditLedger({ ttlMs: 1000 }); - const tok = L.issue(META, 0); - ok(L.claim("not-a-real-token", META, 0) === false, "a guessed/forged token is rejected"); - ok(L.claim(tok, OTHER_BODY, 0) === false, "a token cannot be claimed for a DIFFERENT body"); - ok(L.claim(tok, OTHER_ROUTE, 0) === false, "a token cannot be claimed for a DIFFERENT route"); - ok(L.claim(tok, META, 0) === true, "the mismatched attempts did NOT consume the credit — it is still claimable for its real request"); - ok(L.claim(null, META, 0) === false && L.claim(undefined, META, 0) === false, "missing token → not claimable"); -} - -// --- TTL expiry ------------------------------------------------------------ -{ - const L = createRenderCreditLedger({ ttlMs: 1000 }); - const tok = L.issue(META, 0); - ok(L.valid(tok, META, 999) === true, "valid within the TTL"); - ok(L.claim(tok, META, 1000) === false, "cannot claim at/after the TTL boundary (expired)"); - const tok2 = L.issue(META, 2000); - ok(L.claim(tok2, META, 7000) === false, "claiming past the TTL reports false (expired)"); -} - -// --- capacity refusal: a failed pre-paid retry re-issues, success does not -- -{ - const L = createRenderCreditLedger({ ttlMs: 10_000 }); - const tok = L.issue(META, 0); // paid call 503'd -> token minted - ok(L.claim(tok, META, 100) === true, "retry #1 claims the credit (bypasses the paywall)"); - // retry #1 ALSO 503s -> the server re-issues a fresh token; the OLD one is dead - ok(L.claim(tok, META, 150) === false, "the claimed token is dead even though delivery failed (server re-issues a new one)"); - const tok2 = L.issue(META, 200); // server's re-issue on the failed retry - ok(L.claim(tok2, META, 300) === true, "retry #2 with the re-issued token succeeds -> consumed"); - ok(L.claim(tok2, META, 400) === false, "no free renders after the one paid delivery"); -} - -// --- FIFO eviction at capacity --------------------------------------------- -{ - const L = createRenderCreditLedger({ ttlMs: 10_000, maxEntries: 2 }); - const a = L.issue(META, 0), b = L.issue(META, 1); - const c = L.issue(META, 2); // evicts the oldest (a) - ok(L.valid(a, META, 5) === false, "oldest credit evicted when the ledger is full"); - ok(L.valid(b, META, 5) && L.valid(c, META, 5), "newer credits retained"); -} - -console.log(`\n${fail ? "FAILED" : "OK"}: ${pass} passed, ${fail} failed`); -process.exit(fail ? 1 : 0); diff --git a/scripts/test-tollbooth-verifier.js b/scripts/test-tollbooth-verifier.js new file mode 100644 index 00000000..b9cfdd5d --- /dev/null +++ b/scripts/test-tollbooth-verifier.js @@ -0,0 +1,73 @@ +// P1.5 / FR4-10: x402VerifierFromExpress — a first-party verifyX402 that owns +// timeout + cancellation. Grants on next(), denies on a 402 write, honors the +// gate's AbortSignal (opts.signal), backstops with its own timeout, and rejects +// on a thrown middleware. Offline, no network. +// +// node scripts/test-tollbooth-verifier.js +import { x402VerifierFromExpress } from "../tollbooth/index.js"; + +let pass = 0, fail = 0; +const ok = (c, m) => { c ? pass++ : fail++; console.log(`${c ? "ok" : "FAIL"} - ${m}`); }; +const req = { headers: { "x-payment": "sig" } }; +const settled = (p) => p.then((v) => ({ v }), (e) => ({ e })); + +// grant: middleware calls next() +{ + const v = x402VerifierFromExpress((_req, _res, next) => next()); + ok((await v(req, {})) === true, "grants (true) when the middleware calls next()"); +} +// deny: middleware writes a 402 response +{ + const v = x402VerifierFromExpress((_req, res) => res.status(402).json({ error: "no payment" })); + ok((await v(req, {})) === false, "denies (false) when the middleware writes a 402"); +} +// deny via send()/end() too +{ + const v = x402VerifierFromExpress((_req, res) => res.end()); + ok((await v(req, {})) === false, "denies when the middleware ends the response without next()"); +} +// honors an already-aborted signal +{ + const ac = new AbortController(); ac.abort(); + const v = x402VerifierFromExpress(() => { /* would hang */ }); + ok((await v(req, { signal: ac.signal })) === false, "an already-aborted signal → not verified (no hang)"); +} +// honors an abort that fires mid-verification +{ + const ac = new AbortController(); + const v = x402VerifierFromExpress(() => { /* never resolves */ }); + const p = settled(v(req, { signal: ac.signal })); + setTimeout(() => ac.abort(), 20); + const r = await p; + ok(r.v === false, "an abort during verification → resolves false (gate not left hanging)"); +} +// own timeout backstop when no signal is passed +{ + const v = x402VerifierFromExpress(() => { /* hangs */ }, { timeoutMs: 30 }); + ok((await v(req, {})) === false, "own timeoutMs backstops a hung middleware (no signal needed)"); +} +// a thrown middleware rejects (gate maps it to a 402) +{ + const v = x402VerifierFromExpress(() => { throw new Error("boom"); }); + const r = await settled(v(req, {})); + ok(r.e instanceof Error, "a thrown middleware rejects (gate .catch → 402)"); +} +// a late resolution after abort cannot flip the result back to granted +{ + const ac = new AbortController(); + let stashedNext; + const v = x402VerifierFromExpress((_req, _res, next) => { stashedNext = next; }); + const p = settled(v(req, { signal: ac.signal })); + ac.abort(); + const r1 = await p; + stashedNext?.(); // middleware "grants" late — must be ignored + ok(r1.v === false, "a settlement that completes AFTER the abort cannot grant (no charged-then-denied flip)"); +} +// bad input +{ + let threw = false; try { x402VerifierFromExpress(null); } catch { threw = true; } + ok(threw, "throws if not given a middleware function"); +} + +console.log(`\n${fail ? "FAILED" : "OK"}: ${pass} passed, ${fail} failed`); +process.exit(fail ? 1 : 0); diff --git a/scripts/test-worker-isolation.js b/scripts/test-worker-isolation.js index 3feac05b..2185c6da 100644 --- a/scripts/test-worker-isolation.js +++ b/scripts/test-worker-isolation.js @@ -72,7 +72,9 @@ const listen = (app) => new Promise((res) => { const s = app.listen(0, () => res // --- 3. The boot guard: a forbidden secret in the worker's env aborts start --- { const child = spawn(process.execPath, [join(ROOT, "worker", "server.js")], { - env: { ...process.env, WALLET_ADDRESS: "0xdeadbeef", PORT: "0", RENDER_WORKER_URL: "", RENDER_WORKER_TOKEN: "" }, + // RAILWAY_ENVIRONMENT triggers strict mode (the real runtime): WALLET_ADDRESS + // isn't secret-SHAPED, but strict mode refuses any non-allowlisted var. + env: { ...process.env, RAILWAY_ENVIRONMENT: "production", WALLET_ADDRESS: "0xdeadbeef", PORT: "0", RENDER_WORKER_URL: "", RENDER_WORKER_TOKEN: "" }, stdio: ["ignore", "pipe", "pipe"], }); let out = ""; @@ -106,21 +108,70 @@ const listen = (app) => new Promise((res) => { const s = app.listen(0, () => res // this is the RAILWAY_PRIVATE_DOMAIN regression that took a deploy down. const cleanWorkerEnv = { PORT: "3999", WORKER_MODE: "true", RENDER_WORKER_TOKEN: "abc", + RAILWAY_ENVIRONMENT: "production", // triggers STRICT mode (the real Railway runtime) RAILWAY_PRIVATE_DOMAIN: "agent402-worker.railway.internal", RAILWAY_ENVIRONMENT_NAME: "production", RAILWAY_PROJECT_ID: "x", RAILWAY_SERVICE_ID: "y", RAILWAY_SERVICE_AGENT402_URL: "z", RAILWAY_DOCKERFILE_PATH: "Dockerfile.worker", PATH: "/usr/bin", HOME: "/home/node", + HOSTNAME: "abc", NODE_VERSION: "22", npm_config_cache: "/tmp", RENDER_EGRESS_PROXY_URL: "http://127.0.0.1:5", }; - ok(forbiddenSecretsIn(cleanWorkerEnv).length === 0, "the secretless worker env (incl. RAILWAY_PRIVATE_DOMAIN + its own token) does NOT trip the guard"); + ok(forbiddenSecretsIn(cleanWorkerEnv).length === 0, "the real (strict-mode) Railway worker env passes: only allowlisted / system / RAILWAY_* vars"); // Real secrets the OLD denylist missed must now trip it. for (const k of ["GITHUB_TOKEN", "E2B_API_KEY", "STELLAR_FACILITATOR_KEY", "ALGORAND_BURNER_MNEMONIC", "OPENROUTER_API_KEY", "CDP_API_KEY_SECRET"]) { ok(forbiddenSecretsIn({ [k]: "v", RENDER_WORKER_TOKEN: "t" }).includes(k), `guard catches ${k} (pattern-based, not the old 12-name denylist)`); } // Non-pattern secrets stay covered; the worker's own token stays allowed. - ok(forbiddenSecretsIn({ WALLET_ADDRESS: "0x", DATABASE_URL: "postgres://x" }).length === 2, "non-pattern secrets (WALLET_ADDRESS, DATABASE_URL) still caught"); + ok(forbiddenSecretsIn({ DATABASE_URL: "postgres://x" }).includes("DATABASE_URL"), "a DB connection string (DATABASE_URL) is caught in every mode (secret-shaped)"); + ok(forbiddenSecretsIn({ RAILWAY_ENVIRONMENT: "production", WALLET_ADDRESS: "0x" }).includes("WALLET_ADDRESS"), "a non-secret-shaped operator var (WALLET_ADDRESS) is caught in strict (Railway) mode"); ok(forbiddenSecretsIn({ RENDER_WORKER_TOKEN: "t" }).length === 0, "the worker's OWN inbound-auth token is allowed"); ok(forbiddenSecretsIn({ FOO_KEY: "" }).length === 0, "an empty secret var is ignored (only set values count)"); + // FR4-05: credential-bearing URL/DSN/DB/ledger names (which don't contain + // KEY/SECRET/TOKEN) must now trip the guard too. + for (const k of ["REDIS_URL", "SENTRY_DSN", "REVENUE_LEDGER_URL", "ECONOMY_DATABASE_URL", "MONGO_URI", "SALES_LEDGER"]) { + ok(forbiddenSecretsIn({ [k]: "v", RENDER_WORKER_TOKEN: "t" }).includes(k), `guard catches credential-bearing ${k}`); + } + ok(forbiddenSecretsIn({ RAILWAY_SERVICE_AGENT402_URL: "http://x", RENDER_WORKER_TOKEN: "t" }).length === 0, "benign Railway service-metadata URL is allowed (RAILWAY_* URLs exempt)"); + ok(forbiddenSecretsIn({ RAILWAY_TOKEN: "secret" }).includes("RAILWAY_TOKEN"), "but RAILWAY_TOKEN (a real secret) is still caught"); + // STRICT allowlist (P1.2), enforced in the Railway runtime (RAILWAY_ENVIRONMENT + // present): an UNRECOGNIZED var — even one with no secret-shaped name — refuses + // boot; only explicitly-allowed / system / Railway metadata pass. + ok(forbiddenSecretsIn({ RAILWAY_ENVIRONMENT: "production", SOME_RANDOM_CONFIG: "x", RENDER_WORKER_TOKEN: "t" }).includes("SOME_RANDOM_CONFIG"), "strict (Railway) mode: an unrecognized (non-secret-shaped) var refuses boot"); + ok(forbiddenSecretsIn({ RAILWAY_ENVIRONMENT: "production", PATH: "/usr/bin", NODE_ENV: "production", npm_config_cache: "/tmp", RENDER_WORKER_TIMEOUT_MS: "60000" }).length === 0, "strict mode: known system + exact-allowed vars pass"); + // Dev/CI (no RAILWAY_ENVIRONMENT): benign unknown noise is tolerated so the + // worker still boots locally — but secret-shaped names are STILL blocked. + ok(forbiddenSecretsIn({ SOME_EDITOR_VAR: "x", __CF_USER_TEXT_ENCODING: "0x1F5", RENDER_WORKER_TOKEN: "t" }).length === 0, "dev mode: benign unknown vars are tolerated (no false-positive boot failure)"); + ok(forbiddenSecretsIn({ GITHUB_TOKEN: "ghp_x" }).includes("GITHUB_TOKEN"), "dev mode: a secret-shaped name is STILL blocked (shape denylist applies everywhere)"); +} + +// --- 6. FR4-06: render-worker config is atomic (both URL+token or neither) ---- +{ + const { workerEnabled, assertWorkerConfig } = await import("../src/worker-client.js"); + const save = { u: process.env.RENDER_WORKER_URL, t: process.env.RENDER_WORKER_TOKEN }; + const set = (u, t) => { u == null ? delete process.env.RENDER_WORKER_URL : process.env.RENDER_WORKER_URL = u; t == null ? delete process.env.RENDER_WORKER_TOKEN : process.env.RENDER_WORKER_TOKEN = t; }; + const threw = (fn) => { try { fn(); return false; } catch { return true; } }; + + set("http://w.internal:3999", "tok"); + ok(workerEnabled() === true, "workerEnabled() true only when BOTH url and token are set"); + ok(!threw(assertWorkerConfig), "assertWorkerConfig passes when both are set"); + set("http://w.internal:3999", null); + ok(workerEnabled() === false, "workerEnabled() false when the token is missing (would 401 every paid call)"); + ok(threw(assertWorkerConfig), "assertWorkerConfig THROWS on a partial config (url without token)"); + set(null, "tok"); + ok(threw(assertWorkerConfig), "assertWorkerConfig THROWS on a partial config (token without url)"); + set(null, null); + ok(workerEnabled() === false && !threw(assertWorkerConfig), "neither set -> in-process, no throw"); + // P1.1: RENDER_WORKER_REQUIRED makes a missing worker config a HARD boot failure + // (prod can demand isolation instead of silently running browser/media in-process). + const saveReq = process.env.RENDER_WORKER_REQUIRED; + process.env.RENDER_WORKER_REQUIRED = "true"; + set(null, null); + ok(threw(assertWorkerConfig), "RENDER_WORKER_REQUIRED=true + no worker config -> THROWS (fail closed)"); + set("http://w.internal:3999", "tok"); + ok(!threw(assertWorkerConfig), "RENDER_WORKER_REQUIRED=true + full config -> ok"); + if (saveReq == null) delete process.env.RENDER_WORKER_REQUIRED; else process.env.RENDER_WORKER_REQUIRED = saveReq; + // restore + set(save.u, save.t); } console.log(`\n${fail ? "FAILED" : "OK"}: ${pass} passed, ${fail} failed`); diff --git a/src/render-credit.js b/src/render-credit.js deleted file mode 100644 index 1ec04dbf..00000000 --- a/src/render-credit.js +++ /dev/null @@ -1,69 +0,0 @@ -// F13: idempotent durable credit for capacity-refused paid renders. -// -// x402 settles BEFORE the handler runs. If a paid render/screenshot then hits the -// bounded Chromium pool and is refused (503, security audit A402-08), the buyer -// was charged but not served — and because each x402 authorization nonce is -// single-use, a naive retry pays AGAIN. On such a refusal we mint a one-time -// CREDIT TOKEN — a 256-bit bearer secret returned only in that buyer's own 503 -// response — and record a credit bound to the exact request (route + body). A -// retry that presents the token (`X-Render-Credit`) for the SAME request skips -// the paywall and is served without a second charge; the credit is consumed only -// on successful delivery and survives a repeated 503 for another retry. -// -// Why a server-minted token, NOT the payer address: the token is unguessable and -// is handed only to the paying caller, so a third party who merely knows the -// (public) wallet cannot forge or steal the credit. Binding to route+body means a -// token can't be spent on a different or costlier render than the one paid for. -import { createHash, randomBytes } from "node:crypto"; - -export function bodyHashFor(body) { - return body && typeof body === "object" && Object.keys(body).length - ? createHash("sha256").update(JSON.stringify(body)).digest("hex") - : "-"; -} - -// 256-bit URL-safe bearer secret. Only ever returned to the paying caller. -export function mintCreditToken() { - return randomBytes(32).toString("base64url"); -} - -export function createRenderCreditLedger({ ttlMs = 10 * 60 * 1000, maxEntries = 5000 } = {}) { - const store = new Map(); // token -> { route, bodyHash, at } - const live = (e, now) => !!e && now - e.at < ttlMs; - const matches = (e, meta) => e.route === meta.route && e.bodyHash === meta.bodyHash; - return { - // Record a credit for a settled-but-refused render; returns the bearer token. - // A caller-supplied token is only for deterministic tests. - issue(meta, now = Date.now(), token = mintCreditToken()) { - while (store.size >= maxEntries && store.size > 0) store.delete(store.keys().next().value); - store.set(token, { route: meta.route, bodyHash: meta.bodyHash, at: now }); - return token; - }, - // True iff there is a LIVE credit for this token AND it was issued for this - // exact request (route + body). Read-only; lazily drops an expired token. - valid(token, meta, now = Date.now()) { - if (!token || typeof token !== "string") return false; - const e = store.get(token); - if (!live(e, now)) { if (e) store.delete(token); return false; } - return matches(e, meta); - }, - // ATOMIC single-use admission: validate AND remove in one synchronous step. - // Returns true iff a LIVE credit for this exact request existed (now consumed). - // This is what closes the double-spend: concurrent retries presenting the same - // token race here — only the FIRST claim removes it and returns true; every - // other gets false and falls through to the paywall. A non-matching request - // does NOT consume the credit. Use this at gate admission, never `valid`. - claim(token, meta, now = Date.now()) { - if (!token || typeof token !== "string") return false; - const e = store.get(token); - if (!live(e, now)) { if (e) store.delete(token); return false; } - if (!matches(e, meta)) return false; - store.delete(token); - return true; - }, - prune(now = Date.now()) { - for (const [t, e] of store) if (!live(e, now)) store.delete(t); - }, - size() { return store.size; }, - }; -} diff --git a/src/server.js b/src/server.js index e14aa071..81f18539 100644 --- a/src/server.js +++ b/src/server.js @@ -16,7 +16,7 @@ import { extractArticle, fetchPageMeta } from "./tools/extract.js"; import { dnsLookup } from "./tools/dns.js"; import { pdfToText } from "./tools/pdf.js"; import { renderArticle, screenshotPage, rasterizeSvg } from "./tools/render.js"; -import { workerEnabled, runOnWorker } from "./worker-client.js"; +import { workerEnabled, runOnWorker, assertWorkerConfig } from "./worker-client.js"; import { memoryPut, memoryGet, memoryDelete, memoryIncr, memoryCas, grant, revoke, listGrants, getLog, remember, recall, forget, @@ -111,7 +111,7 @@ import { CRYPTO_HASH_TOOLS } from "./tools/crypto-hash-kit.js"; import { STRING_TOOLS } from "./tools/string-kit.js"; import { CALENDAR_TOOLS } from "./tools/calendar-kit.js"; import { LLM_TOOLS } from "./tools/llm-kit.js"; -import { LLM_GATEWAY_TOOLS, modelsList, promptCacheKey, promptCacheGet, GATEWAY_TIER_BY_PATH, embeddingsCacheKey, EMBEDDINGS_PATH, gatewayCreditsStatus } from "./tools/llm-gateway-kit.js"; +import { LLM_GATEWAY_TOOLS, modelsList, promptCacheKey, promptCacheGet, promptCacheStore, GATEWAY_TIER_BY_PATH, embeddingsCacheKey, EMBEDDINGS_PATH, gatewayCreditsStatus } from "./tools/llm-gateway-kit.js"; // /v1/audio/speech stays behind OPENROUTER_TTS_ENABLED as a rollout gate: // x402 settles before the handler, so a listed-but-broken route charges // buyers for 502s (no route -> no 402 -> no charge). The upstream WAS @@ -176,7 +176,6 @@ import { buildRouteExecuteTool } from "./tools/route-execute.js"; import { issueChallenge, verifySolution, isComputePayable, powInfo, POW_DIFFICULTY, WALLET_ONLY_SLUGS, verifyHeartbeatToken } from "./pow.js"; import { createLimiter as createRateLimiter, LIMITS_LABEL as POW_LIMITS_LABEL } from "./rate-limit.js"; import { sweepStaleTsMap, makeWindowCounter } from "./rate-sweep.js"; -import { bodyHashFor, createRenderCreditLedger } from "./render-credit.js"; // Shared with the MCP free tier (src/mcp-http.js) — same policy, separate // per-IP bucket. PoW redemption on the direct HTTP path goes through here. @@ -191,6 +190,8 @@ const WALLET_ADDRESS = process.env.WALLET_ADDRESS; const WALLET_ENS = process.env.WALLET_ENS || "agent402.base.eth"; const NETWORK = process.env.NETWORK || "base"; const FREE_MODE = process.env.FREE_MODE === "true"; +// FR4-06: fail fast on a partial render-worker config (one of URL/token set). +assertWorkerConfig(); const BASE_URL = process.env.BASE_URL || `http://localhost:${PORT}`; const CATALOG = { @@ -579,7 +580,11 @@ const SKILL_INLINE_HANDLERS = { }, meta: async ({ url } = {}) => fetchPageMeta(url), dns: async ({ name, type } = {}) => dnsLookup(name, type), - render: async ({ url } = {}) => renderArticle(url), + // FR4-03: route the skill-pack render through the secretless worker too — the + // runner prefers this inline map, so calling renderArticle() directly here ran + // Chromium in the main secret-bearing process for packs like content-extraction + // / structured-scrape, defeating the worker isolation the /api/render route has. + render: async ({ url } = {}) => (workerEnabled() ? runOnWorker("render", { url }) : renderArticle(url)), pdf: async ({ url } = {}) => pdfToText(url), }; const SKILL_TOOLS = buildSkillTools({ @@ -2566,29 +2571,37 @@ app.use((req, res, next) => { res.setHeader("X-Idempotent-Replay", "true"); return res.status(200).json(hit.body); } + // Settlement-aware caching (FR4-01). @x402/express (v2.16) runs the handler + // FIRST, then settles, and ONLY on a <400 response; on settlement FAILURE it + // replaces the buffered 200 with a 402. So committing to the cache at + // res.json() time (handler completion, BEFORE settlement) would store a result + // whose payment never settled, and a retry could replay it for free. Capture + // the body at res.json() but COMMIT only on 'finish', when res.statusCode is + // the post-settlement reality: a final 200 means settlement succeeded (the + // paywall would have written a 402 otherwise). PoW (free) requests never enter + // the settle path, so their 200 is final at finish too — cached correctly. + let captured; const origJson = res.json.bind(res); - res.json = (body) => { - if (res.statusCode === 200) { - let bytes = 0; - try { bytes = Buffer.byteLength(JSON.stringify(body), "utf8"); } catch { bytes = 0; } - if (bytes && bytes <= IDEM_MAX_BODY_BYTES) { - // Evict oldest entries (Map preserves insertion order → FIFO ≈ LRU - // for write-heavy access) until we fit by entries AND by bytes. - while ( - (idemStore.size >= IDEM_MAX_ENTRIES || idemBytes + bytes > IDEM_MAX_BYTES) - && idemStore.size > 0 - ) { - const firstKey = idemStore.keys().next().value; - const ev = idemStore.get(firstKey); - if (ev) idemBytes -= ev.bytes; - idemStore.delete(firstKey); - } - idemStore.set(key, { at: Date.now(), body, bytes }); - idemBytes += bytes; - } + res.json = (body) => { captured = body; return origJson(body); }; + res.on("finish", () => { + if (res.statusCode !== 200 || captured === undefined) return; + let bytes = 0; + try { bytes = Buffer.byteLength(JSON.stringify(captured), "utf8"); } catch { bytes = 0; } + if (!bytes || bytes > IDEM_MAX_BODY_BYTES) return; + // Evict oldest entries (Map preserves insertion order → FIFO ≈ LRU for + // write-heavy access) until we fit by entries AND by bytes. + while ( + (idemStore.size >= IDEM_MAX_ENTRIES || idemBytes + bytes > IDEM_MAX_BYTES) + && idemStore.size > 0 + ) { + const firstKey = idemStore.keys().next().value; + const ev = idemStore.get(firstKey); + if (ev) idemBytes -= ev.bytes; + idemStore.delete(firstKey); } - return origJson(body); - }; + idemStore.set(key, { at: Date.now(), body: captured, bytes }); + idemBytes += bytes; + }); next(); }); @@ -2703,66 +2716,15 @@ if (FREE_MODE) { next(); }); - // F13: idempotent durable credit for capacity-refused paid renders. x402 - // settles BEFORE the handler, so a paid render/screenshot that then hits the - // bounded Chromium pool (503) was charged but not served — and the single-use - // nonce means a naive retry pays again. On such a refusal we mint a one-time - // credit TOKEN (256-bit bearer secret) returned only in that buyer's own 503 - // response and bound to the exact request (route+body); a retry presenting the - // token via `X-Render-Credit` skips the gate below (served without a second - // charge) and the credit is consumed only on a successful delivery. The token - // is unguessable and handed only to the payer, so knowing a (public) wallet - // can't forge or steal it. Flagged OFF by default → billing is byte-identical - // until RENDER_CREDIT_ENABLED is set. - const RENDER_CREDIT_ENABLED = /^(1|true|yes|on)$/i.test((process.env.RENDER_CREDIT_ENABLED || "").trim()); - const RENDER_CREDIT_SLUGS = new Set(["render", "screenshot"]); // the A402-08 bounded-pool tools - const renderCredits = createRenderCreditLedger(); - if (RENDER_CREDIT_ENABLED) { - setInterval(() => renderCredits.prune(), 60_000).unref(); - app.use((req, res, next) => { - const def = CATALOG[`${req.method} ${req.path}`]; - if (!def || !RENDER_CREDIT_SLUGS.has(def.slug)) return next(); - const meta = { route: `${req.method} ${req.originalUrl}`, bodyHash: bodyHashFor(req.body) }; - // Retry path — ATOMIC single-use admission. claim() validates AND removes - // the token in one synchronous step, so a burst of concurrent retries with - // the same token can't each be served: only the first claim wins, the rest - // get false and fall through to the paywall. (A validate-then-consume-on- - // finish pattern would let N concurrent retries all pass before any one - // finished — the double-spend this closes.) - const token = req.header("x-render-credit"); - const claimed = !!(token && renderCredits.claim(token, meta)); - if (claimed) { - req.__renderCreditToken = token; - res.setHeader("X-Render-Credit-Replay", "true"); - } - // A capacity 503 only reaches the handler AFTER the paywall passed, so a - // 503 on a request that carried a payment header means the buyer settled - // and was not served → mint a token. And if a CLAIMED credit didn't deliver - // (any non-2xx), give a fresh token BACK so the consumed-but-undelivered - // credit is never lost. The token rides the response headers (not yet - // flushed) so only the paying caller receives it. res.json is the error - // path for both render and screenshot; a screenshot SUCCESS uses res.send, - // so a delivered credit is never re-issued. - const paid = !!(req.header("x-payment") || req.header("payment-signature")); - if (paid || claimed) { - const origJson = res.json.bind(res); - res.json = (body) => { - const delivered = res.statusCode >= 200 && res.statusCode < 300; - const owe = !delivered && (claimed || (paid && res.statusCode === 503)); - if (owe && !res.headersSent) { - const t = renderCredits.issue(meta); - res.setHeader("X-Render-Credit", t); - if (body && typeof body === "object" && !Array.isArray(body)) { - body = { ...body, renderCredit: t, renderCreditHint: "retry this exact request with header 'X-Render-Credit: ' to be served without paying again" }; - } - } - return origJson(body); - }; - } - next(); - }); - console.log("F13 render-credit ENABLED: a capacity-refused paid render returns an X-Render-Credit token; retry with it is served without a second charge"); - } + // NB (FR4-02): the former "render-credit" feature (F13/R-10) was REMOVED. It + // assumed x402 settled BEFORE the handler, so a capacity 503 meant + // charged-but-not-served and warranted a retry credit. The installed + // @x402/express (v2.16) does the opposite — it runs the handler first and + // CANCELS settlement for any >=400 response (`reason: "handler_failed"`), so a + // capacity 503 is NEVER charged and no credit is ever owed. Issuing a bearer + // credit on a 503 was therefore a free-render bypass. If a future facilitator + // ever charges before delivery, reintroduce credits from a settlement-CONFIRMED + // hook, never from payment-header presence or handler status. // Gate: for a compute-payable route, a valid proof-of-work bypasses the x402 // paywall; otherwise the normal USDC paywall applies (and we advertise the @@ -2771,9 +2733,6 @@ if (FREE_MODE) { // hosted MCP free tier (src/rate-limit.js) — otherwise a client exhausted // on /mcp could keep hammering /api/* with fresh PoW solutions for free. app.use((req, res, next) => { - // F13: a valid pre-paid render credit token skips the whole gate (PoW, replay - // guard, and USDC paywall) — the buyer already paid for this exact render. - if (req.__renderCreditToken) return next(); const slug = POW_ROUTES.get(`${req.method} ${req.path}`); if (slug) { const solution = req.header("x-pow-solution"); @@ -3145,11 +3104,20 @@ for (const tool of ALL_KIT) { // paywall settled; never cached (idempotency hooks res.json only). if (result && typeof result.__sse === "function") { await result.__sse(res); return; } - // Cache successful, non-error JSON responses. Errors are never cached — - // an upstream blip shouldn't poison the key for the whole TTL. - if (cacheKey && result && typeof result === "object" && !result.error) { - cacheSet(cacheKey, result, cachePolicy.ttl || 300).catch(() => {}); - } + // Cache successful, non-error JSON responses — but ONLY after settlement. + // @x402/express settles AFTER the handler and rewrites a settlement-failure + // into a 402, so committing here (pre-settlement) would let an UNSETTLED + // response be served free on a byte-identical repeat (same class as + // FR4-01). Commit on res.on("finish") when the FINAL status is 200. This + // covers both the generic route cache AND the LLM-gateway prompt/embeddings + // cache (handlers stash their write on req.__deferredCache). + res.on("finish", () => { + if (res.statusCode !== 200) return; + if (cacheKey && result && typeof result === "object" && !result.error) { + cacheSet(cacheKey, result, cachePolicy.ttl || 300).catch(() => {}); + } + for (const w of req.__deferredCache || []) { try { promptCacheStore(w.key, w.body); } catch { /* cache is best-effort */ } } + }); res.json(result); } catch (err) { errored = true; diff --git a/src/tools/code-run-kit.js b/src/tools/code-run-kit.js index 41054f94..d7b823c6 100644 --- a/src/tools/code-run-kit.js +++ b/src/tools/code-run-kit.js @@ -108,8 +108,13 @@ async function runInSandbox(code, language, tierSlug) { const stdout = take(execution.logs?.stdout?.join("") ?? ""); const stderr = take(execution.logs?.stderr?.join("") ?? ""); const result = execution.text != null ? take(execution.text) : { text: null, truncated: false }; + // FR4-09: the error name AND message (execution.error.value) go through the + // SAME aggregate budget — a multi-megabyte thrown error must not bypass the + // output cap. + const errName = execution.error ? take(execution.error.name ?? "Error") : { text: "Error", truncated: false }; + const errMsg = execution.error ? take(execution.error.value ?? "") : { text: "", truncated: false }; const traceback = execution.error ? take(execution.error.traceback ?? "") : { text: "", truncated: false }; - const truncated = stdout.truncated || stderr.truncated || result.truncated || traceback.truncated; + const truncated = stdout.truncated || stderr.truncated || result.truncated || errName.truncated || errMsg.truncated || traceback.truncated; return { language, @@ -117,7 +122,7 @@ async function runInSandbox(code, language, tierSlug) { stderr: stderr.text, result: result.text, error: execution.error - ? { name: execution.error.name ?? "Error", message: execution.error.value ?? "", traceback: traceback.text } + ? { name: errName.text ?? "Error", message: errMsg.text ?? "", traceback: traceback.text } : null, ...(truncated ? { truncated: true } : {}), }; diff --git a/src/tools/llm-gateway-kit.js b/src/tools/llm-gateway-kit.js index 5b56c913..ccb91ff1 100644 --- a/src/tools/llm-gateway-kit.js +++ b/src/tools/llm-gateway-kit.js @@ -737,7 +737,7 @@ export function embeddingsCacheKey(input) { return createHash("sha256").update(`v1-embeddings\n${stableStringify(body)}`).digest("hex"); } -async function embeddingsHandler(input) { +async function embeddingsHandler(input, req) { const body = validateEmbeddingsRequest(input); const key = OPENAI_KEY(); if (!key) throw bad("Embeddings gateway not configured (OPENAI_API_KEY unset)", 503); @@ -758,8 +758,13 @@ async function embeddingsHandler(input) { try { data = JSON.parse(text); } catch { throw bad("Upstream returned non-JSON", 502); } // Full OpenAI wire shape passes through untouched (object, data[], model, // usage). Store unless the buyer opted out; oversized batches are skipped - // by the store's own per-entry byte cap. - try { promptCacheStore(embeddingsCacheKey(input), data); } catch { /* never fail a served response over the cache */ } + // by the store's own per-entry byte cap. FR4-01 class: defer the write to + // AFTER settlement (the route binder commits on a final 200) so an unsettled + // 200 isn't cached and served free on a repeat; direct write for non-HTTP. + try { + const w = { key: embeddingsCacheKey(input), body: data }; + if (req) (req.__deferredCache ??= []).push(w); else promptCacheStore(w.key, w.body); + } catch { /* never fail a served response over the cache */ } return data; } @@ -1060,12 +1065,14 @@ function countImages(messages) { } function makeHandler(tierSlug) { - return async (input) => { + return async (input, req) => { const body = validateRequest(input, tierSlug); - // Payment settles BEFORE this handler runs, so an upstream provider - // failure must not become the buyer's problem when an equivalent model - // can serve: walk the tier's fallback chain on upstream errors - // (502/503/504) only — our own validation 4xxs pass through untouched. + // NB: @x402/express settles AFTER this handler and cancels settlement for a + // >=400 response, so an upstream failure that we let surface as a 5xx is NOT + // charged. We still walk the tier's fallback chain on upstream errors + // (502/503/504) so an equivalent model can serve rather than fail — better + // UX, and a served 200 only bills if it then settles. Our own validation + // 4xxs pass through untouched. // The response's `model` field discloses which model actually served. // (Origin: openai/gpt-4.1-nano returned persistent provider errors on // 2026-07-08 — two independent paid runs — and buyers were charged $0.003 @@ -1166,7 +1173,14 @@ function makeHandler(tierSlug) { data.agent402_router = { category: routedCategory, quality: routedQuality, served: data.model || model }; } if (input.cache === true) { - try { promptCacheStore(promptCacheKey(tierSlug, input), data); } catch { /* never fail a served response over the cache */ } + // FR4-01 class: defer the cache write to AFTER settlement. @x402/express + // settles after this handler, so writing now would cache an + // unsettled 200. Stash on req; the route binder commits on a final 200. + // Fall back to a direct write for non-HTTP callers (no settlement). + try { + const w = { key: promptCacheKey(tierSlug, input), body: data }; + if (req) (req.__deferredCache ??= []).push(w); else promptCacheStore(w.key, w.body); + } catch { /* never fail a served response over the cache */ } } return data; } catch (e) { diff --git a/src/tools/network-kit2.js b/src/tools/network-kit2.js index bcaa22fe..e1786e51 100644 --- a/src/tools/network-kit2.js +++ b/src/tools/network-kit2.js @@ -240,8 +240,10 @@ async function httpHeadersHandler(body) { } finally { clearTimeout(timer); } - // Drain to free the socket (we only care about headers). - try { await response.arrayBuffer(); } catch {} + // FR4-07: we only want headers — CANCEL the body, don't drain it. Draining + // (arrayBuffer) after the timeout was cleared would buffer an unbounded / + // slow-streamed body from a user-controlled URL with no cap or deadline. + try { await response.body?.cancel(); } catch { /* ignore */ } const headers = {}; for (const [k, v] of response.headers.entries()) headers[k.toLowerCase()] = v; diff --git a/src/tools/x402-kit.js b/src/tools/x402-kit.js index d6a26b99..01b2195c 100644 --- a/src/tools/x402-kit.js +++ b/src/tools/x402-kit.js @@ -16,6 +16,29 @@ import { randomBytes } from "node:crypto"; import sha3 from "js-sha3"; // CommonJS — default import, then destructure const { keccak256 } = sha3; + +// FR4-08: read a response body with a hard byte cap. `AbortSignal.timeout` +// bounds elapsed time, not memory — a fast large response from a user-controlled +// URL would still buffer unbounded via res.json()/res.text(). Cancels the stream +// and throws (502) past the cap. x402 quote/audit metadata is small. +const X402_MAX_BODY_BYTES = 256 * 1024; +async function boundedText(res, maxBytes = X402_MAX_BODY_BYTES) { + const reader = res.body?.getReader?.(); + if (!reader) return await res.text(); + const chunks = []; + let total = 0; + for (;;) { + const { done, value } = await reader.read(); + if (done) break; + total += value.byteLength; + if (total > maxBytes) { + try { await reader.cancel(); } catch { /* ignore */ } + throw Object.assign(new Error(`response body exceeded ${maxBytes} bytes`), { statusCode: 502 }); + } + chunks.push(Buffer.from(value)); + } + return Buffer.concat(chunks).toString("utf8"); +} import { assertPublicUrl, ssrfDispatcher } from "./fetch-guard.js"; import { redactSecrets } from "./redact.js"; @@ -130,7 +153,7 @@ async function rpc(net, method, params, { passes = 2 } = {}) { signal: AbortSignal.timeout(15000), dispatcher: ssrfDispatcher, }); - const text = await r.text(); + const text = await boundedText(r); let j; try { j = JSON.parse(text); } catch { lastErr = new Error(`${url}: non-JSON`); continue; } if (j.result !== undefined) return j.result; lastErr = new Error(`${url}: ${JSON.stringify(j.error ?? j).slice(0, 120)}`); @@ -254,7 +277,7 @@ export const X402_TOOLS = [ } const paymentRequired = res.status === 402; let body = null; - try { body = await res.json(); } catch { /* may be empty/non-JSON */ } + try { const t = await boundedText(res); body = t ? JSON.parse(t) : null; } catch { /* may be empty/non-JSON/oversized */ } // x402 v1 put the payment requirements in the 402 body; v2 moved them to // the base64-encoded PAYMENT-REQUIRED header (the body is `{}`). Decode // whichever the seller speaks — without the header path this tool returns @@ -494,7 +517,7 @@ export const X402_TOOLS = [ const cacheControl = res.headers.get("cache-control") || ""; let bodyText = ""; - try { bodyText = await res.text(); } catch { /* empty/unreadable body */ } + try { bodyText = await boundedText(res); } catch { /* empty/unreadable/oversized body */ } const paymentRequiredHeader = res.headers.get("payment-required") || null; return gradeX402Response({ href: url.href, protocol: url.protocol, status: res.status, cacheControl, bodyText, paymentRequiredHeader }); }, diff --git a/src/worker-client.js b/src/worker-client.js index 3fedeff3..e599a8e6 100644 --- a/src/worker-client.js +++ b/src/worker-client.js @@ -14,8 +14,56 @@ const workerUrl = () => (process.env.RENDER_WORKER_URL || "").trim().replace(/\/$/, ""); const workerToken = () => (process.env.RENDER_WORKER_TOKEN || "").trim(); const WORKER_TIMEOUT_MS = Number(process.env.RENDER_WORKER_TIMEOUT_MS) || 75_000; +// FR4-06: cap the response a (compromised/malfunctioning) worker can make the +// main service buffer. Renders (markdown) and screenshots (base64 PNG) are +// already bounded by render.js's byte budget; this is the belt-and-suspenders. +const WORKER_MAX_BYTES = Number(process.env.RENDER_WORKER_MAX_BYTES) || 40 * 1024 * 1024; -export function workerEnabled() { return Boolean(workerUrl()); } +// FR4-06: config is ATOMIC — the worker is used only when BOTH the URL and the +// token are set. A URL without a token would send unauthenticated calls that the +// (fail-closed) worker 401s, silently breaking every paid render. +export function workerEnabled() { return Boolean(workerUrl() && workerToken()); } + +// FR4-06 / P1.1: fail LOUD at boot on a partial worker config (exactly one of +// URL/token set), and — when isolation is REQUIRED (RENDER_WORKER_REQUIRED +// truthy, e.g. in prod) — on a missing worker config, so browser/media tools can +// never silently run in-process in a deployment that intends isolation. Call once +// at boot. +export function assertWorkerConfig() { + const hasUrl = Boolean(workerUrl()); + const hasToken = Boolean(workerToken()); + if (hasUrl !== hasToken) { + throw new Error( + `Incomplete render-worker config: RENDER_WORKER_URL is ${hasUrl ? "set" : "unset"} but RENDER_WORKER_TOKEN is ${hasToken ? "set" : "unset"}. Set BOTH (worker isolation) or NEITHER (in-process).` + ); + } + const required = /^(1|true|yes|on)$/i.test((process.env.RENDER_WORKER_REQUIRED || "").trim()); + if (required && !(hasUrl && hasToken)) { + throw new Error( + "RENDER_WORKER_REQUIRED is set but the render worker is not configured (need RENDER_WORKER_URL + RENDER_WORKER_TOKEN). Refusing to boot so browser/media tools cannot run in-process and defeat the isolation." + ); + } +} + +// Read a fetch Response body as text with a hard byte cap; aborts the stream and +// throws (502) if the worker returns more than `maxBytes`. +async function readBounded(res, maxBytes) { + const reader = res.body?.getReader?.(); + if (!reader) return await res.text(); + const chunks = []; + let total = 0; + for (;;) { + const { done, value } = await reader.read(); + if (done) break; + total += value.byteLength; + if (total > maxBytes) { + try { await reader.cancel(); } catch { /* ignore */ } + throw Object.assign(new Error(`render worker response exceeded ${maxBytes} bytes`), { statusCode: 502 }); + } + chunks.push(Buffer.from(value)); + } + return Buffer.concat(chunks).toString("utf8"); +} // Dispatch one browser/media job to the worker. Returns the tool's JSON result, // or `{ __binary: Buffer, contentType }` for a binary result (screenshot PNG). @@ -37,10 +85,14 @@ export async function runOnWorker(slug, input, { signal } = {}) { } catch (e) { throw Object.assign(new Error(`render worker unreachable: ${e.name === "AbortError" ? "timeout" : e.message}`), { statusCode: 503 }); } finally { clearTimeout(timer); } - const text = await res.text(); + const text = await readBounded(res, WORKER_MAX_BYTES); let body = null; try { body = JSON.parse(text); } catch { /* non-JSON */ } if (!res.ok) throw Object.assign(new Error(body?.error || `render worker HTTP ${res.status}`), { statusCode: res.status }); - if (body && body.__binary) return { __binary: Buffer.from(body.__binary, "base64"), contentType: body.contentType }; + if (body && body.__binary) { + const buf = Buffer.from(body.__binary, "base64"); + if (buf.byteLength > WORKER_MAX_BYTES) throw Object.assign(new Error("render worker binary too large"), { statusCode: 502 }); + return { __binary: buf, contentType: body.contentType }; + } return body; } diff --git a/tollbooth/README.md b/tollbooth/README.md index 47f645c4..5325031c 100644 --- a/tollbooth/README.md +++ b/tollbooth/README.md @@ -151,19 +151,28 @@ server stack (`@x402/express` / your facilitator) rather than reinventing settlement: ```js +import { createTollbooth, x402VerifierFromExpress } from "agent402-tollbooth"; import { paymentMiddleware } from "x402-express"; // or @x402/express const x402 = paymentMiddleware(/* your wallet + facilitator config */); app.use(createTollbooth({ payTo: "0xYourWallet", network: "base", - // Reuse the standard middleware to verify the X-PAYMENT header: - verifyX402: (req) => new Promise((resolve) => - x402(req, { setHeader() {}, status() { return this; }, json() { resolve(false); } }, () => resolve(true))), + // First-party verifier that OWNS timeout + cancellation: it honors the + // AbortSignal the gate passes on the SECOND argument (opts.signal), so a slow + // middleware can't hang the gate or settle after the gate already returned 402. + verifyX402: x402VerifierFromExpress(x402, { timeoutMs: 9000 }), })); ``` -(PoW is checked first, so an agent without a wallet always has a free path.) +The gate calls `verifyX402(req, opts)` and puts an `AbortSignal` on `opts.signal` +(aborted when its `TOLLBOOTH_VERIFY_TIMEOUT_MS` fires). `x402VerifierFromExpress` +honors it and stops treating a late result as valid. Note @x402/express settles +by broadcasting and has no cancel hook, so keep the timeout above your settle +latency (the abort is a backstop); a verifier that ignores the signal entirely +can still charge a buyer who already received a 402. If you write your own +verifier instead of using the helper, destructure `opts.signal` and reject/stop +on abort. (PoW is checked first, so an agent without a wallet always has a free path.) ## Configuration diff --git a/tollbooth/index.js b/tollbooth/index.js index aecc76ed..3a9a7b6e 100644 --- a/tollbooth/index.js +++ b/tollbooth/index.js @@ -23,6 +23,53 @@ export { createPow, leadingZeroBits } from "./pow.js"; export { memorySink, kvStatsSink, httpStatsSink } from "./sinks.js"; const VERIFY_TIMEOUT_MS = Number(process.env.TOLLBOOTH_VERIFY_TIMEOUT_MS) || 10_000; + +// P1.5 / FR4-10: a first-party `verifyX402` built from a standard @x402/express +// payment middleware, that OWNS timeout + cancellation. The gate calls +// verifyX402(req, opts) and puts an AbortSignal on `opts.signal` (aborted when +// the gate's own verify timeout fires); this wrapper honors it — once the signal +// aborts (or its own optional backstop timeout fires) it settles the verify +// promise immediately and reports NOT-verified, so a slow middleware can't leave +// the gate hanging or produce a charged-then-denied result after the gate has +// returned 402. Grants on next(); denies when the middleware writes a 402. +// +// LIMITATION (documented): @x402/express settles by BROADCASTING and exposes no +// cancel hook, so this wrapper cannot abort an on-chain settle already in flight. +// Keep TOLLBOOTH_VERIFY_TIMEOUT_MS comfortably above your settle latency so the +// abort is a backstop, not the normal path; or front it with a facilitator that +// supports cancellation. +export function x402VerifierFromExpress(paymentMiddleware, { timeoutMs } = {}) { + if (typeof paymentMiddleware !== "function") { + throw new TypeError("x402VerifierFromExpress: paymentMiddleware must be a function"); + } + return (req, opts = {}) => new Promise((resolve, reject) => { + const signal = opts?.signal; + let done = false; + let timer = null; + const finish = (fn, v) => { + if (done) return; + done = true; + if (timer) clearTimeout(timer); + signal?.removeEventListener?.("abort", onAbort); + fn(v); + }; + const onAbort = () => finish(resolve, false); // gate timed out → treat as not verified + if (signal?.aborted) return finish(resolve, false); + signal?.addEventListener?.("abort", onAbort, { once: true }); + if (timeoutMs > 0) timer = setTimeout(() => finish(resolve, false), timeoutMs); + // Minimal response shim: the middleware calls next() to GRANT, or writes a + // 402 via status/json/send/end to DENY. + const res = { + setHeader() { return this; }, getHeader() {}, removeHeader() { return this; }, getHeaders() { return {}; }, + status() { return this; }, json() { finish(resolve, false); return this; }, + send() { finish(resolve, false); return this; }, end() { finish(resolve, false); return this; }, + writeHead() { return this; }, write() { return true; }, flushHeaders() {}, + }; + try { + Promise.resolve(paymentMiddleware(req, res, () => finish(resolve, true))).catch((e) => finish(reject, e)); + } catch (e) { finish(reject, e); } + }); +} // Headers a client must never be able to forge through the proxy: the gate's own // trust signals and forwarding/hop-by-hop headers. const STRIP_INBOUND = new Set([ diff --git a/tollbooth/worker.js b/tollbooth/worker.js index a5fcf224..3663353b 100644 --- a/tollbooth/worker.js +++ b/tollbooth/worker.js @@ -79,14 +79,27 @@ export default { // locations. KV is a best-effort fallback (eventually consistent; concurrent // dupes can both pass). With neither, protection is per-isolate only. const enforcing = env.TOLLBOOTH_OBSERVE !== "true"; + // FR4-04: FAIL CLOSED in enforcing mode without an ATOMIC replay store. A + // Durable Object (TOLLBOOTH_REPLAY) gives strict single-use across isolates + // and locations. KV is eventually consistent (get-then-put, NOT atomic), so + // concurrent duplicate PoW solutions can both pass — a warning does not + // protect a production deployment. Refuse to enforce on a non-atomic (or + // absent) store unless the operator sets an explicit, loud insecure override. + const allowNonAtomic = env.TOLLBOOTH_ALLOW_NON_ATOMIC_REPLAY === "true"; + if (enforcing && !env.TOLLBOOTH_REPLAY && !allowNonAtomic) { + return new Response( + "Tollbooth misconfigured: enforcing mode requires an ATOMIC replay store. Bind a Durable Object as TOLLBOOTH_REPLAY (see deploy/cloudflare/wrangler.toml), run in observe mode (TOLLBOOTH_OBSERVE=true), or accept non-atomic replay explicitly with TOLLBOOTH_ALLOW_NON_ATOMIC_REPLAY=true.", + { status: 500 } + ); + } let replayStore; if (env.TOLLBOOTH_REPLAY) { replayStore = durableObjectStore(env.TOLLBOOTH_REPLAY); } else if (env.TOLLBOOTH_KV) { replayStore = kvStore(env.TOLLBOOTH_KV); - if (enforcing) console.warn("agent402-tollbooth: replay protection is on eventually-consistent KV (get-then-put, NOT atomic) — concurrent duplicate solutions across isolates/locations can both pass. Bind a Durable Object as TOLLBOOTH_REPLAY for strict single-use in enforcement (see wrangler.toml)."); + if (enforcing) console.warn("agent402-tollbooth: ENFORCING on eventually-consistent KV via explicit TOLLBOOTH_ALLOW_NON_ATOMIC_REPLAY override (get-then-put is NOT atomic — concurrent duplicate solutions across isolates/locations can both pass). Bind a Durable Object as TOLLBOOTH_REPLAY for strict single-use."); } else if (enforcing) { - console.warn("agent402-tollbooth: no replay store bound (TOLLBOOTH_REPLAY Durable Object or TOLLBOOTH_KV) — proof-of-work replay protection is per-isolate only. Bind one for production enforcement."); + console.warn("agent402-tollbooth: ENFORCING with NO replay store via explicit TOLLBOOTH_ALLOW_NON_ATOMIC_REPLAY override — proof-of-work replay protection is per-isolate only. Bind a Durable Object for production."); } // Durable stats live in KV if a namespace is bound. Without it, the dashboard // is per-isolate (dies on cold start) — fine for dev, useless for prod. diff --git a/worker/server.js b/worker/server.js index 86f8965c..e68630f8 100644 --- a/worker/server.js +++ b/worker/server.js @@ -12,6 +12,8 @@ // in this process's env — the worker must be secretless by construction. import express from "express"; import { timingSafeEqual } from "node:crypto"; +import { resolve } from "node:path"; +import { fileURLToPath } from "node:url"; import { renderArticle, screenshotPage } from "../src/tools/render.js"; import { MEDIA_TOOLS } from "../src/tools/media-kit.js"; import { startEgressProxy } from "./egress-proxy.js"; @@ -29,29 +31,54 @@ const HANDLERS = { }; for (const t of MEDIA_TOOLS) HANDLERS[t.slug] = (input) => t.handler(input || {}); -// Boot guard: this worker must be SECRETLESS by construction. It refuses to boot -// (see the isMain block) if any secret-bearing env var is present — a -// misconfigured deployment that would defeat the isolation. The check is -// PATTERN-based (KEY/SECRET/TOKEN/…) so it catches current AND future secrets by -// name, not a hand-kept denylist that drifts behind the app's 30+ secret env -// vars; plus a few secret vars whose names don't match the pattern. The worker's -// OWN inbound-auth token is the sole allowed pattern match. Enforced ONLY at boot -// (not at module import) so a test/CI process — which legitimately has its own -// GITHUB_TOKEN etc. — can import the exports without tripping it. Because Railway -// gates a deploy on the healthcheck, a false positive fails the new deploy -// visibly while the old worker keeps serving, rather than causing an outage. -// NB: no bare "PRIVATE" — Railway injects RAILWAY_PRIVATE_DOMAIN (the worker's -// own private-network hostname, not a secret), and a private KEY is already -// caught by "KEY". Matching "PRIVATE" alone would false-positive and refuse to -// boot on a benign infra var. -const SECRET_NAME_RE = /(KEY|SECRET|TOKEN|MNEMONIC|PASSWORD|CREDENTIAL)/i; -const SECRET_NAME_ALLOW = new Set(["RENDER_WORKER_TOKEN"]); -const FORBIDDEN_ENV = ["WALLET_ADDRESS", "WALLET_ENS", "DATABASE_URL", "ANALYTICS_DATABASE_URL"]; -// Pure + exported for tests: the secret-bearing env var names present in `env`. +// Boot guard: this worker must be SECRETLESS by construction. Enforced ONLY at +// boot (see the isMain block), not at module import — so a test/CI process, which +// legitimately has its own GITHUB_TOKEN etc., can import the exports without +// tripping it. +// FR4-05 (strict allowlist, P1.2): the worker boots ONLY if every set env var is +// explicitly allowed or a known non-secret system/Railway var. Anything +// unrecognized — or anything that looks secret-bearing — refuses boot. This is +// the strict form: an unknown var is denied, not just secret-shaped ones. A +// genuinely-new benign var must be added to ALLOWED_EXACT/ALLOWED_SYSTEM. Because +// Railway health-gates the deploy, a false positive fails the NEW deploy (the old +// worker keeps serving) rather than causing an outage. +const ALLOWED_EXACT = new Set([ + "PORT", "NODE_ENV", "WORKER_MODE", "TZ", + "RENDER_WORKER_TOKEN", "RENDER_WORKER_URL", "RENDER_WORKER_TIMEOUT_MS", + "RENDER_WORKER_MAX_BYTES", "RENDER_EGRESS_PROXY_URL", "RENDER_WORKER_REQUIRED", +]); +// Non-secret vars injected by Docker / Node / the shell / Playwright — plus the +// macOS/dev/editor runtime noise a developer would have when running the worker +// locally. Broad on purpose: it's SAFE because secret-shaped names are blocked +// first (SECRET_SHAPE_RE below), so nothing sensitive can slip through here. +const ALLOWED_SYSTEM = /^(PATH|HOME|HOSTNAME|PWD|OLDPWD|SHLVL|SHELL|TERM|COLORTERM|COLORFGBG|LANG|LANGUAGE|LC_[A-Z]+|LS_COLORS|LSCOLORS|USER|LOGNAME|MAIL|EDITOR|PAGER|LESS|TMPDIR|TMP|TEMP|DISPLAY|HISTFILE|HISTSIZE|INFOPATH|MANPATH|COMMAND_MODE|MallocNanoZone|SECURITYSESSIONID|_|NODE_[A-Z_]*|npm_[a-z_0-9]*|YARN_[A-Z_]*|PLAYWRIGHT[_A-Z]*|CHROME[_A-Z]*|CHROMIUM[_A-Z]*|PUPPETEER[_A-Z]*|LD_LIBRARY_PATH|LD_PRELOAD|SSL_CERT_[A-Z]+|NODE_EXTRA_CA_CERTS|DEBIAN_FRONTEND|GPG_[A-Z_]*|container|__CF[A-Za-z_]*|XPC_[A-Z_]*|Apple[A-Za-z_]*|SSH_[A-Z]+|TERM_[A-Z_]*|ITERM_[A-Z_]*|VSCODE_[A-Z_]*|CURSOR_[A-Z_]*|ZSH[A-Z_]*|BASH[A-Z_]*)$/; +// Secret-shaped names are ALWAYS blocked (in every environment), except the +// worker's own inbound-auth token — this is the auditor's URL/DSN/DB/ledger +// concern (those carry credentials) plus keys, tokens, mnemonics, passwords. +// NB: no bare "PRIVATE" — it false-positives on Railway's benign +// RAILWAY_PRIVATE_DOMAIN (took a deploy down once); a private KEY is caught by "KEY". +const SECRET_SHAPE_RE = /(KEY|SECRET|TOKEN|MNEMONIC|PASSWORD|PASSWD|CREDENTIAL|DSN|LEDGER|DATABASE|REDIS|POSTGRES|MONGO|MYSQL|CONN(ECTION)?_?STR(ING)?)/i; +const SECRET_ALLOW = new Set(["RENDER_WORKER_TOKEN"]); +// Pure + exported for tests: env var names that BLOCK a secretless boot. +// +// The STRICT allowlist ("deny anything not explicitly allowed") is enforced only +// in the controlled deployment — on Railway (RAILWAY_ENVIRONMENT present) or when +// forced with WORKER_STRICT_ENV — where the container env is minimal and known. +// In dev/CI the process env is full of benign editor/OS/tooling noise (a real +// run here surfaced 25+ such vars), so applying deny-unknown there just refuses +// to boot on harmless names; instead we enforce the secret-SHAPE denylist, which +// still blocks every credential-shaped name. Railway health-gates the deploy, so +// a strict-mode false positive fails the NEW deploy (old worker keeps serving). export function forbiddenSecretsIn(env) { - return Object.keys(env).filter( - (k) => (env[k] || "").trim() && !SECRET_NAME_ALLOW.has(k) && (SECRET_NAME_RE.test(k) || FORBIDDEN_ENV.includes(k)) - ); + const strict = Boolean((env.RAILWAY_ENVIRONMENT || "").trim()) + || /^(1|true|yes|on)$/i.test((env.WORKER_STRICT_ENV || "").trim()); + return Object.keys(env).filter((k) => { + if (!(env[k] || "").trim()) return false; // unset/empty → ignore + if (SECRET_SHAPE_RE.test(k) && !SECRET_ALLOW.has(k)) return true; // secret-shaped: always block + if (!strict) return false; // dev/CI: shape denylist only + if (ALLOWED_EXACT.has(k) || ALLOWED_SYSTEM.test(k) || /^RAILWAY_/.test(k)) return false; + return true; // Railway strict: unrecognized → deny + }); } const secretsPresent = () => forbiddenSecretsIn(process.env); @@ -90,7 +117,9 @@ app.post("/call", async (req, res) => { // Start listening when run directly (`node worker/server.js`) OR when the shared // image's start.js dispatcher selected worker mode via WORKER_MODE. Imports by a // test (argv is the test file, WORKER_MODE unset) still just read the exports. -const isMain = (process.argv[1] && process.argv[1].endsWith("worker/server.js")) +// FR4-12: compare resolved paths (not a forward-slash suffix) so `node +// worker\server.js` on Windows also triggers the boot guard, not only WORKER_MODE. +const isMain = (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) || /^(1|true|yes|on)$/i.test((process.env.WORKER_MODE || "").trim()); if (isMain) { // Enforce the secretless invariant and the auth requirement ONLY when actually @@ -116,4 +145,4 @@ if (isMain) { app.listen(PORT, "::", () => console.log(`[worker] secretless browser/media worker on :${PORT} — tools: ${Object.keys(HANDLERS).join(", ")}`)); } -export { app, HANDLERS, tokenOk, FORBIDDEN_ENV }; +export { app, HANDLERS, tokenOk };