diff --git a/appstore-meta/data/apps/io.pilot.generallegal.json b/appstore-meta/data/apps/io.pilot.generallegal.json new file mode 100644 index 0000000..a327403 --- /dev/null +++ b/appstore-meta/data/apps/io.pilot.generallegal.json @@ -0,0 +1,309 @@ +{ + "id": "io.pilot.generallegal", + "name": "General Legal", + "tagline": "Attorney-backed contract review and Delaware company formation \u2014 flat-fee legal work from a licensed US law firm", + "description": "General Legal is a Y Combinator-backed law firm. This app puts a licensed\nattorney and a Delaware filing desk behind your agent \u2014 contract review and\ncompany formation, in one namespace.\n\nThe two halves have different requirements. Formation works the moment you\ninstall. Contract review needs a General Legal account and an API key.\n\n### Company formation \u2014 nothing to set up\n\nNo account, no key, no configuration. Call\n`generallegal.formation_options` and it works on a fresh install.\n\nYou will still need to pay: `formation_start_llc` and\n`formation_start_c_corp` return a payment link, and the founder pays there.\nUse the founder's real details \u2014 this files an actual Delaware company.\n\n### Contract review \u2014 bring your own API key\n\n1. Sign up at https://portal.general.legal/signup\n2. Open the account menu and choose **API keys**\n (https://portal.general.legal/api-keys)\n3. Create a key and copy it \u2014 the full value is shown **once**\n4. Import it into the app:\n\n```\nprintf '{\"GENERAL_LEGAL_API_KEY\":\"glk_YOUR_KEY\"}' > ~/.pilot/apps/io.pilot.generallegal/secrets.json\nchmod 600 ~/.pilot/apps/io.pilot.generallegal/secrets.json\npilotctl appstore restart io.pilot.generallegal\n```\n\nThe restart matters: the key is read at startup. Verify with\n`pilotctl appstore call io.pilot.generallegal generallegal.deals_list '{}'`.\nThe key stays on your machine, is never sent to the formation service, and\nscopes you to your own General Legal organization.\n\n### What it costs\n\nContract review is flat-fee, with no hourly billing and no minimums. The fee\ncovers every turn through signature, including negotiation with the\ncounterparty.\n\n| Work | Price |\n| --- | --- |\n| Contract, 3 pages or fewer | $250 |\n| Contract, 3-50 pages | $500 |\n| Contract, 50+ pages | $10 per page |\n| Drafting from scratch | $2,000 |\n| Delaware LLC | $190 instant / $210 standard / $260 next-day / $310 same-day |\n| Delaware C-corp | $218 standard / $268 next-day / $318 same-day |\n\n### What the app does\n\n**Company formation** \u2014 `formation_options` (free), `formation_start_llc`\n(**paid**), `formation_start_c_corp` (**paid**), `formation_status`,\n`formation_update`, `formation_documents` (free).\n\n**Contract review** \u2014 `deal_open` (**paid**), `document_upload` (**paid**),\n`thread_post` (**paid**, covered by the matter's flat fee), plus\n`deals_list`, `deal_get`, `thread_get`, `contracts_list`, `contract_get`,\n`version_download_link` (free) and `upload_begin`, `upload_chunk`,\n`upload_abort` (free \u2014 a document is staged in chunks because a single call\ncannot carry a file).\n\n### What costs money\n\nFive methods spend real money and will not warn you first:\n`formation_start_llc`, `formation_start_c_corp`, `deal_open`,\n`document_upload` and `thread_post`. Every other method is free.\n`generallegal.help` lists them under `billable_methods` with the price.", + "categories": [ + "work" + ], + "primary_category": "work", + "keywords": [ + "legal", + "contracts", + "attorney", + "review", + "nda", + "redline", + "incorporation", + "delaware", + "llc", + "c-corp" + ], + "version": "0.1.0", + "vendor": "General Legal", + "vendor_url": "https://general.legal", + "license": "Apache-2.0", + "source_url": "https://github.com/pilot-protocol/generallegal-app", + "homepage": "https://general.legal", + "methods": [ + { + "name": "generallegal.formation_options", + "summary": "Itemised pricing for every Delaware entity type and filing speed. No account or credentials needed to call it.", + "example": "", + "gated": "", + "billable": "" + }, + { + "name": "generallegal.formation_start_llc", + "summary": "File a sole-member Delaware LLC and get back a payment link plus a formation_id. No account or credentials needed to call it. 'instant' hands over a pre-formed shelf company; the other speeds file under a name you choose.", + "example": "", + "gated": "", + "billable": "Paid \u2014 files a real Delaware LLC. $190 instant / $210 standard / $260 next-day / $310 same-day, paid by the founder at the returned link." + }, + { + "name": "generallegal.formation_start_c_corp", + "summary": "File a Delaware C-corp and get back a payment link plus a formation_id. No account or credentials needed to call it. The founder acts as sole incorporator.", + "example": "", + "gated": "", + "billable": "Paid \u2014 files a real Delaware C-corp. $218 standard / $268 next-day / $318 same-day, paid by the founder at the returned link." + }, + { + "name": "generallegal.formation_status", + "summary": "Poll a filing's progress. No account or credentials needed to call it. Repeat after poll_after_seconds while it keeps coming back.", + "example": "", + "gated": "", + "billable": "" + }, + { + "name": "generallegal.formation_update", + "summary": "Change the company name before the documents are generated. No account or credentials needed to call it. Late changes are rejected.", + "example": "", + "gated": "", + "billable": "" + }, + { + "name": "generallegal.formation_documents", + "summary": "Short-lived links to a completed formation's documents. No account or credentials needed to call it.", + "example": "", + "gated": "", + "billable": "" + }, + { + "name": "generallegal.deals_list", + "summary": "List your matters, paginated, with an optional status filter. Free. Needs your General Legal API key.", + "example": "", + "gated": "", + "billable": "" + }, + { + "name": "generallegal.deal_open", + "summary": "Open a matter from a written request. It reaches a real attorney.", + "example": "", + "gated": "", + "billable": "Paid \u2014 flat fee per contract: $250 (3 pages or fewer), $500 (3\u201350 pages), $10/page (50+), $2,000 to draft from scratch. Covers every turn through signature." + }, + { + "name": "generallegal.deal_get", + "summary": "One matter with its documents and released versions. Free.", + "example": "", + "gated": "", + "billable": "" + }, + { + "name": "generallegal.thread_get", + "summary": "Read the lawyer-client thread on a matter. Free.", + "example": "", + "gated": "", + "billable": "" + }, + { + "name": "generallegal.thread_post", + "summary": "Reply to the attorney on a matter's thread.", + "example": "", + "gated": "", + "billable": "Paid \u2014 covered by the matter's flat fee; General Legal does not bill hourly, so a reply adds no separate charge. Free when target is \"ai\"." + }, + { + "name": "generallegal.contracts_list", + "summary": "List your documents. Free.", + "example": "", + "gated": "", + "billable": "" + }, + { + "name": "generallegal.contract_get", + "summary": "One document with its released versions; the version ids feed downloads. Free.", + "example": "", + "gated": "", + "billable": "" + }, + { + "name": "generallegal.document_upload", + "summary": "Upload a DOCX, PDF, PNG, JPEG or Markdown document for AI + attorney review, up to 20 MiB. Stage the bytes with upload_begin/upload_chunk first.", + "example": "", + "gated": "", + "billable": "Paid \u2014 priced per contract by length: $250 (3 pages or fewer), $500 (3\u201350 pages), $10/page (50+). Flat fee, covering every turn through signature." + }, + { + "name": "generallegal.version_download_link", + "summary": "Issue a short-lived (~15 min) direct download URL for a released version. Free.", + "example": "", + "gated": "", + "billable": "" + }, + { + "name": "generallegal.upload_begin", + "summary": "Start a staged upload and get a blob_id. Free \u2014 a document cannot cross in one call, so declare it here first.", + "example": "", + "gated": "", + "billable": "" + }, + { + "name": "generallegal.upload_chunk", + "summary": "Append the next chunk of a staged upload, at most 512 KiB of raw bytes per call. Free.", + "example": "", + "gated": "", + "billable": "" + }, + { + "name": "generallegal.upload_abort", + "summary": "Discard a staged upload and its bytes. Free.", + "example": "", + "gated": "", + "billable": "" + }, + { + "name": "generallegal.help", + "summary": "Every method with its parameters, duration class, and which calls cost money. Free, local, no backend call.", + "example": "", + "gated": "", + "billable": "" + } + ], + "changelog": [ + { + "version": "0.1.0", + "date": "2026-08-29", + "notes": [ + "Contract review: matters, lawyer thread, document upload and released-version downloads.", + "Delaware company formation: pricing, LLC and C-corp filing, status and documents \u2014 no account needed.", + "Bring your own General Legal API key for contract review; it never reaches the formation service." + ] + } + ], + "grants": [ + "fs.read:$APP/config.json", + "fs.read:$APP/secrets.json", + "fs.read:$APP/blobs", + "fs.write:$APP/blobs", + "net.dial:api.general.legal", + "net.dial:incorp-mcp.general.legal", + "audit.log:*" + ], + "limits": [ + { + "label": "Contract review", + "value": "$250 / $500 / $10 per page \u2014 flat fee" + }, + { + "label": "Delaware LLC", + "value": "$190\u2013$310 depending on speed" + }, + { + "label": "Delaware C-corp", + "value": "$218\u2013$318 depending on speed" + }, + { + "label": "New matters", + "value": "25 / day per organization" + } + ], + "bundles": [ + { + "platform": "darwin-arm64", + "bytes": 0 + }, + { + "platform": "darwin-amd64", + "bytes": 0 + }, + { + "platform": "linux-arm64", + "bytes": 0 + }, + { + "platform": "linux-amd64", + "bytes": 0 + } + ], + "installed_bytes": 0, + "depends": [], + "protection": "shareable", + "featured": false, + "in_catalogue": true, + "icon": { + "mode": "image", + "img": "/appicons/io.pilot.generallegal.png", + "file": "", + "fit": "cover", + "pos": "center", + "color": "#0e1a2b", + "ink": false, + "hue": 45, + "mark": null + }, + "min_pilot_version": "", + "runtimes": [ + "go" + ], + "published_at": "2026-08-29", + "updated_at": "2026-08-29", + "product_demo": { + "skill": "io.pilot.generallegal", + "title": "Full usage demo", + "metered": false, + "when_to_use": "When a contract needs a licensed attorney to review, redline or draft it, or when an agent needs its own Delaware company. Formation works on a fresh install; contract review needs your own General Legal API key.", + "quickstart": { + "goal": "Price a Delaware company \u2014 works on a fresh install, no account, no key", + "command": "pilotctl appstore call io.pilot.generallegal generallegal.formation_options '{\"entity_type\":\"llc\"}'", + "expect": "{\"options\":[{\"filing_speed\":\"instant\",\"total_cents\":19000},{\"filing_speed\":\"standard\",\"total_cents\":21000}]}", + "note": "Free, and it needs no account or key at all \u2014 the fastest way to confirm the app works. Contract review is the half that needs a key; see the examples." + }, + "examples": [ + { + "title": "Form a Delaware company \u2014 no API key needed", + "goal": "Price it, file it, hand the founder a payment link", + "command": "pilotctl appstore call io.pilot.generallegal generallegal.formation_start_llc '{\"filing_speed\":\"standard\",\"company_name\":\"NewCo LLC\",\"founder\":{\"full_name\":\"Ada Lovelace\",\"email\":\"ada@example.com\"},\"principal_address\":{\"street\":\"1 Main St\",\"city\":\"Dover\",\"state\":\"DE\",\"postal_code\":\"19901\"},\"ai_agent_description\":\"Procurement agent\",\"authority_limits\":\"No commitments above $5,000 without sign-off\",\"contract_threshold\":\"$5,000\"}'", + "expect": "{\"formation_id\":\"f-...\",\"payment_url\":\"https://...\",\"status\":\"awaiting_payment\"}", + "note": "BILLABLE - files a real company. $190 instant / $210 standard / $260 next-day / $310 same-day, paid by the founder at the returned link. No account or key needed to call it. formation_id is shown once." + }, + { + "title": "Track the filing and collect the paperwork", + "goal": "Poll to completion, then pull the documents", + "command": "pilotctl appstore call io.pilot.generallegal generallegal.formation_status '{\"formation_id\":\"f-...\"}'", + "expect": "{\"status\":\"filed\",\"poll_after_seconds\":30} then {\"status\":\"complete\"}", + "note": "Free. Repeat only after poll_after_seconds. When complete, generallegal.formation_documents returns short-lived links." + }, + { + "title": "Bring your own key, then confirm it works", + "goal": "Authenticate as your own General Legal organization", + "command": "pilotctl appstore call io.pilot.generallegal generallegal.deals_list '{\"page\":1,\"page_size\":5}'", + "expect": "{\"items\":[...],\"total\":n} once the key is in place; 401 until then", + "note": "Contract review only. Sign up at https://portal.general.legal/signup, then account menu -> API keys (shown once). Import: printf '{\"GENERAL_LEGAL_API_KEY\":\"glk_YOUR_KEY\"}' > ~/.pilot/apps/io.pilot.generallegal/secrets.json && chmod 600 ~/.pilot/apps/io.pilot.generallegal/secrets.json && pilotctl appstore restart io.pilot.generallegal" + }, + { + "title": "Ask an attorney to review a contract", + "goal": "Open a matter a real lawyer picks up", + "command": "pilotctl appstore call io.pilot.generallegal generallegal.deal_open '{\"initial_request\":\"Please review this mutual NDA. We are the disclosing party; flag anything unusual in the confidentiality term.\",\"deal_name\":\"Acme mutual NDA\"}'", + "expect": "{\"deal_id\":\"d-9f3...\",\"status\":\"open\"}", + "note": "BILLABLE - flat fee per contract: $250 (<=3 pages), $500 (3-50), $10/page (50+), $2,000 to draft. Covers every turn through signature. Keep the deal_id." + }, + { + "title": "Stage the document, then send it", + "goal": "Push the bytes in chunks (a file cannot cross in one message), then upload", + "command": "pilotctl appstore call io.pilot.generallegal generallegal.upload_begin '{\"file_name\":\"nda.docx\",\"content_type\":\"application/vnd.openxmlformats-officedocument.wordprocessingml.document\",\"total_bytes\":3145728,\"sha256\":\"\"}'", + "expect": "{\"blob_id\":\"a1b2...\",\"max_chunk_bytes\":524288,\"next_seq\":0}", + "note": "Then generallegal.upload_chunk with seq 0,1,2... and base64 of at most max_chunk_bytes raw bytes. The last returns complete:true. Finally generallegal.document_upload with the blob_id and deal_id (BILLABLE)." + }, + { + "title": "Collect the released redline", + "goal": "Get a downloadable link to counsel's version", + "command": "pilotctl appstore call io.pilot.generallegal generallegal.version_download_link '{\"version_id\":\"v-07...\"}'", + "expect": "{\"file_name\":\"nda-redline.docx\",\"download_url\":\"https://...\",\"download_token_expires_at\":\"...\"}", + "note": "Free. The URL needs no auth and expires in ~15 minutes - fetch it yourself. Find version ids via generallegal.deal_get or generallegal.contract_get." + } + ], + "gotchas": [ + "Two halves, different requirements. Company formation needs no account and no key. Contract review needs a General Legal account and an API key imported.", + "For contract review: sign up at portal.general.legal/signup, mint a key under API keys, write it to $APP/secrets.json, then restart the app - the key is read at startup.", + "Five methods spend money: deal_open and document_upload are flat-fee per contract ($250/$500/$10-per-page/$2,000); formation_start_llc and formation_start_c_corp file a real company ($190-$318); thread_post is covered by the matter's fee.", + "A document cannot ride in one call. Use upload_begin then upload_chunk (<=512 KiB raw each) and pass the blob_id. A blob is single-use and dropped once sent.", + "Matters reach a real attorney and formations file a real company. Neither is a sandbox - confirm before calling a billable method.", + "Contract-review limits are per organization: 120 reads/min, 25 new matters/day, 50 uploads/day. Honour Retry-After on a 429." + ], + "next": [ + "generallegal.formation_options to price a company, or generallegal.deals_list to see the matters your key can reach.", + "generallegal.help lists every method and marks exactly which ones cost money." + ] + } +} \ No newline at end of file diff --git a/appstore-meta/data/index.json b/appstore-meta/data/index.json index a4135d4..f1a52f5 100644 --- a/appstore-meta/data/index.json +++ b/appstore-meta/data/index.json @@ -23,7 +23,7 @@ { "id": "infra", "name": "Infrastructure", - "blurb": "Containers, microVMs, and deploys — the compute layer for agents.", + "blurb": "Containers, microVMs, and deploys \u2014 the compute layer for agents.", "hue": 30 }, { @@ -41,13 +41,13 @@ { "id": "work", "name": "Work & Research", - "blurb": "Put real-world work in motion and get real answers back — human hands on a task, and pricing research with your own customers.", + "blurb": "Put real-world work in motion and get real answers back \u2014 human hands on a task, and pricing research with your own customers.", "hue": 45 }, { "id": "comms", "name": "Communications", - "blurb": "Give an agent its own phone number or email inbox — voice, SMS/iMessage, email, and threaded conversations.", + "blurb": "Give an agent its own phone number or email inbox \u2014 voice, SMS/iMessage, email, and threaded conversations.", "hue": 315 } ], @@ -57,6 +57,7 @@ "io.pilot.docker" ], "app_order": [ + "io.pilot.generallegal", "io.pilot.dial", "io.pilot.deadsimple", "io.pilot.kinetic", @@ -86,4 +87,4 @@ "io.pilot.tldr", "io.pilot.insforge" ] -} +} \ No newline at end of file diff --git a/docs/MULTIPART-UPLOADS.md b/docs/MULTIPART-UPLOADS.md new file mode 100644 index 0000000..50553f2 --- /dev/null +++ b/docs/MULTIPART-UPLOADS.md @@ -0,0 +1,150 @@ +# Multipart uploads — sending a file to a partner API + +Some partner endpoints take a file, not JSON. General Legal's document endpoint is +the shape: `POST /api/v1/documents` is `multipart/form-data` carrying a DOCX or PDF +plus a few scalar fields, one of which names the matter the document belongs to. + +Declare `multipart:` on the route and the generator handles the rest. + +```yaml +- name: gl.document_upload + summary: "Upload a document for attorney review (20 MiB max)." + duration: slow + timeout: 180s + http: + verb: POST + path: /api/v1/documents + multipart: + file_field: file # the form field the partner reads (default "file") + max_bytes: 20971520 # match the partner's own limit + params: + blob_id: "string (required) — from gl.upload_begin" + deal_id: "string (optional) — attach to an existing matter" + context_for_legal: "string (optional)" +``` + +Every param except `blob_id` and the path placeholders becomes a form field, so the +shape an agent reads in `.help` is the shape the partner receives. + +## Why the file is staged instead of sent inline + +The obvious design — base64 the file into the method's JSON payload — does not fit, +and this is worth understanding before reaching for it anyway. + +Pilot IPC is JSON in, JSON out over a framed unix socket, and `ipc.MaxFrameSize` +caps a single envelope at **1 MiB**; an oversize frame is refused and the connection +dropped. base64 inflates by 4/3, so inline encoding tops out around **740 KiB** of +real file. Partner limits are much larger (General Legal: 20 MiB). That is not a +constant to tune — a 20 MiB document needs a 27 MiB envelope, 27× the frame. + +So the file travels in chunks that each fit, and the adapter reassembles it on disk +under `$APP/blobs` before building one multipart body. The IPC layer never carries +more than a chunk and no platform limit has to move. + +`internal/multipartkit` asserts this premise directly +(`TestBase64InOneEnvelopeExceedsIPCFrame`) rather than leaving it as a comment, so +if the frame size ever changes the trade-off gets re-examined instead of silently +becoming wrong. + +## What an agent does + +Three generated methods appear automatically on any app with a multipart route — +`.upload_begin`, `.upload_chunk`, `.upload_abort`. Do not author them. + +```bash +# 1. Declare the file. sha256 is the integrity contract over the whole reassembly. +pilotctl appstore call io.pilot.generallegal gl.upload_begin \ + '{"file_name":"nda.docx","content_type":"application/vnd.openxmlformats-officedocument.wordprocessingml.document","total_bytes":3145728,"sha256":"<64 hex>"}' +# -> {"blob_id":"…","max_chunk_bytes":524288,"next_seq":0} + +# 2. Push the bytes, in order, at most max_chunk_bytes of RAW file per call. +pilotctl appstore call io.pilot.generallegal gl.upload_chunk \ + '{"blob_id":"…","seq":0,"data_base64":"…"}' +# -> {"received":524288,"next_seq":1,"complete":false} +# …the last chunk returns "complete":true once the sha256 verifies. + +# 3. Send it. +pilotctl appstore call io.pilot.generallegal gl.document_upload \ + '{"blob_id":"…","deal_id":"…","context_for_legal":"Standard mutual NDA."}' +``` + +The blob is **single-use**: once the partner has the bytes the adapter drops the +local copy, so a replayed `blob_id` fails rather than uploading twice. Staged +uploads that are never sent are reclaimed on a TTL, and an unfinished staging does +not survive an adapter respawn (its rolling hash and chunk cursor die with the +process, so resuming it could splice a gap into the middle of a document). + +## Rules the store enforces + +Chunked reassembly is a place to get integrity wrong, so the store is strict: + +| Rule | Why | +|---|---| +| `blob_id` is minted, never caller-supplied | a chosen id overwrites someone else's in-flight upload | +| ids are 32 hex chars, validated | an id becomes a filename; nothing that could hold a separator is admitted | +| chunks must be strictly sequential | tolerating a gap or a replay reassembles something the caller never sent | +| declared size is a hard cap as bytes land | not just checked at the end | +| sha256 must match at finalize | the integrity contract over the whole reassembly | +| the file name is reduced to its base name | it is metadata for the form part, never a path | + +## Managed apps: two broker steps + +A multipart app behind the managed-key broker needs two things in its registry +entry beyond the usual (see [`MANAGED-KEY.md`](MANAGED-KEY.md)): + +```json +"forward_content_types": ["multipart/form-data"], +"max_body_bytes": 25165824, +"tenancy": { + "body_refs": {"deal_id": "deal"}, + ... +} +``` + +- **`forward_content_types`** — the broker forces `application/json` by default, + which strips the boundary and makes the body undecodable. This is an allow-list + rather than a passthrough on purpose: the request media type selects which parser + the partner runs, and letting a caller choose that freely is the same lever as the + duplicate-key parser differential tenancy already refuses. +- **`max_body_bytes`** — the broker-wide default (8 MiB) is tuned for JSON calls and + is smaller than an upload partner's limit. Without this an upload the partner + would have accepted is refused with `413`. +- **`tenancy.body_refs`** — an upload names the resource it acts on in a **form + field**, never in the path. An app that forwards multipart while declaring no + `body_refs` would ownership-check nothing on exactly the route that needs it most, + so the registry **fails the boot** rather than serve it. + +The broker parses the multipart form to check those refs, with the same stance as +the JSON path: unparseable bodies, missing boundaries, and over-budget part counts +all deny; a repeated **ref** field is refused as a parser differential (repeats of +fields nobody checks are fine — they are legal multipart); and a ref arriving as a +file part is denied outright, since file parts are not inspected and that shape +would route a ref past the check. + +## Testing an upload app + +`docs/PUBLISHING-PLAYBOOK.md` Step 4 applies unchanged, plus: + +- Upload a file **larger than 1 MiB**. Anything smaller would fit an inline + encoding and proves nothing about the transport this design exists for. +- Verify the partner received the bytes **unchanged** — compare sha256, not size. +- If the app is managed, run it through a real broker, not just socket mode: the + Content-Type forwarding and the form-field ownership check only exist there. + +The reference tests are `internal/scaffold/zz_multipart_e2e_test.go` (generated +adapter, real socket, real chunking) and `zz_multipart_broker_e2e_test.go` (the full +adapter → broker → partner topology, including an upload into an unowned resource +being refused). + +## Limits + +| | | +|---|---| +| chunk | 512 KiB raw per call (`max_chunk_bytes`, reported by `upload_begin`) | +| staged upload | 24 MiB default, or `multipart.max_bytes` | +| parts the broker will parse | 64 | +| ref field value | 4 KiB | +| staging TTL | 30 minutes | + +One file per request. A partner endpoint taking several files at once would need +the route to carry several blob ids; nothing we ship requires it yet. diff --git a/docs/PUBLISHING-PLAYBOOK.md b/docs/PUBLISHING-PLAYBOOK.md index bb8f6ae..377163c 100644 --- a/docs/PUBLISHING-PLAYBOOK.md +++ b/docs/PUBLISHING-PLAYBOOK.md @@ -4,7 +4,8 @@ The end-to-end runbook for getting an app live on the Pilot app-store, for **eve backend and auth mode. It ties together the focused docs ([`PUBLISHING.md`](PUBLISHING.md), [`CLI-ADAPTER.md`](CLI-ADAPTER.md), [`NATIVE-APPS.md`](NATIVE-APPS.md), [`R2-ARTIFACT-REGISTRY.md`](R2-ARTIFACT-REGISTRY.md), -[`MANAGED-KEY.md`](MANAGED-KEY.md), [`CI-AB-REPORT.md`](CI-AB-REPORT.md), +[`MANAGED-KEY.md`](MANAGED-KEY.md), [`MULTIPART-UPLOADS.md`](MULTIPART-UPLOADS.md), +[`CI-AB-REPORT.md`](CI-AB-REPORT.md), [`PRODUCT-DEMOS.md`](PRODUCT-DEMOS.md), [`UPDATING.md`](UPDATING.md), [`UPDATING-BUNDLES.md`](UPDATING-BUNDLES.md), [`APP-PUBLISHING-SPEC.md`](APP-PUBLISHING-SPEC.md)) into one @@ -47,6 +48,14 @@ Two orthogonal choices. Get these right first; everything else follows. | a CLI tool **already on every host** | `cli` | methods → subprocess argv | | a CLI tool **not** on the host | `cli` + `assets[]` | the adapter fetches the binary from the R2 registry at install ([`NATIVE-APPS.md`](NATIVE-APPS.md)) | +> **Edge case — an endpoint takes a file.** A `multipart/form-data` route cannot send +> the file inside the JSON payload: `ipc.MaxFrameSize` caps an envelope at 1 MiB, so +> base64 tops out near 740 KiB of real file. Declare `multipart:` on the route and the +> generator adds the staging methods that push the bytes in frame-sized chunks +> ([`MULTIPART-UPLOADS.md`](MULTIPART-UPLOADS.md)). Managed apps additionally need +> `forward_content_types` and `tenancy.body_refs` in the broker registry, or uploads +> are either undecodable on arrival or ownership-checked against nothing. + **Auth** (`backend.auth`) — only relevant when the backend needs a key: | Key situation | `backend.auth` | What ships | diff --git a/internal/broker/broker.go b/internal/broker/broker.go index 71451ed..32e64e0 100644 --- a/internal/broker/broker.go +++ b/internal/broker/broker.go @@ -247,14 +247,28 @@ func (b *Broker) ServeHTTP(w http.ResponseWriter, r *http.Request) { } maxBody := b.MaxBody + if app.MaxBodyBytes > 0 { + maxBody = app.MaxBodyBytes + } if app.Provision != nil && mpath == app.Provision.PushPath { maxBody = app.Provision.ArtifactMaxBytes } - body, err := io.ReadAll(io.LimitReader(r.Body, maxBody)) + // Read ONE byte past the cap so an oversize body is detectable. Reading + // exactly maxBody silently truncates it instead, and a truncated body is far + // worse than a refused one: the signature no longer matches (401) or, for a + // multipart upload, the closing boundary is missing and the request dies as + // an opaque tenancy refusal — neither of which tells the caller their file + // was too big. + body, err := io.ReadAll(io.LimitReader(r.Body, maxBody+1)) if err != nil { writeJSON(w, http.StatusBadRequest, map[string]string{"error": "read body"}) return } + if int64(len(body)) > maxBody { + writeJSON(w, http.StatusRequestEntityTooLarge, map[string]string{ + "error": fmt.Sprintf("request body exceeds %d bytes", maxBody)}) + return + } // 1. WHO is calling — verified, not asserted. Signed over the full request. caller, sigErr := b.Verify.Verify(r.Header.Get, r.Method, r.URL.Path, body) @@ -300,7 +314,7 @@ func (b *Broker) ServeHTTP(w http.ResponseWriter, r *http.Request) { // indistinguishable, otherwise the broker is an oracle for enumerating // other tenants' resource ids. if app.Tenancy != nil { - if _, ok := app.Tenancy.EnforceRequest(b.ownerStore(), appID, app.allowSegs, r.Method, mpath, r.URL.RawQuery, body, string(caller)); !ok { + if _, ok := app.Tenancy.EnforceRequest(b.ownerStore(), appID, app.allowSegs, r.Method, mpath, r.URL.RawQuery, r.Header.Get("Content-Type"), body, string(caller)); !ok { writeJSON(w, http.StatusNotFound, map[string]string{"error": "not found"}) return } @@ -411,7 +425,7 @@ func (b *Broker) ServeHTTP(w http.ResponseWriter, r *http.Request) { b.internalError(w, http.StatusBadGateway, appID, "build upstream", err) return } - ureq.Header.Set("Content-Type", "application/json") + ureq.Header.Set("Content-Type", app.forwardContentType(r.Header.Get("Content-Type"))) app.injector.Inject(ureq, app.master) resp, err := b.Client.Do(ureq) diff --git a/internal/broker/meter_test.go b/internal/broker/meter_test.go index 33e0ba3..01e6b7f 100644 --- a/internal/broker/meter_test.go +++ b/internal/broker/meter_test.go @@ -100,7 +100,13 @@ func TestMeter_PartialChargeKeepsRunning(t *testing.T) { func TestRunMeter_OneTick(t *testing.T) { st := NewMemStore() - st.Provision("io.pilot.smol", "alice", "ip", 100, 0, 0, time.Unix(1, 0)) + // One micro-dollar of credit against a 4 cpu / 8 GB machine (302_400 + // micro-$/hour) is exhausted by the very first tick, whatever the tick + // interval. Seeding enough credit to survive ~1.2s of metering instead made + // this a race between that arithmetic and the 2s deadline below, with under + // a second of headroom — so any unrelated work added to the package could + // flake it, which is not something a test should be sensitive to. + st.Provision("io.pilot.smol", "alice", "ip", 1, 0, 0, time.Unix(1, 0)) fp := &fakeProvider{stopped: map[string]bool{}, machines: []MachineInfo{{ID: "m1", Owner: "alice", State: "started", Cpus: 4, MemoryMb: 8192}}} app := meterApp(fp) diff --git a/internal/broker/registry.go b/internal/broker/registry.go index 205c430..462a376 100644 --- a/internal/broker/registry.go +++ b/internal/broker/registry.go @@ -3,6 +3,7 @@ package broker import ( "encoding/json" "fmt" + "mime" "os" "sort" "strings" @@ -54,11 +55,37 @@ type AppEntry struct { // otherwise just a self-minted keypair. RequireAccessKey bool `json:"require_access_key"` + // ForwardContentTypes allow-lists request media types the broker may forward + // to the partner VERBATIM instead of forcing application/json. + // + // This exists for multipart uploads: a multipart/form-data body is + // meaningless without its boundary parameter, which lives in the + // Content-Type header. Forcing application/json (the historical behaviour) + // makes the partner reject every upload. + // + // It is an ALLOW-LIST, not a passthrough, and it is deliberately opt-in. + // The broker is the only thing between an untrusted caller and the master + // key, and the request media type selects which PARSER the partner runs. + // Letting a caller choose that freely is the same class of lever as the + // duplicate-key parser differential tenancy already refuses: a body the + // broker validated as one thing could be re-read by the partner as another. + // Unlisted types are forced to application/json exactly as before. + ForwardContentTypes []string `json:"forward_content_types,omitempty"` + + // MaxBodyBytes overrides the broker-wide body cap for this app (0 = use it). + // + // The default is tuned for JSON calls and is smaller than what an upload + // partner accepts, so an app with a multipart route needs to raise it to at + // least the partner's own limit — otherwise the broker refuses (413) uploads + // the partner would have taken. + MaxBodyBytes int64 `json:"max_body_bytes,omitempty"` + master string // resolved from KeyEnv at load (managed: partner key; provisioned: cloud master, e.g. smk_) injector AuthInjector // built from AuthHeader/Scheme allowSet map[string]bool // key = costKey(method, path); method "" = any method allowPatterns []allowPattern // templated allow entries ("{x}" matches any one segment) allowSegs [][]string // every templated allow path (method-independent), for tenancy param extraction + fwdCT map[string]bool // normalised ForwardContentTypes (media type only, lowercased) breaker *Breaker creditSeed int // Credit.SeedCredits (0 ⇒ no budget) @@ -376,6 +403,17 @@ func ParseRegistry(raw []byte, getenv func(string) string) (*Registry, error) { a.allowSet[costKey(method, p)] = true } } + a.fwdCT = map[string]bool{} + for _, ct := range a.ForwardContentTypes { + mt, _, err := mime.ParseMediaType(strings.TrimSpace(ct)) + if err != nil { + return nil, fmt.Errorf("registry: app %s: forward_content_types %q is not a media type: %w", a.ID, ct, err) + } + if mt == "application/json" { + continue // already the default; listing it is a no-op, not an error + } + a.fwdCT[strings.ToLower(mt)] = true + } if a.CostField == "" { a.CostField = "cost_cents" } @@ -518,3 +556,29 @@ func (r *Registry) AppsRequiringAccessKey() []string { sort.Strings(out) return out } + +// forwardContentType decides the Content-Type the broker sends upstream. +// +// Default (and the historical behaviour) is application/json: the caller does +// not get to choose the partner's parser. An app may opt specific media types +// in via forward_content_types, and only then is the caller's header forwarded +// VERBATIM — parameters included, because a multipart body without its +// boundary= parameter is undecodable. +func (a *AppEntry) forwardContentType(incoming string) string { + if incoming == "" || len(a.fwdCT) == 0 { + return "application/json" + } + mt, _, err := mime.ParseMediaType(incoming) + if err != nil { + return "application/json" // unparseable → do not let it through + } + if !a.fwdCT[strings.ToLower(mt)] { + return "application/json" + } + return incoming +} + +// forwardsMultipart reports whether this app may forward multipart bodies. +func (a *AppEntry) forwardsMultipart() bool { + return a.fwdCT["multipart/form-data"] +} diff --git a/internal/broker/tenancy.go b/internal/broker/tenancy.go index 6e5ad71..62fc763 100644 --- a/internal/broker/tenancy.go +++ b/internal/broker/tenancy.go @@ -1,10 +1,13 @@ package broker import ( + "bytes" "encoding/json" "errors" "fmt" "io" + "mime" + "mime/multipart" "net/url" "strconv" "strings" @@ -307,7 +310,7 @@ type refusal struct { // and the JSON body, and requires the caller to own each one. // // It FAILS CLOSED: an unparseable body, an unknown id, or a store error all deny. -func (t *Tenancy) EnforceRequest(s OwnerStore, app string, allowPatterns [][]string, method, path, rawQuery string, body []byte, caller string) (*refusal, bool) { +func (t *Tenancy) EnforceRequest(s OwnerStore, app string, allowPatterns [][]string, method, path, rawQuery, contentType string, body []byte, caller string) (*refusal, bool) { if t == nil { return nil, true } @@ -352,6 +355,13 @@ func (t *Tenancy) EnforceRequest(s OwnerStore, app string, allowPatterns [][]str // acts through in the body must be checked here, or path-level isolation // is decorative. if len(body) > 0 && len(t.BodyRefs) > 0 { + // A multipart body carries its refs as FORM FIELDS, not JSON keys. Without + // this branch such a body is unparseable to the JSON decoder below and the + // request is refused — correct, but it makes uploads impossible. Parse the + // form instead and check the same refs against the field values. + if mt, params, err := mime.ParseMediaType(contentType); err == nil && mt == "multipart/form-data" { + return t.checkMultipartRefs(s, app, body, params["boundary"], caller) + } // PARSER DIFFERENTIAL. The broker validates the body with Go's decoder but // forwards the RAW bytes, so the partner re-parses them with a different // parser. Go keeps the LAST duplicate key; a parser that keeps the FIRST @@ -632,6 +642,16 @@ func validateTenancy(a *AppEntry) error { } } } + // A multipart upload names the resource it acts on in a FORM FIELD, never in + // the path — POST /api/v1/documents carries deal_id in the body and nowhere + // else. So an app that forwards multipart while declaring no body_refs has + // no way to check the one reference that matters, and every caller could + // upload into every other caller's resource. param_types alone satisfies the + // check above, which is exactly how this would slip through unnoticed. + if len(t.BodyRefs) == 0 && a.forwardsMultipart() { + return fmt.Errorf("registry: app %s: forwards multipart/form-data but declares no tenancy.body_refs — an upload names its resource in a form field, so nothing would be ownership-checked", a.ID) + } + // Every claimable type should be reachable: a type referenced by params/body // but never created can never be owned, which would deny the app entirely. created := map[string]bool{} @@ -729,3 +749,96 @@ func hasDuplicateKeys(body []byte) bool { } } } + +// Multipart parsing bounds. A body that needs more than this to describe itself +// is not a legitimate upload; refusing is cheaper than reasoning about it. +const ( + maxMultipartParts = 64 + maxRefFieldBytes = 4 << 10 // a resource id is never larger +) + +// checkMultipartRefs ownership-checks the BodyRefs fields of a multipart body. +// +// It is the multipart twin of the JSON body check and keeps the same three +// properties, because a shared master key sits behind it: +// +// - FAIL-CLOSED. A body that cannot be parsed, a boundary that is missing, a +// part budget that is exceeded — all deny. We never forward a body whose +// refs we could not read. +// - NO PARSER DIFFERENTIAL on a checked field. A REF field that appears twice +// is refused: Go hands us every part in order, other stacks keep the first +// or the last, so a body naming deal_id twice could be validated as one +// value and acted on as another. Repeats of fields we do not check are +// allowed through — they are legal multipart (a multi-file upload posts the +// same name repeatedly) and no security decision rests on them, so banning +// them would break real apps to no end. +// - A REF IS A FIELD, NEVER A FILE. A part that carries a filename is treated +// as file content and its bytes are not inspected. So a caller who smuggles +// "deal_id" in as a file part would slip an unchecked ref past us if we +// simply skipped it — that shape is denied explicitly. +func (t *Tenancy) checkMultipartRefs(s OwnerStore, app string, body []byte, boundary, caller string) (*refusal, bool) { + if boundary == "" { + return &refusal{}, false // no boundary → undecodable → deny + } + mr := multipart.NewReader(bytes.NewReader(body), boundary) + type ref struct { + rtype string + value string + } + found := map[string]ref{} + parts := 0 + for { + part, err := mr.NextPart() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + return &refusal{}, false // malformed → cannot verify → deny + } + // Count only parts we actually received. Checking the budget before + // NextPart rejects a body sitting exactly on the limit, because the call + // that would have reported EOF never happens. + parts++ + if parts > maxMultipartParts { + part.Close() + return &refusal{}, false // more parts than any real upload needs + } + name := part.FormName() + rtype, isRef := t.BodyRefs[name] + if !isRef { + // Not a field we check. Close advances past the part's remaining + // bytes; NextPart would do it anyway, so there is nothing to drain + // by hand. + part.Close() + continue + } + if part.FileName() != "" { + // A ref wearing a filename: file parts are not inspected, so this is + // the shape that routes a ref past the check. + part.Close() + return &refusal{}, false + } + if _, dup := found[name]; dup { + part.Close() + return &refusal{}, false // repeated REF → parser differential → deny + } + val, err := io.ReadAll(io.LimitReader(part, maxRefFieldBytes+1)) + part.Close() + if err != nil || len(val) > maxRefFieldBytes { + return &refusal{}, false + } + found[name] = ref{rtype: rtype, value: strings.TrimSpace(string(val))} + } + for _, r := range found { + // An empty ref names no resource — the field is present but unset, which + // the partner treats as absent. Matches the JSON path, where asID rejects + // the empty string and the ref is skipped rather than checked. + if r.value == "" { + continue + } + if !Owns(s, app, r.rtype, r.value, caller) { + return &refusal{Type: r.rtype, ID: r.value}, false + } + } + return nil, true +} diff --git a/internal/broker/zz_multipart_limits_test.go b/internal/broker/zz_multipart_limits_test.go new file mode 100644 index 0000000..8d126cc --- /dev/null +++ b/internal/broker/zz_multipart_limits_test.go @@ -0,0 +1,261 @@ +package broker + +import ( + "bytes" + "fmt" + "mime/multipart" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +// Regression tests for the boundary conditions of multipart forwarding. Each +// one pins a bug that the happy-path tests in zz_multipart_test.go did not +// catch, which is exactly why they are worth keeping separate. + +// glBrokerWithCap builds the General Legal broker with an explicit body cap so +// the 413 path is testable without pushing megabytes through every run. +func glBrokerWithCap(t *testing.T, got *[]glUpload, cap int64) (*Broker, func()) { + t.Helper() + b, done := glBroker(t, got) + b.Registry().Get("io.pilot.generallegal").MaxBodyBytes = cap + return b, done +} + +// TestOversizeBody_413NotSilentTruncation: the broker used to read exactly +// MaxBody bytes, which truncated a larger body instead of refusing it. The +// caller then saw a signature failure or an opaque tenancy 404 — never "your +// file was too big". A truncated body must never be processed at all. +func TestOversizeBody_413NotSilentTruncation(t *testing.T) { + var got []glUpload + b, done := glBrokerWithCap(t, &got, 64<<10) + defer done() + _, alice := newKey(t) + + body, ct := buildUpload(t, "big.docx", randBytes(t, 256<<10), map[string]string{"deal_id": "d1"}) + rec := httptest.NewRecorder() + b.ServeHTTP(rec, signedCT(t, alice, "POST", "/io.pilot.generallegal/api/v1/documents", ct, body)) + + if rec.Code != http.StatusRequestEntityTooLarge { + t.Fatalf("oversize body: status = %d, want 413 (body: %s)", rec.Code, rec.Body.String()) + } + if len(got) != 0 { + t.Fatal("an oversize body reached the partner") + } +} + +// TestBodyExactlyAtCapIsAccepted is the other half: the cap is inclusive, so a +// body sitting exactly on it must still go through. Reading maxBody+1 to detect +// overflow is easy to get wrong by one byte in this direction. +func TestBodyExactlyAtCapIsAccepted(t *testing.T) { + var got []glUpload + b, done := glBroker(t, &got) + defer done() + _, alice := newKey(t) + dealID := openDeal(t, b, alice) + + body, ct := buildUpload(t, "x.docx", randBytes(t, 4096), map[string]string{"deal_id": dealID}) + b.Registry().Get("io.pilot.generallegal").MaxBodyBytes = int64(len(body)) + + rec := httptest.NewRecorder() + b.ServeHTTP(rec, signedCT(t, alice, "POST", "/io.pilot.generallegal/api/v1/documents", ct, body)) + if rec.Code != 200 { + t.Fatalf("body exactly at the cap: status = %d, want 200 (%s)", rec.Code, rec.Body.String()) + } +} + +// TestPerAppCapOverridesBrokerDefault: the broker default is tuned for JSON and +// is smaller than an upload partner's own limit (8 MiB vs General Legal's 20), +// so without a per-app override the platform silently cannot carry uploads the +// partner would have accepted. Asserted by lowering the default rather than by +// pushing megabytes through, which tests the same branch far more cheaply. +func TestPerAppCapOverridesBrokerDefault(t *testing.T) { + var got []glUpload + b, done := glBroker(t, &got) + defer done() + _, alice := newKey(t) + dealID := openDeal(t, b, alice) + + body, ct := buildUpload(t, "big.pdf", randBytes(t, 96<<10), map[string]string{"deal_id": dealID}) + + // Below the broker-wide cap: refused. + b.MaxBody = 32 << 10 + rec := httptest.NewRecorder() + b.ServeHTTP(rec, signedCT(t, alice, "POST", "/io.pilot.generallegal/api/v1/documents", ct, body)) + if rec.Code != http.StatusRequestEntityTooLarge { + t.Fatalf("under the broker default: status = %d, want 413", rec.Code) + } + + // Same body, same broker, per-app cap raised: accepted. + b.Registry().Get("io.pilot.generallegal").MaxBodyBytes = 1 << 20 + rec = httptest.NewRecorder() + b.ServeHTTP(rec, signedCT(t, alice, "POST", "/io.pilot.generallegal/api/v1/documents", ct, body)) + if rec.Code != 200 { + t.Fatalf("with the per-app cap raised: status = %d, want 200 (%s)", rec.Code, rec.Body.String()) + } + if len(got) != 1 || got[0].FileBytes != 96<<10 { + t.Fatalf("partner got %d uploads; want one of %d bytes", len(got), 96<<10) + } +} + +// buildParts assembles a body with n scalar fields plus one file, so the part +// budget can be exercised from both sides. +func buildParts(t *testing.T, n int, dealID string) ([]byte, string) { + t.Helper() + var buf bytes.Buffer + w := multipart.NewWriter(&buf) + if dealID != "" { + _ = w.WriteField("deal_id", dealID) + n-- + } + fw, _ := w.CreateFormFile("file", "x.docx") + _, _ = fw.Write([]byte("hello")) + n-- + for i := 0; i < n; i++ { + _ = w.WriteField(fmt.Sprintf("pad_%d", i), "v") + } + _ = w.Close() + return buf.Bytes(), w.FormDataContentType() +} + +// TestPartBudget_ExactlyAtLimitAccepted: the guard checked the counter before +// reading, so a body holding exactly maxMultipartParts parts was refused even +// though it was within budget. +func TestPartBudget_ExactlyAtLimitAccepted(t *testing.T) { + var got []glUpload + b, done := glBroker(t, &got) + defer done() + _, alice := newKey(t) + dealID := openDeal(t, b, alice) + + body, ct := buildParts(t, maxMultipartParts, dealID) + rec := httptest.NewRecorder() + b.ServeHTTP(rec, signedCT(t, alice, "POST", "/io.pilot.generallegal/api/v1/documents", ct, body)) + if rec.Code != 200 { + t.Fatalf("body with exactly %d parts: status = %d, want 200 (%s)", maxMultipartParts, rec.Code, rec.Body.String()) + } +} + +func TestPartBudget_OverLimitRefused(t *testing.T) { + var got []glUpload + b, done := glBroker(t, &got) + defer done() + _, alice := newKey(t) + dealID := openDeal(t, b, alice) + + body, ct := buildParts(t, maxMultipartParts+1, dealID) + rec := httptest.NewRecorder() + b.ServeHTTP(rec, signedCT(t, alice, "POST", "/io.pilot.generallegal/api/v1/documents", ct, body)) + if rec.Code != http.StatusNotFound { + t.Fatalf("body with %d parts: status = %d, want 404", maxMultipartParts+1, rec.Code) + } + if len(got) != 0 { + t.Fatal("an over-budget body reached the partner") + } +} + +// TestDuplicateNonRefFieldAllowed: repeating a field name is legal multipart — +// a multi-file upload posts the same name several times. The duplicate ban +// exists to stop a parser differential on a field the broker made a decision +// about, so applying it to fields nobody checks only breaks working apps. +func TestDuplicateNonRefFieldAllowed(t *testing.T) { + var got []glUpload + b, done := glBroker(t, &got) + defer done() + _, alice := newKey(t) + dealID := openDeal(t, b, alice) + + var buf bytes.Buffer + w := multipart.NewWriter(&buf) + _ = w.WriteField("deal_id", dealID) + _ = w.WriteField("tag", "one") // repeated, and not a ref + _ = w.WriteField("tag", "two") + fw, _ := w.CreateFormFile("file", "a.docx") + _, _ = fw.Write([]byte("a")) + fw2, _ := w.CreateFormFile("file", "b.docx") // repeated file part + _, _ = fw2.Write([]byte("b")) + _ = w.Close() + + rec := httptest.NewRecorder() + b.ServeHTTP(rec, signedCT(t, alice, "POST", "/io.pilot.generallegal/api/v1/documents", w.FormDataContentType(), buf.Bytes())) + if rec.Code != 200 { + t.Fatalf("duplicate non-ref fields: status = %d, want 200 (%s)", rec.Code, rec.Body.String()) + } +} + +// TestEmptyRefFieldSkipped: deal_id is optional on this endpoint (omitting it +// opens a fresh matter), so a present-but-empty field must not be looked up as +// a resource id and refused. Mirrors the JSON path, where the empty string is +// not an id. +func TestEmptyRefFieldSkipped(t *testing.T) { + var got []glUpload + b, done := glBroker(t, &got) + defer done() + _, alice := newKey(t) + + body, ct := buildUpload(t, "new.docx", []byte("x"), map[string]string{"deal_id": ""}) + rec := httptest.NewRecorder() + b.ServeHTTP(rec, signedCT(t, alice, "POST", "/io.pilot.generallegal/api/v1/documents", ct, body)) + if rec.Code != 200 { + t.Fatalf("empty deal_id: status = %d, want 200 (%s)", rec.Code, rec.Body.String()) + } +} + +// TestOversizeRefFieldRefused: a ref value far larger than any id is a probe, +// not a document reference. +func TestOversizeRefFieldRefused(t *testing.T) { + var got []glUpload + b, done := glBroker(t, &got) + defer done() + _, alice := newKey(t) + + body, ct := buildUpload(t, "x.docx", []byte("x"), map[string]string{ + "deal_id": string(bytes.Repeat([]byte("a"), maxRefFieldBytes+1)), + }) + rec := httptest.NewRecorder() + b.ServeHTTP(rec, signedCT(t, alice, "POST", "/io.pilot.generallegal/api/v1/documents", ct, body)) + if rec.Code != http.StatusNotFound { + t.Fatalf("oversize ref field: status = %d, want 404", rec.Code) + } + if len(got) != 0 { + t.Fatal("oversize-ref body reached the partner") + } +} + +// TestMultipartWithoutBodyRefsFailsBoot: an upload names its resource in a form +// field, so an app that forwards multipart with only param_types declared would +// ownership-check nothing on exactly the route that needs it most. The registry +// fails the boot rather than serve a spec that does not isolate — the same +// stance the rest of validateTenancy takes. +func TestMultipartWithoutBodyRefsFailsBoot(t *testing.T) { + reg := `[{ + "id":"io.pilot.leaky","upstream":"http://x","key_env":"K", + "auth_header":"Authorization", + "allow":["POST /api/v1/deals","POST /api/v1/documents"], + "forward_content_types":["multipart/form-data"], + "tenancy":{ + "param_types":{"deal_id":"deal"}, + "create":[{"method":"POST","path":"/api/v1/deals","type":"deal","id_field":"deal_id"}] + }}]` + _, err := ParseRegistry([]byte(reg), func(string) string { return "k" }) + if err == nil { + t.Fatal("a multipart app with no body_refs loaded successfully; it would forward unchecked uploads") + } + if !strings.Contains(err.Error(), "body_refs") { + t.Fatalf("error should name the missing field, got: %v", err) + } + + // The same spec WITH body_refs is fine. + ok := strings.Replace(reg, `"param_types":{"deal_id":"deal"},`, + `"param_types":{"deal_id":"deal"},"body_refs":{"deal_id":"deal"},`, 1) + if _, err := ParseRegistry([]byte(ok), func(string) string { return "k" }); err != nil { + t.Fatalf("valid multipart spec rejected: %v", err) + } + + // And an app that does NOT forward multipart is unaffected by the new rule. + noMP := strings.Replace(reg, `"forward_content_types":["multipart/form-data"],`, "", 1) + if _, err := ParseRegistry([]byte(noMP), func(string) string { return "k" }); err != nil { + t.Fatalf("non-multipart app newly rejected: %v", err) + } +} diff --git a/internal/broker/zz_multipart_test.go b/internal/broker/zz_multipart_test.go new file mode 100644 index 0000000..5bb450c --- /dev/null +++ b/internal/broker/zz_multipart_test.go @@ -0,0 +1,436 @@ +package broker + +import ( + "bytes" + "crypto/ed25519" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + mrand "math/rand" + "mime" + "mime/multipart" + "net/http" + "net/http/httptest" + "net/textproto" + "testing" + "time" +) + +// Multipart uploads through a shared master key. +// +// The General Legal API's document upload is multipart/form-data, and the deal +// it uploads into arrives as a FORM FIELD, not a JSON key. That breaks two +// broker assumptions at once: the forward hardcoded application/json, and the +// tenancy body-ref check could only read JSON and so refused any body it could +// not parse. Refusing was the safe failure, but it made uploads impossible. +// +// These tests pin down both halves: the body must survive the hop intact, and +// the ownership check must still hold on a field the broker now has to parse +// out of a multipart envelope. + +const glRegistryJSON = `[{ + "id": "io.pilot.generallegal", + "upstream": "%s", + "key_env": "GL_KEY", + "auth_header": "Authorization", + "auth_scheme": "Bearer", + "quota": 0, + "allow": [ + "POST /api/v1/deals", "GET /api/v1/deals", + "GET /api/v1/deals/{deal_id}", + "POST /api/v1/documents" + ], + "forward_content_types": ["multipart/form-data"], + "tenancy": { + "param_types": {"deal_id": "deal"}, + "body_refs": {"deal_id": "deal"}, + "create": [ + {"method": "POST", "path": "/api/v1/deals", "type": "deal", "id_field": "deal_id"} + ], + "list": [ + {"method": "GET", "path": "/api/v1/deals", "array": "items", + "owner_by": [{"field": "id", "type": "deal"}], "count_fields": ["total"]} + ] + } +}]` + +// glUpload is what the partner actually received, so a test can assert the +// bytes crossed every hop unchanged. +type glUpload struct { + ContentType string + FileName string + FileSHA256 string + FileBytes int + Fields map[string]string + Auth string +} + +// glUpstream fakes api.general.legal's document endpoint: it parses the +// multipart body the way FastAPI would and records what arrived. +func glUpstream(t *testing.T, got *[]glUpload) *httptest.Server { + t.Helper() + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + switch { + case r.Method == "POST" && r.URL.Path == "/api/v1/deals": + fmt.Fprintf(w, `{"deal_id":"deal_%d","status":"open"}`, time.Now().UnixNano()) + + case r.Method == "POST" && r.URL.Path == "/api/v1/documents": + mt, params, err := mime.ParseMediaType(r.Header.Get("Content-Type")) + if err != nil || mt != "multipart/form-data" { + http.Error(w, `{"detail":"expected multipart/form-data"}`, http.StatusUnprocessableEntity) + return + } + u := glUpload{ + ContentType: r.Header.Get("Content-Type"), + Fields: map[string]string{}, + Auth: r.Header.Get("Authorization"), + } + mr := multipart.NewReader(r.Body, params["boundary"]) + for { + p, err := mr.NextPart() + if err == io.EOF { + break + } + if err != nil { + http.Error(w, `{"detail":"malformed multipart"}`, http.StatusBadRequest) + return + } + b, _ := io.ReadAll(p) + if p.FileName() != "" { + sum := sha256.Sum256(b) + u.FileName = p.FileName() + u.FileSHA256 = hex.EncodeToString(sum[:]) + u.FileBytes = len(b) + } else { + u.Fields[p.FormName()] = string(b) + } + p.Close() + } + if got != nil { + *got = append(*got, u) + } + fmt.Fprint(w, `{"deal_id":"d1","contract_id":"c1","version_id":"v1"}`) + + default: + fmt.Fprint(w, `{"ok":true}`) + } + })) +} + +func glBroker(t *testing.T, got *[]glUpload) (*Broker, func()) { + t.Helper() + up := glUpstream(t, got) + reg, err := ParseRegistry([]byte(fmt.Sprintf(glRegistryJSON, up.URL)), + func(string) string { return "glk_master" }) + if err != nil { + t.Fatalf("ParseRegistry: %v", err) + } + b := New(reg, NewMemStore()) + b.Verify = VerifyConfig{Window: time.Hour} + return b, up.Close +} + +// signedCT is signedReq plus an explicit Content-Type. The signature covers +// method, path, timestamp and a hash of the body — not the header — so setting +// it after signing is correct and mirrors what the adapter does. +func signedCT(t *testing.T, priv ed25519.PrivateKey, method, path, ct string, body []byte) *http.Request { + t.Helper() + req := signedReq(t, priv, method, path, body, time.Now()) + req.Header.Set("Content-Type", ct) + return req +} + +// openDeal creates a deal as the given caller and returns its id, so the +// upload tests have a resource that is genuinely owned. +func openDeal(t *testing.T, b *Broker, priv ed25519.PrivateKey) string { + t.Helper() + rec := do(t, b, priv, "POST", "/io.pilot.generallegal/api/v1/deals", []byte(`{"initial_request":"review an NDA"}`)) + if rec.Code != 200 { + t.Fatalf("open deal: status %d (%s)", rec.Code, rec.Body.String()) + } + var out struct { + DealID string `json:"deal_id"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &out); err != nil || out.DealID == "" { + t.Fatalf("open deal: bad response %s", rec.Body.String()) + } + return out.DealID +} + +// buildUpload assembles a document upload body the way the adapter does. +func buildUpload(t *testing.T, fileName string, content []byte, fields map[string]string) ([]byte, string) { + t.Helper() + var buf bytes.Buffer + w := multipart.NewWriter(&buf) + for k, v := range fields { + if err := w.WriteField(k, v); err != nil { + t.Fatalf("write field: %v", err) + } + } + h := make(textproto.MIMEHeader) + h.Set("Content-Disposition", fmt.Sprintf(`form-data; name="file"; filename=%q`, fileName)) + h.Set("Content-Type", "application/vnd.openxmlformats-officedocument.wordprocessingml.document") + part, err := w.CreatePart(h) + if err != nil { + t.Fatalf("create part: %v", err) + } + if _, err := part.Write(content); err != nil { + t.Fatalf("write part: %v", err) + } + if err := w.Close(); err != nil { + t.Fatalf("close: %v", err) + } + return buf.Bytes(), w.FormDataContentType() +} + +// randBytes builds test content from a seeded PRNG rather than crypto/rand. +// The bytes only need to be non-uniform enough that a truncation or a corrupted +// hop changes the sha256; paying for cryptographic randomness by the megabyte +// just makes the package slow. +func randBytes(t *testing.T, n int) []byte { + t.Helper() + b := make([]byte, n) + r := mrand.New(mrand.NewSource(int64(n))) + _, _ = r.Read(b) + return b +} + +// TestMultipart_UploadReachesPartnerIntact is the headline case: a 3 MiB +// document — comfortably past the 1 MiB IPC frame that forced the chunked +// staging design — arrives at the partner byte-identical, with its boundary +// intact and the master key injected. +func TestMultipart_UploadReachesPartnerIntact(t *testing.T) { + var got []glUpload + b, closeUp := glBroker(t, &got) + defer closeUp() + _, alice := newKey(t) + + dealID := openDeal(t, b, alice) + content := randBytes(t, 3<<20) + want := sha256.Sum256(content) + + body, ct := buildUpload(t, "mutual-nda.docx", content, map[string]string{ + "deal_id": dealID, + "context_for_legal": "Standard mutual NDA, we are the disclosing party.", + "last_edit_by": "client", + }) + + rec := httptest.NewRecorder() + b.ServeHTTP(rec, signedCT(t, alice, "POST", "/io.pilot.generallegal/api/v1/documents", ct, body)) + + if rec.Code != 200 { + t.Fatalf("status = %d, want 200 (body: %s)", rec.Code, rec.Body.String()) + } + if len(got) != 1 { + t.Fatalf("partner received %d uploads, want 1", len(got)) + } + u := got[0] + if u.FileSHA256 != hex.EncodeToString(want[:]) { + t.Errorf("file corrupted in transit:\n got sha %s (%d bytes)\nwant sha %s (%d bytes)", + u.FileSHA256, u.FileBytes, hex.EncodeToString(want[:]), len(content)) + } + if u.FileName != "mutual-nda.docx" { + t.Errorf("file name = %q, want mutual-nda.docx", u.FileName) + } + if u.Fields["deal_id"] != dealID { + t.Errorf("deal_id = %q, want %q", u.Fields["deal_id"], dealID) + } + if u.Fields["context_for_legal"] == "" { + t.Error("context_for_legal did not survive the hop") + } + if u.Auth != "Bearer glk_master" { + t.Errorf("Authorization = %q, want the injected master key", u.Auth) + } + if mt, params, _ := mime.ParseMediaType(u.ContentType); mt != "multipart/form-data" || params["boundary"] == "" { + t.Errorf("Content-Type = %q, want multipart/form-data with a boundary", u.ContentType) + } +} + +// TestMultipart_CannotUploadIntoAnotherTenantsDeal is the isolation case the +// JSON path already had and the multipart path must not lose: Mallory names +// Alice's deal in a form field. Before the multipart body-ref check existed, +// this body was simply unparseable — which denied Alice too. +func TestMultipart_CannotUploadIntoAnotherTenantsDeal(t *testing.T) { + var got []glUpload + b, closeUp := glBroker(t, &got) + defer closeUp() + _, alice := newKey(t) + _, mallory := newKey(t) + + aliceDeal := openDeal(t, b, alice) + + body, ct := buildUpload(t, "exfil.docx", randBytes(t, 4096), map[string]string{ + "deal_id": aliceDeal, + }) + rec := httptest.NewRecorder() + b.ServeHTTP(rec, signedCT(t, mallory, "POST", "/io.pilot.generallegal/api/v1/documents", ct, body)) + + if rec.Code != http.StatusNotFound { + t.Fatalf("Mallory uploading into Alice's deal: status = %d, want 404 (body: %s)", rec.Code, rec.Body.String()) + } + if len(got) != 0 { + t.Fatalf("refused upload still reached the partner (%d times)", len(got)) + } + + // Paired assertion: the check denies the impostor without denying the owner. + okBody, okCT := buildUpload(t, "ok.docx", randBytes(t, 4096), map[string]string{"deal_id": aliceDeal}) + rec2 := httptest.NewRecorder() + b.ServeHTTP(rec2, signedCT(t, alice, "POST", "/io.pilot.generallegal/api/v1/documents", okCT, okBody)) + if rec2.Code != 200 { + t.Fatalf("Alice's own upload: status = %d, want 200 (body: %s)", rec2.Code, rec2.Body.String()) + } +} + +// TestMultipart_DuplicateFieldRefused closes the multipart twin of the +// duplicate-JSON-key parser differential: Go hands us every part, other stacks +// keep the first or the last, so a body naming deal_id twice could be validated +// as one value and acted on as another. +func TestMultipart_DuplicateFieldRefused(t *testing.T) { + var got []glUpload + b, closeUp := glBroker(t, &got) + defer closeUp() + _, alice := newKey(t) + _, mallory := newKey(t) + + aliceDeal := openDeal(t, b, alice) + mallDeal := openDeal(t, b, mallory) + + // Mallory's own deal first (which we would validate), Alice's second. + var buf bytes.Buffer + w := multipart.NewWriter(&buf) + _ = w.WriteField("deal_id", mallDeal) + _ = w.WriteField("deal_id", aliceDeal) + fw, _ := w.CreateFormFile("file", "x.docx") + _, _ = fw.Write(randBytes(t, 512)) + _ = w.Close() + + rec := httptest.NewRecorder() + b.ServeHTTP(rec, signedCT(t, mallory, "POST", "/io.pilot.generallegal/api/v1/documents", w.FormDataContentType(), buf.Bytes())) + + if rec.Code != http.StatusNotFound { + t.Fatalf("duplicate deal_id: status = %d, want 404", rec.Code) + } + if len(got) != 0 { + t.Fatal("body with duplicate refs reached the partner") + } +} + +// TestMultipart_RefSmuggledAsFilePartRefused: file parts are not inspected, so +// a ref arriving with a filename would otherwise slip past the ownership check +// while the partner still reads it as a field. +func TestMultipart_RefSmuggledAsFilePartRefused(t *testing.T) { + var got []glUpload + b, closeUp := glBroker(t, &got) + defer closeUp() + _, alice := newKey(t) + _, mallory := newKey(t) + aliceDeal := openDeal(t, b, alice) + + var buf bytes.Buffer + w := multipart.NewWriter(&buf) + fw, _ := w.CreateFormFile("deal_id", "deal_id.txt") // a ref wearing a filename + _, _ = fw.Write([]byte(aliceDeal)) + fw2, _ := w.CreateFormFile("file", "x.docx") + _, _ = fw2.Write(randBytes(t, 512)) + _ = w.Close() + + rec := httptest.NewRecorder() + b.ServeHTTP(rec, signedCT(t, mallory, "POST", "/io.pilot.generallegal/api/v1/documents", w.FormDataContentType(), buf.Bytes())) + + if rec.Code != http.StatusNotFound { + t.Fatalf("ref smuggled as a file part: status = %d, want 404", rec.Code) + } + if len(got) != 0 { + t.Fatal("smuggled-ref body reached the partner") + } +} + +// TestMultipart_MalformedBodyRefused keeps the fail-closed property: a body we +// cannot parse is a body whose refs we cannot check. +func TestMultipart_MalformedBodyRefused(t *testing.T) { + var got []glUpload + b, closeUp := glBroker(t, &got) + defer closeUp() + _, alice := newKey(t) + + junk := []byte("--boundary\r\nnot actually a multipart body") + rec := httptest.NewRecorder() + b.ServeHTTP(rec, signedCT(t, alice, "POST", "/io.pilot.generallegal/api/v1/documents", + "multipart/form-data; boundary=boundary", junk)) + + if rec.Code != http.StatusNotFound { + t.Fatalf("malformed multipart: status = %d, want 404", rec.Code) + } + if len(got) != 0 { + t.Fatal("unparseable body reached the partner") + } +} + +// TestMultipart_MissingBoundaryRefused: without a boundary the body is +// undecodable, so it must not be forwarded on the hope the partner copes. +func TestMultipart_MissingBoundaryRefused(t *testing.T) { + var got []glUpload + b, closeUp := glBroker(t, &got) + defer closeUp() + _, alice := newKey(t) + body, _ := buildUpload(t, "x.docx", randBytes(t, 256), map[string]string{"deal_id": "d"}) + + rec := httptest.NewRecorder() + b.ServeHTTP(rec, signedCT(t, alice, "POST", "/io.pilot.generallegal/api/v1/documents", + "multipart/form-data", body)) + + if rec.Code != http.StatusNotFound { + t.Fatalf("missing boundary: status = %d, want 404", rec.Code) + } + if len(got) != 0 { + t.Fatal("boundary-less body reached the partner") + } +} + +// TestForwardContentType_DefaultsToJSON is the regression guard on the opt-in: +// an app that does NOT list multipart must still see the historical behaviour, +// so enabling this for one partner cannot quietly change every other one. +func TestForwardContentType_DefaultsToJSON(t *testing.T) { + reg, err := ParseRegistry([]byte(`[{ + "id":"io.pilot.plain","upstream":"http://x","key_env":"K", + "auth_header":"Authorization","allow":["POST /p"]}]`), + func(string) string { return "k" }) + if err != nil { + t.Fatalf("ParseRegistry: %v", err) + } + app := reg.Get("io.pilot.plain") + for _, in := range []string{ + "multipart/form-data; boundary=abc", + "application/x-www-form-urlencoded", + "text/plain", + "", + } { + if got := app.forwardContentType(in); got != "application/json" { + t.Errorf("forwardContentType(%q) = %q, want application/json", in, got) + } + } +} + +// TestForwardContentType_AllowListed: only the listed type is passed through, +// and it is passed through VERBATIM so the boundary parameter survives. +func TestForwardContentType_AllowListed(t *testing.T) { + var got []glUpload + b, closeUp := glBroker(t, &got) + defer closeUp() + app := b.Registry().Get("io.pilot.generallegal") + + in := "multipart/form-data; boundary=xyz123" + if out := app.forwardContentType(in); out != in { + t.Errorf("allow-listed type = %q, want it forwarded verbatim (%q)", out, in) + } + // Still an allow-list, not a passthrough: an unlisted type is forced back. + if out := app.forwardContentType("application/x-www-form-urlencoded"); out != "application/json" { + t.Errorf("unlisted type = %q, want application/json", out) + } + if out := app.forwardContentType("!! not a media type"); out != "application/json" { + t.Errorf("unparseable type = %q, want application/json", out) + } +} diff --git a/internal/broker/zz_tenancy_test.go b/internal/broker/zz_tenancy_test.go index 8a20b16..5d43e2a 100644 --- a/internal/broker/zz_tenancy_test.go +++ b/internal/broker/zz_tenancy_test.go @@ -300,10 +300,10 @@ func TestTenancy_TypeConfusion(t *testing.T) { func TestTenancy_FailsClosedWithoutLedger(t *testing.T) { tn := &Tenancy{ParamTypes: map[string]string{"agent_id": "agent"}} tn.compile() - if _, ok := tn.EnforceRequest(nil, "app", nil, "GET", "/v1/agents/a1", "", nil, "alice"); ok { + if _, ok := tn.EnforceRequest(nil, "app", nil, "GET", "/v1/agents/a1", "", "", nil, "alice"); ok { t.Error("nil ledger must deny") } - if _, ok := tn.EnforceRequest(NewMemStore(), "app", nil, "GET", "/v1/agents/a1", "", nil, ""); ok { + if _, ok := tn.EnforceRequest(NewMemStore(), "app", nil, "GET", "/v1/agents/a1", "", "", nil, ""); ok { t.Error("empty caller must deny") } } diff --git a/internal/multipartkit/blob.go b/internal/multipartkit/blob.go new file mode 100644 index 0000000..00cab15 --- /dev/null +++ b/internal/multipartkit/blob.go @@ -0,0 +1,422 @@ +// Package multipartkit lets a Pilot app forward multipart/form-data uploads to +// its partner API despite the JSON-only IPC contract. +// +// # Why this exists +// +// Pilot IPC is JSON in, JSON out, over a framed unix socket, and +// ipc.MaxFrameSize caps a single envelope at 1 MiB. The obvious encoding for a +// file — base64 inside the JSON payload — therefore tops out around 740 KiB of +// real bytes before the frame is refused and the connection dropped. Partner +// upload limits are much larger than that (General Legal accepts 20 MiB), so +// "just base64 it" is not an implementation detail that can be tuned; it is a +// design that does not fit. +// +// So the file does not travel in one envelope. The agent stages it in chunks +// that each fit comfortably inside a frame, the adapter reassembles it on disk +// under $APP, and only then builds one multipart body and sends it. The IPC +// layer never carries more than a chunk, and no platform limit has to move. +// +// .upload_begin {file_name, content_type, total_bytes, sha256} -> {blob_id} +// .upload_chunk {blob_id, seq, data_base64} -> {received} +// .document_upload {blob_id, deal_id, ...} -> partner JSON +// +// # What this package guarantees +// +// Chunked reassembly is a place to get integrity wrong, so the store is strict +// rather than forgiving: ids are minted here and never taken from the caller, +// chunks must arrive in order, the declared size is a hard cap as bytes land, +// and finalize refuses to hand back a blob whose sha256 does not match what the +// caller promised at begin. A partially-written upload is never usable. +// +// # Restarts +// +// The daemon supervises adapters and respawns them, so a store must not keep +// the only record of a finished upload in memory. A sealed blob is fully +// described on disk by its .blob and .meta pair and is recovered on the next +// NewStore. An in-flight .part cannot be recovered — the rolling hash and the +// chunk cursor die with the process — so it is discarded at startup rather than +// left to be silently completed with a gap in the middle. +package multipartkit + +import ( + "crypto/rand" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "hash" + "os" + "path/filepath" + "strings" + "sync" + "time" +) + +// MaxChunkBytes is the largest raw chunk an agent should send in one +// upload_chunk call. base64 inflates by 4/3, so 512 KiB of file becomes ~683 +// KiB of payload and still leaves room for the envelope inside the 1 MiB +// ipc.MaxFrameSize. Chunks larger than this are refused here rather than +// failing later as a dropped connection, which is much harder to diagnose. +const MaxChunkBytes = 512 << 10 + +// DefaultMaxBlobBytes bounds a single staged upload. Partner limits are lower +// (General Legal: 20 MiB); this is the backstop that keeps a caller from +// filling the disk under $APP. +const DefaultMaxBlobBytes int64 = 24 << 20 + +// DefaultTTL is how long a staged upload survives before GC reclaims it. +const DefaultTTL = 30 * time.Minute + +var ( + ErrNotFound = errors.New("multipartkit: unknown blob_id") + ErrSealed = errors.New("multipartkit: blob already finalized") + ErrOutOfOrder = errors.New("multipartkit: chunk out of order") + ErrTooLarge = errors.New("multipartkit: upload exceeds declared size") + ErrChecksum = errors.New("multipartkit: sha256 mismatch") + ErrIncomplete = errors.New("multipartkit: fewer bytes than declared") + ErrChunkTooBig = errors.New("multipartkit: chunk exceeds MaxChunkBytes") + ErrBadBlobID = errors.New("multipartkit: malformed blob_id") + ErrBadDeclared = errors.New("multipartkit: invalid total_bytes") + ErrBadChecksum = errors.New("multipartkit: sha256 must be 64 hex chars") + ErrNameRequired = errors.New("multipartkit: file_name is required") +) + +// Meta describes a staged upload. It is persisted beside a sealed blob so the +// upload survives an adapter respawn. +type Meta struct { + ID string `json:"id"` + FileName string `json:"file_name"` + ContentType string `json:"content_type"` + TotalBytes int64 `json:"total_bytes"` + SHA256 string `json:"sha256"` + Created time.Time `json:"created"` +} + +// staged is an upload in progress. A sealed blob is not held here: it lives on +// disk, which is what lets it outlive the process. +type staged struct { + meta Meta + f *os.File + h hash.Hash + written int64 + nextSeq int +} + +// Store stages chunked uploads under a root directory (in practice $APP/blobs). +// +// All state transitions happen under mu. The lock is held across the file write +// in Append deliberately: releasing it to do I/O would let Abort close the file +// underneath an in-flight chunk, and uploads are not hot enough for that +// contention to matter. +type Store struct { + root string + maxBytes int64 + ttl time.Duration + + mu sync.Mutex + open map[string]*staged +} + +// NewStore prepares a staging area under root and recovers what is recoverable +// from a previous process: sealed blobs stay, unfinishable .part files go. +func NewStore(root string) (*Store, error) { + if err := os.MkdirAll(root, 0o700); err != nil { + return nil, fmt.Errorf("multipartkit: create %s: %w", root, err) + } + s := &Store{root: root, maxBytes: DefaultMaxBlobBytes, ttl: DefaultTTL, open: map[string]*staged{}} + s.sweepOrphanParts() + return s, nil +} + +// sweepOrphanParts removes .part files left by a previous process. Their hash +// state and chunk cursor are gone, so they can never be finalized; keeping them +// would only grow $APP without bound. +func (s *Store) sweepOrphanParts() { + entries, err := os.ReadDir(s.root) + if err != nil { + return + } + for _, e := range entries { + if !e.IsDir() && strings.HasSuffix(e.Name(), ".part") { + _ = os.Remove(filepath.Join(s.root, e.Name())) + } + } +} + +// SetLimits overrides the size cap and TTL (0 keeps the current value). +func (s *Store) SetLimits(maxBytes int64, ttl time.Duration) { + s.mu.Lock() + defer s.mu.Unlock() + if maxBytes > 0 { + s.maxBytes = maxBytes + } + if ttl > 0 { + s.ttl = ttl + } +} + +// Begin reserves a blob id and opens its staging file. +// +// The id is minted here from crypto/rand and is never taken from the caller: a +// caller-chosen id is a path to overwriting somebody else's in-flight upload, +// and on a shared host that is a file-substitution bug rather than a naming +// inconvenience. +func (s *Store) Begin(fileName, contentType string, totalBytes int64, sha256hex string) (Meta, error) { + if strings.TrimSpace(fileName) == "" { + return Meta{}, ErrNameRequired + } + s.mu.Lock() + max := s.maxBytes + s.mu.Unlock() + if totalBytes <= 0 || totalBytes > max { + return Meta{}, fmt.Errorf("%w: %d (max %d)", ErrBadDeclared, totalBytes, max) + } + sha256hex = strings.ToLower(strings.TrimSpace(sha256hex)) + if len(sha256hex) != 64 { + return Meta{}, ErrBadChecksum + } + if _, err := hex.DecodeString(sha256hex); err != nil { + return Meta{}, ErrBadChecksum + } + var raw [16]byte + if _, err := rand.Read(raw[:]); err != nil { + return Meta{}, fmt.Errorf("multipartkit: mint id: %w", err) + } + id := hex.EncodeToString(raw[:]) + + // Only the base name of the caller's file name is ever kept, and it is never + // used to build a path — the on-disk name is the minted id. The original is + // metadata that rides along into the multipart body. + m := Meta{ + ID: id, + FileName: filepath.Base(filepath.Clean("/" + strings.TrimSpace(fileName))), + ContentType: strings.TrimSpace(contentType), + TotalBytes: totalBytes, + SHA256: sha256hex, + Created: time.Now().UTC(), + } + if m.ContentType == "" { + m.ContentType = "application/octet-stream" + } + f, err := os.OpenFile(s.partPath(id), os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600) + if err != nil { + return Meta{}, fmt.Errorf("multipartkit: open staging file: %w", err) + } + s.mu.Lock() + s.open[id] = &staged{meta: m, f: f, h: sha256.New()} + s.mu.Unlock() + return m, nil +} + +// Append writes the next chunk. seq must be exactly the next index (0-based): +// an upload that silently tolerates a gap or a replay reassembles to something +// the caller never sent, and the sha256 check at the end would then be the only +// thing standing between that and the partner. +func (s *Store) Append(id string, seq int, data []byte) (int64, error) { + if !validID(id) { + return 0, ErrBadBlobID + } + if len(data) > MaxChunkBytes { + return 0, ErrChunkTooBig + } + s.mu.Lock() + defer s.mu.Unlock() + st, ok := s.open[id] + if !ok { + if s.sealedOnDisk(id) { + return 0, ErrSealed + } + return 0, ErrNotFound + } + if seq != st.nextSeq { + return st.written, fmt.Errorf("%w: want %d, got %d", ErrOutOfOrder, st.nextSeq, seq) + } + if st.written+int64(len(data)) > st.meta.TotalBytes { + return st.written, ErrTooLarge + } + if _, err := st.f.Write(data); err != nil { + return st.written, fmt.Errorf("multipartkit: write chunk: %w", err) + } + _, _ = st.h.Write(data) + st.written += int64(len(data)) + st.nextSeq++ + return st.written, nil +} + +// Finalize verifies the reassembled bytes and seals the blob for sending. +// It is idempotent: finalizing an already-sealed blob returns its metadata. +func (s *Store) Finalize(id string) (Meta, error) { + if !validID(id) { + return Meta{}, ErrBadBlobID + } + s.mu.Lock() + defer s.mu.Unlock() + st, ok := s.open[id] + if !ok { + if m, err := s.readMeta(id); err == nil { + return m, nil // already sealed by us or by a previous process + } + return Meta{}, ErrNotFound + } + if st.written != st.meta.TotalBytes { + return Meta{}, fmt.Errorf("%w: got %d of %d", ErrIncomplete, st.written, st.meta.TotalBytes) + } + if got := hex.EncodeToString(st.h.Sum(nil)); got != st.meta.SHA256 { + return Meta{}, fmt.Errorf("%w: declared %s, reassembled %s", ErrChecksum, st.meta.SHA256, got) + } + if err := st.f.Close(); err != nil { + return Meta{}, fmt.Errorf("multipartkit: close staging file: %w", err) + } + // Write the metadata BEFORE the rename. The .blob appearing is what makes a + // blob look complete to a later process, so it must never be observable + // without the metadata that describes it. + b, err := json.Marshal(st.meta) + if err != nil { + return Meta{}, fmt.Errorf("multipartkit: encode meta: %w", err) + } + if err := os.WriteFile(s.metaPath(id), b, 0o600); err != nil { + return Meta{}, fmt.Errorf("multipartkit: write meta: %w", err) + } + if err := os.Rename(s.partPath(id), s.blobPath(id)); err != nil { + _ = os.Remove(s.metaPath(id)) + return Meta{}, fmt.Errorf("multipartkit: seal: %w", err) + } + delete(s.open, id) // sealed state lives on disk from here on + return st.meta, nil +} + +// Open returns the sealed blob's bytes for sending. A blob sealed before the +// adapter was respawned opens just the same, because sealed state is on disk. +func (s *Store) Open(id string) (*os.File, Meta, error) { + if !validID(id) { + return nil, Meta{}, ErrBadBlobID + } + s.mu.Lock() + _, inFlight := s.open[id] + s.mu.Unlock() + if inFlight { + return nil, Meta{}, ErrIncomplete // staged but not finalized + } + meta, err := s.readMeta(id) + if err != nil { + return nil, Meta{}, ErrNotFound + } + f, err := os.Open(s.blobPath(id)) + if err != nil { + return nil, Meta{}, ErrNotFound + } + return f, meta, nil +} + +// Abort discards an upload and its bytes, in flight or sealed. +func (s *Store) Abort(id string) error { + if !validID(id) { + return ErrBadBlobID + } + s.mu.Lock() + st, wasOpen := s.open[id] + if wasOpen { + _ = st.f.Close() + delete(s.open, id) + } + s.mu.Unlock() + + _ = os.Remove(s.partPath(id)) + sealed := s.sealedOnDisk(id) + _ = os.Remove(s.blobPath(id)) + _ = os.Remove(s.metaPath(id)) + if !wasOpen && !sealed { + return ErrNotFound + } + return nil +} + +// GC reclaims uploads past the TTL, in flight and sealed alike. An agent that +// stages a document and never sends it must not leave bytes under $APP forever. +// It walks the directory rather than the map so blobs sealed by a previous +// process are reclaimed too. +func (s *Store) GC() int { + s.mu.Lock() + cutoff := time.Now().UTC().Add(-s.ttl) + var stale []string + for id, st := range s.open { + if st.meta.Created.Before(cutoff) { + stale = append(stale, id) + } + } + s.mu.Unlock() + + seen := map[string]bool{} + for _, id := range stale { + seen[id] = true + } + if entries, err := os.ReadDir(s.root); err == nil { + for _, e := range entries { + name := e.Name() + if !strings.HasSuffix(name, ".blob") { + continue + } + id := strings.TrimSuffix(name, ".blob") + if seen[id] || !validID(id) { + continue + } + info, err := e.Info() + if err != nil || !info.ModTime().UTC().Before(cutoff) { + continue + } + seen[id] = true + stale = append(stale, id) + } + } + for _, id := range stale { + _ = s.Abort(id) + } + return len(stale) +} + +// readMeta loads a sealed blob's metadata, and only reports success when the +// bytes it describes are actually there. +func (s *Store) readMeta(id string) (Meta, error) { + b, err := os.ReadFile(s.metaPath(id)) + if err != nil { + return Meta{}, err + } + var m Meta + if err := json.Unmarshal(b, &m); err != nil { + return Meta{}, err + } + info, err := os.Stat(s.blobPath(id)) + if err != nil { + return Meta{}, err + } + if info.Size() != m.TotalBytes { + return Meta{}, fmt.Errorf("%w: on-disk blob is %d bytes, meta says %d", ErrIncomplete, info.Size(), m.TotalBytes) + } + return m, nil +} + +func (s *Store) sealedOnDisk(id string) bool { + _, err := os.Stat(s.blobPath(id)) + return err == nil +} + +func (s *Store) partPath(id string) string { return filepath.Join(s.root, id+".part") } +func (s *Store) blobPath(id string) string { return filepath.Join(s.root, id+".blob") } +func (s *Store) metaPath(id string) string { return filepath.Join(s.root, id+".meta") } + +// validID accepts only what Begin mints: 32 lowercase hex characters. This is +// the whole path-traversal defence — an id is concatenated into a filename, so +// nothing that could contain a separator or a dot segment is ever admitted. +func validID(id string) bool { + if len(id) != 32 { + return false + } + for i := 0; i < len(id); i++ { + c := id[i] + if (c < '0' || c > '9') && (c < 'a' || c > 'f') { + return false + } + } + return true +} diff --git a/internal/multipartkit/blob_test.go b/internal/multipartkit/blob_test.go new file mode 100644 index 0000000..0d121f1 --- /dev/null +++ b/internal/multipartkit/blob_test.go @@ -0,0 +1,348 @@ +package multipartkit + +import ( + "bytes" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "encoding/json" + "errors" + "io" + mrand "math/rand" + "mime" + "mime/multipart" + "strings" + "testing" + + "github.com/pilot-protocol/app-store/pkg/ipc" +) + +// randN builds test content from a seeded PRNG. Integrity here is checked by +// sha256, which only needs bytes that change when the content changes — +// crypto/rand by the megabyte buys nothing and costs seconds. +func randN(t *testing.T, n int) []byte { + t.Helper() + b := make([]byte, n) + r := mrand.New(mrand.NewSource(int64(n) * 2654435761)) + _, _ = r.Read(b) + return b +} + +func sha(b []byte) string { + s := sha256.Sum256(b) + return hex.EncodeToString(s[:]) +} + +// stage pushes content through the chunked API the way an agent would. +func stage(t *testing.T, s *Store, name, ct string, content []byte) string { + t.Helper() + m, err := s.Begin(name, ct, int64(len(content)), sha(content)) + if err != nil { + t.Fatalf("Begin: %v", err) + } + for seq, off := 0, 0; off < len(content); seq, off = seq+1, off+MaxChunkBytes { + end := off + MaxChunkBytes + if end > len(content) { + end = len(content) + } + if _, err := s.Append(m.ID, seq, content[off:end]); err != nil { + t.Fatalf("Append seq %d: %v", seq, err) + } + } + if _, err := s.Finalize(m.ID); err != nil { + t.Fatalf("Finalize: %v", err) + } + return m.ID +} + +// TestBase64InOneEnvelopeExceedsIPCFrame is the premise the whole chunked +// design rests on, asserted rather than assumed: the obvious encoding — the +// whole file base64'd into one JSON payload — does not fit in an IPC frame for +// any document-sized file, so no amount of tuning makes the simple version work. +func TestBase64InOneEnvelopeExceedsIPCFrame(t *testing.T) { + for _, size := range []int{1 << 20, 3 << 20, 20 << 20} { + payload, err := json.Marshal(map[string]string{ + "file_name": "contract.docx", + "content_base64": base64.StdEncoding.EncodeToString(randN(t, size)), + }) + if err != nil { + t.Fatalf("marshal: %v", err) + } + if len(payload) <= ipc.MaxFrameSize { + t.Errorf("a %d-byte file base64s to a %d-byte envelope, which FITS in the %d-byte frame — the chunking premise needs revisiting", + size, len(payload), ipc.MaxFrameSize) + } + } + // And the converse: one chunk must comfortably fit, or chunking is useless. + chunk, _ := json.Marshal(map[string]any{ + "blob_id": strings.Repeat("a", 32), "seq": 0, + "data_base64": base64.StdEncoding.EncodeToString(randN(t, MaxChunkBytes)), + }) + if len(chunk) > ipc.MaxFrameSize { + t.Fatalf("one MaxChunkBytes chunk encodes to %d bytes, over the %d-byte frame", len(chunk), ipc.MaxFrameSize) + } +} + +// TestChunkedReassemblyIsExact: a file far larger than one frame comes back +// byte-identical after being pushed through in chunks. +func TestChunkedReassemblyIsExact(t *testing.T) { + s, err := NewStore(t.TempDir()) + if err != nil { + t.Fatalf("NewStore: %v", err) + } + content := randN(t, 3<<20) + id := stage(t, s, "mutual-nda.docx", "application/vnd.openxmlformats-officedocument.wordprocessingml.document", content) + + f, meta, err := s.Open(id) + if err != nil { + t.Fatalf("Open: %v", err) + } + defer f.Close() + got, err := io.ReadAll(f) + if err != nil { + t.Fatalf("read: %v", err) + } + if !bytes.Equal(got, content) { + t.Fatalf("reassembled %d bytes, want %d, and they differ", len(got), len(content)) + } + if meta.FileName != "mutual-nda.docx" { + t.Errorf("FileName = %q", meta.FileName) + } +} + +func TestChunkTooBigRefused(t *testing.T) { + s, _ := NewStore(t.TempDir()) + content := randN(t, MaxChunkBytes+1) + m, err := s.Begin("x.bin", "", int64(len(content)), sha(content)) + if err != nil { + t.Fatalf("Begin: %v", err) + } + if _, err := s.Append(m.ID, 0, content); !errors.Is(err, ErrChunkTooBig) { + t.Fatalf("Append oversize chunk: err = %v, want ErrChunkTooBig", err) + } +} + +// TestOutOfOrderChunkRefused: tolerating a gap or a replay would reassemble +// something the caller never sent. +func TestOutOfOrderChunkRefused(t *testing.T) { + s, _ := NewStore(t.TempDir()) + content := randN(t, 4096) + m, _ := s.Begin("x.bin", "", int64(len(content)), sha(content)) + if _, err := s.Append(m.ID, 0, content[:2048]); err != nil { + t.Fatalf("Append 0: %v", err) + } + if _, err := s.Append(m.ID, 2, content[2048:]); !errors.Is(err, ErrOutOfOrder) { + t.Fatalf("skipped seq: err = %v, want ErrOutOfOrder", err) + } + if _, err := s.Append(m.ID, 0, content[2048:]); !errors.Is(err, ErrOutOfOrder) { + t.Fatalf("replayed seq: err = %v, want ErrOutOfOrder", err) + } +} + +// TestChecksumMismatchRefused: the declared sha is the integrity contract over +// the whole reassembly, so a body that does not match it must never be sendable. +func TestChecksumMismatchRefused(t *testing.T) { + s, _ := NewStore(t.TempDir()) + content := randN(t, 4096) + tampered := append([]byte(nil), content...) + tampered[0] ^= 0xff + + m, _ := s.Begin("x.bin", "", int64(len(content)), sha(content)) + if _, err := s.Append(m.ID, 0, tampered); err != nil { + t.Fatalf("Append: %v", err) + } + if _, err := s.Finalize(m.ID); !errors.Is(err, ErrChecksum) { + t.Fatalf("Finalize: err = %v, want ErrChecksum", err) + } + if _, _, err := s.Open(m.ID); err == nil { + t.Fatal("a blob that failed its checksum is still openable") + } +} + +func TestOversizeAndIncompleteRefused(t *testing.T) { + s, _ := NewStore(t.TempDir()) + content := randN(t, 4096) + m, _ := s.Begin("x.bin", "", int64(len(content)), sha(content)) + if _, err := s.Append(m.ID, 0, randN(t, 8192)); !errors.Is(err, ErrTooLarge) { + t.Fatalf("over-declared write: err = %v, want ErrTooLarge", err) + } + if _, err := s.Finalize(m.ID); !errors.Is(err, ErrIncomplete) { + t.Fatalf("short blob: err = %v, want ErrIncomplete", err) + } +} + +// TestBlobIDIsNotCallerControlled: ids are minted, and anything that could +// contain a path separator is refused outright. +func TestBlobIDIsNotCallerControlled(t *testing.T) { + s, _ := NewStore(t.TempDir()) + for _, bad := range []string{ + "../../etc/passwd", "..", "/abs", "short", "", + strings.Repeat("g", 32), // right length, not hex + strings.Repeat("a", 33), + } { + if _, err := s.Append(bad, 0, []byte("x")); err == nil { + t.Errorf("Append(%q) was accepted", bad) + } + if _, err := s.Finalize(bad); err == nil { + t.Errorf("Finalize(%q) was accepted", bad) + } + } + // Two Begins never collide on an id. + seen := map[string]bool{} + for i := 0; i < 50; i++ { + m, err := s.Begin("x", "", 8, sha([]byte("12345678"))) + if err != nil { + t.Fatalf("Begin: %v", err) + } + if seen[m.ID] { + t.Fatalf("minted a duplicate blob id: %s", m.ID) + } + seen[m.ID] = true + } +} + +// TestFileNameCannotEscape: the caller's file name is metadata, never a path. +func TestFileNameCannotEscape(t *testing.T) { + s, _ := NewStore(t.TempDir()) + m, err := s.Begin("../../../etc/passwd", "", 4, sha([]byte("abcd"))) + if err != nil { + t.Fatalf("Begin: %v", err) + } + if strings.ContainsAny(m.FileName, `/\`) { + t.Fatalf("FileName = %q, still carries path separators", m.FileName) + } + if m.FileName != "passwd" { + t.Errorf("FileName = %q, want the base name only", m.FileName) + } +} + +func TestBadDeclaredSizeAndChecksumRefused(t *testing.T) { + s, _ := NewStore(t.TempDir()) + if _, err := s.Begin("x", "", 0, sha([]byte("x"))); !errors.Is(err, ErrBadDeclared) { + t.Errorf("zero size: %v", err) + } + if _, err := s.Begin("x", "", DefaultMaxBlobBytes+1, sha([]byte("x"))); !errors.Is(err, ErrBadDeclared) { + t.Errorf("over cap: %v", err) + } + if _, err := s.Begin("x", "", 4, "nothex"); !errors.Is(err, ErrBadChecksum) { + t.Errorf("bad sha: %v", err) + } + if _, err := s.Begin(" ", "", 4, sha([]byte("abcd"))); !errors.Is(err, ErrNameRequired) { + t.Errorf("blank name: %v", err) + } +} + +// TestBuildFormRoundTrip: what the adapter assembles is what a FastAPI-shaped +// parser reads back, including the real content type of the file part. +func TestBuildFormRoundTrip(t *testing.T) { + s, _ := NewStore(t.TempDir()) + content := randN(t, 2<<20) + docxCT := "application/vnd.openxmlformats-officedocument.wordprocessingml.document" + id := stage(t, s, "nda.docx", docxCT, content) + + f, meta, err := s.Open(id) + if err != nil { + t.Fatalf("Open: %v", err) + } + defer f.Close() + + body, ct, err := BuildForm("file", meta.FileName, meta.ContentType, f, map[string]string{ + "deal_id": "deal_123", + "context_for_legal": "Standard mutual NDA.", + }) + if err != nil { + t.Fatalf("BuildForm: %v", err) + } + + mt, params, err := mime.ParseMediaType(ct) + if err != nil || mt != "multipart/form-data" || params["boundary"] == "" { + t.Fatalf("Content-Type = %q (mt %q, boundary %q)", ct, mt, params["boundary"]) + } + + mr := multipart.NewReader(bytes.NewReader(body), params["boundary"]) + fields := map[string]string{} + var fileSHA, fileName, filePartCT string + for { + p, err := mr.NextPart() + if err == io.EOF { + break + } + if err != nil { + t.Fatalf("NextPart: %v", err) + } + b, _ := io.ReadAll(p) + if p.FileName() != "" { + fileName, fileSHA, filePartCT = p.FileName(), sha(b), p.Header.Get("Content-Type") + } else { + fields[p.FormName()] = string(b) + } + p.Close() + } + if fileSHA != sha(content) { + t.Error("file bytes changed passing through BuildForm") + } + if fileName != "nda.docx" { + t.Errorf("file name = %q", fileName) + } + if filePartCT != docxCT { + t.Errorf("file part Content-Type = %q, want %q (octet-stream loses what the partner accepts on)", filePartCT, docxCT) + } + if fields["deal_id"] != "deal_123" { + t.Errorf("deal_id = %q", fields["deal_id"]) + } +} + +// TestBuildFormEscapesFileName: a quote in the file name must not break out of +// the Content-Disposition header and forge a part. +func TestBuildFormEscapesFileName(t *testing.T) { + evil := `a".docx"; name="deal_id` + body, ct, err := BuildForm("file", evil, "text/plain", strings.NewReader("x"), map[string]string{"deal_id": "real"}) + if err != nil { + t.Fatalf("BuildForm: %v", err) + } + _, params, _ := mime.ParseMediaType(ct) + mr := multipart.NewReader(bytes.NewReader(body), params["boundary"]) + names := map[string]int{} + for { + p, err := mr.NextPart() + if err == io.EOF { + break + } + if err != nil { + t.Fatalf("NextPart: %v", err) + } + names[p.FormName()]++ + p.Close() + } + if names["deal_id"] != 1 { + t.Fatalf("deal_id appears %d times — the file name forged a part", names["deal_id"]) + } + if names["file"] != 1 { + t.Fatalf("file part appears %d times", names["file"]) + } +} + +func TestAbortAndGC(t *testing.T) { + s, _ := NewStore(t.TempDir()) + content := randN(t, 1024) + m, _ := s.Begin("x.bin", "", int64(len(content)), sha(content)) + if _, err := s.Append(m.ID, 0, content); err != nil { + t.Fatalf("Append: %v", err) + } + if err := s.Abort(m.ID); err != nil { + t.Fatalf("Abort: %v", err) + } + if _, err := s.Append(m.ID, 1, content); !errors.Is(err, ErrNotFound) { + t.Fatalf("after Abort: err = %v, want ErrNotFound", err) + } + + s2, _ := NewStore(t.TempDir()) + s2.SetLimits(0, 1) // 1ns TTL: everything is immediately stale + m2, _ := s2.Begin("y.bin", "", 16, sha(randN(t, 16))) + if n := s2.GC(); n != 1 { + t.Fatalf("GC reclaimed %d, want 1", n) + } + if _, err := s2.Append(m2.ID, 0, []byte("x")); !errors.Is(err, ErrNotFound) { + t.Fatalf("after GC: err = %v, want ErrNotFound", err) + } +} diff --git a/internal/multipartkit/chain_test.go b/internal/multipartkit/chain_test.go new file mode 100644 index 0000000..5cd49c9 --- /dev/null +++ b/internal/multipartkit/chain_test.go @@ -0,0 +1,268 @@ +package multipartkit_test + +// The whole path, in one test: an agent stages a document in IPC-sized chunks, +// the adapter reassembles it and builds one multipart body, signs it, and the +// real broker verifies the caller, checks that the deal in the form field is +// theirs, injects the master key, and forwards it to the partner. +// +// Each half is covered on its own (blob_test.go, internal/broker/zz_multipart_test.go). +// This exists for the seams between them, which is where a design like this +// actually breaks: a boundary lost at a hop, a body hashed before it was +// finished, a field the tenancy layer reads differently from the partner. + +import ( + "bytes" + "crypto/ed25519" + "crypto/rand" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + mrand "math/rand" + "mime" + "mime/multipart" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/pilot-protocol/app-template/internal/broker" + "github.com/pilot-protocol/app-template/internal/multipartkit" +) + +const chainRegistry = `[{ + "id": "io.pilot.generallegal", + "upstream": "%s", + "key_env": "GL_KEY", + "auth_header": "Authorization", + "auth_scheme": "Bearer", + "quota": 0, + "allow": ["POST /api/v1/deals", "POST /api/v1/documents"], + "forward_content_types": ["multipart/form-data"], + "tenancy": { + "param_types": {"deal_id": "deal"}, + "body_refs": {"deal_id": "deal"}, + "create": [{"method":"POST","path":"/api/v1/deals","type":"deal","id_field":"deal_id"}], + "list": [] + } +}]` + +type received struct { + sha string + bytes int + name string + fields map[string]string + auth string +} + +func partnerServer(t *testing.T, got *received) *httptest.Server { + t.Helper() + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + if r.URL.Path == "/api/v1/deals" { + fmt.Fprintf(w, `{"deal_id":"deal_%d"}`, time.Now().UnixNano()) + return + } + mt, params, err := mime.ParseMediaType(r.Header.Get("Content-Type")) + if err != nil || mt != "multipart/form-data" { + http.Error(w, `{"detail":"not multipart"}`, http.StatusUnprocessableEntity) + return + } + rec := received{fields: map[string]string{}, auth: r.Header.Get("Authorization")} + mr := multipart.NewReader(r.Body, params["boundary"]) + for { + p, err := mr.NextPart() + if err == io.EOF { + break + } + if err != nil { + http.Error(w, `{"detail":"bad multipart"}`, http.StatusBadRequest) + return + } + b, _ := io.ReadAll(p) + if p.FileName() != "" { + s := sha256.Sum256(b) + rec.sha, rec.bytes, rec.name = hex.EncodeToString(s[:]), len(b), p.FileName() + } else { + rec.fields[p.FormName()] = string(b) + } + p.Close() + } + *got = rec + fmt.Fprint(w, `{"deal_id":"d1","contract_id":"c1","version_id":"v1"}`) + })) +} + +// send signs a request the way the generated adapter does and runs it through +// the broker. The signature covers the body bytes, so it has to be taken after +// the multipart body is fully assembled. +func send(t *testing.T, b *broker.Broker, priv ed25519.PrivateKey, method, path, ct string, body []byte) *httptest.ResponseRecorder { + t.Helper() + req := httptest.NewRequest(method, path, bytes.NewReader(body)) + for k, v := range broker.Sign(priv, method, path, body, time.Now()) { + req.Header.Set(k, v) + } + if ct != "" { + req.Header.Set("Content-Type", ct) + } + rec := httptest.NewRecorder() + b.ServeHTTP(rec, req) + return rec +} + +func TestFullChain_ChunkedStagingToPartner(t *testing.T) { + var got received + up := partnerServer(t, &got) + defer up.Close() + + reg, err := broker.ParseRegistry([]byte(fmt.Sprintf(chainRegistry, up.URL)), + func(string) string { return "glk_master" }) + if err != nil { + t.Fatalf("ParseRegistry: %v", err) + } + bk := broker.New(reg, broker.NewMemStore()) + bk.Verify = broker.VerifyConfig{Window: time.Hour} + + _, alice, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatalf("keygen: %v", err) + } + _, mallory, _ := ed25519.GenerateKey(rand.Reader) + + // 1. Alice opens a matter. The broker claims the deal for her. + rec := send(t, bk, alice, "POST", "/io.pilot.generallegal/api/v1/deals", "application/json", + []byte(`{"initial_request":"Review this mutual NDA"}`)) + if rec.Code != 200 { + t.Fatalf("open deal: %d (%s)", rec.Code, rec.Body.String()) + } + var deal struct { + DealID string `json:"deal_id"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &deal); err != nil { + t.Fatalf("decode deal: %v", err) + } + + // 2. The agent stages a 5 MiB document in IPC-sized chunks. Five megabytes + // is five times the frame that made the naive encoding impossible. + // Seeded, not cryptographic: the size is the point (five times the 1 MiB + // frame), and sha256 catches any corruption regardless of where the bytes + // came from. + content := make([]byte, 5<<20) + _, _ = mrand.New(mrand.NewSource(1)).Read(content) + sum := sha256.Sum256(content) + want := hex.EncodeToString(sum[:]) + + store, err := multipartkit.NewStore(t.TempDir()) + if err != nil { + t.Fatalf("NewStore: %v", err) + } + docxCT := "application/vnd.openxmlformats-officedocument.wordprocessingml.document" + meta, err := store.Begin("mutual-nda.docx", docxCT, int64(len(content)), want) + if err != nil { + t.Fatalf("Begin: %v", err) + } + chunks := 0 + for seq, off := 0, 0; off < len(content); seq, off = seq+1, off+multipartkit.MaxChunkBytes { + end := off + multipartkit.MaxChunkBytes + if end > len(content) { + end = len(content) + } + if _, err := store.Append(meta.ID, seq, content[off:end]); err != nil { + t.Fatalf("Append %d: %v", seq, err) + } + chunks++ + } + if _, err := store.Finalize(meta.ID); err != nil { + t.Fatalf("Finalize: %v", err) + } + if chunks < 5 { + t.Fatalf("staged in %d chunks; expected the file to span several frames", chunks) + } + + // 3. The adapter builds one multipart body from the staged blob. + f, m, err := store.Open(meta.ID) + if err != nil { + t.Fatalf("Open: %v", err) + } + body, ct, err := multipartkit.BuildForm("file", m.FileName, m.ContentType, f, map[string]string{ + "deal_id": deal.DealID, + "context_for_legal": "Standard mutual NDA, we are the disclosing party.", + "last_edit_by": "client", + }) + f.Close() + if err != nil { + t.Fatalf("BuildForm: %v", err) + } + + // 4. Signed, brokered, forwarded. + rec = send(t, bk, alice, "POST", "/io.pilot.generallegal/api/v1/documents", ct, body) + if rec.Code != 200 { + t.Fatalf("upload: %d (%s)", rec.Code, rec.Body.String()) + } + if got.sha != want { + t.Errorf("document corrupted end to end:\n got %s (%d bytes)\nwant %s (%d bytes)", + got.sha, got.bytes, want, len(content)) + } + if got.name != "mutual-nda.docx" { + t.Errorf("file name = %q", got.name) + } + if got.fields["deal_id"] != deal.DealID { + t.Errorf("deal_id = %q, want %q", got.fields["deal_id"], deal.DealID) + } + if got.auth != "Bearer glk_master" { + t.Errorf("Authorization = %q — the master key was not injected", got.auth) + } + + // 5. And the isolation the shared key makes necessary still holds on the far + // side of all that machinery: Mallory reuses Alice's staged bytes and her + // deal id, signs as herself, and is refused before the partner is touched. + before := got + rec = send(t, bk, mallory, "POST", "/io.pilot.generallegal/api/v1/documents", ct, body) + if rec.Code != http.StatusNotFound { + t.Fatalf("Mallory uploading into Alice's deal: %d, want 404", rec.Code) + } + if got.sha != before.sha || got.fields["deal_id"] != before.fields["deal_id"] { + t.Fatal("the refused upload still reached the partner") + } +} + +// TestFullChain_TamperedBodyFailsSignature: the signature is taken over the +// assembled multipart bytes, so flipping one byte of the file after signing +// must be rejected as an identity failure rather than silently forwarded. +func TestFullChain_TamperedBodyFailsSignature(t *testing.T) { + var got received + up := partnerServer(t, &got) + defer up.Close() + + reg, _ := broker.ParseRegistry([]byte(fmt.Sprintf(chainRegistry, up.URL)), + func(string) string { return "glk_master" }) + bk := broker.New(reg, broker.NewMemStore()) + bk.Verify = broker.VerifyConfig{Window: time.Hour} + _, alice, _ := ed25519.GenerateKey(rand.Reader) + + body, ct, err := multipartkit.BuildForm("file", "x.docx", "text/plain", + bytes.NewReader([]byte("original contents")), map[string]string{"deal_id": "d1"}) + if err != nil { + t.Fatalf("BuildForm: %v", err) + } + + path := "/io.pilot.generallegal/api/v1/documents" + req := httptest.NewRequest("POST", path, nil) + for k, v := range broker.Sign(alice, "POST", path, body, time.Now()) { + req.Header.Set(k, v) + } + req.Header.Set("Content-Type", ct) + + tampered := bytes.Replace(body, []byte("original"), []byte("modified"), 1) + if bytes.Equal(tampered, body) { + t.Fatal("test setup: body was not actually tampered with") + } + req.Body = io.NopCloser(bytes.NewReader(tampered)) + + rec := httptest.NewRecorder() + bk.ServeHTTP(rec, req) + if rec.Code != http.StatusUnauthorized { + t.Fatalf("tampered multipart body: %d, want 401", rec.Code) + } +} diff --git a/internal/multipartkit/durability_test.go b/internal/multipartkit/durability_test.go new file mode 100644 index 0000000..59095e1 --- /dev/null +++ b/internal/multipartkit/durability_test.go @@ -0,0 +1,266 @@ +package multipartkit + +import ( + "errors" + "io" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" +) + +// The daemon supervises adapters and respawns them, so "the process restarted" +// is an ordinary event rather than a disaster case. These tests pin what has to +// survive that and what must not. + +// TestSealedBlobSurvivesRestart: an upload staged and finalized before a respawn +// is still sendable afterwards. Keeping the only record of a finished upload in +// a map means an agent that staged 20 MiB has to stage it all over again for +// reasons it cannot see. +func TestSealedBlobSurvivesRestart(t *testing.T) { + dir := t.TempDir() + content := randN(t, 3<<20) + + s1, err := NewStore(dir) + if err != nil { + t.Fatalf("NewStore: %v", err) + } + id := stage(t, s1, "nda.docx", "application/pdf", content) + + // A new Store over the same root is what the respawned adapter gets. + s2, err := NewStore(dir) + if err != nil { + t.Fatalf("NewStore (restart): %v", err) + } + f, meta, err := s2.Open(id) + if err != nil { + t.Fatalf("Open after restart: %v", err) + } + defer f.Close() + got, err := io.ReadAll(f) + if err != nil { + t.Fatalf("read: %v", err) + } + if string(got) != string(content) { + t.Fatal("blob bytes changed across the restart") + } + if meta.FileName != "nda.docx" || meta.ContentType != "application/pdf" { + t.Errorf("metadata lost across restart: %+v", meta) + } +} + +// TestInFlightPartSweptOnRestart: a half-written .part cannot be finished — the +// rolling hash and the chunk cursor died with the process. Leaving it on disk +// grows $APP forever and invites a later resume that would splice a gap into +// the middle of a document. +func TestInFlightPartSweptOnRestart(t *testing.T) { + dir := t.TempDir() + content := randN(t, 4096) + + s1, err := NewStore(dir) + if err != nil { + t.Fatalf("NewStore: %v", err) + } + m, err := s1.Begin("half.docx", "", int64(len(content)), sha(content)) + if err != nil { + t.Fatalf("Begin: %v", err) + } + if _, err := s1.Append(m.ID, 0, content[:2048]); err != nil { + t.Fatalf("Append: %v", err) + } + if _, err := os.Stat(filepath.Join(dir, m.ID+".part")); err != nil { + t.Fatalf("test setup: .part missing before restart: %v", err) + } + + s2, err := NewStore(dir) + if err != nil { + t.Fatalf("NewStore (restart): %v", err) + } + if _, err := os.Stat(filepath.Join(dir, m.ID+".part")); !os.IsNotExist(err) { + t.Errorf("orphan .part survived the restart (err = %v)", err) + } + if _, err := s2.Append(m.ID, 1, content[2048:]); !errors.Is(err, ErrNotFound) { + t.Errorf("resuming a swept upload: err = %v, want ErrNotFound", err) + } + if _, _, err := s2.Open(m.ID); !errors.Is(err, ErrNotFound) { + t.Errorf("opening a swept upload: err = %v, want ErrNotFound", err) + } +} + +// TestOpenBeforeFinalizeRefused: a staged-but-unsealed blob must not be +// sendable — its bytes have not been checked against the declared sha yet. +func TestOpenBeforeFinalizeRefused(t *testing.T) { + s, _ := NewStore(t.TempDir()) + content := randN(t, 4096) + m, _ := s.Begin("x.docx", "", int64(len(content)), sha(content)) + if _, err := s.Append(m.ID, 0, content); err != nil { + t.Fatalf("Append: %v", err) + } + if _, _, err := s.Open(m.ID); !errors.Is(err, ErrIncomplete) { + t.Fatalf("Open before Finalize: err = %v, want ErrIncomplete", err) + } +} + +// TestAppendAfterFinalizeIsSealed: once sealed, the bytes behind a blob id are +// fixed. Appending must report that clearly rather than as "unknown id", or a +// caller cannot tell a finished upload from a typo. +func TestAppendAfterFinalizeIsSealed(t *testing.T) { + s, _ := NewStore(t.TempDir()) + content := randN(t, 1024) + id := stage(t, s, "x.docx", "", content) + if _, err := s.Append(id, 1, []byte("more")); !errors.Is(err, ErrSealed) { + t.Fatalf("Append after Finalize: err = %v, want ErrSealed", err) + } +} + +// TestFinalizeIsIdempotent: a retried finalize (an agent that lost the reply) +// must not fail. +func TestFinalizeIsIdempotent(t *testing.T) { + s, _ := NewStore(t.TempDir()) + content := randN(t, 1024) + id := stage(t, s, "x.docx", "", content) + m, err := s.Finalize(id) + if err != nil { + t.Fatalf("second Finalize: %v", err) + } + if m.ID != id { + t.Errorf("Finalize returned id %q, want %q", m.ID, id) + } +} + +// TestGCReclaimsSealedBlobsOnDisk: GC used to walk only the in-memory map, so +// blobs sealed before a restart were never reclaimed and $APP grew without +// bound. +func TestGCReclaimsSealedBlobsOnDisk(t *testing.T) { + dir := t.TempDir() + s1, _ := NewStore(dir) + id := stage(t, s1, "x.docx", "", randN(t, 2048)) + + // Age the sealed blob past any TTL. + old := time.Now().Add(-2 * time.Hour) + for _, suffix := range []string{".blob", ".meta"} { + if err := os.Chtimes(filepath.Join(dir, id+suffix), old, old); err != nil { + t.Fatalf("chtimes: %v", err) + } + } + + s2, _ := NewStore(dir) // restart: nothing in the map + s2.SetLimits(0, time.Minute) + if n := s2.GC(); n != 1 { + t.Fatalf("GC reclaimed %d, want 1", n) + } + if _, _, err := s2.Open(id); !errors.Is(err, ErrNotFound) { + t.Errorf("blob still open after GC: %v", err) + } + left, _ := os.ReadDir(dir) + for _, e := range left { + if strings.HasPrefix(e.Name(), id) { + t.Errorf("GC left %s behind", e.Name()) + } + } +} + +// TestGCKeepsFreshBlobs: reclaiming aggressively would delete a document the +// agent is about to send. +func TestGCKeepsFreshBlobs(t *testing.T) { + dir := t.TempDir() + s, _ := NewStore(dir) + id := stage(t, s, "x.docx", "", randN(t, 1024)) + if n := s.GC(); n != 0 { + t.Fatalf("GC reclaimed %d fresh blobs, want 0", n) + } + f, _, err := s.Open(id) + if err != nil { + t.Fatalf("fresh blob was reclaimed: %v", err) + } + f.Close() +} + +// TestConcurrentAppendAndAbort: Append writes to a file that Abort closes, so +// the two must be serialised. Run under -race this catches the window where +// Append looked the blob up, released the lock, and then wrote. +func TestConcurrentAppendAndAbort(t *testing.T) { + s, _ := NewStore(t.TempDir()) + var wg sync.WaitGroup + for i := 0; i < 24; i++ { + content := randN(t, 8192) + m, err := s.Begin("x.bin", "", int64(len(content)), sha(content)) + if err != nil { + t.Fatalf("Begin: %v", err) + } + wg.Add(2) + go func(id string, data []byte) { + defer wg.Done() + // Errors are expected and fine here — a torn write or a crash is not. + _, _ = s.Append(id, 0, data[:4096]) + _, _ = s.Append(id, 1, data[4096:]) + _, _ = s.Finalize(id) + }(m.ID, content) + go func(id string) { + defer wg.Done() + _ = s.Abort(id) + }(m.ID) + } + wg.Wait() +} + +// TestConcurrentBeginsDoNotCollide: ids are minted under contention too. +func TestConcurrentBeginsDoNotCollide(t *testing.T) { + s, _ := NewStore(t.TempDir()) + const n = 64 + ids := make(chan string, n) + var wg sync.WaitGroup + for i := 0; i < n; i++ { + wg.Add(1) + go func() { + defer wg.Done() + m, err := s.Begin("x", "", 8, sha([]byte("12345678"))) + if err != nil { + t.Errorf("Begin: %v", err) + return + } + ids <- m.ID + }() + } + wg.Wait() + close(ids) + seen := map[string]bool{} + for id := range ids { + if seen[id] { + t.Fatalf("duplicate blob id under contention: %s", id) + } + seen[id] = true + } + if len(seen) != n { + t.Fatalf("minted %d ids, want %d", len(seen), n) + } +} + +// TestMetaWithoutBlobIsNotOpenable: a .meta whose bytes are missing or the +// wrong length must not present as a usable upload. Sealed state is two files +// and both have to agree. +func TestMetaWithoutBlobIsNotOpenable(t *testing.T) { + dir := t.TempDir() + s, _ := NewStore(dir) + id := stage(t, s, "x.docx", "", randN(t, 2048)) + + if err := os.Remove(filepath.Join(dir, id+".blob")); err != nil { + t.Fatalf("remove blob: %v", err) + } + s2, _ := NewStore(dir) + if _, _, err := s2.Open(id); !errors.Is(err, ErrNotFound) { + t.Errorf("meta without blob: err = %v, want ErrNotFound", err) + } + + // And a blob truncated behind the metadata's back is not usable either. + id2 := stage(t, s, "y.docx", "", randN(t, 2048)) + if err := os.Truncate(filepath.Join(dir, id2+".blob"), 10); err != nil { + t.Fatalf("truncate: %v", err) + } + s3, _ := NewStore(dir) + if _, _, err := s3.Open(id2); err == nil { + t.Error("a truncated blob opened successfully") + } +} diff --git a/internal/multipartkit/form.go b/internal/multipartkit/form.go new file mode 100644 index 0000000..3583019 --- /dev/null +++ b/internal/multipartkit/form.go @@ -0,0 +1,78 @@ +package multipartkit + +import ( + "bytes" + "fmt" + "io" + "mime/multipart" + "net/textproto" + "sort" + "strings" +) + +// BuildForm assembles one multipart/form-data body: the staged file under +// fileField, plus the scalar fields. +// +// The body is materialised into memory rather than streamed, and that is forced +// by the security model, not laziness. A managed adapter signs +// base64(sha256(BODY)) so the broker can prove who is calling; you cannot hash +// a body you have not finished producing. DefaultMaxBlobBytes is what keeps +// that bounded. +// +// Returns the body and the Content-Type header value — which carries the +// generated boundary and is therefore not optional to propagate. Every hop +// between here and the partner has to preserve it verbatim or the body is +// undecodable on arrival. +func BuildForm(fileField, fileName, fileContentType string, content io.Reader, fields map[string]string) ([]byte, string, error) { + if strings.TrimSpace(fileField) == "" { + return nil, "", fmt.Errorf("multipartkit: file field name is required") + } + var buf bytes.Buffer + w := multipart.NewWriter(&buf) + + // Deterministic field order. Not cosmetic: it makes a signed body + // reproducible, so a failure can be replayed byte-for-byte when diagnosing + // a signature or a partner-side parse. + names := make([]string, 0, len(fields)) + for k := range fields { + names = append(names, k) + } + sort.Strings(names) + for _, k := range names { + if err := w.WriteField(k, fields[k]); err != nil { + return nil, "", fmt.Errorf("multipartkit: write field %q: %w", k, err) + } + } + + // The file part is written with an explicit Content-Type. multipart's own + // CreateFormFile hardcodes application/octet-stream, which loses the real + // type the partner uses to decide whether it will accept the document at + // all. + h := make(textproto.MIMEHeader) + h.Set("Content-Disposition", fmt.Sprintf(`form-data; name=%q; filename=%q`, + escapeQuotes(fileField), escapeQuotes(fileName))) + ct := strings.TrimSpace(fileContentType) + if ct == "" { + ct = "application/octet-stream" + } + h.Set("Content-Type", ct) + + part, err := w.CreatePart(h) + if err != nil { + return nil, "", fmt.Errorf("multipartkit: create file part: %w", err) + } + if _, err := io.Copy(part, content); err != nil { + return nil, "", fmt.Errorf("multipartkit: copy file content: %w", err) + } + if err := w.Close(); err != nil { + return nil, "", fmt.Errorf("multipartkit: close writer: %w", err) + } + return buf.Bytes(), w.FormDataContentType(), nil +} + +// escapeQuotes matches mime/multipart's own escaper so a file name containing a +// quote or a backslash cannot break out of the Content-Disposition header and +// forge part boundaries. +func escapeQuotes(s string) string { + return strings.NewReplacer("\\", "\\\\", `"`, "\\\"").Replace(s) +} diff --git a/internal/scaffold/config.go b/internal/scaffold/config.go index 624797f..bf15347 100644 --- a/internal/scaffold/config.go +++ b/internal/scaffold/config.go @@ -11,6 +11,7 @@ package scaffold import ( "encoding/json" "fmt" + "net" "net/url" "regexp" "sort" @@ -278,6 +279,11 @@ type Backend struct { // publisher uploads the master key to Pilot once; users bring nothing. Auth string `yaml:"auth"` + // MCP, when set, adds a JSON-RPC tool-server backend alongside base_url for + // methods that declare an `mcp:` route. Credentials are never shared between + // the two. + MCP *MCPBackend `yaml:"mcp"` + // X402 enables transparent, capped payment for paid (x402) APIs: on a // backend HTTP 402 the adapter asks the Pilot wallet to satisfy the charge // and retries with X-PAYMENT. Presence enables it; it adds an ipc.call @@ -479,6 +485,89 @@ const DefaultProvisionPath = "/_provision" // broker's pilotBalancePath can never drift apart. const BalanceMetaPath = pilotpath.Balance +// Blob staging defaults for multipart uploads. +const ( + // DefaultFileField is the form field a file is sent under when the route + // does not name one. + DefaultFileField = "file" + // DefaultBlobParam is the payload key carrying the staged blob id. + DefaultBlobParam = "blob_id" + // DefaultMaxBlobBytes bounds one staged upload. Sized above the largest + // partner limit we ship against (20 MiB) with room for a manifest bump. + DefaultMaxBlobBytes int64 = 24 << 20 + // BlobDir is where staged uploads live, under the app's own directory so the + // manifest can grant exactly it and nothing wider. + BlobDir = "$APP/blobs" +) + +// HasMCP reports whether any method is a tool-server call. +func (c *Config) HasMCP() bool { + for i := range c.Methods { + if c.Methods[i].MCP != nil { + return true + } + } + return false +} + +// MCPHost is the tool server's hostname, for the net.dial grant. +func (c *Config) MCPHost() string { + if c.Backend.MCP == nil { + return "" + } + if u, err := url.Parse(c.Backend.MCP.URL); err == nil { + return u.Hostname() + } + return "" +} + +// HasBillable reports whether any method costs the user real money. +// +// This is deliberately NOT `gated`. Gated means "your plan does not include +// this" — the call is unavailable until the account is upgraded. Billable means +// the call works fine and charges you for it. Conflating them tells an agent +// something false in both directions: that a working method is locked, or that +// a free-to-attempt method is free to use. +// +// It matters most for a byo app fronting a partner who bills out-of-band, where +// Pilot has no price to meter and the only honest thing to do is say which +// calls spend money and let the agent decide. +func (c *Config) HasBillable() bool { + for i := range c.Methods { + if strings.TrimSpace(c.Methods[i].Billable) != "" { + return true + } + } + return false +} + +// HasMultipart reports whether any method sends multipart/form-data. It drives +// the blob-staging runtime, the injected upload methods, and the $APP/blobs +// grants — all three or none, since a staged upload with no way to stage is +// just a method that always fails. +func (c *Config) HasMultipart() bool { + for i := range c.Methods { + if c.Methods[i].HTTP != nil && c.Methods[i].HTTP.Multipart != nil { + return true + } + } + return false +} + +// MaxBlobBytes is the largest staged upload any multipart method allows. +func (c *Config) MaxBlobBytes() int64 { + max := int64(0) + for i := range c.Methods { + if h := c.Methods[i].HTTP; h != nil && h.Multipart != nil && h.Multipart.MaxBytes > max { + max = h.Multipart.MaxBytes + } + } + if max == 0 { + return DefaultMaxBlobBytes + } + return max +} + // ProvisionPath is the reserved provision route the generated adapter recognizes. func (c *Config) ProvisionPath() string { return DefaultProvisionPath } @@ -574,9 +663,12 @@ type Method struct { CLI *CLIRoute `yaml:"cli"` // cli backend route Local *LocalRoute `yaml:"local"` // local metadata route (no backend call) Signup *SignupRoute `yaml:"signup"` // no-broker self-signup route (mints + saves a per-user key) + Upload *UploadRoute `yaml:"-"` // injected blob-staging step for a multipart app (never authored) + MCP *MCPRoute `yaml:"mcp"` // tool-server route: a JSON-RPC tool call on backend.mcp, sent with NO credentials Params map[string]string `yaml:"params"` // name -> human description, for help Roundtrip string `yaml:"roundtrip"` // measured warm roundtrip, for help Gated string `yaml:"gated"` // non-empty = usable only after an account upgrade; the string is the reason, surfaced in a disclaimer at the bottom of .help + Billable string `yaml:"billable"` // non-empty = this call costs the user real money; the string says what it costs, surfaced in .help } // SignupRoute makes a method self-provision a per-user API key WITHOUT a Pilot @@ -668,6 +760,69 @@ type LocalRoute struct { // // Omitting a param's location keeps the historical default, so existing specs // build identically. +// MCPRoute maps a method onto a tool exposed by a JSON-RPC tool server +// (backend.mcp), rather than onto a REST endpoint on backend.base_url. +// +// Tool servers are their own host with their own auth story — usually none — +// so these methods deliberately do NOT go through the REST client and never +// carry its credentials. See MCPBackend. +type MCPRoute struct { + // Tool is the tool name as the server advertises it in tools/list. + Tool string `yaml:"tool"` + // Wrap, when set, nests the method payload under this single argument name + // (some servers take one object argument, e.g. {"intake": {...}}). + Wrap string `yaml:"wrap"` + // Keep lists payload keys that stay at the TOP level when Wrap is set. + // Some tools take a mix — an id beside a nested update object, e.g. + // {"formation_id": "...", "update": {...}} — and wrapping the id along with + // everything else silently produces a call the server cannot satisfy. + Keep []string `yaml:"keep"` +} + +// MCPBackend points at a JSON-RPC tool server reachable over Streamable HTTP. +// +// It is a SECOND backend, alongside base_url, and the separation is the point: +// an app can front a credentialed REST API and an open tool server at once +// without the REST key ever reaching the tool server's host. The generated +// client for these routes is built with no auth headers at all, so there is no +// path by which the user's key leaks to a different origin. +type MCPBackend struct { + // URL is the tool server endpoint (the one that answers initialize). + URL string `yaml:"url"` + // Auth is reserved; only "" (no credentials) is supported today. It exists + // so adding a credentialed tool server later is a config change and not a + // silent behaviour change for every app already shipping. + Auth string `yaml:"auth"` +} + +// UploadRoute marks one of the three blob-staging methods injected into any app +// with a multipart route. They are never authored in pilot.app.yaml — an author +// declares the upload itself and gets the staging steps that make it reachable. +// +// Step is "begin", "chunk", or "abort". +type UploadRoute struct { + Step string `yaml:"step"` +} + +// MultipartRoute makes a method send multipart/form-data instead of a JSON body. +// +// The file does not arrive in the IPC payload — it cannot, since ipc.MaxFrameSize +// caps one envelope at 1 MiB and partner upload limits are far larger. The agent +// stages it first through .upload_begin / upload_chunk, and this method takes +// the resulting blob id. Every other param becomes a form field. +type MultipartRoute struct { + // FileField is the form field the file is sent under (default "file"). + // It must match what the partner's endpoint expects. + FileField string `yaml:"file_field"` + // BlobParam is the payload key carrying the staged blob id (default + // "blob_id"). It is consumed by the adapter and never sent as a form field. + BlobParam string `yaml:"blob_param"` + // MaxBytes caps a staged upload (default DefaultMaxBlobBytes). Set it to the + // partner's own documented limit so an oversize file is refused locally + // rather than after the bytes have crossed the network. + MaxBytes int64 `yaml:"max_bytes"` +} + type HTTPRoute struct { Verb string `yaml:"verb"` // GET (default) | POST | PATCH | PUT | DELETE Path string `yaml:"path"` // e.g. /current or /v1/calls/{call_id} @@ -680,6 +835,10 @@ type HTTPRoute struct { // the provisioned number to ~/.pilot/.agentphone. CaptureTo string `yaml:"capture_to"` + // Multipart, when set, sends this method as multipart/form-data built from a + // staged blob rather than as a JSON body. See MultipartRoute. + Multipart *MultipartRoute `yaml:"multipart"` + // ParamIn carries each param's explicit location (one of the five values // above); a param absent from the map takes the verb/path default. Set from // the submission in ToConfig (the YAML form uses per-param `in`). @@ -874,6 +1033,54 @@ func (c *Config) Resolve() { // method that always 403s. A provisioned app that wants a balance method // must author one itself, pointed at its actual balance path (see // io.pilot.smol's hand-authored `smol.balance` -> "/_balance"). + // Multipart defaults, then the staging methods that make an upload reachable. + // Injected before the normalization loop below so they pick up the same + // Kind/Duration/Timeout defaults as an authored method and flow through + // registration, the manifest `exposes` list, and .help like any other. + for i := range c.Methods { + h := c.Methods[i].HTTP + if h == nil || h.Multipart == nil { + continue + } + if h.Multipart.FileField == "" { + h.Multipart.FileField = DefaultFileField + } + if h.Multipart.BlobParam == "" { + h.Multipart.BlobParam = DefaultBlobParam + } + if h.Multipart.MaxBytes == 0 { + h.Multipart.MaxBytes = DefaultMaxBlobBytes + } + } + if c.HasMultipart() { + for _, step := range []struct{ name, step, kind, dur, summary string }{ + {"upload_begin", "begin", "utility", "fast", + "Start a staged upload and get a blob_id. The file does NOT travel in one IPC message — an envelope is capped at 1 MiB — so declare it here ({\"file_name\",\"content_type\",\"total_bytes\",\"sha256\"}), push the bytes with " + c.Namespace + ".upload_chunk, then pass the blob_id to the upload method."}, + {"upload_chunk", "chunk", "utility", "fast", + "Append the next chunk of a staged upload: {\"blob_id\",\"seq\" (0-based, strictly sequential),\"data_base64\"}. Send at most 512 KiB of raw bytes per call. Returns {\"received\",\"total_bytes\",\"complete\"}; when complete, the sha256 you declared has been verified and the blob is ready to send."}, + {"upload_abort", "abort", "utility", "fast", + "Discard a staged upload and its bytes: {\"blob_id\"}. Staged uploads are also reclaimed automatically once they go stale."}, + } { + name := c.Namespace + "." + step.name + exists := false + for i := range c.Methods { + if c.Methods[i].Name == name { + exists = true + break + } + } + if exists { + continue + } + c.Methods = append(c.Methods, Method{ + Name: name, + Summary: step.summary, + Kind: step.kind, + Duration: step.dur, + Upload: &UploadRoute{Step: step.step}, + }) + } + } if c.Managed() && !c.Provisioned() { balName := c.Namespace + ".balance" has := false @@ -1084,6 +1291,11 @@ func (c *Config) Validate() []error { errs = append(errs, fmt.Errorf("methods[%d] (%s): a method cannot declare both a local and an http route", i, m.Name)) case m.Signup != nil && (m.HTTP != nil || m.Local != nil || m.CLI != nil): errs = append(errs, fmt.Errorf("methods[%d] (%s): a signup method must not also declare an http/cli/local route", i, m.Name)) + case m.Upload != nil: + // Blob-staging steps are generated, never authored: they run + // entirely in the adapter and have no backend route to validate. + case m.MCP != nil: + errs = append(errs, c.validateMCPMethod(i, m)...) case m.Signup != nil: errs = append(errs, c.validateSignupMethod(i, m)...) case m.Local != nil: @@ -1112,6 +1324,71 @@ func (c *Config) Validate() []error { return errs } +// validateMCPMethod checks one tool-server route. +func (c *Config) validateMCPMethod(i int, m Method) []error { + var errs []error + if c.Backend.MCP == nil || strings.TrimSpace(c.Backend.MCP.URL) == "" { + errs = append(errs, fmt.Errorf("methods[%d] (%s): declares an mcp route but backend.mcp.url is not set", i, m.Name)) + } else if u, err := url.Parse(c.Backend.MCP.URL); err != nil || u.Hostname() == "" || (u.Scheme != "https" && !isLoopbackHost(u.Hostname())) { + errs = append(errs, fmt.Errorf("methods[%d] (%s): backend.mcp.url %q must be an https URL", i, m.Name, c.Backend.MCP.URL)) + } + if len(m.MCP.Keep) > 0 && strings.TrimSpace(m.MCP.Wrap) == "" { + errs = append(errs, fmt.Errorf("methods[%d] (%s): mcp.keep only means something with mcp.wrap set", i, m.Name)) + } + if strings.TrimSpace(m.MCP.Tool) == "" { + errs = append(errs, fmt.Errorf("methods[%d] (%s): mcp.tool is required (the tool name the server advertises)", i, m.Name)) + } + if m.HTTP != nil || m.CLI != nil || m.Local != nil || m.Signup != nil { + errs = append(errs, fmt.Errorf("methods[%d] (%s): an mcp method must not also declare an http/cli/local/signup route", i, m.Name)) + } + // Only the no-credential mode is implemented. Failing loudly here beats an + // app shipping with an `auth:` its client silently ignores. + if c.Backend.MCP != nil && strings.TrimSpace(c.Backend.MCP.Auth) != "" { + errs = append(errs, fmt.Errorf("methods[%d] (%s): backend.mcp.auth %q is not supported — tool-server routes send no credentials", i, m.Name, c.Backend.MCP.Auth)) + } + return errs +} + +// validateMultipartMethod checks one method's multipart route. +// +// Every rule here turns a spec that would generate a broken adapter into a +// build-time error, because the failure it prevents is otherwise only visible +// as a partner-side 422 long after publish. +func (c *Config) validateMultipartMethod(i int, m Method) []error { + mp := m.HTTP.Multipart + var errs []error + if c.Backend.Type != "http" { + errs = append(errs, fmt.Errorf("methods[%d] (%s): multipart is an http-backend route; backend.type is %q", i, m.Name, c.Backend.Type)) + } + if !m.HTTP.BodyVerb() { + errs = append(errs, fmt.Errorf("methods[%d] (%s): multipart needs a body verb (POST/PUT/PATCH), got %s — there is nowhere to put the form", i, m.Name, m.HTTP.Verb)) + } + if strings.TrimSpace(mp.FileField) == "" { + errs = append(errs, fmt.Errorf("methods[%d] (%s): multipart.file_field must not be empty", i, m.Name)) + } + if strings.TrimSpace(mp.BlobParam) == "" { + errs = append(errs, fmt.Errorf("methods[%d] (%s): multipart.blob_param must not be empty", i, m.Name)) + } + // The blob id is consumed by the adapter to find the staged bytes. If it were + // also the file field, the field would be written twice with different + // meanings — once as the id, once as the file. + if mp.BlobParam != "" && mp.BlobParam == mp.FileField { + errs = append(errs, fmt.Errorf("methods[%d] (%s): multipart.blob_param and file_field are both %q; the blob id names the staged file, it is not the file", i, m.Name, mp.FileField)) + } + // A path/query/header-located param never reaches the form, so pointing the + // blob id at one silently loses it and the upload sends an empty file. + if loc := m.HTTP.ParamIn[mp.BlobParam]; loc != "" && loc != "body" { + errs = append(errs, fmt.Errorf("methods[%d] (%s): multipart.blob_param %q is located in %q; it must stay in the body, where the adapter consumes it", i, m.Name, mp.BlobParam, loc)) + } + if mp.MaxBytes <= 0 { + errs = append(errs, fmt.Errorf("methods[%d] (%s): multipart.max_bytes must be positive", i, m.Name)) + } + if m.HTTP.BodyRawParam != "" { + errs = append(errs, fmt.Errorf("methods[%d] (%s): multipart cannot be combined with a raw body param — the body is the form", i, m.Name)) + } + return errs +} + // validateLocalMethod checks one method's local metadata route: it needs a // store path and must not also declare an http/cli route. func (c *Config) validateLocalMethod(i int, m Method) []error { @@ -1183,6 +1460,9 @@ func (c *Config) validateSignupMethod(i int, m Method) []error { // per-param locations). Shared by http and hybrid apps. func (c *Config) validateHTTPMethod(i int, m Method) []error { var errs []error + if m.HTTP != nil && m.HTTP.Multipart != nil { + errs = append(errs, c.validateMultipartMethod(i, m)...) + } if m.HTTP == nil { return append(errs, fmt.Errorf("methods[%d] (%s): http backend requires an http: route", i, m.Name)) } @@ -1434,3 +1714,15 @@ func (m Method) SortedParamKeys() []string { sort.Strings(keys) return keys } + +// isLoopbackHost reports whether a host never leaves this machine, which is the +// only case where plaintext http is acceptable for a backend URL. It exists so +// a test (or a local development server) can be pointed at without loosening +// the https requirement that protects every real deployment. +func isLoopbackHost(host string) bool { + if host == "localhost" { + return true + } + ip := net.ParseIP(host) + return ip != nil && ip.IsLoopback() +} diff --git a/internal/scaffold/scaffold.go b/internal/scaffold/scaffold.go index a47a108..d00a8bf 100644 --- a/internal/scaffold/scaffold.go +++ b/internal/scaffold/scaffold.go @@ -48,6 +48,20 @@ func Generate(cfg *Config, outDir string) ([]string, error) { if cfg.HasSignup() { files = append(files, file{filepath.Join("cmd", cfg.BinaryName, "signup.go"), "signup.go.tmpl"}) } + // Multipart uploads: the blob store + form builder go beside the client + // (package backend), the IPC staging handlers beside main.go. All three + // or none — a staged upload with no way to stage is a method that always + // fails. + if cfg.HasMCP() { + files = append(files, file{filepath.Join("internal", "backend", "mcp.go"), "mcp.go.tmpl"}) + } + if cfg.HasMultipart() { + files = append(files, + file{filepath.Join("internal", "backend", "blob.go"), "blob.go.tmpl"}, + file{filepath.Join("internal", "backend", "multipartform.go"), "multipartform.go.tmpl"}, + file{filepath.Join("cmd", cfg.BinaryName, "upload.go"), "upload.go.tmpl"}, + ) + } // The broker-call runtime is a separate file emitted only for broker apps // (it imports the signer, absent from a plain register/verify app). if cfg.HasBrokerSignup() { diff --git a/internal/scaffold/templates/blob.go.tmpl b/internal/scaffold/templates/blob.go.tmpl new file mode 100644 index 0000000..d5050f8 --- /dev/null +++ b/internal/scaffold/templates/blob.go.tmpl @@ -0,0 +1,422 @@ +// Package multipartkit lets a Pilot app forward multipart/form-data uploads to +// its partner API despite the JSON-only IPC contract. +// +// # Why this exists +// +// Pilot IPC is JSON in, JSON out, over a framed unix socket, and +// ipc.MaxFrameSize caps a single envelope at 1 MiB. The obvious encoding for a +// file — base64 inside the JSON payload — therefore tops out around 740 KiB of +// real bytes before the frame is refused and the connection dropped. Partner +// upload limits are much larger than that (General Legal accepts 20 MiB), so +// "just base64 it" is not an implementation detail that can be tuned; it is a +// design that does not fit. +// +// So the file does not travel in one envelope. The agent stages it in chunks +// that each fit comfortably inside a frame, the adapter reassembles it on disk +// under $APP, and only then builds one multipart body and sends it. The IPC +// layer never carries more than a chunk, and no platform limit has to move. +// +// .upload_begin {file_name, content_type, total_bytes, sha256} -> {blob_id} +// .upload_chunk {blob_id, seq, data_base64} -> {received} +// .document_upload {blob_id, deal_id, ...} -> partner JSON +// +// # What this package guarantees +// +// Chunked reassembly is a place to get integrity wrong, so the store is strict +// rather than forgiving: ids are minted here and never taken from the caller, +// chunks must arrive in order, the declared size is a hard cap as bytes land, +// and finalize refuses to hand back a blob whose sha256 does not match what the +// caller promised at begin. A partially-written upload is never usable. +// +// # Restarts +// +// The daemon supervises adapters and respawns them, so a store must not keep +// the only record of a finished upload in memory. A sealed blob is fully +// described on disk by its .blob and .meta pair and is recovered on the next +// NewStore. An in-flight .part cannot be recovered — the rolling hash and the +// chunk cursor die with the process — so it is discarded at startup rather than +// left to be silently completed with a gap in the middle. +package backend + +import ( + "crypto/rand" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "hash" + "os" + "path/filepath" + "strings" + "sync" + "time" +) + +// MaxChunkBytes is the largest raw chunk an agent should send in one +// upload_chunk call. base64 inflates by 4/3, so 512 KiB of file becomes ~683 +// KiB of payload and still leaves room for the envelope inside the 1 MiB +// ipc.MaxFrameSize. Chunks larger than this are refused here rather than +// failing later as a dropped connection, which is much harder to diagnose. +const MaxChunkBytes = 512 << 10 + +// DefaultMaxBlobBytes bounds a single staged upload. Partner limits are lower +// (General Legal: 20 MiB); this is the backstop that keeps a caller from +// filling the disk under $APP. +const DefaultMaxBlobBytes int64 = 24 << 20 + +// DefaultTTL is how long a staged upload survives before GC reclaims it. +const DefaultTTL = 30 * time.Minute + +var ( + ErrNotFound = errors.New("multipartkit: unknown blob_id") + ErrSealed = errors.New("multipartkit: blob already finalized") + ErrOutOfOrder = errors.New("multipartkit: chunk out of order") + ErrTooLarge = errors.New("multipartkit: upload exceeds declared size") + ErrChecksum = errors.New("multipartkit: sha256 mismatch") + ErrIncomplete = errors.New("multipartkit: fewer bytes than declared") + ErrChunkTooBig = errors.New("multipartkit: chunk exceeds MaxChunkBytes") + ErrBadBlobID = errors.New("multipartkit: malformed blob_id") + ErrBadDeclared = errors.New("multipartkit: invalid total_bytes") + ErrBadChecksum = errors.New("multipartkit: sha256 must be 64 hex chars") + ErrNameRequired = errors.New("multipartkit: file_name is required") +) + +// Meta describes a staged upload. It is persisted beside a sealed blob so the +// upload survives an adapter respawn. +type Meta struct { + ID string `json:"id"` + FileName string `json:"file_name"` + ContentType string `json:"content_type"` + TotalBytes int64 `json:"total_bytes"` + SHA256 string `json:"sha256"` + Created time.Time `json:"created"` +} + +// staged is an upload in progress. A sealed blob is not held here: it lives on +// disk, which is what lets it outlive the process. +type staged struct { + meta Meta + f *os.File + h hash.Hash + written int64 + nextSeq int +} + +// Store stages chunked uploads under a root directory (in practice $APP/blobs). +// +// All state transitions happen under mu. The lock is held across the file write +// in Append deliberately: releasing it to do I/O would let Abort close the file +// underneath an in-flight chunk, and uploads are not hot enough for that +// contention to matter. +type Store struct { + root string + maxBytes int64 + ttl time.Duration + + mu sync.Mutex + open map[string]*staged +} + +// NewStore prepares a staging area under root and recovers what is recoverable +// from a previous process: sealed blobs stay, unfinishable .part files go. +func NewStore(root string) (*Store, error) { + if err := os.MkdirAll(root, 0o700); err != nil { + return nil, fmt.Errorf("multipartkit: create %s: %w", root, err) + } + s := &Store{root: root, maxBytes: DefaultMaxBlobBytes, ttl: DefaultTTL, open: map[string]*staged{}} + s.sweepOrphanParts() + return s, nil +} + +// sweepOrphanParts removes .part files left by a previous process. Their hash +// state and chunk cursor are gone, so they can never be finalized; keeping them +// would only grow $APP without bound. +func (s *Store) sweepOrphanParts() { + entries, err := os.ReadDir(s.root) + if err != nil { + return + } + for _, e := range entries { + if !e.IsDir() && strings.HasSuffix(e.Name(), ".part") { + _ = os.Remove(filepath.Join(s.root, e.Name())) + } + } +} + +// SetLimits overrides the size cap and TTL (0 keeps the current value). +func (s *Store) SetLimits(maxBytes int64, ttl time.Duration) { + s.mu.Lock() + defer s.mu.Unlock() + if maxBytes > 0 { + s.maxBytes = maxBytes + } + if ttl > 0 { + s.ttl = ttl + } +} + +// Begin reserves a blob id and opens its staging file. +// +// The id is minted here from crypto/rand and is never taken from the caller: a +// caller-chosen id is a path to overwriting somebody else's in-flight upload, +// and on a shared host that is a file-substitution bug rather than a naming +// inconvenience. +func (s *Store) Begin(fileName, contentType string, totalBytes int64, sha256hex string) (Meta, error) { + if strings.TrimSpace(fileName) == "" { + return Meta{}, ErrNameRequired + } + s.mu.Lock() + max := s.maxBytes + s.mu.Unlock() + if totalBytes <= 0 || totalBytes > max { + return Meta{}, fmt.Errorf("%w: %d (max %d)", ErrBadDeclared, totalBytes, max) + } + sha256hex = strings.ToLower(strings.TrimSpace(sha256hex)) + if len(sha256hex) != 64 { + return Meta{}, ErrBadChecksum + } + if _, err := hex.DecodeString(sha256hex); err != nil { + return Meta{}, ErrBadChecksum + } + var raw [16]byte + if _, err := rand.Read(raw[:]); err != nil { + return Meta{}, fmt.Errorf("multipartkit: mint id: %w", err) + } + id := hex.EncodeToString(raw[:]) + + // Only the base name of the caller's file name is ever kept, and it is never + // used to build a path — the on-disk name is the minted id. The original is + // metadata that rides along into the multipart body. + m := Meta{ + ID: id, + FileName: filepath.Base(filepath.Clean("/" + strings.TrimSpace(fileName))), + ContentType: strings.TrimSpace(contentType), + TotalBytes: totalBytes, + SHA256: sha256hex, + Created: time.Now().UTC(), + } + if m.ContentType == "" { + m.ContentType = "application/octet-stream" + } + f, err := os.OpenFile(s.partPath(id), os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600) + if err != nil { + return Meta{}, fmt.Errorf("multipartkit: open staging file: %w", err) + } + s.mu.Lock() + s.open[id] = &staged{meta: m, f: f, h: sha256.New()} + s.mu.Unlock() + return m, nil +} + +// Append writes the next chunk. seq must be exactly the next index (0-based): +// an upload that silently tolerates a gap or a replay reassembles to something +// the caller never sent, and the sha256 check at the end would then be the only +// thing standing between that and the partner. +func (s *Store) Append(id string, seq int, data []byte) (int64, error) { + if !validID(id) { + return 0, ErrBadBlobID + } + if len(data) > MaxChunkBytes { + return 0, ErrChunkTooBig + } + s.mu.Lock() + defer s.mu.Unlock() + st, ok := s.open[id] + if !ok { + if s.sealedOnDisk(id) { + return 0, ErrSealed + } + return 0, ErrNotFound + } + if seq != st.nextSeq { + return st.written, fmt.Errorf("%w: want %d, got %d", ErrOutOfOrder, st.nextSeq, seq) + } + if st.written+int64(len(data)) > st.meta.TotalBytes { + return st.written, ErrTooLarge + } + if _, err := st.f.Write(data); err != nil { + return st.written, fmt.Errorf("multipartkit: write chunk: %w", err) + } + _, _ = st.h.Write(data) + st.written += int64(len(data)) + st.nextSeq++ + return st.written, nil +} + +// Finalize verifies the reassembled bytes and seals the blob for sending. +// It is idempotent: finalizing an already-sealed blob returns its metadata. +func (s *Store) Finalize(id string) (Meta, error) { + if !validID(id) { + return Meta{}, ErrBadBlobID + } + s.mu.Lock() + defer s.mu.Unlock() + st, ok := s.open[id] + if !ok { + if m, err := s.readMeta(id); err == nil { + return m, nil // already sealed by us or by a previous process + } + return Meta{}, ErrNotFound + } + if st.written != st.meta.TotalBytes { + return Meta{}, fmt.Errorf("%w: got %d of %d", ErrIncomplete, st.written, st.meta.TotalBytes) + } + if got := hex.EncodeToString(st.h.Sum(nil)); got != st.meta.SHA256 { + return Meta{}, fmt.Errorf("%w: declared %s, reassembled %s", ErrChecksum, st.meta.SHA256, got) + } + if err := st.f.Close(); err != nil { + return Meta{}, fmt.Errorf("multipartkit: close staging file: %w", err) + } + // Write the metadata BEFORE the rename. The .blob appearing is what makes a + // blob look complete to a later process, so it must never be observable + // without the metadata that describes it. + b, err := json.Marshal(st.meta) + if err != nil { + return Meta{}, fmt.Errorf("multipartkit: encode meta: %w", err) + } + if err := os.WriteFile(s.metaPath(id), b, 0o600); err != nil { + return Meta{}, fmt.Errorf("multipartkit: write meta: %w", err) + } + if err := os.Rename(s.partPath(id), s.blobPath(id)); err != nil { + _ = os.Remove(s.metaPath(id)) + return Meta{}, fmt.Errorf("multipartkit: seal: %w", err) + } + delete(s.open, id) // sealed state lives on disk from here on + return st.meta, nil +} + +// Open returns the sealed blob's bytes for sending. A blob sealed before the +// adapter was respawned opens just the same, because sealed state is on disk. +func (s *Store) Open(id string) (*os.File, Meta, error) { + if !validID(id) { + return nil, Meta{}, ErrBadBlobID + } + s.mu.Lock() + _, inFlight := s.open[id] + s.mu.Unlock() + if inFlight { + return nil, Meta{}, ErrIncomplete // staged but not finalized + } + meta, err := s.readMeta(id) + if err != nil { + return nil, Meta{}, ErrNotFound + } + f, err := os.Open(s.blobPath(id)) + if err != nil { + return nil, Meta{}, ErrNotFound + } + return f, meta, nil +} + +// Abort discards an upload and its bytes, in flight or sealed. +func (s *Store) Abort(id string) error { + if !validID(id) { + return ErrBadBlobID + } + s.mu.Lock() + st, wasOpen := s.open[id] + if wasOpen { + _ = st.f.Close() + delete(s.open, id) + } + s.mu.Unlock() + + _ = os.Remove(s.partPath(id)) + sealed := s.sealedOnDisk(id) + _ = os.Remove(s.blobPath(id)) + _ = os.Remove(s.metaPath(id)) + if !wasOpen && !sealed { + return ErrNotFound + } + return nil +} + +// GC reclaims uploads past the TTL, in flight and sealed alike. An agent that +// stages a document and never sends it must not leave bytes under $APP forever. +// It walks the directory rather than the map so blobs sealed by a previous +// process are reclaimed too. +func (s *Store) GC() int { + s.mu.Lock() + cutoff := time.Now().UTC().Add(-s.ttl) + var stale []string + for id, st := range s.open { + if st.meta.Created.Before(cutoff) { + stale = append(stale, id) + } + } + s.mu.Unlock() + + seen := map[string]bool{} + for _, id := range stale { + seen[id] = true + } + if entries, err := os.ReadDir(s.root); err == nil { + for _, e := range entries { + name := e.Name() + if !strings.HasSuffix(name, ".blob") { + continue + } + id := strings.TrimSuffix(name, ".blob") + if seen[id] || !validID(id) { + continue + } + info, err := e.Info() + if err != nil || !info.ModTime().UTC().Before(cutoff) { + continue + } + seen[id] = true + stale = append(stale, id) + } + } + for _, id := range stale { + _ = s.Abort(id) + } + return len(stale) +} + +// readMeta loads a sealed blob's metadata, and only reports success when the +// bytes it describes are actually there. +func (s *Store) readMeta(id string) (Meta, error) { + b, err := os.ReadFile(s.metaPath(id)) + if err != nil { + return Meta{}, err + } + var m Meta + if err := json.Unmarshal(b, &m); err != nil { + return Meta{}, err + } + info, err := os.Stat(s.blobPath(id)) + if err != nil { + return Meta{}, err + } + if info.Size() != m.TotalBytes { + return Meta{}, fmt.Errorf("%w: on-disk blob is %d bytes, meta says %d", ErrIncomplete, info.Size(), m.TotalBytes) + } + return m, nil +} + +func (s *Store) sealedOnDisk(id string) bool { + _, err := os.Stat(s.blobPath(id)) + return err == nil +} + +func (s *Store) partPath(id string) string { return filepath.Join(s.root, id+".part") } +func (s *Store) blobPath(id string) string { return filepath.Join(s.root, id+".blob") } +func (s *Store) metaPath(id string) string { return filepath.Join(s.root, id+".meta") } + +// validID accepts only what Begin mints: 32 lowercase hex characters. This is +// the whole path-traversal defence — an id is concatenated into a filename, so +// nothing that could contain a separator or a dot segment is ever admitted. +func validID(id string) bool { + if len(id) != 32 { + return false + } + for i := 0; i < len(id); i++ { + c := id[i] + if (c < '0' || c > '9') && (c < 'a' || c > 'f') { + return false + } + } + return true +} diff --git a/internal/scaffold/templates/client_http.go.tmpl b/internal/scaffold/templates/client_http.go.tmpl index 73db3d1..216722e 100644 --- a/internal/scaffold/templates/client_http.go.tmpl +++ b/internal/scaffold/templates/client_http.go.tmpl @@ -170,6 +170,25 @@ func (c *Client) PostRaw(ctx context.Context, path string, payload json.RawMessa return c.do(req) } +// DoMultipart sends a pre-assembled multipart/form-data body. +// +// The body arrives already built (see backend.BuildForm) rather than being +// streamed, because a managed adapter signs base64(sha256(BODY)) and you cannot +// hash a body you have not finished producing. contentType carries the +// generated boundary and MUST be sent verbatim — without it the body is +// undecodable on arrival, so it is a parameter rather than a constant. +func (c *Client) DoMultipart(ctx context.Context, method, path string, body []byte, contentType string) (json.RawMessage, error) { + req, err := http.NewRequestWithContext(ctx, method, c.base()+path, bytes.NewReader(body)) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", contentType) +{{- if .Managed}} + c.applySig(req, body) +{{- end}} + return c.do(req) +} + // Do issues an arbitrary method on with an optional query string, extra // per-call headers, and JSON body, and returns the raw JSON body. Path // placeholders are already filled by the caller; query carries the query-located diff --git a/internal/scaffold/templates/example.pilot.app.yaml b/internal/scaffold/templates/example.pilot.app.yaml index 09e8e7d..ff63c00 100644 --- a/internal/scaffold/templates/example.pilot.app.yaml +++ b/internal/scaffold/templates/example.pilot.app.yaml @@ -102,6 +102,27 @@ listing: # date: "2026-06-16" # notes: ["Initial release"] +# --- a multipart upload (see docs/MULTIPART-UPLOADS.md) --- +# A file cannot ride inside the JSON payload: an IPC envelope is capped at 1 MiB, +# so base64 tops out around 740 KiB of real file. Declare `multipart:` and the +# generator adds .upload_begin / upload_chunk / upload_abort, which stage the +# bytes in frame-sized pieces; this method then takes the resulting blob_id. +# Every other param becomes a form field. +# +# methods: +# - name: weather.report_upload +# summary: "Attach a file to a report." +# duration: slow +# http: +# verb: POST # multipart needs a body verb +# path: /reports +# multipart: +# file_field: file # the form field the backend reads +# max_bytes: 20971520 # match the backend's own limit +# params: +# blob_id: "string (required) — from weather.upload_begin" +# report_id: "string (optional) — attach to an existing report" + # --- a cli backend instead (see docs/CLI-ADAPTER.md) --- # backend: # type: cli diff --git a/internal/scaffold/templates/main.go.tmpl b/internal/scaffold/templates/main.go.tmpl index c3f235b..b0d53c9 100644 --- a/internal/scaffold/templates/main.go.tmpl +++ b/internal/scaffold/templates/main.go.tmpl @@ -13,6 +13,9 @@ package main import ( +{{- if .HasMCP}} + "bytes" +{{- end}} "context" "encoding/json" "flag" @@ -122,7 +125,7 @@ func main() { d := ipc.NewDispatcher() {{- if eq .Backend.Type "http"}} - registerHandlers(d, client, readAppVersion(*manifestPath), cfg.BackendURL{{if .HasSignup}}, *manifestPath{{end}}{{if .HasBrokerSignup}}, signer{{end}}) + registerHandlers(d, client, readAppVersion(*manifestPath), cfg.BackendURL{{if or .HasSignup .HasMultipart}}, *manifestPath{{end}}{{if .HasBrokerSignup}}, signer{{end}}{{if .HasMCP}}, backend.NewToolClient({{printf "%q" .Backend.MCP.URL}}){{end}}) {{- else if eq .Backend.Type "hybrid"}} registerHandlers(d, runner, cloudCli, readAppVersion(*manifestPath), cfg.BackendURL) {{- else}} @@ -175,10 +178,28 @@ func serve(ctx context.Context, socketPath string, d *ipc.Dispatcher) error { // registerHandlers wires each IPC method to its backend call. Method names MUST // match the manifest's `exposes` list, or the daemon won't broker them. {{- if eq .Backend.Type "http"}} -func registerHandlers(d *ipc.Dispatcher, c *backend.Client, version, backendURL string{{if .HasSignup}}, manifestPath string{{end}}{{if .HasBrokerSignup}}, signer backend.Signer{{end}}) { +func registerHandlers(d *ipc.Dispatcher, c *backend.Client, version, backendURL string{{if or .HasSignup .HasMultipart}}, manifestPath string{{end}}{{if .HasBrokerSignup}}, signer backend.Signer{{end}}{{if .HasMCP}}, tools *backend.ToolClient{{end}}) { {{- range .Methods}} {{- if .Local}} d.Register("{{.Name}}", localRead(expandHome("{{.Local.Store}}"))) // {{.Duration}} (local metadata) +{{- else if .Upload}} +{{- if eq .Upload.Step "begin"}} + d.Register("{{.Name}}", uploadBeginHandler(manifestPath)) // {{.Duration}} (local: stage a file for upload) +{{- else if eq .Upload.Step "chunk"}} + d.Register("{{.Name}}", uploadChunkHandler(manifestPath)) // {{.Duration}} (local: push one chunk) +{{- else}} + d.Register("{{.Name}}", uploadAbortHandler(manifestPath)) // {{.Duration}} (local: discard a staged file) +{{- end}} +{{- else if .MCP}} + d.Register("{{.Name}}", forwardTool(tools, {{printf "%q" .MCP.Tool}}, {{printf "%q" .MCP.Wrap}}, []string{ {{- range $k := .MCP.Keep}}{{printf "%q" $k}}, {{end -}} }, dur("{{.TimeoutFor}}"))) // {{.Duration}} (tool server, no credentials) +{{- else if and .HTTP .HTTP.Multipart}} + d.Register("{{.Name}}", forwardMultipart(c, multipartRoute{ + method: "{{.HTTP.Verb}}", pathTmpl: "{{.HTTP.Path}}", + fileField: {{printf "%q" .HTTP.Multipart.FileField}}, + blobParam: {{printf "%q" .HTTP.Multipart.BlobParam}}, + pathParams: []string{ {{- range $p := .HTTP.PathParams}}{{printf "%q" $p}}, {{end -}} }, + rawPathParams: []string{ {{- range $p := .HTTP.RawPathParams}}{{printf "%q" $p}}, {{end -}} }, + }, manifestPath, dur("{{.TimeoutFor}}"))) // {{.Duration}} (multipart upload) {{- else if .Signup}} {{- if eq .Signup.Step "account"}} d.Register("{{.Name}}", accountHandler(accountConfig{ @@ -365,6 +386,48 @@ type route struct { // and sends the REMAINING (residual) fields as a JSON body for a body verb or a // query string otherwise. One generic forwarder covers every verb + placement, // so adding a method is pure pilot.app.yaml. +{{if .HasMCP}} +// forwardTool calls one tool on the JSON-RPC tool server. The payload becomes +// the tool's arguments, optionally nested under a single wrapper key for +// servers that take one object argument. +// +// This path never touches the REST client, so the API key that authenticates +// the REST backend is not in scope here and cannot reach this host. +func forwardTool(t *backend.ToolClient, tool, wrap string, keep []string, timeout time.Duration) ipc.Handler { + return func(ctx context.Context, req *ipc.Envelope) (json.RawMessage, error) { + args := req.Payload + if len(bytes.TrimSpace(args)) == 0 { + args = json.RawMessage(`{}`) + } + if wrap != "" { + fields, err := objectFields(args) + if err != nil { + return nil, err + } + outer := map[string]json.RawMessage{} + for _, k := range keep { + if v, ok := fields[k]; ok { + outer[k] = v + delete(fields, k) + } + } + inner, err := json.Marshal(fields) + if err != nil { + return nil, err + } + outer[wrap] = inner + wrapped, err := json.Marshal(outer) + if err != nil { + return nil, err + } + args = wrapped + } + cctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + return t.Call(cctx, tool, args) + } +} +{{end}} func forward(c *backend.Client, r route, timeout time.Duration) ipc.Handler { return func(ctx context.Context, req *ipc.Envelope) (json.RawMessage, error) { fields, err := objectFields(req.Payload) @@ -675,6 +738,7 @@ type methodDoc struct { Cost string `json:"cost,omitempty"` // what a cloud method debits (empty = free) Roundtrip string `json:"typical_roundtrip,omitempty"` Gated string `json:"requires,omitempty"` // non-empty = needs an account upgrade; the reason it names is shown in the free-plan disclaimer + Billable string `json:"billable,omitempty"` // non-empty = this call costs real money; the reason says what it costs } var helpMethods = []methodDoc{ @@ -684,6 +748,9 @@ var helpMethods = []methodDoc{ {{- if $m.Gated}} Gated: {{printf "%q" $m.Gated}}, {{- end}} +{{- if $m.Billable}} + Billable: {{printf "%q" $m.Billable}}, +{{- end}} {{- if and $.Pricing (index $.Pricing.CreditCost $m.Name)}} Cost: "{{index $.Pricing.CreditCost $m.Name}} credit(s)", {{- end}} @@ -749,6 +816,8 @@ type helpDoc struct { // of methods that need an account upgrade before they work. GatedNote string `json:"gated_note,omitempty"` GatedMethods []methodDoc `json:"gated_methods,omitempty"` + BillableNote string `json:"billable_note,omitempty"` + BillableMethods []methodDoc `json:"billable_methods,omitempty"` } // splitPlanes groups methods into local vs cloud so .help reads clearly. @@ -795,6 +864,18 @@ func splitPlanes(ms []methodDoc) (local, cloud []methodDoc) { doc.GatedMethods = append(doc.GatedMethods, m) } } + for _, m := range helpMethods { + if m.Billable != "" { + doc.BillableMethods = append(doc.BillableMethods, m) + } + } + if len(doc.BillableMethods) > 0 { + doc.BillableNote = "────────────── THESE CALLS COST MONEY ────────────── " + + "The methods listed under billable_methods spend real money on the account whose API key this app is using. " + + "They are not plan-gated and they will not warn you first — they work, and you are charged. Every other method is " + + "free to call. Each one's `billable` field says exactly what it costs. Confirm with whoever owns the account before " + + "calling one on their behalf." + } if len(doc.GatedMethods) > 0 { doc.GatedNote = "────────────── NOT AVAILABLE ON THE FREE PLAN ────────────── " + "The methods listed under gated_methods require an account upgrade the default free plan does not have " + diff --git a/internal/scaffold/templates/manifest.json.tmpl b/internal/scaffold/templates/manifest.json.tmpl index c20d25d..314b5e5 100644 --- a/internal/scaffold/templates/manifest.json.tmpl +++ b/internal/scaffold/templates/manifest.json.tmpl @@ -21,6 +21,10 @@ {{- if or .Provisioned .HasSignup}} {"cap": "fs.write", "target": "$APP/secrets.json"}, {{- end}} +{{- if .HasMultipart}} + {"cap": "fs.read", "target": "$APP/blobs"}, + {"cap": "fs.write", "target": "$APP/blobs"}, +{{- end}} {{- if or .Managed .HasBrokerSignup}} {"cap": "key.sign", "target": "self"}, {{- end}} @@ -28,6 +32,10 @@ {"cap": "net.dial", "target": "{{.AdapterBackendHost}}", "if": {"kind": "rate", "params": {"per": "min", "limit": {{.Grants.RatePerMin}}}}}, {{- end}} +{{- if .HasMCP}} + {"cap": "net.dial", "target": "{{.MCPHost}}", + "if": {"kind": "rate", "params": {"per": "min", "limit": {{.Grants.RatePerMin}}}}}, +{{- end}} {{- range .SignupHosts}} {"cap": "net.dial", "target": "{{.}}", "if": {"kind": "rate", "params": {"per": "min", "limit": {{$.Grants.RatePerMin}}}}}, diff --git a/internal/scaffold/templates/mcp.go.tmpl b/internal/scaffold/templates/mcp.go.tmpl new file mode 100644 index 0000000..d2d89d6 --- /dev/null +++ b/internal/scaffold/templates/mcp.go.tmpl @@ -0,0 +1,287 @@ +// JSON-RPC tool-server client for {{.ID}}. +// +// This is a SECOND backend, separate from the REST client in client.go, and the +// separation is the whole point: it is built with no credentials at all, so the +// API key that authenticates the REST side has no path by which it could reach +// this server's host. Tool servers are usually a different origin with a +// different (often absent) auth story; sending one backend's key to the other +// because they share a vendor is exactly the mistake this shape prevents. +// +// The transport is Streamable HTTP: a session is established once with +// `initialize`, the server's session id rides on every later request, and a +// reply arrives either as plain JSON or as a single Server-Sent Event. +// +// GENERATED by pilot-app; edit pilot.app.yaml and re-generate. +package backend + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net" + "net/http" + "strings" + "sync" + "time" +) + +// ToolClient calls tools on a JSON-RPC tool server over Streamable HTTP. +type ToolClient struct { + url string + http *http.Client + + mu sync.Mutex + session string // server-assigned; empty until the first initialize + nextID int64 +} + +// NewToolClient returns a client for the tool server at url. +// +// It deliberately takes no credentials and has no way to accept any. +func NewToolClient(url string) *ToolClient { + return &ToolClient{ + url: url, + nextID: 1, + http: &http.Client{ + Timeout: 120 * time.Second, + Transport: &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: (&net.Dialer{Timeout: 5 * time.Second, KeepAlive: 30 * time.Second}).DialContext, + ForceAttemptHTTP2: true, + MaxIdleConns: 8, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + }, + }, + } +} + +// Call invokes one tool and returns its result as JSON. +// +// The session is established lazily and reused. If the server has forgotten it +// (sessions expire, and servers restart), the call is retried once against a +// fresh session rather than surfacing a transport detail the agent cannot act +// on. +func (t *ToolClient) Call(ctx context.Context, tool string, args json.RawMessage) (json.RawMessage, error) { + out, err := t.call(ctx, tool, args) + if err != nil && isSessionLost(err) { + t.resetSession() + out, err = t.call(ctx, tool, args) + } + return out, err +} + +func (t *ToolClient) call(ctx context.Context, tool string, args json.RawMessage) (json.RawMessage, error) { + if err := t.ensureSession(ctx); err != nil { + return nil, err + } + if len(bytes.TrimSpace(args)) == 0 { + args = json.RawMessage(`{}`) + } + params, err := json.Marshal(map[string]any{ + "name": tool, + "arguments": args, + }) + if err != nil { + return nil, fmt.Errorf("tool %s: encode arguments: %w", tool, err) + } + res, err := t.rpc(ctx, "tools/call", params) + if err != nil { + return nil, fmt.Errorf("tool %s: %w", tool, err) + } + + // A tool result carries structuredContent when the server declares an output + // schema, and a content[] block otherwise. Prefer the structured form: it is + // already JSON, where content[].text is a string that usually holds JSON and + // would otherwise reach the agent double-encoded. + var envelope struct { + StructuredContent json.RawMessage `json:"structuredContent"` + IsError bool `json:"isError"` + Content []struct { + Type string `json:"type"` + Text string `json:"text"` + } `json:"content"` + } + if err := json.Unmarshal(res, &envelope); err != nil { + return res, nil // not the shape we expected; hand it back untouched + } + var text string + for _, c := range envelope.Content { + if c.Type == "text" || c.Text != "" { + text = c.Text + break + } + } + if envelope.IsError { + if text == "" { + text = string(res) + } + return nil, fmt.Errorf("tool %s: %s", tool, strings.TrimSpace(text)) + } + if len(bytes.TrimSpace(envelope.StructuredContent)) > 0 { + return envelope.StructuredContent, nil + } + if text != "" { + if json.Valid([]byte(text)) { + return json.RawMessage(text), nil + } + b, _ := json.Marshal(map[string]string{"text": text}) + return json.RawMessage(b), nil + } + return res, nil +} + +// ensureSession performs the initialize handshake once. +func (t *ToolClient) ensureSession(ctx context.Context) error { + t.mu.Lock() + have := t.session != "" + t.mu.Unlock() + if have { + return nil + } + params, _ := json.Marshal(map[string]any{ + "protocolVersion": "2025-06-18", + "capabilities": map[string]any{}, + "clientInfo": map[string]any{"name": "{{.ID}}", "version": "{{.AppVersion}}"}, + }) + if _, err := t.rpc(ctx, "initialize", params); err != nil { + return fmt.Errorf("tool server handshake: %w", err) + } + // The spec requires this notification before any tool call; servers are + // entitled to reject calls made before it. + _ = t.notify(ctx, "notifications/initialized") + return nil +} + +func (t *ToolClient) resetSession() { + t.mu.Lock() + t.session = "" + t.mu.Unlock() +} + +// rpc sends one JSON-RPC request and returns its `result`. +func (t *ToolClient) rpc(ctx context.Context, method string, params json.RawMessage) (json.RawMessage, error) { + t.mu.Lock() + id := t.nextID + t.nextID++ + session := t.session + t.mu.Unlock() + + body, err := json.Marshal(map[string]any{ + "jsonrpc": "2.0", "id": id, "method": method, "params": params, + }) + if err != nil { + return nil, err + } + req, err := http.NewRequestWithContext(ctx, "POST", t.url, bytes.NewReader(body)) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Accept", "application/json, text/event-stream") + if session != "" { + req.Header.Set("Mcp-Session-Id", session) + } + resp, err := t.http.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + // The server assigns the session on the initialize response. + if s := resp.Header.Get("Mcp-Session-Id"); s != "" { + t.mu.Lock() + t.session = s + t.mu.Unlock() + } + raw, err := io.ReadAll(io.LimitReader(resp.Body, 8<<20)) + if err != nil { + return nil, err + } + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return nil, fmt.Errorf("tool server %s -> %d: %s", method, resp.StatusCode, strings.TrimSpace(string(raw))) + } + payload, err := decodeMaybeSSE(raw) + if err != nil { + return nil, err + } + var out struct { + Result json.RawMessage `json:"result"` + Error *struct { + Code int `json:"code"` + Message string `json:"message"` + } `json:"error"` + } + if err := json.Unmarshal(payload, &out); err != nil { + return nil, fmt.Errorf("tool server %s: bad JSON-RPC reply: %w", method, err) + } + if out.Error != nil { + return nil, fmt.Errorf("tool server %s: %s (code %d)", method, out.Error.Message, out.Error.Code) + } + return out.Result, nil +} + +// notify sends a JSON-RPC notification (no id, no reply expected). +func (t *ToolClient) notify(ctx context.Context, method string) error { + t.mu.Lock() + session := t.session + t.mu.Unlock() + body, _ := json.Marshal(map[string]any{"jsonrpc": "2.0", "method": method}) + req, err := http.NewRequestWithContext(ctx, "POST", t.url, bytes.NewReader(body)) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Accept", "application/json, text/event-stream") + if session != "" { + req.Header.Set("Mcp-Session-Id", session) + } + resp, err := t.http.Do(req) + if err != nil { + return err + } + _, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 1<<20)) + return resp.Body.Close() +} + +// decodeMaybeSSE returns the JSON payload from either a plain JSON body or a +// Server-Sent Event stream. Streamable HTTP lets the server choose per +// response, so a client that only understands one of the two works until the +// day it does not. +func decodeMaybeSSE(raw []byte) ([]byte, error) { + trimmed := bytes.TrimSpace(raw) + if len(trimmed) > 0 && (trimmed[0] == '{' || trimmed[0] == '[') { + return trimmed, nil + } + sc := bufio.NewScanner(bytes.NewReader(raw)) + sc.Buffer(make([]byte, 0, 64<<10), 8<<20) + var data []string + for sc.Scan() { + line := sc.Text() + if v, ok := strings.CutPrefix(line, "data:"); ok { + data = append(data, strings.TrimSpace(v)) + } + } + if err := sc.Err(); err != nil { + return nil, fmt.Errorf("read event stream: %w", err) + } + if len(data) == 0 { + return nil, fmt.Errorf("tool server returned neither JSON nor an event stream") + } + return []byte(strings.Join(data, "\n")), nil +} + +// isSessionLost recognises the server having forgotten our session, which is +// recoverable by handshaking again rather than a real failure. +func isSessionLost(err error) bool { + if err == nil { + return false + } + s := err.Error() + return strings.Contains(s, "-> 404") || strings.Contains(s, "-> 400") || + strings.Contains(strings.ToLower(s), "session") +} diff --git a/internal/scaffold/templates/multipartform.go.tmpl b/internal/scaffold/templates/multipartform.go.tmpl new file mode 100644 index 0000000..5f5d211 --- /dev/null +++ b/internal/scaffold/templates/multipartform.go.tmpl @@ -0,0 +1,78 @@ +package backend + +import ( + "bytes" + "fmt" + "io" + "mime/multipart" + "net/textproto" + "sort" + "strings" +) + +// BuildForm assembles one multipart/form-data body: the staged file under +// fileField, plus the scalar fields. +// +// The body is materialised into memory rather than streamed, and that is forced +// by the security model, not laziness. A managed adapter signs +// base64(sha256(BODY)) so the broker can prove who is calling; you cannot hash +// a body you have not finished producing. DefaultMaxBlobBytes is what keeps +// that bounded. +// +// Returns the body and the Content-Type header value — which carries the +// generated boundary and is therefore not optional to propagate. Every hop +// between here and the partner has to preserve it verbatim or the body is +// undecodable on arrival. +func BuildForm(fileField, fileName, fileContentType string, content io.Reader, fields map[string]string) ([]byte, string, error) { + if strings.TrimSpace(fileField) == "" { + return nil, "", fmt.Errorf("multipartkit: file field name is required") + } + var buf bytes.Buffer + w := multipart.NewWriter(&buf) + + // Deterministic field order. Not cosmetic: it makes a signed body + // reproducible, so a failure can be replayed byte-for-byte when diagnosing + // a signature or a partner-side parse. + names := make([]string, 0, len(fields)) + for k := range fields { + names = append(names, k) + } + sort.Strings(names) + for _, k := range names { + if err := w.WriteField(k, fields[k]); err != nil { + return nil, "", fmt.Errorf("multipartkit: write field %q: %w", k, err) + } + } + + // The file part is written with an explicit Content-Type. multipart's own + // CreateFormFile hardcodes application/octet-stream, which loses the real + // type the partner uses to decide whether it will accept the document at + // all. + h := make(textproto.MIMEHeader) + h.Set("Content-Disposition", fmt.Sprintf(`form-data; name=%q; filename=%q`, + escapeQuotes(fileField), escapeQuotes(fileName))) + ct := strings.TrimSpace(fileContentType) + if ct == "" { + ct = "application/octet-stream" + } + h.Set("Content-Type", ct) + + part, err := w.CreatePart(h) + if err != nil { + return nil, "", fmt.Errorf("multipartkit: create file part: %w", err) + } + if _, err := io.Copy(part, content); err != nil { + return nil, "", fmt.Errorf("multipartkit: copy file content: %w", err) + } + if err := w.Close(); err != nil { + return nil, "", fmt.Errorf("multipartkit: close writer: %w", err) + } + return buf.Bytes(), w.FormDataContentType(), nil +} + +// escapeQuotes matches mime/multipart's own escaper so a file name containing a +// quote or a backslash cannot break out of the Content-Disposition header and +// forge part boundaries. +func escapeQuotes(s string) string { + return strings.NewReplacer("\\", "\\\\", `"`, "\\\"").Replace(s) +} diff --git a/internal/scaffold/templates/upload.go.tmpl b/internal/scaffold/templates/upload.go.tmpl new file mode 100644 index 0000000..c635b33 --- /dev/null +++ b/internal/scaffold/templates/upload.go.tmpl @@ -0,0 +1,264 @@ +// Staged multipart uploads for {{.ID}}. +// +// Pilot IPC is JSON in, JSON out, and ipc.MaxFrameSize caps one envelope at +// 1 MiB — so a file base64'd into a single payload tops out around 740 KiB of +// real bytes, well under what this backend accepts. The file therefore does not +// travel in one message: the agent stages it in chunks, this adapter reassembles +// it under $APP/blobs, and only then builds one multipart/form-data body. +// +// {{.Namespace}}.upload_begin {file_name, content_type, total_bytes, sha256} -> {blob_id} +// {{.Namespace}}.upload_chunk {blob_id, seq, data_base64} -> {received, complete} +// {{.Namespace}}.upload_abort {blob_id} -> {aborted} +// +// GENERATED by pilot-app; edit pilot.app.yaml and re-generate. +package main + +import ( + "context" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "path/filepath" + "strings" + "sync" + "time" + + "github.com/pilot-protocol/app-store/pkg/ipc" + "{{.GoModule}}/internal/backend" +) + +// The blob store is created on first use, not at startup: an adapter whose +// agent never uploads anything should not create directories under $APP. +var ( + blobOnce sync.Once + blobs *backend.Store + blobErr error +) + +func uploadStore(manifestPath string) (*backend.Store, error) { + blobOnce.Do(func() { + blobs, blobErr = backend.NewStore(filepath.Join(appDir(manifestPath), "blobs")) + if blobErr != nil { + return + } + blobs.SetLimits({{.MaxBlobBytes}}, 0) + // Reclaim stale stagings. An agent that begins an upload and wanders off + // must not leave bytes under $APP forever. + go func() { + t := time.NewTicker(5 * time.Minute) + defer t.Stop() + for range t.C { + blobs.GC() + } + }() + }) + return blobs, blobErr +} + +// appDir resolves $APP from the manifest path the daemon passes in: the +// manifest sits at the root of the app's own directory. +func appDir(manifestPath string) string { + if manifestPath == "" { + return "." + } + return filepath.Dir(manifestPath) +} + +func uploadBeginHandler(manifestPath string) ipc.Handler { + return func(_ context.Context, req *ipc.Envelope) (json.RawMessage, error) { + st, err := uploadStore(manifestPath) + if err != nil { + return nil, err + } + var in struct { + FileName string `json:"file_name"` + ContentType string `json:"content_type"` + TotalBytes int64 `json:"total_bytes"` + SHA256 string `json:"sha256"` + } + if err := decodeUpload(req.Payload, &in); err != nil { + return nil, err + } + m, err := st.Begin(in.FileName, in.ContentType, in.TotalBytes, in.SHA256) + if err != nil { + return nil, err + } + return marshalUpload(map[string]any{ + "blob_id": m.ID, + "file_name": m.FileName, + "content_type": m.ContentType, + "total_bytes": m.TotalBytes, + "max_chunk_bytes": backend.MaxChunkBytes, + "next_seq": 0, + }) + } +} + +func uploadChunkHandler(manifestPath string) ipc.Handler { + return func(_ context.Context, req *ipc.Envelope) (json.RawMessage, error) { + st, err := uploadStore(manifestPath) + if err != nil { + return nil, err + } + var in struct { + BlobID string `json:"blob_id"` + Seq int `json:"seq"` + Data string `json:"data_base64"` + } + if err := decodeUpload(req.Payload, &in); err != nil { + return nil, err + } + raw, err := decodeChunkB64(in.Data) + if err != nil { + return nil, fmt.Errorf("data_base64: %w", err) + } + received, err := st.Append(in.BlobID, in.Seq, raw) + if err != nil { + return nil, err + } + out := map[string]any{ + "blob_id": in.BlobID, + "received": received, + "next_seq": in.Seq + 1, + "complete": false, + } + // Seal as soon as the declared size is met, so the agent needs no separate + // finalize call. A checksum mismatch surfaces here, before anything is sent. + m, ferr := st.Finalize(in.BlobID) + switch { + case ferr == nil: + out["complete"] = true + out["total_bytes"] = m.TotalBytes + case errors.Is(ferr, backend.ErrIncomplete): + // More chunks expected — the ordinary state between calls. + default: + return nil, ferr + } + return marshalUpload(out) + } +} + +func uploadAbortHandler(manifestPath string) ipc.Handler { + return func(_ context.Context, req *ipc.Envelope) (json.RawMessage, error) { + st, err := uploadStore(manifestPath) + if err != nil { + return nil, err + } + var in struct { + BlobID string `json:"blob_id"` + } + if err := decodeUpload(req.Payload, &in); err != nil { + return nil, err + } + if err := st.Abort(in.BlobID); err != nil { + return nil, err + } + return marshalUpload(map[string]any{"blob_id": in.BlobID, "aborted": true}) + } +} + +// multipartRoute describes one upload method: where it posts, which form field +// carries the file, and which payload key names the staged blob. +type multipartRoute struct { + method string + pathTmpl string + fileField string + blobParam string + pathParams []string + rawPathParams []string +} + +// forwardMultipart sends a staged blob as multipart/form-data. Every payload +// field except the blob id and the path params becomes a form field, so the +// shape an agent sees in .help is the shape the partner receives. +func forwardMultipart(c *backend.Client, r multipartRoute, manifestPath string, timeout time.Duration) ipc.Handler { + return func(ctx context.Context, req *ipc.Envelope) (json.RawMessage, error) { + st, err := uploadStore(manifestPath) + if err != nil { + return nil, err + } + fields, err := objectFields(req.Payload) + if err != nil { + return nil, err + } + blobRaw, ok := fields[r.blobParam] + if !ok { + return nil, fmt.Errorf("missing required parameter %q — stage the file with %s.upload_begin and %s.upload_chunk first", r.blobParam, "{{.Namespace}}", "{{.Namespace}}") + } + blobID := jsonScalar(blobRaw) + delete(fields, r.blobParam) + + // fillPath consumes the path params; deleting them afterwards keeps them + // out of the form, exactly as the JSON path keeps them out of the body. + path, err := fillPath(r.pathTmpl, r.pathParams, r.rawPathParams, fields) + if err != nil { + return nil, err + } + for _, p := range r.pathParams { + delete(fields, p) + } + for _, p := range r.rawPathParams { + delete(fields, p) + } + + form := make(map[string]string, len(fields)) + for k, v := range fields { + form[k] = jsonScalar(v) + } + + f, meta, err := st.Open(blobID) + if err != nil { + return nil, fmt.Errorf("blob %q: %w", blobID, err) + } + defer f.Close() + + body, contentType, err := backend.BuildForm(r.fileField, meta.FileName, meta.ContentType, f, form) + if err != nil { + return nil, err + } + cctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + out, err := c.DoMultipart(cctx, r.method, path, body, contentType) + if err != nil { + return nil, err + } + // The bytes are with the partner now. Keeping the local copy would let + // $APP grow by one document per upload. + _ = st.Abort(blobID) + return out, nil + } +} + +// decodeChunkB64 accepts standard and URL-safe base64, padded or not: an agent +// assembling a chunk by hand should not have to guess the dialect. +func decodeChunkB64(s string) ([]byte, error) { + s = strings.TrimSpace(s) + for _, enc := range []*base64.Encoding{ + base64.StdEncoding, base64.RawStdEncoding, + base64.URLEncoding, base64.RawURLEncoding, + } { + if b, err := enc.DecodeString(s); err == nil { + return b, nil + } + } + return nil, errors.New("not valid base64") +} + +func decodeUpload(payload json.RawMessage, dst any) error { + if len(payload) == 0 { + return errors.New("payload is required") + } + if err := json.Unmarshal(payload, dst); err != nil { + return fmt.Errorf("payload: %w", err) + } + return nil +} + +func marshalUpload(v any) (json.RawMessage, error) { + b, err := json.Marshal(v) + if err != nil { + return nil, err + } + return json.RawMessage(b), nil +} diff --git a/internal/scaffold/zz_mcp_e2e_test.go b/internal/scaffold/zz_mcp_e2e_test.go new file mode 100644 index 0000000..4e41c35 --- /dev/null +++ b/internal/scaffold/zz_mcp_e2e_test.go @@ -0,0 +1,340 @@ +//go:build !windows + +package scaffold + +import ( + "encoding/json" + "fmt" + "io" + "net" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/pilot-protocol/app-store/pkg/ipc" +) + +// An app can front a credentialed REST API and an open tool server at the same +// time. The property that matters is that those two backends never share a +// credential: the REST key belongs to one origin, and shipping it to another +// because the same vendor happens to run both is how a key ends up somewhere +// its owner never agreed to. +// +// The generated ToolClient has no field to hold a credential, which makes the +// property structural rather than a habit — but structure is easy to erode, so +// this asserts it from the outside, against a real running adapter. +const mcpSpec = ` +id: io.pilot.toolx +app_version: 0.1.0 +description: "App with a credentialed REST backend and an open tool server." +namespace: toolx +backend: + base_url: https://placeholder.invalid + headers: + authorization: "Bearer ${TOOLX_API_KEY}" + mcp: + url: https://placeholder.invalid/mcp/ +methods: + - name: toolx.rest_read + summary: "A REST call that DOES carry the key." + http: { verb: GET, path: /things } + - name: toolx.options + summary: "A tool call that must carry nothing." + mcp: { tool: get_filing_options } + params: { entity_type: "llc or c-corp" } + - name: toolx.start + summary: "A tool call taking one wrapped object argument." + duration: slow + mcp: { tool: start_llc_formation, wrap: intake } + params: { company_name: "the name" } + - name: toolx.update + summary: "A tool call mixing a top-level id with a nested object." + mcp: { tool: update_formation, wrap: update, keep: [formation_id] } + params: { formation_id: "the id", company_name: "the new name" } +` + +// toolServer records every header it is sent and answers the JSON-RPC calls the +// generated client makes. It replies over SSE, the harder of the two transports. +type toolServer struct { + mu sync.Mutex + headers []http.Header + lastArgs map[string]json.RawMessage + lastTool string + calls int +} + +func newToolServer(t *testing.T) (*httptest.Server, *toolServer) { + t.Helper() + rec := &toolServer{} + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + rec.mu.Lock() + rec.headers = append(rec.headers, r.Header.Clone()) + rec.mu.Unlock() + + var req struct { + ID json.RawMessage `json:"id"` + Method string `json:"method"` + Params struct { + Name string `json:"name"` + Arguments map[string]json.RawMessage `json:"arguments"` + } `json:"params"` + } + _ = json.Unmarshal(body, &req) + + switch req.Method { + case "initialize": + w.Header().Set("Mcp-Session-Id", "sess-123") + w.Header().Set("Content-Type", "text/event-stream") + fmt.Fprintf(w, "event: message\ndata: {\"jsonrpc\":\"2.0\",\"id\":%s,\"result\":{\"protocolVersion\":\"2025-06-18\",\"serverInfo\":{\"name\":\"mock\"}}}\n\n", req.ID) + case "notifications/initialized": + w.WriteHeader(http.StatusAccepted) + case "tools/call": + rec.mu.Lock() + rec.lastTool, rec.lastArgs, rec.calls = req.Params.Name, req.Params.Arguments, rec.calls+1 + rec.mu.Unlock() + w.Header().Set("Content-Type", "text/event-stream") + // structuredContent is the preferred shape; return real-looking data. + fmt.Fprintf(w, "event: message\ndata: {\"jsonrpc\":\"2.0\",\"id\":%s,\"result\":{\"structuredContent\":{\"tool\":%q,\"ok\":true}}}\n\n", + req.ID, req.Params.Name) + default: + w.WriteHeader(http.StatusBadRequest) + } + })) + return srv, rec +} + +func TestGeneratedToolServerNeverSeesTheRestKeyE2E(t *testing.T) { + if testing.Short() { + t.Skip("builds and runs a real adapter binary; skipped under -short") + } + if _, err := exec.LookPath("go"); err != nil { + t.Skip("go toolchain not available") + } + + tools, rec := newToolServer(t) + defer tools.Close() + + restHit := make(chan http.Header, 4) + rest := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + restHit <- r.Header.Clone() + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"ok":true}`)) + })) + defer rest.Close() + + root := t.TempDir() + cfg := parseSpec(t, strings.Replace(mcpSpec, "https://placeholder.invalid/mcp/", tools.URL+"/mcp/", 1)) + if errs := cfg.Validate(); len(errs) != 0 { + t.Fatalf("spec invalid: %v", errs) + } + proj := filepath.Join(root, "proj") + if _, err := Generate(cfg, proj); err != nil { + t.Fatalf("generate: %v", err) + } + if sum, err := os.ReadFile(filepath.Join("..", "..", "go.sum")); err == nil { + _ = os.WriteFile(filepath.Join(proj, "go.sum"), sum, 0o644) + } + bin := filepath.Join(root, "adapter") + build := exec.Command("go", "build", "-o", bin, "./cmd/"+cfg.BinaryName) + build.Dir = proj + build.Env = append(os.Environ(), "GOFLAGS=-mod=mod") + if out, err := build.CombinedOutput(); err != nil { + t.Fatalf("build adapter: %v\n%s", err, out) + } + + sockDir, err := os.MkdirTemp("", "tlsk") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(sockDir) + sock := filepath.Join(sockDir, "a.sock") + adapter := exec.Command(bin, "--socket", sock, "--manifest", filepath.Join(proj, "manifest.json")) + adapter.Stderr = os.Stderr + // A real, secret-looking key on the REST side. It must never leave that origin. + adapter.Env = append(os.Environ(), + "TOOLX_BACKEND_URL="+rest.URL, + "TOOLX_API_KEY=glk_super_secret_do_not_leak") + if err := adapter.Start(); err != nil { + t.Fatalf("start adapter: %v", err) + } + defer func() { _ = adapter.Process.Kill(); _, _ = adapter.Process.Wait() }() + + deadline := time.Now().Add(15 * time.Second) + for time.Now().Before(deadline) { + if _, err := os.Stat(sock); err == nil { + break + } + time.Sleep(20 * time.Millisecond) + } + call := func(method, args string) json.RawMessage { + t.Helper() + conn, err := net.DialTimeout("unix", sock, 5*time.Second) + if err != nil { + t.Fatalf("dial: %v", err) + } + defer conn.Close() + var out json.RawMessage + if err := ipc.Call(conn, method, json.RawMessage(args), &out); err != nil { + t.Fatalf("call %s: %v", method, err) + } + return out + } + + // The REST side really does carry the key — otherwise the negative below + // would pass for the wrong reason. + call("toolx.rest_read", `{}`) + select { + case h := <-restHit: + if h.Get("Authorization") != "Bearer glk_super_secret_do_not_leak" { + t.Fatalf("REST backend did not receive the key (got %q); the leak check below would be vacuous", h.Get("Authorization")) + } + case <-time.After(10 * time.Second): + t.Fatal("REST backend was never called") + } + + // Now exercise every tool route. + out := call("toolx.options", `{"entity_type":"llc"}`) + if !strings.Contains(string(out), `"ok":true`) { + t.Errorf("tool result not unwrapped from structuredContent: %s", out) + } + call("toolx.start", `{"company_name":"NewCo"}`) + call("toolx.update", `{"formation_id":"f-1","company_name":"Renamed"}`) + + // THE ASSERTION: nothing credential-shaped ever reached the tool server. + rec.mu.Lock() + defer rec.mu.Unlock() + if len(rec.headers) < 3 { + t.Fatalf("tool server saw %d requests, expected the handshake plus three calls", len(rec.headers)) + } + for i, h := range rec.headers { + for _, name := range []string{"Authorization", "X-Api-Key", "Api-Key", "Proxy-Authorization", "Cookie"} { + if v := h.Get(name); v != "" { + t.Errorf("request %d carried %s: %q — the REST key must never reach the tool server", i, name, v) + } + } + for name, vals := range h { + for _, v := range vals { + if strings.Contains(v, "glk_super_secret") { + t.Errorf("request %d leaked the REST key in header %s: %q", i, name, v) + } + } + } + } + + // The session is established once and reused, not renegotiated per call. + inits := 0 + for _, h := range rec.headers { + if h.Get("Mcp-Session-Id") == "" { + inits++ + } + } + if inits > 2 { // initialize + the initialized notification + t.Errorf("tool server saw %d sessionless requests; the session should be established once and reused", inits) + } + + // wrap/keep must shape the arguments the way the tool declares them. + if rec.lastTool != "update_formation" { + t.Fatalf("last tool = %q", rec.lastTool) + } + if got := string(rec.lastArgs["formation_id"]); got != `"f-1"` { + t.Errorf("formation_id should stay top-level, got %q", got) + } + upd, ok := rec.lastArgs["update"] + if !ok { + t.Fatal("the wrapped `update` argument is missing") + } + if !strings.Contains(string(upd), "Renamed") { + t.Errorf("update = %s, want it to carry company_name", upd) + } + if strings.Contains(string(upd), "formation_id") { + t.Errorf("update = %s, want formation_id kept OUT of the wrapped object", upd) + } +} + +// TestMCPValidationRules: a spec that would generate a broken or credential- +// leaking tool client must fail at build time. +func TestMCPValidationRules(t *testing.T) { + base := ` +id: io.pilot.mcpv +app_version: 0.1.0 +description: "fixture" +namespace: mcpv +backend: + base_url: https://placeholder.invalid +%s +methods: + - name: mcpv.t + summary: "tool" +%s +` + cases := []struct{ name, backend, method, want string }{ + {"tool is required", + " mcp:\n url: https://example.com/mcp/", " mcp: { tool: \"\" }", "mcp.tool is required"}, + {"backend.mcp.url must be set", + "", " mcp: { tool: x }", "backend.mcp.url is not set"}, + {"url must be https", + " mcp:\n url: http://example.com/mcp/", " mcp: { tool: x }", "must be an https URL"}, + {"no credentialed tool servers yet", + " mcp:\n url: https://example.com/mcp/\n auth: bearer", " mcp: { tool: x }", "not supported"}, + {"keep needs wrap", + " mcp:\n url: https://example.com/mcp/", " mcp: { tool: x, keep: [a] }", "only means something with mcp.wrap"}, + {"cannot mix routes", + " mcp:\n url: https://example.com/mcp/", " mcp: { tool: x }\n http: { verb: GET, path: /y }", "must not also declare"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := validateSpec(t, fmt.Sprintf(base, tc.backend, tc.method)) + if !strings.Contains(got, tc.want) { + t.Fatalf("want an error mentioning %q, got: %s", tc.want, got) + } + }) + } +} + +// TestMCPGrantsAndFiles: the tool server's host needs its own net.dial grant, +// and the client is only emitted for an app that actually has a tool route. +func TestMCPGrantsAndFiles(t *testing.T) { + c := parseSpec(t, strings.Replace(mcpSpec, "https://placeholder.invalid/mcp/", "https://tools.example.com/mcp/", 1)) + if errs := c.Validate(); len(errs) != 0 { + t.Fatalf("spec invalid: %v", errs) + } + dir := t.TempDir() + if _, err := Generate(c, dir); err != nil { + t.Fatalf("generate: %v", err) + } + man, _ := os.ReadFile(filepath.Join(dir, "manifest.json")) + if !strings.Contains(string(man), `"net.dial", "target": "tools.example.com"`) { + t.Errorf("manifest is missing the tool-server net.dial grant:\n%s", man) + } + if _, err := os.Stat(filepath.Join(dir, "internal", "backend", "mcp.go")); err != nil { + t.Errorf("mcp.go not emitted: %v", err) + } + + plain, _ := Parse([]byte(` +id: io.pilot.notools +app_version: 0.1.0 +description: "no tools" +namespace: notools +backend: + base_url: https://placeholder.invalid +methods: + - name: notools.ping + summary: "ping" + http: { verb: GET, path: /ping } +`)) + plain.Resolve() + d2 := t.TempDir() + if _, err := Generate(plain, d2); err != nil { + t.Fatalf("generate: %v", err) + } + if _, err := os.Stat(filepath.Join(d2, "internal", "backend", "mcp.go")); err == nil { + t.Error("mcp.go emitted for an app with no tool routes") + } +} diff --git a/internal/scaffold/zz_multipart_broker_e2e_test.go b/internal/scaffold/zz_multipart_broker_e2e_test.go new file mode 100644 index 0000000..b7880cd --- /dev/null +++ b/internal/scaffold/zz_multipart_broker_e2e_test.go @@ -0,0 +1,316 @@ +//go:build !windows + +package scaffold + +import ( + "crypto/ed25519" + "crypto/rand" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "encoding/json" + "fmt" + "io" + mrand "math/rand" + "mime" + "mime/multipart" + "net" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/pilot-protocol/app-store/pkg/ipc" + "github.com/pilot-protocol/app-template/internal/broker" +) + +// The production topology for a managed upload app, assembled for real: +// +// generated adapter --signed--> broker (master key + tenancy) --> partner +// +// The adapter and the broker are each covered on their own. This is the seam: +// the adapter signs a multipart body it assembled, and the broker has to verify +// that signature, parse the SAME body to ownership-check a form field, forward +// it with the boundary intact, and inject a key the adapter never sees. Those +// two halves were written against each other's contract, and this is the only +// test where that contract is actually exercised rather than assumed. +const managedUploadSpec = ` +id: io.pilot.uploadz +app_version: 0.1.0 +description: "Managed multipart upload app." +namespace: uploadz +backend: + base_url: https://placeholder.invalid + auth: managed +methods: + - name: uploadz.deal_open + summary: "Open a matter." + http: { verb: POST, path: /api/v1/deals } + params: { initial_request: "what you need" } + - name: uploadz.document_upload + summary: "Upload a document into a matter." + duration: slow + http: + verb: POST + path: /api/v1/documents + multipart: { file_field: file } + params: + blob_id: "staged blob" + deal_id: "matter to attach to" +` + +const managedUploadRegistry = `[{ + "id": "io.pilot.uploadz", + "upstream": "%s", + "key_env": "UPLOADZ_KEY", + "auth_header": "Authorization", + "auth_scheme": "Bearer", + "quota": 0, + "allow": ["POST /api/v1/deals", "POST /api/v1/documents"], + "forward_content_types": ["multipart/form-data"], + "max_body_bytes": 16777216, + "tenancy": { + "param_types": {"deal_id": "deal"}, + "body_refs": {"deal_id": "deal"}, + "create": [{"method":"POST","path":"/api/v1/deals","type":"deal","id_field":"deal_id"}], + "list": [] + } +}]` + +func TestGeneratedManagedUploadThroughBrokerE2E(t *testing.T) { + if testing.Short() { + t.Skip("builds and runs a real adapter binary; skipped under -short") + } + if _, err := exec.LookPath("go"); err != nil { + t.Skip("go toolchain not available") + } + + type arrived struct { + sha string + size int + name string + auth string + fields map[string]string + } + got := make(chan arrived, 4) + + partner := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + if r.URL.Path == "/api/v1/deals" { + fmt.Fprintf(w, `{"deal_id":"deal_%d"}`, time.Now().UnixNano()) + return + } + mt, params, err := mime.ParseMediaType(r.Header.Get("Content-Type")) + if err != nil || mt != "multipart/form-data" { + http.Error(w, `{"detail":"not multipart: `+r.Header.Get("Content-Type")+`"}`, http.StatusUnprocessableEntity) + return + } + a := arrived{fields: map[string]string{}, auth: r.Header.Get("Authorization")} + mr := multipart.NewReader(r.Body, params["boundary"]) + for { + p, err := mr.NextPart() + if err == io.EOF { + break + } + if err != nil { + http.Error(w, `{"detail":"bad multipart"}`, http.StatusBadRequest) + return + } + b, _ := io.ReadAll(p) + if p.FileName() != "" { + s := sha256.Sum256(b) + a.sha, a.size, a.name = hex.EncodeToString(s[:]), len(b), p.FileName() + } else { + a.fields[p.FormName()] = string(b) + } + p.Close() + } + got <- a + fmt.Fprint(w, `{"deal_id":"d1","contract_id":"c1","version_id":"v1"}`) + })) + defer partner.Close() + + reg, err := broker.ParseRegistry([]byte(fmt.Sprintf(managedUploadRegistry, partner.URL)), + func(string) string { return "glk_master" }) + if err != nil { + t.Fatalf("ParseRegistry: %v", err) + } + bk := broker.New(reg, broker.NewMemStore()) + bk.Verify = broker.VerifyConfig{Window: time.Hour} + brokerSrv := httptest.NewServer(bk) + defer brokerSrv.Close() + + root := t.TempDir() + cfg := parseSpec(t, managedUploadSpec) + if errs := cfg.Validate(); len(errs) != 0 { + t.Fatalf("spec invalid: %v", errs) + } + proj := filepath.Join(root, "proj") + if _, err := Generate(cfg, proj); err != nil { + t.Fatalf("generate: %v", err) + } + if sum, err := os.ReadFile(filepath.Join("..", "..", "go.sum")); err == nil { + _ = os.WriteFile(filepath.Join(proj, "go.sum"), sum, 0o644) + } + + bin := filepath.Join(root, "adapter") + build := exec.Command("go", "build", "-o", bin, "./cmd/"+cfg.BinaryName) + build.Dir = proj + build.Env = append(os.Environ(), "GOFLAGS=-mod=mod") + if out, err := build.CombinedOutput(); err != nil { + t.Fatalf("build adapter: %v\n%s", err, out) + } + + // The daemon provisions a per-app ed25519 identity; stand in for it. + _, priv, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatalf("keygen: %v", err) + } + idFile := filepath.Join(root, "identity.key") + if err := os.WriteFile(idFile, []byte(base64.StdEncoding.EncodeToString(priv.Seed())), 0o600); err != nil { + t.Fatalf("write identity: %v", err) + } + + sockDir, err := os.MkdirTemp("", "mbsk") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(sockDir) + sock := filepath.Join(sockDir, "a.sock") + + adapter := exec.Command(bin, + "--socket", sock, + "--manifest", filepath.Join(proj, "manifest.json"), + "--identity", idFile) + adapter.Stderr = os.Stderr + adapter.Env = append(os.Environ(), "UPLOADZ_BACKEND_URL="+brokerSrv.URL+"/io.pilot.uploadz") + if err := adapter.Start(); err != nil { + t.Fatalf("start adapter: %v", err) + } + defer func() { _ = adapter.Process.Kill(); _, _ = adapter.Process.Wait() }() + + deadline := time.Now().Add(15 * time.Second) + for time.Now().Before(deadline) { + if _, err := os.Stat(sock); err == nil { + break + } + time.Sleep(20 * time.Millisecond) + } + + call := func(method, args string) (json.RawMessage, error) { + conn, err := net.DialTimeout("unix", sock, 5*time.Second) + if err != nil { + t.Fatalf("dial: %v", err) + } + defer conn.Close() + var out json.RawMessage + err = ipc.Call(conn, method, json.RawMessage(args), &out) + return out, err + } + mustCall := func(method, args string) json.RawMessage { + t.Helper() + out, err := call(method, args) + if err != nil { + t.Fatalf("call %s: %v", method, err) + } + return out + } + + // 1. Open a matter. The broker claims the deal for this adapter's identity. + var deal struct { + DealID string `json:"deal_id"` + } + if err := json.Unmarshal(mustCall("uploadz.deal_open", `{"initial_request":"review an NDA"}`), &deal); err != nil { + t.Fatalf("decode deal: %v", err) + } + if deal.DealID == "" { + t.Fatal("no deal_id returned") + } + + // 2. Stage a 2 MiB document through the frame-limited transport. + content := make([]byte, 2<<20) + _, _ = mrand.New(mrand.NewSource(11)).Read(content) + sum := sha256.Sum256(content) + wantSHA := hex.EncodeToString(sum[:]) + + beginArgs, _ := json.Marshal(map[string]any{ + "file_name": "nda.docx", "content_type": "application/pdf", + "total_bytes": len(content), "sha256": wantSHA, + }) + var begin struct { + BlobID string `json:"blob_id"` + MaxChunk int `json:"max_chunk_bytes"` + } + if err := json.Unmarshal(mustCall("uploadz.upload_begin", string(beginArgs)), &begin); err != nil { + t.Fatalf("decode begin: %v", err) + } + for seq, off := 0, 0; off < len(content); seq, off = seq+1, off+begin.MaxChunk { + end := off + begin.MaxChunk + if end > len(content) { + end = len(content) + } + a, _ := json.Marshal(map[string]any{ + "blob_id": begin.BlobID, "seq": seq, + "data_base64": base64.StdEncoding.EncodeToString(content[off:end]), + }) + mustCall("uploadz.upload_chunk", string(a)) + } + + // 3. Upload it into the matter, through the broker. + upArgs, _ := json.Marshal(map[string]any{"blob_id": begin.BlobID, "deal_id": deal.DealID}) + mustCall("uploadz.document_upload", string(upArgs)) + + select { + case a := <-got: + if a.sha != wantSHA { + t.Errorf("document corrupted adapter->broker->partner:\n got %s (%d bytes)\nwant %s (%d bytes)", + a.sha, a.size, wantSHA, len(content)) + } + if a.name != "nda.docx" { + t.Errorf("file name = %q", a.name) + } + if a.fields["deal_id"] != deal.DealID { + t.Errorf("deal_id = %q, want %q", a.fields["deal_id"], deal.DealID) + } + if a.auth != "Bearer glk_master" { + t.Errorf("Authorization = %q — the broker did not inject the master key", a.auth) + } + case <-time.After(25 * time.Second): + t.Fatal("partner never received the upload") + } + + // 4. The isolation the shared key makes necessary, asserted through the real + // adapter: a deal this identity does not own is refused by the broker + // even though the adapter happily assembled and signed the body. + content2 := []byte("someone else's matter") + s2 := sha256.Sum256(content2) + b2, _ := json.Marshal(map[string]any{ + "file_name": "x.txt", "content_type": "text/plain", + "total_bytes": len(content2), "sha256": hex.EncodeToString(s2[:]), + }) + var begin2 struct { + BlobID string `json:"blob_id"` + } + _ = json.Unmarshal(mustCall("uploadz.upload_begin", string(b2)), &begin2) + c2, _ := json.Marshal(map[string]any{ + "blob_id": begin2.BlobID, "seq": 0, + "data_base64": base64.StdEncoding.EncodeToString(content2), + }) + mustCall("uploadz.upload_chunk", string(c2)) + + foreign, _ := json.Marshal(map[string]any{"blob_id": begin2.BlobID, "deal_id": "deal_not_mine"}) + if _, err := call("uploadz.document_upload", string(foreign)); err == nil { + t.Error("uploading into an unowned deal succeeded") + } else if !strings.Contains(err.Error(), "404") && !strings.Contains(strings.ToLower(err.Error()), "not found") { + t.Errorf("want the broker's opaque 404 for an unowned resource, got: %v", err) + } + select { + case a := <-got: + t.Fatalf("the refused upload still reached the partner: %+v", a) + case <-time.After(500 * time.Millisecond): + } +} diff --git a/internal/scaffold/zz_multipart_config_test.go b/internal/scaffold/zz_multipart_config_test.go new file mode 100644 index 0000000..da97ac5 --- /dev/null +++ b/internal/scaffold/zz_multipart_config_test.go @@ -0,0 +1,248 @@ +package scaffold + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +// specWith builds a one-method multipart spec with the given http: block, so a +// validation rule can be exercised without repeating the whole document. +func specWith(httpBlock string) string { + return ` +id: io.pilot.mpx +app_version: 0.1.0 +description: "multipart validation fixture" +namespace: mpx +backend: + base_url: https://placeholder.invalid +methods: + - name: mpx.up + summary: "upload" + http:` + httpBlock + ` + params: + blob_id: "staged blob" +` +} + +// validateSpec parses + resolves + validates, returning the joined errors. +func validateSpec(t *testing.T, doc string) string { + t.Helper() + c, err := Parse([]byte(doc)) + if err != nil { + return err.Error() + } + c.Resolve() + errs := c.Validate() + msgs := make([]string, 0, len(errs)) + for _, e := range errs { + msgs = append(msgs, e.Error()) + } + return strings.Join(msgs, "; ") +} + +// TestMultipartValidationRules: each of these specs would generate an adapter +// that fails only when a real upload is attempted against the partner, so they +// have to fail at build time instead. +func TestMultipartValidationRules(t *testing.T) { + cases := []struct { + name string + http string + want string + valid bool + }{ + { + name: "a body verb is fine", + http: " { verb: POST, path: /docs, multipart: {} }", + valid: true, + }, + { + name: "GET has nowhere to put a form", + http: " { verb: GET, path: /docs, multipart: {} }", + want: "needs a body verb", + }, + { + name: "blob_param must not be the file field", + http: " { verb: POST, path: /docs, multipart: { file_field: blob_id } }", + want: "the blob id names the staged file", + }, + { + name: "blob_param must stay in the body", + http: " { verb: POST, path: /docs, multipart: {}, param_in: { blob_id: query } }", + want: "must stay in the body", + }, + { + name: "max_bytes must be positive", + http: " { verb: POST, path: /docs, multipart: { max_bytes: -1 } }", + want: "max_bytes must be positive", + }, + { + name: "a raw body param conflicts with the form", + http: " { verb: POST, path: /docs, multipart: {}, param_in: { blob_id: body_raw } }", + want: "cannot be combined with a raw body param", + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := validateSpec(t, specWith(tc.http)) + if tc.valid { + if got != "" { + t.Fatalf("valid spec rejected: %s", got) + } + return + } + if !strings.Contains(got, tc.want) { + t.Fatalf("want an error mentioning %q, got: %s", tc.want, got) + } + }) + } +} + +// TestMultipartDefaultsResolved: an author writing `multipart: {}` gets the +// conventional field names rather than empty strings that would produce a form +// the partner cannot read. +func TestMultipartDefaultsResolved(t *testing.T) { + c, err := Parse([]byte(specWith(" { verb: POST, path: /docs, multipart: {} }"))) + if err != nil { + t.Fatalf("parse: %v", err) + } + c.Resolve() + mp := c.Methods[0].HTTP.Multipart + if mp.FileField != DefaultFileField { + t.Errorf("file_field = %q, want %q", mp.FileField, DefaultFileField) + } + if mp.BlobParam != DefaultBlobParam { + t.Errorf("blob_param = %q, want %q", mp.BlobParam, DefaultBlobParam) + } + if mp.MaxBytes != DefaultMaxBlobBytes { + t.Errorf("max_bytes = %d, want %d", mp.MaxBytes, DefaultMaxBlobBytes) + } + if !c.HasMultipart() { + t.Error("HasMultipart() = false for a spec with a multipart route") + } + if c.MaxBlobBytes() != DefaultMaxBlobBytes { + t.Errorf("MaxBlobBytes() = %d", c.MaxBlobBytes()) + } +} + +// TestStagingMethodsInjectedOnceAndOnlyWhenNeeded: the three staging steps are +// what make an upload reachable at all, so they must appear for a multipart app +// — and must NOT appear for an app without one, where they would be three dead +// methods on the store page. +func TestStagingMethodsInjectedOnceAndOnlyWhenNeeded(t *testing.T) { + c, _ := Parse([]byte(specWith(" { verb: POST, path: /docs, multipart: {} }"))) + c.Resolve() + c.Resolve() // idempotent: re-resolving must not duplicate the injected methods + counts := map[string]int{} + for i := range c.Methods { + counts[c.Methods[i].Name]++ + } + for _, want := range []string{"mpx.upload_begin", "mpx.upload_chunk", "mpx.upload_abort"} { + if counts[want] != 1 { + t.Errorf("%s appears %d times, want exactly 1", want, counts[want]) + } + } + + plain, _ := Parse([]byte(` +id: io.pilot.plainx +app_version: 0.1.0 +description: "no uploads here" +namespace: plainx +backend: + base_url: https://placeholder.invalid +methods: + - name: plainx.ping + summary: "ping" + http: { verb: GET, path: /ping } +`)) + plain.Resolve() + if plain.HasMultipart() { + t.Fatal("HasMultipart() = true for a spec with no multipart route") + } + for i := range plain.Methods { + if strings.Contains(plain.Methods[i].Name, "upload_") { + t.Errorf("staging method %q injected into an app with no multipart route", plain.Methods[i].Name) + } + } +} + +// TestMultipartGrantsAndFiles: the manifest must grant exactly $APP/blobs (not +// $APP, which native delivery uses and which is far wider than an upload needs), +// and the staging runtime must actually be emitted. +func TestMultipartGrantsAndFiles(t *testing.T) { + c, _ := Parse([]byte(specWith(" { verb: POST, path: /docs, multipart: {} }"))) + c.Resolve() + if errs := c.Validate(); len(errs) != 0 { + t.Fatalf("spec invalid: %v", errs) + } + dir := t.TempDir() + if _, err := Generate(c, dir); err != nil { + t.Fatalf("generate: %v", err) + } + + man, err := os.ReadFile(filepath.Join(dir, "manifest.json")) + if err != nil { + t.Fatalf("read manifest: %v", err) + } + for _, want := range []string{ + `{"cap": "fs.read", "target": "$APP/blobs"}`, + `{"cap": "fs.write", "target": "$APP/blobs"}`, + } { + if !strings.Contains(string(man), want) { + t.Errorf("manifest is missing grant %s", want) + } + } + if strings.Contains(string(man), `"fs.write", "target": "$APP"}`) { + t.Error("manifest grants fs.write on all of $APP; uploads only need $APP/blobs") + } + for _, want := range []string{"mpx.upload_begin", "mpx.upload_chunk", "mpx.upload_abort", "mpx.up"} { + if !strings.Contains(string(man), `"`+want+`"`) { + t.Errorf("manifest exposes list is missing %q", want) + } + } + + for _, f := range []string{ + filepath.Join("internal", "backend", "blob.go"), + filepath.Join("internal", "backend", "multipartform.go"), + filepath.Join("cmd", c.BinaryName, "upload.go"), + } { + if _, err := os.Stat(filepath.Join(dir, f)); err != nil { + t.Errorf("expected %s to be emitted: %v", f, err) + } + } +} + +// TestNonMultipartAppEmitsNoStagingRuntime: the blob store creates directories +// and starts a GC goroutine, so an app that never uploads should not carry it. +func TestNonMultipartAppEmitsNoStagingRuntime(t *testing.T) { + c, _ := Parse([]byte(` +id: io.pilot.plainy +app_version: 0.1.0 +description: "no uploads" +namespace: plainy +backend: + base_url: https://placeholder.invalid +methods: + - name: plainy.ping + summary: "ping" + http: { verb: GET, path: /ping } +`)) + c.Resolve() + dir := t.TempDir() + if _, err := Generate(c, dir); err != nil { + t.Fatalf("generate: %v", err) + } + for _, f := range []string{ + filepath.Join("internal", "backend", "blob.go"), + filepath.Join("cmd", c.BinaryName, "upload.go"), + } { + if _, err := os.Stat(filepath.Join(dir, f)); err == nil { + t.Errorf("%s was emitted for an app with no multipart route", f) + } + } + man, _ := os.ReadFile(filepath.Join(dir, "manifest.json")) + if strings.Contains(string(man), "$APP/blobs") { + t.Error("manifest grants $APP/blobs for an app that never stages anything") + } +} diff --git a/internal/scaffold/zz_multipart_drift_test.go b/internal/scaffold/zz_multipart_drift_test.go new file mode 100644 index 0000000..6147f9a --- /dev/null +++ b/internal/scaffold/zz_multipart_drift_test.go @@ -0,0 +1,63 @@ +package scaffold + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +// The blob store and form builder emitted into every multipart app are copies +// of internal/multipartkit, which is where they are unit-tested. Two copies of +// security-relevant code is a drift hazard — the emitted one is what actually +// runs on a user's host, and a fix applied only to the tested copy would be a +// fix that never ships. +// +// This is the same stance canonical_golden_test.go takes for the signer, whose +// bytes must match the broker's verifier. The templates are derived from the +// reference by changing the package clause and nothing else, so the check is an +// exact comparison rather than a spot-check of a few constants. +var driftPairs = []struct{ ref, tmpl string }{ + {filepath.Join("..", "multipartkit", "blob.go"), filepath.Join("templates", "blob.go.tmpl")}, + {filepath.Join("..", "multipartkit", "form.go"), filepath.Join("templates", "multipartform.go.tmpl")}, +} + +func TestEmittedMultipartCodeMatchesReference(t *testing.T) { + for _, p := range driftPairs { + ref, err := os.ReadFile(p.ref) + if err != nil { + t.Fatalf("read reference %s: %v", p.ref, err) + } + got, err := os.ReadFile(p.tmpl) + if err != nil { + t.Fatalf("read template %s: %v", p.tmpl, err) + } + want := strings.Replace(string(ref), "package multipartkit", "package backend", 1) + if string(got) != want { + t.Errorf("%s has drifted from %s.\n"+ + "The emitted copy is what runs on a user's host, so a change to one must be made to both.\n"+ + "Regenerate with:\n"+ + " sed 's/^package multipartkit$/package backend/' %s > internal/scaffold/%s", + p.tmpl, p.ref, p.ref, p.tmpl) + } + } +} + +// TestEmittedMultipartTemplatesHaveNoTemplateDelimiters: these two files are +// rendered through text/template like every other template, so a `{{` appearing +// in the reference source would be interpreted as an action and either fail the +// render or silently delete code. Nothing in Go source needs `{{`, but a future +// edit could introduce one (a nested composite literal written without a space), +// and the failure would be confusing far from its cause. +func TestEmittedMultipartTemplatesHaveNoTemplateDelimiters(t *testing.T) { + for _, p := range driftPairs { + b, err := os.ReadFile(p.tmpl) + if err != nil { + t.Fatalf("read %s: %v", p.tmpl, err) + } + if i := strings.Index(string(b), "{{"); i >= 0 { + line := 1 + strings.Count(string(b[:i]), "\n") + t.Errorf("%s line %d contains `{{`, which text/template will interpret as an action; separate the braces", p.tmpl, line) + } + } +} diff --git a/internal/scaffold/zz_multipart_e2e_test.go b/internal/scaffold/zz_multipart_e2e_test.go new file mode 100644 index 0000000..1fd3bad --- /dev/null +++ b/internal/scaffold/zz_multipart_e2e_test.go @@ -0,0 +1,365 @@ +//go:build !windows + +package scaffold + +import ( + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "encoding/json" + "io" + mrand "math/rand" + "mime" + "mime/multipart" + "net" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/pilot-protocol/app-store/pkg/ipc" +) + +// multipartSpec mirrors the General Legal document endpoint: a multipart POST +// whose file is staged out-of-band and whose deal_id rides along as a form +// field, plus a path-param variant to prove placeholders still resolve. +const multipartSpec = ` +id: io.pilot.uploadx +app_version: 0.1.0 +description: "App exercising multipart uploads." +namespace: uploadx +backend: + base_url: https://placeholder.invalid +methods: + - name: uploadx.document_upload + summary: "Upload a document for review." + duration: slow + http: + verb: POST + path: /api/v1/documents + multipart: { file_field: file, max_bytes: 8388608 } + params: + blob_id: "staged blob" + deal_id: "matter to attach to" + context_for_legal: "note for counsel" + - name: uploadx.version_replace + summary: "Upload into a path-scoped resource." + duration: slow + http: + verb: POST + path: "/api/v1/contracts/{contract_id}/versions" + multipart: { file_field: document } + params: + blob_id: "staged blob" + contract_id: "path param" +` + +type gotUpload struct { + path string + contentType string + fileField string + fileName string + filePartCT string + sha string + size int + fields map[string]string +} + +// TestGeneratedMultipartUploadE2E is the end-to-end proof of the whole design: +// it scaffolds a real adapter, builds it, runs it as the daemon would, stages a +// file through the IPC chunk methods in pieces that each fit inside +// ipc.MaxFrameSize, then calls the upload method and asserts the partner +// received one well-formed multipart body with the bytes intact. +// +// It is deliberately driven over the real IPC socket rather than by calling the +// handlers directly: the frame limit is the constraint the whole design exists +// for, so a test that bypasses the transport would prove nothing. +func TestGeneratedMultipartUploadE2E(t *testing.T) { + if testing.Short() { + t.Skip("builds and runs a real adapter binary; skipped under -short") + } + if _, err := exec.LookPath("go"); err != nil { + t.Skip("go toolchain not available") + } + + var mu = make(chan *gotUpload, 8) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mt, params, err := mime.ParseMediaType(r.Header.Get("Content-Type")) + if err != nil || mt != "multipart/form-data" { + http.Error(w, `{"detail":"expected multipart/form-data, got `+r.Header.Get("Content-Type")+`"}`, http.StatusUnprocessableEntity) + return + } + g := &gotUpload{path: r.URL.Path, contentType: r.Header.Get("Content-Type"), fields: map[string]string{}} + mr := multipart.NewReader(r.Body, params["boundary"]) + for { + p, err := mr.NextPart() + if err == io.EOF { + break + } + if err != nil { + http.Error(w, `{"detail":"bad multipart"}`, http.StatusBadRequest) + return + } + b, _ := io.ReadAll(p) + if p.FileName() != "" { + sum := sha256.Sum256(b) + g.fileField, g.fileName = p.FormName(), p.FileName() + g.sha, g.size = hex.EncodeToString(sum[:]), len(b) + g.filePartCT = p.Header.Get("Content-Type") + } else { + g.fields[p.FormName()] = string(b) + } + p.Close() + } + mu <- g + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"deal_id":"d1","contract_id":"c1","version_id":"v1"}`)) + })) + defer srv.Close() + + root := t.TempDir() + cfg := parseSpec(t, multipartSpec) + if errs := cfg.Validate(); len(errs) != 0 { + t.Fatalf("spec invalid: %v", errs) + } + // The staging methods must be generated from the multipart route alone. + for _, want := range []string{"uploadx.upload_begin", "uploadx.upload_chunk", "uploadx.upload_abort"} { + found := false + for i := range cfg.Methods { + if cfg.Methods[i].Name == want { + found = true + } + } + if !found { + t.Fatalf("%s was not injected; a multipart route with no way to stage is unusable", want) + } + } + + proj := filepath.Join(root, "proj") + if _, err := Generate(cfg, proj); err != nil { + t.Fatalf("generate: %v", err) + } + if sum, err := os.ReadFile(filepath.Join("..", "..", "go.sum")); err == nil { + _ = os.WriteFile(filepath.Join(proj, "go.sum"), sum, 0o644) + } + + bin := filepath.Join(root, "adapter") + build := exec.Command("go", "build", "-o", bin, "./cmd/"+cfg.BinaryName) + build.Dir = proj + build.Env = append(os.Environ(), "GOFLAGS=-mod=mod") + if out, err := build.CombinedOutput(); err != nil { + t.Fatalf("build adapter: %v\n%s", err, out) + } + + sockDir, err := os.MkdirTemp("", "upsk") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(sockDir) + sock := filepath.Join(sockDir, "a.sock") + adapter := exec.Command(bin, "--socket", sock, "--manifest", filepath.Join(proj, "manifest.json")) + adapter.Stderr = os.Stderr + adapter.Env = append(os.Environ(), "UPLOADX_BACKEND_URL="+srv.URL) + if err := adapter.Start(); err != nil { + t.Fatalf("start adapter: %v", err) + } + defer func() { _ = adapter.Process.Kill(); _, _ = adapter.Process.Wait() }() + + deadline := time.Now().Add(15 * time.Second) + for time.Now().Before(deadline) { + if _, err := os.Stat(sock); err == nil { + break + } + time.Sleep(20 * time.Millisecond) + } + + call := func(t *testing.T, method, args string) (json.RawMessage, error) { + t.Helper() + conn, err := net.DialTimeout("unix", sock, 5*time.Second) + if err != nil { + t.Fatalf("dial: %v", err) + } + defer conn.Close() + var out json.RawMessage + err = ipc.Call(conn, method, json.RawMessage(args), &out) + return out, err + } + mustCall := func(t *testing.T, method, args string) json.RawMessage { + t.Helper() + out, err := call(t, method, args) + if err != nil { + t.Fatalf("call %s: %v", method, err) + } + return out + } + + // A 3 MiB document: three times the 1 MiB frame, so it provably cannot have + // arrived in one envelope. + content := make([]byte, 3<<20) + _, _ = mrand.New(mrand.NewSource(7)).Read(content) + sum := sha256.Sum256(content) + wantSHA := hex.EncodeToString(sum[:]) + + stageFile := func(t *testing.T, name, ctype string, body []byte) string { + t.Helper() + s := sha256.Sum256(body) + beginArgs, _ := json.Marshal(map[string]any{ + "file_name": name, "content_type": ctype, + "total_bytes": len(body), "sha256": hex.EncodeToString(s[:]), + }) + var begin struct { + BlobID string `json:"blob_id"` + MaxChunk int `json:"max_chunk_bytes"` + } + if err := json.Unmarshal(mustCall(t, "uploadx.upload_begin", string(beginArgs)), &begin); err != nil { + t.Fatalf("decode upload_begin: %v", err) + } + if begin.BlobID == "" || begin.MaxChunk <= 0 { + t.Fatalf("upload_begin returned %+v", begin) + } + chunks := 0 + for seq, off := 0, 0; off < len(body); seq, off = seq+1, off+begin.MaxChunk { + end := off + begin.MaxChunk + if end > len(body) { + end = len(body) + } + args, _ := json.Marshal(map[string]any{ + "blob_id": begin.BlobID, "seq": seq, + "data_base64": base64.StdEncoding.EncodeToString(body[off:end]), + }) + // Every staged envelope must fit the frame, or the design does not work. + if len(args) > ipc.MaxFrameSize { + t.Fatalf("chunk envelope is %d bytes, over the %d-byte frame", len(args), ipc.MaxFrameSize) + } + var got struct { + Complete bool `json:"complete"` + Received int64 `json:"received"` + } + if err := json.Unmarshal(mustCall(t, "uploadx.upload_chunk", string(args)), &got); err != nil { + t.Fatalf("decode upload_chunk: %v", err) + } + chunks++ + wantComplete := end == len(body) + if got.Complete != wantComplete { + t.Fatalf("chunk %d: complete = %v, want %v (received %d of %d)", seq, got.Complete, wantComplete, got.Received, len(body)) + } + } + if len(body) > ipc.MaxFrameSize && chunks < 2 { + t.Fatalf("a %d-byte file staged in %d chunk(s); it cannot have fit the frame", len(body), chunks) + } + return begin.BlobID + } + + // --- the headline path ------------------------------------------------- + blobID := stageFile(t, "mutual-nda.docx", + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", content) + + args, _ := json.Marshal(map[string]any{ + "blob_id": blobID, + "deal_id": "deal_abc", + "context_for_legal": "Standard mutual NDA, we are the disclosing party.", + }) + mustCall(t, "uploadx.document_upload", string(args)) + + select { + case g := <-mu: + if g.path != "/api/v1/documents" { + t.Errorf("path = %q", g.path) + } + if g.sha != wantSHA { + t.Errorf("document corrupted end to end:\n got %s (%d bytes)\nwant %s (%d bytes)", g.sha, g.size, wantSHA, len(content)) + } + if g.fileField != "file" { + t.Errorf("file field = %q, want the configured %q", g.fileField, "file") + } + if g.fileName != "mutual-nda.docx" { + t.Errorf("file name = %q", g.fileName) + } + if !strings.Contains(g.filePartCT, "wordprocessingml") { + t.Errorf("file part Content-Type = %q, want the declared docx type", g.filePartCT) + } + if g.fields["deal_id"] != "deal_abc" { + t.Errorf("deal_id = %q", g.fields["deal_id"]) + } + if g.fields["context_for_legal"] == "" { + t.Error("context_for_legal did not reach the partner") + } + if _, leaked := g.fields["blob_id"]; leaked { + t.Error("blob_id was forwarded as a form field; it is adapter-internal") + } + case <-time.After(20 * time.Second): + t.Fatal("partner never received the upload") + } + + // --- a path param must still resolve, and not become a form field ------ + small := []byte("second document, much smaller") + blob2 := stageFile(t, "v2.txt", "text/plain", small) + args2, _ := json.Marshal(map[string]any{"blob_id": blob2, "contract_id": "c-42"}) + mustCall(t, "uploadx.version_replace", string(args2)) + + select { + case g := <-mu: + if g.path != "/api/v1/contracts/c-42/versions" { + t.Errorf("path param not substituted: %q", g.path) + } + if _, leaked := g.fields["contract_id"]; leaked { + t.Error("contract_id went into the form as well as the path") + } + if g.fileField != "document" { + t.Errorf("file field = %q, want the per-route %q", g.fileField, "document") + } + case <-time.After(20 * time.Second): + t.Fatal("partner never received the second upload") + } + + // --- failure paths, over the real transport ---------------------------- + + // A blob is single-use: the adapter drops the local copy once the partner + // has the bytes, so replaying the same id must fail rather than re-send. + if _, err := call(t, "uploadx.document_upload", string(args)); err == nil { + t.Error("re-using a consumed blob_id succeeded; the staged bytes should be gone") + } + + // A checksum that does not match the staged bytes must be caught locally. + badArgs, _ := json.Marshal(map[string]any{ + "file_name": "x.txt", "content_type": "text/plain", + "total_bytes": 4, "sha256": strings.Repeat("a", 64), + }) + var bad struct { + BlobID string `json:"blob_id"` + } + _ = json.Unmarshal(mustCall(t, "uploadx.upload_begin", string(badArgs)), &bad) + chunkArgs, _ := json.Marshal(map[string]any{ + "blob_id": bad.BlobID, "seq": 0, + "data_base64": base64.StdEncoding.EncodeToString([]byte("abcd")), + }) + if _, err := call(t, "uploadx.upload_chunk", string(chunkArgs)); err == nil { + t.Error("a chunk completing a blob whose sha256 does not match was accepted") + } + + // Uploading without staging anything must say so, not send an empty file. + if _, err := call(t, "uploadx.document_upload", `{"deal_id":"d"}`); err == nil { + t.Error("upload with no blob_id succeeded") + } + + // abort makes the blob unusable. + blob3 := stageFile(t, "gone.txt", "text/plain", []byte("temporary")) + mustCall(t, "uploadx.upload_abort", `{"blob_id":"`+blob3+`"}`) + a3, _ := json.Marshal(map[string]any{"blob_id": blob3, "deal_id": "d"}) + if _, err := call(t, "uploadx.document_upload", string(a3)); err == nil { + t.Error("uploading an aborted blob succeeded") + } + + // The generated help must advertise the staging steps, or an agent that + // installs this app has no way to discover how to upload at all. + help := mustCall(t, "uploadx.help", `{}`) + for _, want := range []string{"upload_begin", "upload_chunk", "document_upload"} { + if !strings.Contains(string(help), want) { + t.Errorf("uploadx.help does not mention %q", want) + } + } +} diff --git a/submissions/io.pilot.generallegal/ab-commands.json b/submissions/io.pilot.generallegal/ab-commands.json new file mode 100644 index 0000000..5df0f20 --- /dev/null +++ b/submissions/io.pilot.generallegal/ab-commands.json @@ -0,0 +1,4 @@ +{ + "note": "http-backend app: there is no vanilla CLI to diff against, so the A/B job has nothing to compare. The runtime proof is internal/scaffold/zz_multipart_e2e_test.go and zz_mcp_e2e_test.go, which build and run this adapter for real.", + "commands": [] +} diff --git a/submissions/io.pilot.generallegal/io.pilot.generallegal-0.1.0-linux-amd64.tar.gz b/submissions/io.pilot.generallegal/io.pilot.generallegal-0.1.0-linux-amd64.tar.gz new file mode 100644 index 0000000..a79159c Binary files /dev/null and b/submissions/io.pilot.generallegal/io.pilot.generallegal-0.1.0-linux-amd64.tar.gz differ diff --git a/submissions/io.pilot.generallegal/metadata.json b/submissions/io.pilot.generallegal/metadata.json new file mode 100644 index 0000000..36eae25 --- /dev/null +++ b/submissions/io.pilot.generallegal/metadata.json @@ -0,0 +1,119 @@ +{ + "schema_version": 1, + "id": "io.pilot.generallegal", + "display_name": "General Legal", + "tagline": "Attorney-backed contract review and Delaware company formation for agents", + "description_md": "General Legal is a Y Combinator-backed law firm. This app puts a licensed\nattorney and a Delaware filing desk behind your agent — contract review and\ncompany formation, in one namespace.\n\nThe two halves have different requirements. Formation works the moment you\ninstall. Contract review needs a General Legal account and an API key.\n\n### Company formation — nothing to set up\n\nNo account, no key, no configuration. Call\n`generallegal.formation_options` and it works on a fresh install.\n\nYou will still need to pay: `formation_start_llc` and\n`formation_start_c_corp` return a payment link, and the founder pays there.\nUse the founder's real details — this files an actual Delaware company.\n\n### Contract review — bring your own API key\n\n1. Sign up at https://portal.general.legal/signup\n2. Open the account menu and choose **API keys**\n (https://portal.general.legal/api-keys)\n3. Create a key and copy it — the full value is shown **once**\n4. Import it into the app:\n\n```\nprintf '{\"GENERAL_LEGAL_API_KEY\":\"glk_YOUR_KEY\"}' \u003e ~/.pilot/apps/io.pilot.generallegal/secrets.json\nchmod 600 ~/.pilot/apps/io.pilot.generallegal/secrets.json\npilotctl appstore restart io.pilot.generallegal\n```\n\nThe restart matters: the key is read at startup. Verify with\n`pilotctl appstore call io.pilot.generallegal generallegal.deals_list '{}'`.\nThe key stays on your machine, is never sent to the formation service, and\nscopes you to your own General Legal organization.\n\n### What it costs\n\nContract review is flat-fee, with no hourly billing and no minimums. The fee\ncovers every turn through signature, including negotiation with the\ncounterparty.\n\n| Work | Price |\n| --- | --- |\n| Contract, 3 pages or fewer | $250 |\n| Contract, 3-50 pages | $500 |\n| Contract, 50+ pages | $10 per page |\n| Drafting from scratch | $2,000 |\n| Delaware LLC | $190 instant / $210 standard / $260 next-day / $310 same-day |\n| Delaware C-corp | $218 standard / $268 next-day / $318 same-day |\n\n### What the app does\n\n**Company formation** — `formation_options` (free), `formation_start_llc`\n(**paid**), `formation_start_c_corp` (**paid**), `formation_status`,\n`formation_update`, `formation_documents` (free).\n\n**Contract review** — `deal_open` (**paid**), `document_upload` (**paid**),\n`thread_post` (**paid**, covered by the matter's flat fee), plus\n`deals_list`, `deal_get`, `thread_get`, `contracts_list`, `contract_get`,\n`version_download_link` (free) and `upload_begin`, `upload_chunk`,\n`upload_abort` (free — a document is staged in chunks because a single call\ncannot carry a file).\n\n### What costs money\n\nFive methods spend real money and will not warn you first:\n`formation_start_llc`, `formation_start_c_corp`, `deal_open`,\n`document_upload` and `thread_post`. Every other method is free.\n`generallegal.help` lists them under `billable_methods` with the price.\n", + "vendor": { + "name": "" + }, + "license": "Apache-2.0", + "categories": [ + "legal", + "business" + ], + "keywords": [ + "legal", + "contracts", + "attorney", + "review", + "nda", + "redline" + ], + "size": { + "bundle_bytes": 0, + "installed_bytes": 0 + }, + "compat": { + "min_pilot_version": "1.0.0", + "runtimes": [ + "go" + ] + }, + "methods": [ + { + "name": "generallegal.deals_list", + "summary": "List your matters (paginated, optional status filter)." + }, + { + "name": "generallegal.deal_open", + "summary": "Open a new matter from a written request. Reaches a real attorney." + }, + { + "name": "generallegal.deal_get", + "summary": "One matter with its documents and released versions." + }, + { + "name": "generallegal.thread_get", + "summary": "Read the lawyer-client thread on a matter." + }, + { + "name": "generallegal.thread_post", + "summary": "Reply to the attorney on a matter's thread." + }, + { + "name": "generallegal.contracts_list", + "summary": "List your documents." + }, + { + "name": "generallegal.contract_get", + "summary": "One document with its released versions (version ids feed downloads)." + }, + { + "name": "generallegal.document_upload", + "summary": "Upload a DOCX, PDF, PNG, JPEG, or Markdown document for AI + attorney review (20 MiB max). Stage the bytes with generallegal.upload_begin/generallegal.upload_chunk first." + }, + { + "name": "generallegal.version_download_link", + "summary": "Issue a short-lived (~15 min) direct download URL for a released version." + }, + { + "name": "generallegal.formation_options", + "summary": "Price every Delaware company type and filing speed, itemised. No account and no credentials needed — this works on a fresh install. Call it first and present every option and total to the founder before filing anything." + }, + { + "name": "generallegal.formation_start_llc", + "summary": "Start a sole-member Delaware LLC and get back a payment link plus a formation_id. No account or credentials needed to call it. The founder acts as member, manager and AI oversight officer. filing_speed 'instant' hands over a pre-formed 'GL AgentCo \u003cn\u003e, LLC' and requires company_name to be omitted; the other speeds file a new LLC under a name you choose. Call generallegal.formation_options first and let the founder pick." + }, + { + "name": "generallegal.formation_start_c_corp", + "summary": "Start a Delaware C-corp and get back a payment link plus a formation_id. No account or credentials needed to call it. The founder acts as sole incorporator. Suggest an LLC first unless corporate stock or fundraising justifies a C-corp. Call generallegal.formation_options first and let the founder pick." + }, + { + "name": "generallegal.formation_status", + "summary": "Poll a formation's progress. No account or credentials needed to call it. Repeat after poll_after_seconds for as long as that field comes back." + }, + { + "name": "generallegal.formation_update", + "summary": "Change the company name before the documents are generated. No account or credentials needed to call it. Late changes are rejected." + }, + { + "name": "generallegal.formation_documents", + "summary": "Get short-lived links to a completed formation's documents. No account or credentials needed to call it. Call again once they expire." + }, + { + "name": "generallegal.upload_begin", + "summary": "Start a staged upload and get a blob_id. The file does NOT travel in one IPC message — an envelope is capped at 1 MiB — so declare it here ({\"file_name\",\"content_type\",\"total_bytes\",\"sha256\"}), push the bytes with generallegal.upload_chunk, then pass the blob_id to the upload method." + }, + { + "name": "generallegal.upload_chunk", + "summary": "Append the next chunk of a staged upload: {\"blob_id\",\"seq\" (0-based, strictly sequential),\"data_base64\"}. Send at most 512 KiB of raw bytes per call. Returns {\"received\",\"total_bytes\",\"complete\"}; when complete, the sha256 you declared has been verified and the blob is ready to send." + }, + { + "name": "generallegal.upload_abort", + "summary": "Discard a staged upload and its bytes: {\"blob_id\"}. Staged uploads are also reclaimed automatically once they go stale." + }, + { + "name": "generallegal.help", + "summary": "Discovery: every method with params, kind, and latency class." + } + ], + "changelog": [ + { + "version": "0.1.0", + "notes": [ + "Released v0.1.0" + ] + } + ] +} diff --git a/submissions/io.pilot.generallegal/submission.json b/submissions/io.pilot.generallegal/submission.json new file mode 100644 index 0000000..de6d62e --- /dev/null +++ b/submissions/io.pilot.generallegal/submission.json @@ -0,0 +1,551 @@ +{ + "product_demo": { + "skill": "io.pilot.generallegal", + "title": "Full usage demo", + "metered": false, + "when_to_use": "When a contract needs a licensed attorney to review, redline or draft it, or when an agent needs its own Delaware company. Formation works on a fresh install; contract review needs your own General Legal API key.", + "quickstart": { + "goal": "Price a Delaware company \u2014 works on a fresh install, no account, no key", + "command": "pilotctl appstore call io.pilot.generallegal generallegal.formation_options '{\"entity_type\":\"llc\"}'", + "expect": "{\"options\":[{\"filing_speed\":\"instant\",\"total_cents\":19000},{\"filing_speed\":\"standard\",\"total_cents\":21000}]}", + "note": "Free, and it needs no account or key at all \u2014 the fastest way to confirm the app works. Contract review is the half that needs a key; see the examples." + }, + "examples": [ + { + "title": "Form a Delaware company \u2014 no API key needed", + "goal": "Price it, file it, hand the founder a payment link", + "command": "pilotctl appstore call io.pilot.generallegal generallegal.formation_start_llc '{\"filing_speed\":\"standard\",\"company_name\":\"NewCo LLC\",\"founder\":{\"full_name\":\"Ada Lovelace\",\"email\":\"ada@example.com\"},\"principal_address\":{\"street\":\"1 Main St\",\"city\":\"Dover\",\"state\":\"DE\",\"postal_code\":\"19901\"},\"ai_agent_description\":\"Procurement agent\",\"authority_limits\":\"No commitments above $5,000 without sign-off\",\"contract_threshold\":\"$5,000\"}'", + "expect": "{\"formation_id\":\"f-...\",\"payment_url\":\"https://...\",\"status\":\"awaiting_payment\"}", + "note": "BILLABLE - files a real company. $190 instant / $210 standard / $260 next-day / $310 same-day, paid by the founder at the returned link. No account or key needed to call it. formation_id is shown once." + }, + { + "title": "Track the filing and collect the paperwork", + "goal": "Poll to completion, then pull the documents", + "command": "pilotctl appstore call io.pilot.generallegal generallegal.formation_status '{\"formation_id\":\"f-...\"}'", + "expect": "{\"status\":\"filed\",\"poll_after_seconds\":30} then {\"status\":\"complete\"}", + "note": "Free. Repeat only after poll_after_seconds. When complete, generallegal.formation_documents returns short-lived links." + }, + { + "title": "Bring your own key, then confirm it works", + "goal": "Authenticate as your own General Legal organization", + "command": "pilotctl appstore call io.pilot.generallegal generallegal.deals_list '{\"page\":1,\"page_size\":5}'", + "expect": "{\"items\":[...],\"total\":n} once the key is in place; 401 until then", + "note": "Contract review only. Sign up at https://portal.general.legal/signup, then account menu -> API keys (shown once). Import: printf '{\"GENERAL_LEGAL_API_KEY\":\"glk_YOUR_KEY\"}' > ~/.pilot/apps/io.pilot.generallegal/secrets.json && chmod 600 ~/.pilot/apps/io.pilot.generallegal/secrets.json && pilotctl appstore restart io.pilot.generallegal" + }, + { + "title": "Ask an attorney to review a contract", + "goal": "Open a matter a real lawyer picks up", + "command": "pilotctl appstore call io.pilot.generallegal generallegal.deal_open '{\"initial_request\":\"Please review this mutual NDA. We are the disclosing party; flag anything unusual in the confidentiality term.\",\"deal_name\":\"Acme mutual NDA\"}'", + "expect": "{\"deal_id\":\"d-9f3...\",\"status\":\"open\"}", + "note": "BILLABLE - flat fee per contract: $250 (<=3 pages), $500 (3-50), $10/page (50+), $2,000 to draft. Covers every turn through signature. Keep the deal_id." + }, + { + "title": "Stage the document, then send it", + "goal": "Push the bytes in chunks (a file cannot cross in one message), then upload", + "command": "pilotctl appstore call io.pilot.generallegal generallegal.upload_begin '{\"file_name\":\"nda.docx\",\"content_type\":\"application/vnd.openxmlformats-officedocument.wordprocessingml.document\",\"total_bytes\":3145728,\"sha256\":\"\"}'", + "expect": "{\"blob_id\":\"a1b2...\",\"max_chunk_bytes\":524288,\"next_seq\":0}", + "note": "Then generallegal.upload_chunk with seq 0,1,2... and base64 of at most max_chunk_bytes raw bytes. The last returns complete:true. Finally generallegal.document_upload with the blob_id and deal_id (BILLABLE)." + }, + { + "title": "Collect the released redline", + "goal": "Get a downloadable link to counsel's version", + "command": "pilotctl appstore call io.pilot.generallegal generallegal.version_download_link '{\"version_id\":\"v-07...\"}'", + "expect": "{\"file_name\":\"nda-redline.docx\",\"download_url\":\"https://...\",\"download_token_expires_at\":\"...\"}", + "note": "Free. The URL needs no auth and expires in ~15 minutes - fetch it yourself. Find version ids via generallegal.deal_get or generallegal.contract_get." + } + ], + "gotchas": [ + "Two halves, different requirements. Company formation needs no account and no key. Contract review needs a General Legal account and an API key imported.", + "For contract review: sign up at portal.general.legal/signup, mint a key under API keys, write it to $APP/secrets.json, then restart the app - the key is read at startup.", + "Five methods spend money: deal_open and document_upload are flat-fee per contract ($250/$500/$10-per-page/$2,000); formation_start_llc and formation_start_c_corp file a real company ($190-$318); thread_post is covered by the matter's fee.", + "A document cannot ride in one call. Use upload_begin then upload_chunk (<=512 KiB raw each) and pass the blob_id. A blob is single-use and dropped once sent.", + "Matters reach a real attorney and formations file a real company. Neither is a sandbox - confirm before calling a billable method.", + "Contract-review limits are per organization: 120 reads/min, 25 new matters/day, 50 uploads/day. Honour Retry-After on a 429." + ], + "next": [ + "generallegal.formation_options to price a company, or generallegal.deals_list to see the matters your key can reach.", + "generallegal.help lists every method and marks exactly which ones cost money." + ] + }, + "next_steps": { + "schema": 1, + "app": "io.pilot.generallegal", + "edges": [ + { + "from": "*", + "on": "err", + "match": "401|[Uu]nauthorized", + "why": "contract review needs your own General Legal API key; company formation needs none and works right now", + "then": [ + { + "cmd": "pilotctl appstore view io.pilot.generallegal", + "why": "the store page carries the exact key-import command; mint the key at portal.general.legal/api-keys first", + "kind": "gateway" + }, + { + "cmd": "pilotctl appstore call io.pilot.generallegal generallegal.formation_options '{\"entity_type\":\"llc\"}'", + "why": "company formation is the other half of this app and needs no account or key at all \u2014 free to explore any time", + "kind": "gateway" + } + ] + }, + { + "from": "generallegal.deals_list", + "on": "ok", + "why": "your key works; open a new matter or inspect one you already have", + "then": [ + { + "cmd": "pilotctl appstore call io.pilot.generallegal generallegal.deal_open '{\"initial_request\":\"Please review this mutual NDA.\"}'", + "why": "opens a matter with a licensed attorney \u2014 billable, so confirm with whoever owns the account", + "kind": "flow" + }, + { + "cmd": "pilotctl appstore call io.pilot.generallegal generallegal.deal_get '{\"deal_id\":\"\"}'", + "why": "free, and it lists a matter's documents and released versions", + "kind": "flow" + }, + { + "cmd": "pilotctl appstore call io.pilot.generallegal generallegal.formation_options '{\"entity_type\":\"llc\"}'", + "why": "company formation is the other half of this app and needs no account or key at all \u2014 free to explore any time", + "kind": "gateway" + } + ] + }, + { + "from": "generallegal.deal_open", + "on": "ok", + "why": "the matter exists; give counsel the contract, or read what they have already asked", + "then": [ + { + "cmd": "pilotctl appstore call io.pilot.generallegal generallegal.upload_begin '{\"file_name\":\"nda.docx\",\"content_type\":\"application/vnd.openxmlformats-officedocument.wordprocessingml.document\",\"total_bytes\":3145728,\"sha256\":\"\"}'", + "why": "a document cannot cross in one message, so stage it first", + "kind": "gateway" + }, + { + "cmd": "pilotctl appstore call io.pilot.generallegal generallegal.thread_get '{\"deal_id\":\"\"}'", + "why": "free polling; counsel often asks a clarifying question before delivering", + "kind": "flow" + }, + { + "cmd": "pilotctl appstore call io.pilot.generallegal generallegal.formation_options '{\"entity_type\":\"llc\"}'", + "why": "company formation is the other half of this app and needs no account or key at all \u2014 free to explore any time", + "kind": "gateway" + } + ] + }, + { + "from": "generallegal.deal_get", + "on": "ok", + "why": "you can see the matter's documents; walk one's versions or answer counsel", + "then": [ + { + "cmd": "pilotctl appstore call io.pilot.generallegal generallegal.contract_get '{\"contract_id\":\"\"}'", + "why": "free, and its version ids are what the download link needs", + "kind": "flow" + }, + { + "cmd": "pilotctl appstore call io.pilot.generallegal generallegal.thread_get '{\"deal_id\":\"\"}'", + "why": "free, and it shows whether counsel is waiting on you", + "kind": "flow" + }, + { + "cmd": "pilotctl appstore call io.pilot.generallegal generallegal.formation_options '{\"entity_type\":\"llc\"}'", + "why": "company formation is the other half of this app and needs no account or key at all \u2014 free to explore any time", + "kind": "gateway" + } + ] + }, + { + "from": "generallegal.thread_get", + "on": "ok", + "why": "you have counsel's messages; answer them or go back to the matter itself", + "then": [ + { + "cmd": "pilotctl appstore call io.pilot.generallegal generallegal.thread_post '{\"deal_id\":\"\",\"text\":\"Yes, proceed on that basis.\"}'", + "why": "replies are billable attorney time; free when target is \"ai\"", + "kind": "flow" + }, + { + "cmd": "pilotctl appstore call io.pilot.generallegal generallegal.deal_get '{\"deal_id\":\"\"}'", + "why": "free, and it shows any version released since you last looked", + "kind": "flow" + }, + { + "cmd": "pilotctl appstore call io.pilot.generallegal generallegal.formation_options '{\"entity_type\":\"llc\"}'", + "why": "company formation is the other half of this app and needs no account or key at all \u2014 free to explore any time", + "kind": "gateway" + } + ] + }, + { + "from": "generallegal.thread_post", + "on": "ok", + "why": "your reply is with counsel; watch for their answer and for a released version", + "then": [ + { + "cmd": "pilotctl appstore call io.pilot.generallegal generallegal.thread_get '{\"deal_id\":\"\"}'", + "why": "free polling for the reply", + "kind": "flow" + }, + { + "cmd": "pilotctl appstore call io.pilot.generallegal generallegal.deal_get '{\"deal_id\":\"\"}'", + "why": "free, and a redline usually lands here rather than in the thread", + "kind": "flow" + }, + { + "cmd": "pilotctl appstore call io.pilot.generallegal generallegal.formation_options '{\"entity_type\":\"llc\"}'", + "why": "company formation is the other half of this app and needs no account or key at all \u2014 free to explore any time", + "kind": "gateway" + } + ] + }, + { + "from": "generallegal.contracts_list", + "on": "ok", + "why": "you have your documents; open one to reach its versions", + "then": [ + { + "cmd": "pilotctl appstore call io.pilot.generallegal generallegal.contract_get '{\"contract_id\":\"\"}'", + "why": "free, and the version ids feed the download link", + "kind": "flow" + }, + { + "cmd": "pilotctl appstore call io.pilot.generallegal generallegal.deals_list '{\"page\":1,\"page_size\":20}'", + "why": "free, and it shows which matter each document belongs to", + "kind": "flow" + }, + { + "cmd": "pilotctl appstore call io.pilot.generallegal generallegal.formation_options '{\"entity_type\":\"llc\"}'", + "why": "company formation is the other half of this app and needs no account or key at all \u2014 free to explore any time", + "kind": "gateway" + } + ] + }, + { + "from": "generallegal.contract_get", + "on": "ok", + "why": "you have the version ids, so you can pull the released document", + "then": [ + { + "cmd": "pilotctl appstore call io.pilot.generallegal generallegal.version_download_link '{\"version_id\":\"\"}'", + "why": "free; returns a short-lived unauthenticated URL you fetch yourself", + "kind": "flow" + }, + { + "cmd": "pilotctl appstore call io.pilot.generallegal generallegal.deals_list '{\"page\":1,\"page_size\":20}'", + "why": "free, and it takes you back to everything currently open", + "kind": "flow" + }, + { + "cmd": "pilotctl appstore call io.pilot.generallegal generallegal.formation_options '{\"entity_type\":\"llc\"}'", + "why": "company formation is the other half of this app and needs no account or key at all \u2014 free to explore any time", + "kind": "gateway" + } + ] + }, + { + "from": "generallegal.version_download_link", + "on": "ok", + "why": "the link expires in about 15 minutes, so fetch it now, then carry on with the matter", + "then": [ + { + "cmd": "pilotctl appstore call io.pilot.generallegal generallegal.thread_get '{\"deal_id\":\"\"}'", + "why": "free, and counsel's notes usually explain what changed in the version", + "kind": "flow" + }, + { + "cmd": "pilotctl appstore call io.pilot.generallegal generallegal.deals_list '{\"page\":1,\"page_size\":20}'", + "why": "free, and it shows what else is still open", + "kind": "flow" + }, + { + "cmd": "pilotctl appstore call io.pilot.generallegal generallegal.formation_options '{\"entity_type\":\"llc\"}'", + "why": "company formation is the other half of this app and needs no account or key at all \u2014 free to explore any time", + "kind": "gateway" + } + ] + }, + { + "from": "generallegal.upload_begin", + "on": "ok", + "why": "you have a blob_id and the chunk size; push the bytes in order until complete is true", + "then": [ + { + "cmd": "pilotctl appstore call io.pilot.generallegal generallegal.upload_chunk '{\"blob_id\":\"\",\"seq\":0,\"data_base64\":\"\"}'", + "why": "chunks must be strictly sequential; the last one verifies your sha256 and seals the blob", + "kind": "flow" + }, + { + "cmd": "pilotctl appstore call io.pilot.generallegal generallegal.formation_options '{\"entity_type\":\"llc\"}'", + "why": "company formation is the other half of this app and needs no account or key at all \u2014 free to explore any time", + "kind": "gateway" + } + ] + }, + { + "from": "generallegal.upload_chunk", + "on": "ok", + "why": "the bytes are staged and the checksum verified, so the document can be sent", + "then": [ + { + "cmd": "pilotctl appstore call io.pilot.generallegal generallegal.document_upload '{\"blob_id\":\"\",\"deal_id\":\"\",\"context_for_legal\":\"Standard mutual NDA.\"}'", + "why": "sends it into AI + attorney review \u2014 billable on your engagement", + "kind": "flow" + }, + { + "cmd": "pilotctl appstore call io.pilot.generallegal generallegal.formation_options '{\"entity_type\":\"llc\"}'", + "why": "company formation is the other half of this app and needs no account or key at all \u2014 free to explore any time", + "kind": "gateway" + } + ], + "match": "\"complete\"\\s*:\\s*true" + }, + { + "from": "generallegal.document_upload", + "on": "ok", + "why": "review has started; watch the thread and the matter for what counsel returns", + "then": [ + { + "cmd": "pilotctl appstore call io.pilot.generallegal generallegal.thread_get '{\"deal_id\":\"\"}'", + "why": "free polling; counsel often asks something before delivering", + "kind": "flow" + }, + { + "cmd": "pilotctl appstore call io.pilot.generallegal generallegal.deal_get '{\"deal_id\":\"\"}'", + "why": "free, and released versions appear here as they land", + "kind": "flow" + }, + { + "cmd": "pilotctl appstore call io.pilot.generallegal generallegal.formation_options '{\"entity_type\":\"llc\"}'", + "why": "company formation is the other half of this app and needs no account or key at all \u2014 free to explore any time", + "kind": "gateway" + } + ] + }, + { + "from": "generallegal.formation_options", + "on": "ok", + "why": "you have every option and its total; present them and let the founder choose before filing anything", + "then": [ + { + "cmd": "pilotctl appstore call io.pilot.generallegal generallegal.formation_start_llc '{\"filing_speed\":\"standard\",\"company_name\":\"NewCo LLC\",\"founder\":{\"full_name\":\"...\",\"email\":\"...\"},\"principal_address\":{\"street\":\"...\",\"city\":\"...\",\"state\":\"DE\",\"postal_code\":\"...\"},\"ai_agent_description\":\"...\",\"authority_limits\":\"...\",\"contract_threshold\":\"...\"}'", + "why": "the simpler default; files a real company and returns a payment link", + "kind": "flow" + }, + { + "cmd": "pilotctl appstore call io.pilot.generallegal generallegal.formation_start_c_corp '{\"filing_speed\":\"standard\",\"founder\":{\"full_name\":\"...\",\"email\":\"...\"},\"principal_address\":{\"street\":\"...\",\"city\":\"...\",\"state\":\"DE\",\"postal_code\":\"...\"},\"ai_agent_description\":\"...\",\"authority_limits\":\"...\",\"contract_threshold\":\"...\",\"director\":{\"full_name\":\"...\",\"email\":\"...\"},\"officer\":{\"full_name\":\"...\",\"email\":\"...\"}}'", + "why": "only when corporate stock or fundraising justifies it", + "kind": "flow" + } + ] + }, + { + "from": "generallegal.formation_start_llc", + "on": "ok", + "why": "the filing is open and the founder has a payment link; store formation_id, it grants full access", + "then": [ + { + "cmd": "pilotctl appstore call io.pilot.generallegal generallegal.formation_status '{\"formation_id\":\"\"}'", + "why": "poll while poll_after_seconds keeps coming back", + "kind": "gateway" + }, + { + "cmd": "pilotctl appstore call io.pilot.generallegal generallegal.formation_update '{\"formation_id\":\"\",\"company_name\":\"Better Name LLC\"}'", + "why": "renaming is only possible before the documents are generated", + "kind": "flow" + } + ] + }, + { + "from": "generallegal.formation_start_c_corp", + "on": "ok", + "why": "the filing is open and the founder has a payment link; store formation_id, it grants full access", + "then": [ + { + "cmd": "pilotctl appstore call io.pilot.generallegal generallegal.formation_status '{\"formation_id\":\"\"}'", + "why": "poll while poll_after_seconds keeps coming back", + "kind": "gateway" + }, + { + "cmd": "pilotctl appstore call io.pilot.generallegal generallegal.formation_update '{\"formation_id\":\"\",\"company_name\":\"Better Name Inc\"}'", + "why": "renaming is only possible before the documents are generated", + "kind": "flow" + } + ] + }, + { + "from": "generallegal.formation_update", + "on": "ok", + "why": "the name is changed; confirm it took and carry on watching the filing", + "then": [ + { + "cmd": "pilotctl appstore call io.pilot.generallegal generallegal.formation_status '{\"formation_id\":\"\"}'", + "why": "confirms the new name is the one being filed", + "kind": "flow" + }, + { + "cmd": "pilotctl appstore call io.pilot.generallegal generallegal.formation_options '{\"entity_type\":\"llc\"}'", + "why": "company formation is the other half of this app and needs no account or key at all \u2014 free to explore any time", + "kind": "flow" + } + ] + }, + { + "from": "generallegal.formation_status", + "on": "ok", + "why": "the filing is still moving; wait the interval it gave you rather than polling tightly", + "then": [ + { + "cmd": "pilotctl appstore call io.pilot.generallegal generallegal.formation_status '{\"formation_id\":\"\"}'", + "why": "repeat only after poll_after_seconds", + "kind": "recovery" + } + ], + "match": "poll_after_seconds" + }, + { + "from": "generallegal.formation_status", + "on": "ok", + "why": "the company exists; collect the paperwork", + "then": [ + { + "cmd": "pilotctl appstore call io.pilot.generallegal generallegal.formation_documents '{\"formation_id\":\"\"}'", + "why": "returns short-lived links; call again once they expire", + "kind": "flow" + } + ], + "match": "complete|filed|ready" + }, + { + "from": "generallegal.formation_documents", + "on": "ok", + "why": "the entity is formed and papered; the other half of this app puts an attorney on its contracts", + "then": [ + { + "cmd": "pilotctl appstore call io.pilot.generallegal generallegal.deal_open '{\"initial_request\":\"We just incorporated. Please review our standard mutual NDA.\"}'", + "why": "needs a General Legal account and an imported API key; billable attorney work", + "kind": "gateway" + }, + { + "cmd": "pilotctl appstore call io.pilot.generallegal generallegal.formation_status '{\"formation_id\":\"\"}'", + "why": "free, and it re-checks state if a document link looked wrong", + "kind": "flow" + } + ] + }, + { + "from": "*", + "on": "err", + "match": "429", + "why": "the organization's rate budget is spent; every key on the account shares it", + "then": [ + { + "cmd": "pilotctl appstore call io.pilot.generallegal generallegal.deals_list '{\"page\":1,\"page_size\":20}'", + "why": "wait out Retry-After, then resume with a paginated read instead of a tight poll", + "kind": "recovery" + }, + { + "cmd": "pilotctl appstore call io.pilot.generallegal generallegal.formation_options '{\"entity_type\":\"llc\"}'", + "why": "company formation is the other half of this app and needs no account or key at all \u2014 free to explore any time", + "kind": "gateway" + } + ] + } + ] + }, + "id": "io.pilot.generallegal", + "version": "0.1.0", + "namespace": "generallegal", + "description": "Attorney-backed contract review and Delaware company formation for agents. Flat-fee legal work from a licensed US law firm.", + "email": "alex@vulturelabs.io", + "bundle": "io.pilot.generallegal-0.1.0-linux-amd64.tar.gz", + "bundle_sha256": "9136d92a1d0fbe99067f2413cecf17466373cf432b3be92b264124ed8cb9c0c0", + "bundles": { + "darwin-amd64": { + "url": "https://artifacts.pilotprotocol.network/bundles/io.pilot.generallegal/0.1.0/darwin-amd64/io.pilot.generallegal-0.1.0-darwin-amd64.tar.gz", + "sha256": "85314abb6e36f95995eadc36ea259046d6ed4dc270b04cfd8e300855392a7554", + "bytes": 5232940 + }, + "darwin-arm64": { + "url": "https://artifacts.pilotprotocol.network/bundles/io.pilot.generallegal/0.1.0/darwin-arm64/io.pilot.generallegal-0.1.0-darwin-arm64.tar.gz", + "sha256": "38b511c449d815e846eadac47a239d4e5a6999b1db4ccc5e2c38a7addb6f7791", + "bytes": 4882961 + }, + "linux-amd64": { + "url": "https://artifacts.pilotprotocol.network/bundles/io.pilot.generallegal/0.1.0/linux-amd64/io.pilot.generallegal-0.1.0-linux-amd64.tar.gz", + "sha256": "9136d92a1d0fbe99067f2413cecf17466373cf432b3be92b264124ed8cb9c0c0", + "bytes": 5074207 + }, + "linux-arm64": { + "url": "https://artifacts.pilotprotocol.network/bundles/io.pilot.generallegal/0.1.0/linux-arm64/io.pilot.generallegal-0.1.0-linux-arm64.tar.gz", + "sha256": "6c02a65f12a11498762d42da705d1a8b46d70efe5955a6d0797cc69cd2482d19", + "bytes": 4630175 + } + }, + "publisher": "ed25519:Tl5OtWqZUPSGJkt7+Z2uOk+0YjvlxBsQdN9Rp6ZI3lE=", + "exposes": [ + "generallegal.deals_list", + "generallegal.deal_open", + "generallegal.deal_get", + "generallegal.thread_get", + "generallegal.thread_post", + "generallegal.contracts_list", + "generallegal.contract_get", + "generallegal.document_upload", + "generallegal.version_download_link", + "generallegal.formation_options", + "generallegal.formation_start_llc", + "generallegal.formation_start_c_corp", + "generallegal.formation_status", + "generallegal.formation_update", + "generallegal.formation_documents", + "generallegal.upload_begin", + "generallegal.upload_chunk", + "generallegal.upload_abort", + "generallegal.help" + ], + "grants": [ + { + "cap": "fs.read", + "target": "$APP/config.json" + }, + { + "cap": "fs.read", + "target": "$APP/secrets.json" + }, + { + "cap": "fs.read", + "target": "$APP/blobs" + }, + { + "cap": "fs.write", + "target": "$APP/blobs" + }, + { + "cap": "net.dial", + "target": "api.general.legal", + "if": { + "kind": "rate", + "params": { + "per": "min", + "limit": 120 + } + } + }, + { + "cap": "net.dial", + "target": "incorp-mcp.general.legal", + "if": { + "kind": "rate", + "params": { + "per": "min", + "limit": 120 + } + } + }, + { + "cap": "audit.log", + "target": "*" + } + ] +} \ No newline at end of file