From b31e0d68a17fff2a93518a85977da597cf14014f Mon Sep 17 00:00:00 2001 From: Traun Leyden Date: Thu, 9 Jul 2026 12:57:00 +0200 Subject: [PATCH 1/4] initial plan --- ...09-claude-ai-audio-transcription-upload.md | 182 ++++++++++++++++++ 1 file changed, 182 insertions(+) create mode 100644 docs/superpowers/plans/2026-07-09-claude-ai-audio-transcription-upload.md diff --git a/docs/superpowers/plans/2026-07-09-claude-ai-audio-transcription-upload.md b/docs/superpowers/plans/2026-07-09-claude-ai-audio-transcription-upload.md new file mode 100644 index 0000000..07547a4 --- /dev/null +++ b/docs/superpowers/plans/2026-07-09-claude-ai-audio-transcription-upload.md @@ -0,0 +1,182 @@ +# Plan: make `transcribe_audio_file` work from Claude.ai, not just ChatGPT + +## Verified problem statement + +`transcribe_audio_file` (`crates/platform/src/native_mcp_tools/whisper_transcribe.rs`) +only accepts an OpenAI-style file reference: + +```json +{ "audio_file": { "download_url": "...", "file_id": "...", ... } } +``` + +and `meta()` advertises `{"openai/fileParams": ["audio_file"]}` — an OpenAI +Apps SDK convention that tells ChatGPT to auto-bind an uploaded file to the +`audio_file` argument and hand the tool a temporary authorized +`download_url` it can `GET`. + +Claude.ai has no equivalent convention. When a user attaches an audio file +in a Claude.ai chat with "Code execution and file creation" enabled, the +bytes land on the code-execution **sandbox's own filesystem**, not behind a +fetchable `download_url`. There is no `_meta` hint Claude.ai honors that +would cause it to synthesize one. So `transcribe_audio_file` is currently +unreachable from Claude.ai — confirmed, not a misconfiguration. + +## Verified: is the proposed presigned-upload-URL pattern real? + +Yes, cross-checked against three independent sources (not just the +assistant-generated script, which I don't treat as authoritative on its +own): + +- **Official docs** (`platform.claude.com/docs/.../code-execution-tool`): + confirms user file uploads made through the **raw Messages API** use a + `container_upload` block + `file_id` from the Files API — that's the + API-level mechanism and it's a red herring for Claude.ai chat. +- **Claude Help Center** ("Create and edit files with Claude"): confirms + Settings → Capabilities → Code execution and file creation has an + "Additional allowed domains" network allowlist, and that this exists for + **individual Free/Pro/Max accounts**, not just Team/Enterprise. Also + confirms: *"If MCP integrations are enabled, network communication + remains possible through those connections regardless of the network + egress setting"* — i.e. the egress toggle only gates code running inside + the sandbox (a `curl` call), not MCP tool calls themselves. +- **Independent third-party implementation** (futuresearch.ai blog, "How to + Upload Large Files to an MCP Server Without Filling the Context Window") + describes the exact same three-step flow already in production: request a + presigned URL from an MCP tool → Claude runs `curl -X PUT` inside the + sandbox → a second MCP tool call exchanges an opaque ID for the parsed + result. Confirms presigned URLs need a short TTL (theirs: 5 min) and that + omitting the domain from "Additional allowed domains" fails the `curl` + with a network error even though the URL is valid. + +One caveat found and worth noting, not blocking: a filed GitHub issue +(`anthropics/claude-code#63182`) reports the **Team-plan org-level** +allowlist not reaching the sandbox proxy. Doesn't affect individual +accounts, which is presumably how this user runs Claude.ai. + +**Conclusion: the pattern is real and is the documented way to get sandbox-local +bytes into an MCP server. The assistant-generated script is not usable +as-is** — it's a standalone Python/FastMCP/faster-whisper server with no +auth, no framework fit, and it duplicates whisper-rs inference brain3 already +has natively in Rust. It also introduces an unauthenticated public HTTP +upload endpoint, which conflicts directly with this repo's security rule +("gateway is intentionally closed to preregistered clients only") and the +AGENTS.MD rule that new ingress points require a `SECURITY_AUDIT.md` threat +model update first. + +## Design goal + +Reuse brain3's existing native Rust transcription path (`whisper-rs`, +existing `max_audio_bytes` cap, existing audio decode/size-limit logic) and +add a second, Claude.ai-shaped way to *get bytes into* that path, without +weakening the "only the preregistered client reaches protected data" +guarantee. + +Two new pieces, one path reused: + +1. **New native MCP tool `request_audio_upload`** (mirrors + `WhisperTranscribeTool`'s shape in + `crates/platform/src/native_mcp_tools/`). Only reachable the same way + every other tool is: through the OAuth-protected `/mcp` route, so only the + preregistered client can ever obtain an upload token. Returns: + - a short-TTL (5 min, matching the pattern found in research), single-use, + cryptographically random `upload_id` + - the `curl -X PUT` command string Claude should run in its sandbox + - the size cap, sourced from the *same* `max_audio_bytes` config + (`config.native_audio_transcription.max_audio_bytes`) already used by + `transcribe_audio_file` — no second, divergent limit. + +2. **New unauthenticated-but-capability-scoped HTTP route** + `PUT /uploads/{upload_id}` in `crates/platform/src/http/` (new + `uploads.rs` handler, wired into both `router.rs` builders). Unauthenticated + because the sandbox's `curl` can't carry the OAuth bearer token — but the + `upload_id` itself is the capability: unguessable, single-use, expires in + 5 minutes, and can only ever have been minted by a call to + `request_audio_upload` that already passed OAuth. Streams the body to a + temp file with the same byte-cap-while-streaming behavior already used in + `download_to_temp_file`, rejecting early (413) rather than buffering + unbounded bytes. + +3. **Extend `transcribe_audio_file`'s input schema** to a `oneOf` / + discriminated union: either the existing + `{ download_url, file_id, ... }` (ChatGPT path, unchanged, existing tests + keep passing) or a new `{ upload_id }` (Claude.ai path) that reads the + already-landed temp file instead of doing an HTTP GET. This keeps the + tool count at +1 total (`request_audio_upload`), consistent with + AGENTS.MD's "keep number of tools as few as possible." + +No changes needed in `brain3-mcp-vault-tools` (Python container) — +transcription is a native in-gateway Rust tool, unrelated to that codebase. + +## Required threat-model update (must happen before/with implementation) + +`SECURITY_AUDIT.md`'s Threat Model section needs new entries before this +ships, per AGENTS.MD: + +- **New asset**: in-flight uploaded audio bytes sitting in the upload + registry/temp dir between `request_audio_upload` and `transcribe_audio_file`. +- **New trust boundary**: `PUT /uploads/{upload_id}` is reachable by anyone + who can reach the public gateway origin, without an OAuth bearer token — + the security boundary shifts from "bearer token" to "possession of a + live, unguessable, single-use, 5-minute-TTL token." Needs explicit + reasoning for why that's an acceptable trade-off (same style as the + existing `download_url` SSRF-adjacent acceptance already documented for + Finding-adjacent assumptions). +- **New attacker capability**: an attacker who can observe/guess an + `upload_id` within its 5-minute window could PUT arbitrary bytes that get + whisper-decoded — mitigated by the existing size cap + audio decode + validation, same as today's `download_url` path. +- Note the token is generated with a CSPRNG (e.g. reuse whatever the + existing OAuth `RandomGenerator`/token generation already uses in this + codebase, don't hand-roll). + +## Operational dependency worth flagging to the user (not a code change) + +The Cloudflare **quick tunnel** (SECURITY_AUDIT.md Finding [3]) gets a new +URL on every gateway restart. Claude.ai's "Additional allowed domains" +allowlist is keyed by hostname, so a quick tunnel would force re-whitelisting +the domain after every restart for the sandbox `curl` to succeed. This flow +works far better with a **named/persistent** Cloudflare tunnel domain. Worth +calling out to the user as a prerequisite, not something to silently work +around in code. + +## Implementation steps + +1. Update `SECURITY_AUDIT.md` Threat Model (assets / trust boundaries / + attacker capabilities) per above. +2. Add an in-memory upload registry (`Mutex>` or + similar), `Entry { expires_at, used: bool, path: Option }`, + likely as a small adapter under `crates/platform/src/` with lazy + expiry-sweep on access (mirrors the Python script's `_uploads` dict but + safe and bounded — needs eventual cleanup of orphaned temp files for + uploads that were reserved but never PUT to, and never transcribed). +3. Add `request_audio_upload` `NativeMcpTool` impl, registered alongside + `WhisperTranscribeTool` in `native_mcp_tools_from_config` (`apps/gateway/src/server.rs`). +4. Add `PUT /uploads/{upload_id}` handler in `crates/platform/src/http/`, + wired into `router.rs`'s `build_router` (must be reachable through the + public tunnel-facing router, same as `/oauth/*`). +5. Extend `WhisperToolArguments`/`OpenAIFileReferenceInput` deserialization + in `whisper_transcribe.rs` to accept the `upload_id` variant, sharing the + existing decode/size-limit/inference path; update `input_schema()` and + the OpenAI `_meta` handling so it's additive, not replacing, the existing + ChatGPT shape. +6. Tests (per AGENTS.MD: core behavior only, no tests on tool description + strings): upload happy path, size-limit rejection while streaming, + unknown/expired upload_id, single-use enforcement (second PUT and second + transcribe both rejected after first success), `transcribe_audio_file` + accepting `upload_id` end-to-end with a fake transcriber (same pattern as + existing tests in `whisper_transcribe.rs`). +7. Local verification per AGENTS.MD: `cargo test -p brain3 --no-run`, then + `cargo test`. Given this touches the gateway's public HTTP surface, also + run the e2e smoke suite (`uv run scripts/e2e_smoke.py`) since AGENTS.MD + calls that out explicitly for gateway/proxy changes. + +## Open questions for you before I implement + +- Confirm you're on an individual Claude.ai plan (Free/Pro/Max) — the Team-plan + allowlist bug (`claude-code#63182`) would make this unreliable on Team. +- Confirm you're willing to move to a named Cloudflare tunnel if not already + (needed so "Additional allowed domains" survives restarts). +- OK with the upload endpoint being unauthenticated-but-capability-scoped + (token-gated) rather than bearer-token gated? That's the only way to make + the sandbox's `curl` work at all, but it is a new ingress shape worth your + explicit sign-off given the security posture of this repo. From 0905d1f50cac6c40659160c904631555c292788f Mon Sep 17 00:00:00 2001 From: Traun Leyden Date: Thu, 9 Jul 2026 14:52:22 +0200 Subject: [PATCH 2/4] - **Option A (new, try first)**: add an inline base64 `audio_data`/`mime_type` --- ...09-claude-ai-audio-transcription-upload.md | 374 ++++++++++-------- 1 file changed, 217 insertions(+), 157 deletions(-) diff --git a/docs/superpowers/plans/2026-07-09-claude-ai-audio-transcription-upload.md b/docs/superpowers/plans/2026-07-09-claude-ai-audio-transcription-upload.md index 07547a4..4a52122 100644 --- a/docs/superpowers/plans/2026-07-09-claude-ai-audio-transcription-upload.md +++ b/docs/superpowers/plans/2026-07-09-claude-ai-audio-transcription-upload.md @@ -21,162 +21,222 @@ fetchable `download_url`. There is no `_meta` hint Claude.ai honors that would cause it to synthesize one. So `transcribe_audio_file` is currently unreachable from Claude.ai — confirmed, not a misconfiguration. -## Verified: is the proposed presigned-upload-URL pattern real? - -Yes, cross-checked against three independent sources (not just the -assistant-generated script, which I don't treat as authoritative on its -own): - -- **Official docs** (`platform.claude.com/docs/.../code-execution-tool`): - confirms user file uploads made through the **raw Messages API** use a - `container_upload` block + `file_id` from the Files API — that's the - API-level mechanism and it's a red herring for Claude.ai chat. -- **Claude Help Center** ("Create and edit files with Claude"): confirms - Settings → Capabilities → Code execution and file creation has an - "Additional allowed domains" network allowlist, and that this exists for - **individual Free/Pro/Max accounts**, not just Team/Enterprise. Also - confirms: *"If MCP integrations are enabled, network communication - remains possible through those connections regardless of the network - egress setting"* — i.e. the egress toggle only gates code running inside - the sandbox (a `curl` call), not MCP tool calls themselves. -- **Independent third-party implementation** (futuresearch.ai blog, "How to - Upload Large Files to an MCP Server Without Filling the Context Window") - describes the exact same three-step flow already in production: request a - presigned URL from an MCP tool → Claude runs `curl -X PUT` inside the - sandbox → a second MCP tool call exchanges an opaque ID for the parsed - result. Confirms presigned URLs need a short TTL (theirs: 5 min) and that - omitting the domain from "Additional allowed domains" fails the `curl` - with a network error even though the URL is valid. - -One caveat found and worth noting, not blocking: a filed GitHub issue -(`anthropics/claude-code#63182`) reports the **Team-plan org-level** -allowlist not reaching the sandbox proxy. Doesn't affect individual -accounts, which is presumably how this user runs Claude.ai. - -**Conclusion: the pattern is real and is the documented way to get sandbox-local -bytes into an MCP server. The assistant-generated script is not usable -as-is** — it's a standalone Python/FastMCP/faster-whisper server with no -auth, no framework fit, and it duplicates whisper-rs inference brain3 already -has natively in Rust. It also introduces an unauthenticated public HTTP -upload endpoint, which conflicts directly with this repo's security rule -("gateway is intentionally closed to preregistered clients only") and the -AGENTS.MD rule that new ingress points require a `SECURITY_AUDIT.md` threat -model update first. - -## Design goal - -Reuse brain3's existing native Rust transcription path (`whisper-rs`, -existing `max_audio_bytes` cap, existing audio decode/size-limit logic) and -add a second, Claude.ai-shaped way to *get bytes into* that path, without -weakening the "only the preregistered client reaches protected data" -guarantee. - -Two new pieces, one path reused: - -1. **New native MCP tool `request_audio_upload`** (mirrors - `WhisperTranscribeTool`'s shape in - `crates/platform/src/native_mcp_tools/`). Only reachable the same way - every other tool is: through the OAuth-protected `/mcp` route, so only the - preregistered client can ever obtain an upload token. Returns: - - a short-TTL (5 min, matching the pattern found in research), single-use, - cryptographically random `upload_id` - - the `curl -X PUT` command string Claude should run in its sandbox - - the size cap, sourced from the *same* `max_audio_bytes` config - (`config.native_audio_transcription.max_audio_bytes`) already used by - `transcribe_audio_file` — no second, divergent limit. - -2. **New unauthenticated-but-capability-scoped HTTP route** - `PUT /uploads/{upload_id}` in `crates/platform/src/http/` (new - `uploads.rs` handler, wired into both `router.rs` builders). Unauthenticated - because the sandbox's `curl` can't carry the OAuth bearer token — but the - `upload_id` itself is the capability: unguessable, single-use, expires in - 5 minutes, and can only ever have been minted by a call to - `request_audio_upload` that already passed OAuth. Streams the body to a - temp file with the same byte-cap-while-streaming behavior already used in - `download_to_temp_file`, rejecting early (413) rather than buffering - unbounded bytes. - -3. **Extend `transcribe_audio_file`'s input schema** to a `oneOf` / - discriminated union: either the existing - `{ download_url, file_id, ... }` (ChatGPT path, unchanged, existing tests - keep passing) or a new `{ upload_id }` (Claude.ai path) that reads the - already-landed temp file instead of doing an HTTP GET. This keeps the - tool count at +1 total (`request_audio_upload`), consistent with - AGENTS.MD's "keep number of tools as few as possible." - -No changes needed in `brain3-mcp-vault-tools` (Python container) — -transcription is a native in-gateway Rust tool, unrelated to that codebase. - -## Required threat-model update (must happen before/with implementation) - -`SECURITY_AUDIT.md`'s Threat Model section needs new entries before this -ships, per AGENTS.MD: - -- **New asset**: in-flight uploaded audio bytes sitting in the upload - registry/temp dir between `request_audio_upload` and `transcribe_audio_file`. -- **New trust boundary**: `PUT /uploads/{upload_id}` is reachable by anyone - who can reach the public gateway origin, without an OAuth bearer token — - the security boundary shifts from "bearer token" to "possession of a - live, unguessable, single-use, 5-minute-TTL token." Needs explicit - reasoning for why that's an acceptable trade-off (same style as the - existing `download_url` SSRF-adjacent acceptance already documented for - Finding-adjacent assumptions). -- **New attacker capability**: an attacker who can observe/guess an - `upload_id` within its 5-minute window could PUT arbitrary bytes that get - whisper-decoded — mitigated by the existing size cap + audio decode - validation, same as today's `download_url` path. -- Note the token is generated with a CSPRNG (e.g. reuse whatever the - existing OAuth `RandomGenerator`/token generation already uses in this - codebase, don't hand-roll). - -## Operational dependency worth flagging to the user (not a code change) - -The Cloudflare **quick tunnel** (SECURITY_AUDIT.md Finding [3]) gets a new -URL on every gateway restart. Claude.ai's "Additional allowed domains" -allowlist is keyed by hostname, so a quick tunnel would force re-whitelisting -the domain after every restart for the sandbox `curl` to succeed. This flow -works far better with a **named/persistent** Cloudflare tunnel domain. Worth -calling out to the user as a prerequisite, not something to silently work -around in code. - -## Implementation steps - -1. Update `SECURITY_AUDIT.md` Threat Model (assets / trust boundaries / - attacker capabilities) per above. -2. Add an in-memory upload registry (`Mutex>` or - similar), `Entry { expires_at, used: bool, path: Option }`, - likely as a small adapter under `crates/platform/src/` with lazy - expiry-sweep on access (mirrors the Python script's `_uploads` dict but - safe and bounded — needs eventual cleanup of orphaned temp files for - uploads that were reserved but never PUT to, and never transcribed). -3. Add `request_audio_upload` `NativeMcpTool` impl, registered alongside - `WhisperTranscribeTool` in `native_mcp_tools_from_config` (`apps/gateway/src/server.rs`). -4. Add `PUT /uploads/{upload_id}` handler in `crates/platform/src/http/`, - wired into `router.rs`'s `build_router` (must be reachable through the - public tunnel-facing router, same as `/oauth/*`). -5. Extend `WhisperToolArguments`/`OpenAIFileReferenceInput` deserialization - in `whisper_transcribe.rs` to accept the `upload_id` variant, sharing the - existing decode/size-limit/inference path; update `input_schema()` and - the OpenAI `_meta` handling so it's additive, not replacing, the existing - ChatGPT shape. -6. Tests (per AGENTS.MD: core behavior only, no tests on tool description - strings): upload happy path, size-limit rejection while streaming, +## Correction: what the MCP spec actually says about binary tool *inputs* + +A second assistant-generated message claimed "the MCP spec defines +`AudioContent` with base64 `data`... the same pattern applies to tool +inputs." I checked this against the spec directly +(`modelcontextprotocol.io/specification/draft/server/tools` and `.../schema`) +and it's **half right, stated too strongly**: + +- `AudioContent`/`ImageContent` (`{ "type": "audio", "data": "", + "mimeType": "..." }`) are real, spec-defined types — but they're part of + `ContentBlock`, used for **tool results** (and prompts/resources), not + tool call *arguments*. +- Tool **inputs** are governed purely by a server's own `inputSchema` (plain + JSON Schema). The spec places zero constraints on how a server author + models a "file" input. A base64 string property is a legal and common + choice, but it is exactly as much of an "application-level convention" as + brain3's existing `{ download_url, file_id }` shape (which is OpenAI's + convention) or the presigned-upload-URL shape (a pattern several MCP + servers have independently converged on, per the earlier research pass) — + none of the three is more "spec-native" than the others. The spec is + silent on file input modeling by design. + +This matters for the plan because it means base64-inline is not free of +design trade-offs just because it "is in the spec" — it isn't, in the input +direction. It's still worth strongly considering on its own merits (below), +just not on the basis of spec compliance. + +## Revised design: two viable options, staged by effort/risk + +### Option A — inline base64 argument (new, recommended to try first) + +Add a third accepted shape to `transcribe_audio_file`'s existing +`audio_file` argument: + +```json +{ "audio_file": { "audio_data": "", "mime_type": "audio/wav", "file_name": "memo.wav" } } +``` + +Server-side: `base64::decode`, then feed straight into the *existing* +`decode_audio_file_to_whisper_pcm` path (same size-limit enforcement, same +whisper-rs inference) — no HTTP GET, no new HTTP route, no new registry, no +new tool. + +**Why this is attractive:** +- Zero new ingress. The bytes travel over the *existing* OAuth-protected + `/mcp` JSON-RPC channel — no new trust boundary, no `SECURITY_AUDIT.md` + rewrite beyond noting the input variant exists. +- No dependency on Cloudflare tunnel naming, "Additional allowed domains," + or the sandbox's `curl` working at all. +- Much smaller code change: one new struct variant + one new schema branch + in a file that already exists, reusing 100% of the decode/limit/inference + pipeline. + +**Why it's genuinely uncertain whether it works from Claude.ai chat, and +needs to be tested empirically before committing to it as the answer:** +- For Claude.ai to call the tool with `audio_data` populated, it needs to + get the attached file's raw bytes into that JSON argument somehow. Two + paths exist, and I could not confirm from docs which (if either) applies + to the **Claude.ai consumer chat product** (as opposed to the raw + Developer Platform/Messages API, which is a different product surface): + 1. *Without code execution*: the model would have to literally emit the + base64 text as tokens in its own response. Per-turn output token caps + (commonly in the thousands, model/config-dependent) make this + impractical past a very short clip, and — per the earlier research + pass — Claude.ai doesn't appear to give the model raw byte access to + an attached audio file without code execution in the first place. + 2. *With code execution enabled*, Anthropic's documented "[programmatic + tool calling](https://www.anthropic.com/engineering/code-execution-with-mcp)" + pattern lets sandboxed code call an MCP tool directly — the tool + argument (including a base64 blob it constructs from a sandbox-local + file) is assembled and sent by the *script*, not typed out by the + model, so it doesn't consume output tokens or context. This is real + and documented for the API, but I found no confirmation it's exposed + in Claude.ai's consumer chat UI today rather than being an + API/agent-harness-only capability. +- Practical size ceiling either way is much smaller than the tool's current + 50MB (`max_audio_bytes`) download cap — a 50MB file would be ~67MB of + base64 text, unworkable as a JSON-RPC argument or (if per case 1 above) + model output. Recommend a separate, much smaller cap specifically for the + inline-base64 path (e.g. single-digit MB of raw audio — enough for a + multi-minute voice memo, not a long recording) rather than reusing + `max_audio_bytes` as-is. + +**Verification step before writing any code**: attach a short (~10-30s) +audio clip in an actual Claude.ai chat with code execution + this gateway's +MCP connector enabled, and ask Claude to transcribe it via a temporary test +tool that just echoes back `len(audio_data)`. This directly answers "can +Claude.ai get sandbox-local bytes into a tool argument at all" before +investing in either Option A's real implementation or Option B below. + +### Option B — presigned-upload-URL + sandbox `curl` (fallback, kept from prior plan draft, demoted not dropped) + +If Option A's verification step shows Claude.ai chat cannot practically get +file bytes into a tool argument (no programmatic-tool-calling access, or +sizes that matter for real recordings blow past output-token limits), fall +back to the previously researched, independently-confirmed pattern: a +`request_audio_upload` tool hands back a short-TTL single-use presigned +`PUT` URL, Claude's sandbox `curl`s the file to it directly (bypassing the +model's context entirely), then `transcribe_audio_file` is called with an +`upload_id`. + +This option is real (confirmed against official docs, the Claude Help +Center, and an independent third-party MCP server doing exactly this) but +costs meaningfully more: +- A genuinely new, only-token-gated (not bearer-gated) public HTTP ingress + point — requires a `SECURITY_AUDIT.md` Threat Model update per AGENTS.MD + before/with implementation (new asset: in-flight uploaded bytes; new + trust boundary: possession of an unguessable single-use 5-minute token + substitutes for the OAuth bearer token on this one route; new attacker + capability: anyone who can reach the public origin and guess/observe a + live token within its TTL can PUT bytes that get whisper-decoded, + mitigated by the same size cap + decode validation already in place). +- Operational dependency: requires a **named/persistent** Cloudflare + tunnel, since Claude.ai's "Additional allowed domains" allowlist is + hostname-keyed and the default quick tunnel's URL rotates on every + gateway restart (SECURITY_AUDIT.md Finding [3]). +- More moving parts: new upload-token registry with TTL/single-use/cleanup + semantics, a new HTTP handler, a new `NativeMcpTool`. + +Design details (registry shape, route placement, size-cap-while-streaming, +reuse of `max_audio_bytes`, test list) are unchanged from the prior version +of this plan and are kept below in case Option A doesn't pan out. + +
+Option B implementation detail (collapsed — only needed if Option A fails verification) + +1. Add an in-memory upload registry (`Mutex>` or + similar), `Entry { expires_at, used: bool, path: Option }`, with + lazy expiry-sweep on access, and cleanup of orphaned temp files for + uploads that were reserved but never PUT to or never transcribed. +2. Add `request_audio_upload` `NativeMcpTool` impl (mirrors + `WhisperTranscribeTool`'s shape), registered alongside it in + `native_mcp_tools_from_config` (`apps/gateway/src/server.rs`). Returns + the `upload_id`, the `curl -X PUT` command, TTL, and the size cap + (sourced from the same `max_audio_bytes` config already used by + `transcribe_audio_file` — no second, divergent limit). +3. Add `PUT /uploads/{upload_id}` handler in `crates/platform/src/http/` + (new `uploads.rs`), wired into `router.rs`'s `build_router` (must be + reachable through the public tunnel-facing router, same as `/oauth/*`). + Streams the body to a temp file with the same byte-cap-while-streaming + behavior already used in `download_to_temp_file`, rejecting early (413) + rather than buffering unbounded bytes. +4. Extend `WhisperToolArguments` deserialization to accept the `upload_id` + variant, sharing the existing decode/size-limit/inference path. +5. Tests: upload happy path, size-limit rejection while streaming, unknown/expired upload_id, single-use enforcement (second PUT and second transcribe both rejected after first success), `transcribe_audio_file` - accepting `upload_id` end-to-end with a fake transcriber (same pattern as - existing tests in `whisper_transcribe.rs`). -7. Local verification per AGENTS.MD: `cargo test -p brain3 --no-run`, then - `cargo test`. Given this touches the gateway's public HTTP surface, also - run the e2e smoke suite (`uv run scripts/e2e_smoke.py`) since AGENTS.MD - calls that out explicitly for gateway/proxy changes. - -## Open questions for you before I implement - -- Confirm you're on an individual Claude.ai plan (Free/Pro/Max) — the Team-plan - allowlist bug (`claude-code#63182`) would make this unreliable on Team. -- Confirm you're willing to move to a named Cloudflare tunnel if not already - (needed so "Additional allowed domains" survives restarts). -- OK with the upload endpoint being unauthenticated-but-capability-scoped - (token-gated) rather than bearer-token gated? That's the only way to make - the sandbox's `curl` work at all, but it is a new ingress shape worth your - explicit sign-off given the security posture of this repo. + accepting `upload_id` end-to-end with a fake transcriber. + +
+ +No changes needed in `brain3-mcp-vault-tools` (Python container) under +either option — transcription is a native in-gateway Rust tool, unrelated +to that codebase. + +## Wording fix: stop branding the existing shape as OpenAI-only + +Today's `input_schema()` describes the `audio_file` object as *"OpenAI file +reference for the uploaded audio file"* and `file_id` as *"OpenAI file +identifier."* That's misleading in the same direction as the base64 claim +above: the `{ download_url, file_id }` **shape** is generic JSON Schema — +any MCP client capable of handing the tool a temporary authorized download +URL could use it. What's actually OpenAI-specific is only the `_meta` +`openai/fileParams` hint that tells *ChatGPT specifically* to populate it +automatically. + +Reword (implementation detail, to land alongside whichever option ships): + +- `audio_file` description: from *"OpenAI file reference for the uploaded + audio file"* to something like *"Reference to an uploaded audio file + accessible via a temporary authorized download URL. Populated + automatically by OpenAI Apps SDK clients via the `openai/fileParams` + hint; any MCP client that can supply a fetchable `download_url` may use + this shape."* +- `file_id` description: from *"OpenAI file identifier"* to *"Client-issued + identifier for the uploaded file (opaque to this tool; used for + logging/correlation only)."* + +This is purely descriptive-string wording — no behavior change, low risk, +worth doing regardless of which option (A or B) also ships, since it +currently mis-documents the tool for any future non-OpenAI client author +(including whichever shape we add for Claude.ai). + +## Implementation order + +1. Reword the existing `audio_file`/`file_id` schema descriptions (above) — + independent, no-risk, do any time. +2. Run the empirical verification step under Option A (attach a short clip + in a real Claude.ai chat, confirm whether bytes reach a tool argument at + all, and by which mechanism). +3. If verification succeeds: implement Option A (new `audio_data`/ + `mime_type` schema branch + decode path + a separate, smaller size cap + for this path + tests). No `SECURITY_AUDIT.md` change expected beyond a + one-line note that this input variant exists, since no new trust + boundary is created. +4. If verification fails or sizes are impractical: implement Option B per + the collapsed detail above, including the required `SECURITY_AUDIT.md` + Threat Model update *before* the new route ships, and confirm with you + first that you're willing to move to a named Cloudflare tunnel (needed + for "Additional allowed domains" to survive restarts) and that you're OK + with a token-gated-rather-than-bearer-gated upload route. +5. Local verification per AGENTS.MD either way: `cargo test -p brain3 + --no-run`, then `cargo test`. If Option B ships (new gateway HTTP + surface), also run `uv run scripts/e2e_smoke.py` per AGENTS.MD's + explicit call-out for gateway/proxy changes. + +## Open questions for you before I implement anything + +- OK with doing the manual Claude.ai verification step (step 2) before + committing to either option's full implementation? It's the cheapest way + to avoid building the wrong one. +- If it comes to Option B: confirm you're on an individual Claude.ai plan + (Free/Pro/Max) — a filed GitHub issue (`anthropics/claude-code#63182`) + reports the Team-plan org-level allowlist not reaching the sandbox proxy, + so Option B would be unreliable on Team. +- If it comes to Option B: confirm you're willing to move to a named + Cloudflare tunnel, and sign off on the token-gated (not bearer-gated) + upload endpoint as a new ingress shape, given this repo's security + posture. From 3ba3a9a84d20ac5c6be413ab551b989c5c3b1e98 Mon Sep 17 00:00:00 2001 From: Traun Leyden Date: Thu, 9 Jul 2026 14:54:09 +0200 Subject: [PATCH 3/4] clarificaiont of plan --- ...09-claude-ai-audio-transcription-upload.md | 34 ++++++++++++++++--- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/docs/superpowers/plans/2026-07-09-claude-ai-audio-transcription-upload.md b/docs/superpowers/plans/2026-07-09-claude-ai-audio-transcription-upload.md index 4a52122..c30bde5 100644 --- a/docs/superpowers/plans/2026-07-09-claude-ai-audio-transcription-upload.md +++ b/docs/superpowers/plans/2026-07-09-claude-ai-audio-transcription-upload.md @@ -48,12 +48,31 @@ design trade-offs just because it "is in the spec" — it isn't, in the input direction. It's still worth strongly considering on its own merits (below), just not on the basis of spec compliance. +## ⚠️ Non-negotiable constraint: do not touch the working OpenAI path + +**The existing `{ download_url, file_id }` shape is live, tested, and +confirmed working end-to-end with ChatGPT today.** Nothing in this plan +should modify its behavior, its wire format, its `_meta` hint, or the +existing passing tests for it +(`whisper_transcribe.rs::call_downloads_decodes_resamples_and_returns_transcript` +and friends). Both options below are strictly *additive*: every code path +that already ships must keep working, byte-for-byte, exactly as it does +today. If an implementation of either option requires changing how the +`download_url`/`file_id` shape is parsed or validated, that's a sign the +approach is wrong — go back and find a purely-additive way instead. + ## Revised design: two viable options, staged by effort/risk ### Option A — inline base64 argument (new, recommended to try first) -Add a third accepted shape to `transcribe_audio_file`'s existing -`audio_file` argument: +**This is not a new tool.** It's the *same* `transcribe_audio_file` tool, +same registration, same name, same OpenAI shape untouched — just one more +accepted shape for its existing `audio_file` argument, sitting alongside +`{ download_url, file_id }` as an alternative (e.g. a `oneOf` branch in the +JSON Schema, or simply extra optional fields with server-side logic that +picks a path based on which fields are present). A client sends *either* +the existing OpenAI-style reference *or* this new one — never both, and the +OpenAI path is completely unaffected by this addition: ```json { "audio_file": { "audio_data": "", "mime_type": "audio/wav", "file_name": "memo.wav" } } @@ -213,9 +232,14 @@ currently mis-documents the tool for any future non-OpenAI client author all, and by which mechanism). 3. If verification succeeds: implement Option A (new `audio_data`/ `mime_type` schema branch + decode path + a separate, smaller size cap - for this path + tests). No `SECURITY_AUDIT.md` change expected beyond a - one-line note that this input variant exists, since no new trust - boundary is created. + for this path + tests) as a **strictly additive** change — do not modify + `OpenAIFileReferenceInput`, the `download_url`/`file_id` required fields, + or the existing `_meta` hint. Run the full existing + `whisper_transcribe.rs` test suite before and after and confirm every + existing test still passes unmodified — that's the regression signal + that the working ChatGPT path is intact. No `SECURITY_AUDIT.md` change + expected beyond a one-line note that this input variant exists, since no + new trust boundary is created. 4. If verification fails or sizes are impractical: implement Option B per the collapsed detail above, including the required `SECURITY_AUDIT.md` Threat Model update *before* the new route ships, and confirm with you From 04fc3d12097ce504d4db65109b8848f00679f190 Mon Sep 17 00:00:00 2001 From: Traun Leyden Date: Thu, 9 Jul 2026 15:00:45 +0200 Subject: [PATCH 4/4] implement claude code fix attempt. untested --- Cargo.lock | 1 + SECURITY_AUDIT.md | 2 + crates/platform/Cargo.toml | 1 + .../native_mcp_tools/whisper_transcribe.rs | 318 +++++++++++++++--- 4 files changed, 280 insertions(+), 42 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 69af654..ca031a8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -357,6 +357,7 @@ dependencies = [ "async-trait", "axum", "axum-test", + "base64 0.22.1", "brain3-core", "chrono", "dotenvy", diff --git a/SECURITY_AUDIT.md b/SECURITY_AUDIT.md index 07186ab..16cf5e6 100644 --- a/SECURITY_AUDIT.md +++ b/SECURITY_AUDIT.md @@ -64,6 +64,7 @@ Brain3’s highest-risk boundary is the optional public gateway/tunnel that fron - The boundary between the Rust gateway and the `brain3-mcp-vault-tools` container - The native transcription boundary where the Rust gateway downloads and parses untrusted audio bytes in-process instead of forwarding them to the container - Gateway-initiated internet egress for temporary audio `download_url` fetches and setup-time Whisper model downloads +- Inline base64 audio bytes sent by authorized MCP clients over the existing OAuth-protected JSON-RPC channel for native transcription - Vault content that may be user-controlled or, in some deployments, third-party-controlled ### Attacker Capabilities @@ -71,6 +72,7 @@ Brain3’s highest-risk boundary is the optional public gateway/tunnel that fron - Send arbitrary HTTP requests and headers to public gateway routes when tunneling is enabled - Operate or compromise the preregistered OAuth client after it is provisioned with Brain3 credentials - Supply a `transcribe_audio_file` audio `download_url` through an authorized MCP client, causing the gateway to issue an outbound GET and parse the returned bytes as audio +- Supply bounded inline base64 audio bytes to `transcribe_audio_file` through an authorized MCP client, causing the gateway to parse those bytes as audio - Read local files or logs available to the current OS principal or broader local principals - Supply hostile vault content when the user does not fully control imported or shared notes diff --git a/crates/platform/Cargo.toml b/crates/platform/Cargo.toml index 3f85903..c94e6b5 100644 --- a/crates/platform/Cargo.toml +++ b/crates/platform/Cargo.toml @@ -6,6 +6,7 @@ edition = "2021" [dependencies] brain3-core = { path = "../core" } async-trait = "0.1" +base64 = "0.22" libc = "0.2" axum = "0.8" tower-http = { version = "0.7", features = ["trace"] } diff --git a/crates/platform/src/native_mcp_tools/whisper_transcribe.rs b/crates/platform/src/native_mcp_tools/whisper_transcribe.rs index 3a0ec87..f0f7bc3 100644 --- a/crates/platform/src/native_mcp_tools/whisper_transcribe.rs +++ b/crates/platform/src/native_mcp_tools/whisper_transcribe.rs @@ -3,6 +3,7 @@ use std::path::{Path, PathBuf}; use std::sync::{Arc, Mutex}; use std::time::{Duration, Instant}; +use base64::{engine::general_purpose::STANDARD, Engine as _}; use brain3_core::ports::native_mcp_tool::{NativeMcpTool, NativeMcpToolError, NativeMcpToolOutput}; use serde::Deserialize; use serde_json::{json, Value}; @@ -21,6 +22,7 @@ const TARGET_SAMPLE_RATE: u32 = 16_000; const PCM_SAMPLE_BYTES: u64 = std::mem::size_of::() as u64; const HTTP_CONNECT_TIMEOUT: Duration = Duration::from_secs(10); const HTTP_TOTAL_TIMEOUT: Duration = Duration::from_secs(300); +const INLINE_AUDIO_MAX_BYTES: u64 = 8 * 1024 * 1024; #[derive(Debug, Clone, PartialEq, Eq)] pub struct WhisperTranscribeConfig { @@ -77,7 +79,19 @@ impl WhisperTranscribeTool { ) -> Result { let args: WhisperToolArguments = serde_json::from_value(arguments) .map_err(|error| WhisperTranscribeError::InvalidArguments(error.to_string()))?; - let download_url = reqwest::Url::parse(&args.audio_file.download_url) + match args.audio_file { + AudioFileInput::DownloadUrl(audio_file) => { + self.execute_download_reference(audio_file).await + } + AudioFileInput::InlineBase64(audio_file) => self.execute_inline_audio(audio_file).await, + } + } + + async fn execute_download_reference( + &self, + audio_file: OpenAIFileReferenceInput, + ) -> Result { + let download_url = reqwest::Url::parse(&audio_file.download_url) .map_err(|error| WhisperTranscribeError::InvalidDownloadUrl(error.to_string()))?; if !matches!(download_url.scheme(), "http" | "https") { return Err(WhisperTranscribeError::InvalidDownloadUrl( @@ -87,9 +101,9 @@ impl WhisperTranscribeTool { let host = download_url.host_str().unwrap_or("unknown").to_string(); tracing::info!( - file_id = %args.audio_file.file_id, - file_name = ?args.audio_file.file_name, - mime_type = ?args.audio_file.mime_type, + file_id = %audio_file.file_id, + file_name = ?audio_file.file_name, + mime_type = ?audio_file.mime_type, host = %host, max_audio_bytes = self.config.max_audio_bytes, "native whisper transcription: downloading audio" @@ -97,10 +111,10 @@ impl WhisperTranscribeTool { let start = Instant::now(); let (_temp_dir, audio_path, bytes_written) = self - .download_to_temp_file(download_url, &args.audio_file) + .download_to_temp_file(download_url, &audio_file) .await?; tracing::info!( - file_id = %args.audio_file.file_id, + file_id = %audio_file.file_id, bytes_written, elapsed_ms = start.elapsed().as_millis() as u64, "native whisper transcription: audio download complete" @@ -109,7 +123,7 @@ impl WhisperTranscribeTool { let decode_start = Instant::now(); let samples = decode_audio_file_to_whisper_pcm(&audio_path, self.config.max_audio_bytes)?; tracing::info!( - file_id = %args.audio_file.file_id, + file_id = %audio_file.file_id, sample_count = samples.len(), duration_ms = samples.len() as u64 * 1000 / TARGET_SAMPLE_RATE as u64, elapsed_ms = decode_start.elapsed().as_millis() as u64, @@ -122,7 +136,7 @@ impl WhisperTranscribeTool { .await .map_err(|error| WhisperTranscribeError::Inference(error.to_string()))??; tracing::info!( - file_id = %args.audio_file.file_id, + file_id = %audio_file.file_id, transcript_chars = transcript.chars().count(), elapsed_ms = inference_start.elapsed().as_millis() as u64, "native whisper transcription: inference complete" @@ -131,6 +145,71 @@ impl WhisperTranscribeTool { Ok(NativeMcpToolOutput::text(transcript)) } + async fn execute_inline_audio( + &self, + audio_file: InlineBase64AudioInput, + ) -> Result { + let inline_limit = self.inline_audio_byte_limit(); + if encoded_base64_may_exceed_decoded_limit(audio_file.audio_data.len(), inline_limit) { + return Err(WhisperTranscribeError::InlineSizeLimitExceeded { + max_audio_bytes: inline_limit, + }); + } + + let audio_bytes = STANDARD + .decode(&audio_file.audio_data) + .map_err(|error| WhisperTranscribeError::InvalidInlineAudioData(error.to_string()))?; + let bytes_written = audio_bytes.len() as u64; + if bytes_written > inline_limit { + return Err(WhisperTranscribeError::InlineSizeLimitExceeded { + max_audio_bytes: inline_limit, + }); + } + + tracing::info!( + file_name = ?audio_file.file_name, + mime_type = ?audio_file.mime_type, + bytes = bytes_written, + inline_max_audio_bytes = inline_limit, + "native whisper transcription: received inline audio" + ); + + let start = Instant::now(); + let (_temp_dir, audio_path) = + write_inline_audio_to_temp_file(&audio_file, &audio_bytes).await?; + tracing::info!( + bytes_written, + elapsed_ms = start.elapsed().as_millis() as u64, + "native whisper transcription: inline audio staged" + ); + + let decode_start = Instant::now(); + let samples = decode_audio_file_to_whisper_pcm(&audio_path, self.config.max_audio_bytes)?; + tracing::info!( + sample_count = samples.len(), + duration_ms = samples.len() as u64 * 1000 / TARGET_SAMPLE_RATE as u64, + elapsed_ms = decode_start.elapsed().as_millis() as u64, + "native whisper transcription: inline audio decoded and resampled" + ); + + let transcriber = Arc::clone(&self.transcriber); + let inference_start = Instant::now(); + let transcript = tokio::task::spawn_blocking(move || transcriber.transcribe(samples)) + .await + .map_err(|error| WhisperTranscribeError::Inference(error.to_string()))??; + tracing::info!( + transcript_chars = transcript.chars().count(), + elapsed_ms = inference_start.elapsed().as_millis() as u64, + "native whisper transcription: inline audio inference complete" + ); + + Ok(NativeMcpToolOutput::text(transcript)) + } + + fn inline_audio_byte_limit(&self) -> u64 { + self.config.max_audio_bytes.min(INLINE_AUDIO_MAX_BYTES) + } + async fn download_to_temp_file( &self, download_url: reqwest::Url, @@ -206,35 +285,65 @@ impl NativeMcpTool for WhisperTranscribeTool { "additionalProperties": false, "properties": { "audio_file": { - "type": "object", - "additionalProperties": false, - "description": "OpenAI file reference for the uploaded audio file", - "properties": { - "download_url": { - "type": "string", - "format": "uri", - "description": "Temporary authorized download URL for the uploaded file" - }, - "file_id": { - "type": "string", - "minLength": 1, - "maxLength": 200, - "description": "OpenAI file identifier" - }, - "mime_type": { - "type": "string", - "minLength": 1, - "maxLength": 200, - "description": "Optional MIME type for the uploaded file" + "description": "Uploaded audio file input. Use either a temporary authorized download URL or inline base64 audio bytes.", + "oneOf": [ + { + "type": "object", + "additionalProperties": false, + "description": "Reference to an uploaded audio file accessible via a temporary authorized download URL. Populated automatically by OpenAI Apps SDK clients via the openai/fileParams hint; any MCP client that can supply a fetchable download_url may use this shape.", + "properties": { + "download_url": { + "type": "string", + "format": "uri", + "description": "Temporary authorized download URL for the uploaded file" + }, + "file_id": { + "type": "string", + "minLength": 1, + "maxLength": 200, + "description": "Client-issued identifier for the uploaded file. Opaque to this tool and used for logging/correlation only." + }, + "mime_type": { + "type": "string", + "minLength": 1, + "maxLength": 200, + "description": "Optional MIME type for the uploaded file" + }, + "file_name": { + "type": "string", + "minLength": 1, + "maxLength": 500, + "description": "Optional original filename for the uploaded file" + } + }, + "required": ["download_url", "file_id"] }, - "file_name": { - "type": "string", - "minLength": 1, - "maxLength": 500, - "description": "Optional original filename for the uploaded file" + { + "type": "object", + "additionalProperties": false, + "description": "Inline base64-encoded audio bytes for MCP clients that can read an attached file locally but cannot provide a temporary download URL.", + "properties": { + "audio_data": { + "type": "string", + "contentEncoding": "base64", + "description": "Base64-encoded audio bytes. Limited to 8 MiB before base64 encoding, and also bounded by the configured audio byte limit." + }, + "mime_type": { + "type": "string", + "minLength": 1, + "maxLength": 200, + "description": "Optional MIME type for the uploaded file" + }, + "file_name": { + "type": "string", + "minLength": 1, + "maxLength": 500, + "description": "Optional original filename for the uploaded file" + } + }, + "required": ["audio_data"] } - }, - "required": ["download_url", "file_id"] + ] } }, "required": ["audio_file"] @@ -323,7 +432,14 @@ impl AudioTranscriber for CachedWhisperTranscriber { #[derive(Debug, Deserialize)] #[serde(deny_unknown_fields)] struct WhisperToolArguments { - audio_file: OpenAIFileReferenceInput, + audio_file: AudioFileInput, +} + +#[derive(Debug, Deserialize)] +#[serde(untagged)] +enum AudioFileInput { + DownloadUrl(OpenAIFileReferenceInput), + InlineBase64(InlineBase64AudioInput), } #[derive(Debug, Deserialize)] @@ -335,16 +451,28 @@ struct OpenAIFileReferenceInput { file_name: Option, } +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct InlineBase64AudioInput { + audio_data: String, + mime_type: Option, + file_name: Option, +} + #[derive(Debug, thiserror::Error)] enum WhisperTranscribeError { #[error("invalid arguments: {0}")] InvalidArguments(String), #[error("invalid download_url: {0}")] InvalidDownloadUrl(String), + #[error("invalid inline audio_data: {0}")] + InvalidInlineAudioData(String), #[error("audio download failed: {0}")] Download(String), #[error("audio download exceeds configured limit of {max_audio_bytes} bytes")] SizeLimitExceeded { max_audio_bytes: u64 }, + #[error("inline audio exceeds configured limit of {max_audio_bytes} bytes")] + InlineSizeLimitExceeded { max_audio_bytes: u64 }, #[error("decoded audio exceeds configured limit of {max_audio_bytes} bytes")] DecodedAudioLimitExceeded { max_audio_bytes: u64 }, #[error("temporary audio file error: {0}")] @@ -361,7 +489,8 @@ impl From for NativeMcpToolError { fn from(error: WhisperTranscribeError) -> Self { match error { WhisperTranscribeError::InvalidArguments(message) - | WhisperTranscribeError::InvalidDownloadUrl(message) => { + | WhisperTranscribeError::InvalidDownloadUrl(message) + | WhisperTranscribeError::InvalidInlineAudioData(message) => { NativeMcpToolError::InvalidArguments(message) } other => NativeMcpToolError::ExecutionFailed(other.to_string()), @@ -369,6 +498,32 @@ impl From for NativeMcpToolError { } } +async fn write_inline_audio_to_temp_file( + audio_file: &InlineBase64AudioInput, + audio_bytes: &[u8], +) -> Result<(TempDir, PathBuf), WhisperTranscribeError> { + let temp_dir = tempfile::Builder::new() + .prefix("brain3_audio_inline_") + .tempdir() + .map_err(|error| WhisperTranscribeError::TempFile(error.to_string()))?; + let audio_path = temp_dir.path().join(destination_filename_from_name( + audio_file.file_name.as_deref(), + )); + let mut output = tokio::fs::File::create(&audio_path) + .await + .map_err(|error| WhisperTranscribeError::TempFile(error.to_string()))?; + output + .write_all(audio_bytes) + .await + .map_err(|error| WhisperTranscribeError::TempFile(error.to_string()))?; + output + .flush() + .await + .map_err(|error| WhisperTranscribeError::TempFile(error.to_string()))?; + + Ok((temp_dir, audio_path)) +} + fn decode_audio_file_to_whisper_pcm( path: &Path, max_decoded_audio_bytes: u64, @@ -535,9 +690,11 @@ fn resample_linear(samples: &[f32], source_rate: u32, target_rate: u32) -> Vec String { - let mut name = audio_file - .file_name - .as_deref() + destination_filename_from_name(audio_file.file_name.as_deref()) +} + +fn destination_filename_from_name(file_name: Option<&str>) -> String { + let mut name = file_name .and_then(|name| Path::new(name).file_name()) .and_then(|name| name.to_str()) .unwrap_or("audio") @@ -549,6 +706,11 @@ fn destination_filename(audio_file: &OpenAIFileReferenceInput) -> String { name } +fn encoded_base64_may_exceed_decoded_limit(encoded_len: usize, max_decoded_bytes: u64) -> bool { + let max_encoded_len = max_decoded_bytes.div_ceil(3).saturating_mul(4); + encoded_len as u64 > max_encoded_len +} + fn whisper_backend_label() -> &'static str { #[cfg(target_os = "macos")] { @@ -566,6 +728,7 @@ mod tests { use std::sync::{Arc, Mutex}; use axum::{body::Body, http::StatusCode, response::Response, routing::get, Router}; + use base64::{engine::general_purpose::STANDARD, Engine as _}; use brain3_core::application::native_mcp_tool_registry::NativeMcpToolRegistry; use brain3_core::ports::native_mcp_tool::NativeMcpTool; use serde_json::json; @@ -623,6 +786,73 @@ mod tests { assert_eq!(calls[0].len(), 1_600); } + #[tokio::test] + async fn call_decodes_inline_base64_audio_and_returns_transcript() { + let wav = pcm_wav(8_000, 1, 800); + let encoded_audio = STANDARD.encode(wav); + let fake = Arc::new(FakeTranscriber::default()); + let tool = WhisperTranscribeTool::with_transcriber( + WhisperTranscribeConfig { + model_path: "/tmp/model.bin".into(), + max_audio_bytes: 1_000_000, + }, + fake.clone(), + ); + + let output = tool + .call(json!({ + "audio_file": { + "audio_data": encoded_audio, + "mime_type": "audio/wav", + "file_name": "sample.wav" + } + })) + .await + .expect("tool call should succeed"); + + assert!(!output.is_error); + assert_eq!(output.content[0].text, "hello from native whisper"); + let calls = fake + .calls + .lock() + .expect("fake transcriber lock should succeed"); + assert_eq!(calls.len(), 1); + assert_eq!(calls[0].len(), 1_600); + } + + #[tokio::test] + async fn call_rejects_inline_audio_larger_than_inline_cap() { + let encoded_audio = STANDARD.encode(vec![0u8; INLINE_AUDIO_MAX_BYTES as usize + 1]); + let fake = Arc::new(FakeTranscriber::default()); + let tool = WhisperTranscribeTool::with_transcriber( + WhisperTranscribeConfig { + model_path: "/tmp/model.bin".into(), + max_audio_bytes: INLINE_AUDIO_MAX_BYTES + 1_000_000, + }, + fake.clone(), + ); + + let error = tool + .call(json!({ + "audio_file": { + "audio_data": encoded_audio, + "mime_type": "audio/wav", + "file_name": "oversized.wav" + } + })) + .await + .expect_err("oversized inline audio should fail"); + + assert!(error + .to_string() + .contains("inline audio exceeds configured limit")); + assert!(fake + .calls + .lock() + .expect("fake transcriber lock should succeed") + .is_empty()); + } + #[tokio::test] async fn call_rejects_downloads_larger_than_configured_cap() { let download_url = serve_once(vec![0u8; 128]).await; @@ -727,7 +957,7 @@ mod tests { } #[test] - fn input_schema_declares_openai_audio_file_reference() { + fn input_schema_declares_download_url_and_inline_audio_file_references() { let fake = Arc::new(FakeTranscriber::default()); let tool = WhisperTranscribeTool::with_transcriber( WhisperTranscribeConfig { @@ -742,9 +972,13 @@ mod tests { assert_eq!(schema["required"], json!(["audio_file"])); assert_eq!( - schema["properties"]["audio_file"]["required"], + schema["properties"]["audio_file"]["oneOf"][0]["required"], json!(["download_url", "file_id"]) ); + assert_eq!( + schema["properties"]["audio_file"]["oneOf"][1]["required"], + json!(["audio_data"]) + ); } #[test]