diff --git a/Cargo.lock b/Cargo.lock index 69af654..ca031a8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -357,6 +357,7 @@ dependencies = [ "async-trait", "axum", "axum-test", + "base64 0.22.1", "brain3-core", "chrono", "dotenvy", diff --git a/SECURITY_AUDIT.md b/SECURITY_AUDIT.md index 07186ab..16cf5e6 100644 --- a/SECURITY_AUDIT.md +++ b/SECURITY_AUDIT.md @@ -64,6 +64,7 @@ Brain3’s highest-risk boundary is the optional public gateway/tunnel that fron - The boundary between the Rust gateway and the `brain3-mcp-vault-tools` container - The native transcription boundary where the Rust gateway downloads and parses untrusted audio bytes in-process instead of forwarding them to the container - Gateway-initiated internet egress for temporary audio `download_url` fetches and setup-time Whisper model downloads +- Inline base64 audio bytes sent by authorized MCP clients over the existing OAuth-protected JSON-RPC channel for native transcription - Vault content that may be user-controlled or, in some deployments, third-party-controlled ### Attacker Capabilities @@ -71,6 +72,7 @@ Brain3’s highest-risk boundary is the optional public gateway/tunnel that fron - Send arbitrary HTTP requests and headers to public gateway routes when tunneling is enabled - Operate or compromise the preregistered OAuth client after it is provisioned with Brain3 credentials - Supply a `transcribe_audio_file` audio `download_url` through an authorized MCP client, causing the gateway to issue an outbound GET and parse the returned bytes as audio +- Supply bounded inline base64 audio bytes to `transcribe_audio_file` through an authorized MCP client, causing the gateway to parse those bytes as audio - Read local files or logs available to the current OS principal or broader local principals - Supply hostile vault content when the user does not fully control imported or shared notes diff --git a/crates/platform/Cargo.toml b/crates/platform/Cargo.toml index 3f85903..c94e6b5 100644 --- a/crates/platform/Cargo.toml +++ b/crates/platform/Cargo.toml @@ -6,6 +6,7 @@ edition = "2021" [dependencies] brain3-core = { path = "../core" } async-trait = "0.1" +base64 = "0.22" libc = "0.2" axum = "0.8" tower-http = { version = "0.7", features = ["trace"] } diff --git a/crates/platform/src/native_mcp_tools/whisper_transcribe.rs b/crates/platform/src/native_mcp_tools/whisper_transcribe.rs index 3a0ec87..f0f7bc3 100644 --- a/crates/platform/src/native_mcp_tools/whisper_transcribe.rs +++ b/crates/platform/src/native_mcp_tools/whisper_transcribe.rs @@ -3,6 +3,7 @@ use std::path::{Path, PathBuf}; use std::sync::{Arc, Mutex}; use std::time::{Duration, Instant}; +use base64::{engine::general_purpose::STANDARD, Engine as _}; use brain3_core::ports::native_mcp_tool::{NativeMcpTool, NativeMcpToolError, NativeMcpToolOutput}; use serde::Deserialize; use serde_json::{json, Value}; @@ -21,6 +22,7 @@ const TARGET_SAMPLE_RATE: u32 = 16_000; const PCM_SAMPLE_BYTES: u64 = std::mem::size_of::() as u64; const HTTP_CONNECT_TIMEOUT: Duration = Duration::from_secs(10); const HTTP_TOTAL_TIMEOUT: Duration = Duration::from_secs(300); +const INLINE_AUDIO_MAX_BYTES: u64 = 8 * 1024 * 1024; #[derive(Debug, Clone, PartialEq, Eq)] pub struct WhisperTranscribeConfig { @@ -77,7 +79,19 @@ impl WhisperTranscribeTool { ) -> Result { let args: WhisperToolArguments = serde_json::from_value(arguments) .map_err(|error| WhisperTranscribeError::InvalidArguments(error.to_string()))?; - let download_url = reqwest::Url::parse(&args.audio_file.download_url) + match args.audio_file { + AudioFileInput::DownloadUrl(audio_file) => { + self.execute_download_reference(audio_file).await + } + AudioFileInput::InlineBase64(audio_file) => self.execute_inline_audio(audio_file).await, + } + } + + async fn execute_download_reference( + &self, + audio_file: OpenAIFileReferenceInput, + ) -> Result { + let download_url = reqwest::Url::parse(&audio_file.download_url) .map_err(|error| WhisperTranscribeError::InvalidDownloadUrl(error.to_string()))?; if !matches!(download_url.scheme(), "http" | "https") { return Err(WhisperTranscribeError::InvalidDownloadUrl( @@ -87,9 +101,9 @@ impl WhisperTranscribeTool { let host = download_url.host_str().unwrap_or("unknown").to_string(); tracing::info!( - file_id = %args.audio_file.file_id, - file_name = ?args.audio_file.file_name, - mime_type = ?args.audio_file.mime_type, + file_id = %audio_file.file_id, + file_name = ?audio_file.file_name, + mime_type = ?audio_file.mime_type, host = %host, max_audio_bytes = self.config.max_audio_bytes, "native whisper transcription: downloading audio" @@ -97,10 +111,10 @@ impl WhisperTranscribeTool { let start = Instant::now(); let (_temp_dir, audio_path, bytes_written) = self - .download_to_temp_file(download_url, &args.audio_file) + .download_to_temp_file(download_url, &audio_file) .await?; tracing::info!( - file_id = %args.audio_file.file_id, + file_id = %audio_file.file_id, bytes_written, elapsed_ms = start.elapsed().as_millis() as u64, "native whisper transcription: audio download complete" @@ -109,7 +123,7 @@ impl WhisperTranscribeTool { let decode_start = Instant::now(); let samples = decode_audio_file_to_whisper_pcm(&audio_path, self.config.max_audio_bytes)?; tracing::info!( - file_id = %args.audio_file.file_id, + file_id = %audio_file.file_id, sample_count = samples.len(), duration_ms = samples.len() as u64 * 1000 / TARGET_SAMPLE_RATE as u64, elapsed_ms = decode_start.elapsed().as_millis() as u64, @@ -122,7 +136,7 @@ impl WhisperTranscribeTool { .await .map_err(|error| WhisperTranscribeError::Inference(error.to_string()))??; tracing::info!( - file_id = %args.audio_file.file_id, + file_id = %audio_file.file_id, transcript_chars = transcript.chars().count(), elapsed_ms = inference_start.elapsed().as_millis() as u64, "native whisper transcription: inference complete" @@ -131,6 +145,71 @@ impl WhisperTranscribeTool { Ok(NativeMcpToolOutput::text(transcript)) } + async fn execute_inline_audio( + &self, + audio_file: InlineBase64AudioInput, + ) -> Result { + let inline_limit = self.inline_audio_byte_limit(); + if encoded_base64_may_exceed_decoded_limit(audio_file.audio_data.len(), inline_limit) { + return Err(WhisperTranscribeError::InlineSizeLimitExceeded { + max_audio_bytes: inline_limit, + }); + } + + let audio_bytes = STANDARD + .decode(&audio_file.audio_data) + .map_err(|error| WhisperTranscribeError::InvalidInlineAudioData(error.to_string()))?; + let bytes_written = audio_bytes.len() as u64; + if bytes_written > inline_limit { + return Err(WhisperTranscribeError::InlineSizeLimitExceeded { + max_audio_bytes: inline_limit, + }); + } + + tracing::info!( + file_name = ?audio_file.file_name, + mime_type = ?audio_file.mime_type, + bytes = bytes_written, + inline_max_audio_bytes = inline_limit, + "native whisper transcription: received inline audio" + ); + + let start = Instant::now(); + let (_temp_dir, audio_path) = + write_inline_audio_to_temp_file(&audio_file, &audio_bytes).await?; + tracing::info!( + bytes_written, + elapsed_ms = start.elapsed().as_millis() as u64, + "native whisper transcription: inline audio staged" + ); + + let decode_start = Instant::now(); + let samples = decode_audio_file_to_whisper_pcm(&audio_path, self.config.max_audio_bytes)?; + tracing::info!( + sample_count = samples.len(), + duration_ms = samples.len() as u64 * 1000 / TARGET_SAMPLE_RATE as u64, + elapsed_ms = decode_start.elapsed().as_millis() as u64, + "native whisper transcription: inline audio decoded and resampled" + ); + + let transcriber = Arc::clone(&self.transcriber); + let inference_start = Instant::now(); + let transcript = tokio::task::spawn_blocking(move || transcriber.transcribe(samples)) + .await + .map_err(|error| WhisperTranscribeError::Inference(error.to_string()))??; + tracing::info!( + transcript_chars = transcript.chars().count(), + elapsed_ms = inference_start.elapsed().as_millis() as u64, + "native whisper transcription: inline audio inference complete" + ); + + Ok(NativeMcpToolOutput::text(transcript)) + } + + fn inline_audio_byte_limit(&self) -> u64 { + self.config.max_audio_bytes.min(INLINE_AUDIO_MAX_BYTES) + } + async fn download_to_temp_file( &self, download_url: reqwest::Url, @@ -206,35 +285,65 @@ impl NativeMcpTool for WhisperTranscribeTool { "additionalProperties": false, "properties": { "audio_file": { - "type": "object", - "additionalProperties": false, - "description": "OpenAI file reference for the uploaded audio file", - "properties": { - "download_url": { - "type": "string", - "format": "uri", - "description": "Temporary authorized download URL for the uploaded file" - }, - "file_id": { - "type": "string", - "minLength": 1, - "maxLength": 200, - "description": "OpenAI file identifier" - }, - "mime_type": { - "type": "string", - "minLength": 1, - "maxLength": 200, - "description": "Optional MIME type for the uploaded file" + "description": "Uploaded audio file input. Use either a temporary authorized download URL or inline base64 audio bytes.", + "oneOf": [ + { + "type": "object", + "additionalProperties": false, + "description": "Reference to an uploaded audio file accessible via a temporary authorized download URL. Populated automatically by OpenAI Apps SDK clients via the openai/fileParams hint; any MCP client that can supply a fetchable download_url may use this shape.", + "properties": { + "download_url": { + "type": "string", + "format": "uri", + "description": "Temporary authorized download URL for the uploaded file" + }, + "file_id": { + "type": "string", + "minLength": 1, + "maxLength": 200, + "description": "Client-issued identifier for the uploaded file. Opaque to this tool and used for logging/correlation only." + }, + "mime_type": { + "type": "string", + "minLength": 1, + "maxLength": 200, + "description": "Optional MIME type for the uploaded file" + }, + "file_name": { + "type": "string", + "minLength": 1, + "maxLength": 500, + "description": "Optional original filename for the uploaded file" + } + }, + "required": ["download_url", "file_id"] }, - "file_name": { - "type": "string", - "minLength": 1, - "maxLength": 500, - "description": "Optional original filename for the uploaded file" + { + "type": "object", + "additionalProperties": false, + "description": "Inline base64-encoded audio bytes for MCP clients that can read an attached file locally but cannot provide a temporary download URL.", + "properties": { + "audio_data": { + "type": "string", + "contentEncoding": "base64", + "description": "Base64-encoded audio bytes. Limited to 8 MiB before base64 encoding, and also bounded by the configured audio byte limit." + }, + "mime_type": { + "type": "string", + "minLength": 1, + "maxLength": 200, + "description": "Optional MIME type for the uploaded file" + }, + "file_name": { + "type": "string", + "minLength": 1, + "maxLength": 500, + "description": "Optional original filename for the uploaded file" + } + }, + "required": ["audio_data"] } - }, - "required": ["download_url", "file_id"] + ] } }, "required": ["audio_file"] @@ -323,7 +432,14 @@ impl AudioTranscriber for CachedWhisperTranscriber { #[derive(Debug, Deserialize)] #[serde(deny_unknown_fields)] struct WhisperToolArguments { - audio_file: OpenAIFileReferenceInput, + audio_file: AudioFileInput, +} + +#[derive(Debug, Deserialize)] +#[serde(untagged)] +enum AudioFileInput { + DownloadUrl(OpenAIFileReferenceInput), + InlineBase64(InlineBase64AudioInput), } #[derive(Debug, Deserialize)] @@ -335,16 +451,28 @@ struct OpenAIFileReferenceInput { file_name: Option, } +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct InlineBase64AudioInput { + audio_data: String, + mime_type: Option, + file_name: Option, +} + #[derive(Debug, thiserror::Error)] enum WhisperTranscribeError { #[error("invalid arguments: {0}")] InvalidArguments(String), #[error("invalid download_url: {0}")] InvalidDownloadUrl(String), + #[error("invalid inline audio_data: {0}")] + InvalidInlineAudioData(String), #[error("audio download failed: {0}")] Download(String), #[error("audio download exceeds configured limit of {max_audio_bytes} bytes")] SizeLimitExceeded { max_audio_bytes: u64 }, + #[error("inline audio exceeds configured limit of {max_audio_bytes} bytes")] + InlineSizeLimitExceeded { max_audio_bytes: u64 }, #[error("decoded audio exceeds configured limit of {max_audio_bytes} bytes")] DecodedAudioLimitExceeded { max_audio_bytes: u64 }, #[error("temporary audio file error: {0}")] @@ -361,7 +489,8 @@ impl From for NativeMcpToolError { fn from(error: WhisperTranscribeError) -> Self { match error { WhisperTranscribeError::InvalidArguments(message) - | WhisperTranscribeError::InvalidDownloadUrl(message) => { + | WhisperTranscribeError::InvalidDownloadUrl(message) + | WhisperTranscribeError::InvalidInlineAudioData(message) => { NativeMcpToolError::InvalidArguments(message) } other => NativeMcpToolError::ExecutionFailed(other.to_string()), @@ -369,6 +498,32 @@ impl From for NativeMcpToolError { } } +async fn write_inline_audio_to_temp_file( + audio_file: &InlineBase64AudioInput, + audio_bytes: &[u8], +) -> Result<(TempDir, PathBuf), WhisperTranscribeError> { + let temp_dir = tempfile::Builder::new() + .prefix("brain3_audio_inline_") + .tempdir() + .map_err(|error| WhisperTranscribeError::TempFile(error.to_string()))?; + let audio_path = temp_dir.path().join(destination_filename_from_name( + audio_file.file_name.as_deref(), + )); + let mut output = tokio::fs::File::create(&audio_path) + .await + .map_err(|error| WhisperTranscribeError::TempFile(error.to_string()))?; + output + .write_all(audio_bytes) + .await + .map_err(|error| WhisperTranscribeError::TempFile(error.to_string()))?; + output + .flush() + .await + .map_err(|error| WhisperTranscribeError::TempFile(error.to_string()))?; + + Ok((temp_dir, audio_path)) +} + fn decode_audio_file_to_whisper_pcm( path: &Path, max_decoded_audio_bytes: u64, @@ -535,9 +690,11 @@ fn resample_linear(samples: &[f32], source_rate: u32, target_rate: u32) -> Vec String { - let mut name = audio_file - .file_name - .as_deref() + destination_filename_from_name(audio_file.file_name.as_deref()) +} + +fn destination_filename_from_name(file_name: Option<&str>) -> String { + let mut name = file_name .and_then(|name| Path::new(name).file_name()) .and_then(|name| name.to_str()) .unwrap_or("audio") @@ -549,6 +706,11 @@ fn destination_filename(audio_file: &OpenAIFileReferenceInput) -> String { name } +fn encoded_base64_may_exceed_decoded_limit(encoded_len: usize, max_decoded_bytes: u64) -> bool { + let max_encoded_len = max_decoded_bytes.div_ceil(3).saturating_mul(4); + encoded_len as u64 > max_encoded_len +} + fn whisper_backend_label() -> &'static str { #[cfg(target_os = "macos")] { @@ -566,6 +728,7 @@ mod tests { use std::sync::{Arc, Mutex}; use axum::{body::Body, http::StatusCode, response::Response, routing::get, Router}; + use base64::{engine::general_purpose::STANDARD, Engine as _}; use brain3_core::application::native_mcp_tool_registry::NativeMcpToolRegistry; use brain3_core::ports::native_mcp_tool::NativeMcpTool; use serde_json::json; @@ -623,6 +786,73 @@ mod tests { assert_eq!(calls[0].len(), 1_600); } + #[tokio::test] + async fn call_decodes_inline_base64_audio_and_returns_transcript() { + let wav = pcm_wav(8_000, 1, 800); + let encoded_audio = STANDARD.encode(wav); + let fake = Arc::new(FakeTranscriber::default()); + let tool = WhisperTranscribeTool::with_transcriber( + WhisperTranscribeConfig { + model_path: "/tmp/model.bin".into(), + max_audio_bytes: 1_000_000, + }, + fake.clone(), + ); + + let output = tool + .call(json!({ + "audio_file": { + "audio_data": encoded_audio, + "mime_type": "audio/wav", + "file_name": "sample.wav" + } + })) + .await + .expect("tool call should succeed"); + + assert!(!output.is_error); + assert_eq!(output.content[0].text, "hello from native whisper"); + let calls = fake + .calls + .lock() + .expect("fake transcriber lock should succeed"); + assert_eq!(calls.len(), 1); + assert_eq!(calls[0].len(), 1_600); + } + + #[tokio::test] + async fn call_rejects_inline_audio_larger_than_inline_cap() { + let encoded_audio = STANDARD.encode(vec![0u8; INLINE_AUDIO_MAX_BYTES as usize + 1]); + let fake = Arc::new(FakeTranscriber::default()); + let tool = WhisperTranscribeTool::with_transcriber( + WhisperTranscribeConfig { + model_path: "/tmp/model.bin".into(), + max_audio_bytes: INLINE_AUDIO_MAX_BYTES + 1_000_000, + }, + fake.clone(), + ); + + let error = tool + .call(json!({ + "audio_file": { + "audio_data": encoded_audio, + "mime_type": "audio/wav", + "file_name": "oversized.wav" + } + })) + .await + .expect_err("oversized inline audio should fail"); + + assert!(error + .to_string() + .contains("inline audio exceeds configured limit")); + assert!(fake + .calls + .lock() + .expect("fake transcriber lock should succeed") + .is_empty()); + } + #[tokio::test] async fn call_rejects_downloads_larger_than_configured_cap() { let download_url = serve_once(vec![0u8; 128]).await; @@ -727,7 +957,7 @@ mod tests { } #[test] - fn input_schema_declares_openai_audio_file_reference() { + fn input_schema_declares_download_url_and_inline_audio_file_references() { let fake = Arc::new(FakeTranscriber::default()); let tool = WhisperTranscribeTool::with_transcriber( WhisperTranscribeConfig { @@ -742,9 +972,13 @@ mod tests { assert_eq!(schema["required"], json!(["audio_file"])); assert_eq!( - schema["properties"]["audio_file"]["required"], + schema["properties"]["audio_file"]["oneOf"][0]["required"], json!(["download_url", "file_id"]) ); + assert_eq!( + schema["properties"]["audio_file"]["oneOf"][1]["required"], + json!(["audio_data"]) + ); } #[test] diff --git a/docs/superpowers/plans/2026-07-09-claude-ai-audio-transcription-upload.md b/docs/superpowers/plans/2026-07-09-claude-ai-audio-transcription-upload.md new file mode 100644 index 0000000..c30bde5 --- /dev/null +++ b/docs/superpowers/plans/2026-07-09-claude-ai-audio-transcription-upload.md @@ -0,0 +1,266 @@ +# Plan: make `transcribe_audio_file` work from Claude.ai, not just ChatGPT + +## Verified problem statement + +`transcribe_audio_file` (`crates/platform/src/native_mcp_tools/whisper_transcribe.rs`) +only accepts an OpenAI-style file reference: + +```json +{ "audio_file": { "download_url": "...", "file_id": "...", ... } } +``` + +and `meta()` advertises `{"openai/fileParams": ["audio_file"]}` — an OpenAI +Apps SDK convention that tells ChatGPT to auto-bind an uploaded file to the +`audio_file` argument and hand the tool a temporary authorized +`download_url` it can `GET`. + +Claude.ai has no equivalent convention. When a user attaches an audio file +in a Claude.ai chat with "Code execution and file creation" enabled, the +bytes land on the code-execution **sandbox's own filesystem**, not behind a +fetchable `download_url`. There is no `_meta` hint Claude.ai honors that +would cause it to synthesize one. So `transcribe_audio_file` is currently +unreachable from Claude.ai — confirmed, not a misconfiguration. + +## Correction: what the MCP spec actually says about binary tool *inputs* + +A second assistant-generated message claimed "the MCP spec defines +`AudioContent` with base64 `data`... the same pattern applies to tool +inputs." I checked this against the spec directly +(`modelcontextprotocol.io/specification/draft/server/tools` and `.../schema`) +and it's **half right, stated too strongly**: + +- `AudioContent`/`ImageContent` (`{ "type": "audio", "data": "", + "mimeType": "..." }`) are real, spec-defined types — but they're part of + `ContentBlock`, used for **tool results** (and prompts/resources), not + tool call *arguments*. +- Tool **inputs** are governed purely by a server's own `inputSchema` (plain + JSON Schema). The spec places zero constraints on how a server author + models a "file" input. A base64 string property is a legal and common + choice, but it is exactly as much of an "application-level convention" as + brain3's existing `{ download_url, file_id }` shape (which is OpenAI's + convention) or the presigned-upload-URL shape (a pattern several MCP + servers have independently converged on, per the earlier research pass) — + none of the three is more "spec-native" than the others. The spec is + silent on file input modeling by design. + +This matters for the plan because it means base64-inline is not free of +design trade-offs just because it "is in the spec" — it isn't, in the input +direction. It's still worth strongly considering on its own merits (below), +just not on the basis of spec compliance. + +## ⚠️ Non-negotiable constraint: do not touch the working OpenAI path + +**The existing `{ download_url, file_id }` shape is live, tested, and +confirmed working end-to-end with ChatGPT today.** Nothing in this plan +should modify its behavior, its wire format, its `_meta` hint, or the +existing passing tests for it +(`whisper_transcribe.rs::call_downloads_decodes_resamples_and_returns_transcript` +and friends). Both options below are strictly *additive*: every code path +that already ships must keep working, byte-for-byte, exactly as it does +today. If an implementation of either option requires changing how the +`download_url`/`file_id` shape is parsed or validated, that's a sign the +approach is wrong — go back and find a purely-additive way instead. + +## Revised design: two viable options, staged by effort/risk + +### Option A — inline base64 argument (new, recommended to try first) + +**This is not a new tool.** It's the *same* `transcribe_audio_file` tool, +same registration, same name, same OpenAI shape untouched — just one more +accepted shape for its existing `audio_file` argument, sitting alongside +`{ download_url, file_id }` as an alternative (e.g. a `oneOf` branch in the +JSON Schema, or simply extra optional fields with server-side logic that +picks a path based on which fields are present). A client sends *either* +the existing OpenAI-style reference *or* this new one — never both, and the +OpenAI path is completely unaffected by this addition: + +```json +{ "audio_file": { "audio_data": "", "mime_type": "audio/wav", "file_name": "memo.wav" } } +``` + +Server-side: `base64::decode`, then feed straight into the *existing* +`decode_audio_file_to_whisper_pcm` path (same size-limit enforcement, same +whisper-rs inference) — no HTTP GET, no new HTTP route, no new registry, no +new tool. + +**Why this is attractive:** +- Zero new ingress. The bytes travel over the *existing* OAuth-protected + `/mcp` JSON-RPC channel — no new trust boundary, no `SECURITY_AUDIT.md` + rewrite beyond noting the input variant exists. +- No dependency on Cloudflare tunnel naming, "Additional allowed domains," + or the sandbox's `curl` working at all. +- Much smaller code change: one new struct variant + one new schema branch + in a file that already exists, reusing 100% of the decode/limit/inference + pipeline. + +**Why it's genuinely uncertain whether it works from Claude.ai chat, and +needs to be tested empirically before committing to it as the answer:** +- For Claude.ai to call the tool with `audio_data` populated, it needs to + get the attached file's raw bytes into that JSON argument somehow. Two + paths exist, and I could not confirm from docs which (if either) applies + to the **Claude.ai consumer chat product** (as opposed to the raw + Developer Platform/Messages API, which is a different product surface): + 1. *Without code execution*: the model would have to literally emit the + base64 text as tokens in its own response. Per-turn output token caps + (commonly in the thousands, model/config-dependent) make this + impractical past a very short clip, and — per the earlier research + pass — Claude.ai doesn't appear to give the model raw byte access to + an attached audio file without code execution in the first place. + 2. *With code execution enabled*, Anthropic's documented "[programmatic + tool calling](https://www.anthropic.com/engineering/code-execution-with-mcp)" + pattern lets sandboxed code call an MCP tool directly — the tool + argument (including a base64 blob it constructs from a sandbox-local + file) is assembled and sent by the *script*, not typed out by the + model, so it doesn't consume output tokens or context. This is real + and documented for the API, but I found no confirmation it's exposed + in Claude.ai's consumer chat UI today rather than being an + API/agent-harness-only capability. +- Practical size ceiling either way is much smaller than the tool's current + 50MB (`max_audio_bytes`) download cap — a 50MB file would be ~67MB of + base64 text, unworkable as a JSON-RPC argument or (if per case 1 above) + model output. Recommend a separate, much smaller cap specifically for the + inline-base64 path (e.g. single-digit MB of raw audio — enough for a + multi-minute voice memo, not a long recording) rather than reusing + `max_audio_bytes` as-is. + +**Verification step before writing any code**: attach a short (~10-30s) +audio clip in an actual Claude.ai chat with code execution + this gateway's +MCP connector enabled, and ask Claude to transcribe it via a temporary test +tool that just echoes back `len(audio_data)`. This directly answers "can +Claude.ai get sandbox-local bytes into a tool argument at all" before +investing in either Option A's real implementation or Option B below. + +### Option B — presigned-upload-URL + sandbox `curl` (fallback, kept from prior plan draft, demoted not dropped) + +If Option A's verification step shows Claude.ai chat cannot practically get +file bytes into a tool argument (no programmatic-tool-calling access, or +sizes that matter for real recordings blow past output-token limits), fall +back to the previously researched, independently-confirmed pattern: a +`request_audio_upload` tool hands back a short-TTL single-use presigned +`PUT` URL, Claude's sandbox `curl`s the file to it directly (bypassing the +model's context entirely), then `transcribe_audio_file` is called with an +`upload_id`. + +This option is real (confirmed against official docs, the Claude Help +Center, and an independent third-party MCP server doing exactly this) but +costs meaningfully more: +- A genuinely new, only-token-gated (not bearer-gated) public HTTP ingress + point — requires a `SECURITY_AUDIT.md` Threat Model update per AGENTS.MD + before/with implementation (new asset: in-flight uploaded bytes; new + trust boundary: possession of an unguessable single-use 5-minute token + substitutes for the OAuth bearer token on this one route; new attacker + capability: anyone who can reach the public origin and guess/observe a + live token within its TTL can PUT bytes that get whisper-decoded, + mitigated by the same size cap + decode validation already in place). +- Operational dependency: requires a **named/persistent** Cloudflare + tunnel, since Claude.ai's "Additional allowed domains" allowlist is + hostname-keyed and the default quick tunnel's URL rotates on every + gateway restart (SECURITY_AUDIT.md Finding [3]). +- More moving parts: new upload-token registry with TTL/single-use/cleanup + semantics, a new HTTP handler, a new `NativeMcpTool`. + +Design details (registry shape, route placement, size-cap-while-streaming, +reuse of `max_audio_bytes`, test list) are unchanged from the prior version +of this plan and are kept below in case Option A doesn't pan out. + +
+Option B implementation detail (collapsed — only needed if Option A fails verification) + +1. Add an in-memory upload registry (`Mutex>` or + similar), `Entry { expires_at, used: bool, path: Option }`, with + lazy expiry-sweep on access, and cleanup of orphaned temp files for + uploads that were reserved but never PUT to or never transcribed. +2. Add `request_audio_upload` `NativeMcpTool` impl (mirrors + `WhisperTranscribeTool`'s shape), registered alongside it in + `native_mcp_tools_from_config` (`apps/gateway/src/server.rs`). Returns + the `upload_id`, the `curl -X PUT` command, TTL, and the size cap + (sourced from the same `max_audio_bytes` config already used by + `transcribe_audio_file` — no second, divergent limit). +3. Add `PUT /uploads/{upload_id}` handler in `crates/platform/src/http/` + (new `uploads.rs`), wired into `router.rs`'s `build_router` (must be + reachable through the public tunnel-facing router, same as `/oauth/*`). + Streams the body to a temp file with the same byte-cap-while-streaming + behavior already used in `download_to_temp_file`, rejecting early (413) + rather than buffering unbounded bytes. +4. Extend `WhisperToolArguments` deserialization to accept the `upload_id` + variant, sharing the existing decode/size-limit/inference path. +5. Tests: upload happy path, size-limit rejection while streaming, + unknown/expired upload_id, single-use enforcement (second PUT and second + transcribe both rejected after first success), `transcribe_audio_file` + accepting `upload_id` end-to-end with a fake transcriber. + +
+ +No changes needed in `brain3-mcp-vault-tools` (Python container) under +either option — transcription is a native in-gateway Rust tool, unrelated +to that codebase. + +## Wording fix: stop branding the existing shape as OpenAI-only + +Today's `input_schema()` describes the `audio_file` object as *"OpenAI file +reference for the uploaded audio file"* and `file_id` as *"OpenAI file +identifier."* That's misleading in the same direction as the base64 claim +above: the `{ download_url, file_id }` **shape** is generic JSON Schema — +any MCP client capable of handing the tool a temporary authorized download +URL could use it. What's actually OpenAI-specific is only the `_meta` +`openai/fileParams` hint that tells *ChatGPT specifically* to populate it +automatically. + +Reword (implementation detail, to land alongside whichever option ships): + +- `audio_file` description: from *"OpenAI file reference for the uploaded + audio file"* to something like *"Reference to an uploaded audio file + accessible via a temporary authorized download URL. Populated + automatically by OpenAI Apps SDK clients via the `openai/fileParams` + hint; any MCP client that can supply a fetchable `download_url` may use + this shape."* +- `file_id` description: from *"OpenAI file identifier"* to *"Client-issued + identifier for the uploaded file (opaque to this tool; used for + logging/correlation only)."* + +This is purely descriptive-string wording — no behavior change, low risk, +worth doing regardless of which option (A or B) also ships, since it +currently mis-documents the tool for any future non-OpenAI client author +(including whichever shape we add for Claude.ai). + +## Implementation order + +1. Reword the existing `audio_file`/`file_id` schema descriptions (above) — + independent, no-risk, do any time. +2. Run the empirical verification step under Option A (attach a short clip + in a real Claude.ai chat, confirm whether bytes reach a tool argument at + all, and by which mechanism). +3. If verification succeeds: implement Option A (new `audio_data`/ + `mime_type` schema branch + decode path + a separate, smaller size cap + for this path + tests) as a **strictly additive** change — do not modify + `OpenAIFileReferenceInput`, the `download_url`/`file_id` required fields, + or the existing `_meta` hint. Run the full existing + `whisper_transcribe.rs` test suite before and after and confirm every + existing test still passes unmodified — that's the regression signal + that the working ChatGPT path is intact. No `SECURITY_AUDIT.md` change + expected beyond a one-line note that this input variant exists, since no + new trust boundary is created. +4. If verification fails or sizes are impractical: implement Option B per + the collapsed detail above, including the required `SECURITY_AUDIT.md` + Threat Model update *before* the new route ships, and confirm with you + first that you're willing to move to a named Cloudflare tunnel (needed + for "Additional allowed domains" to survive restarts) and that you're OK + with a token-gated-rather-than-bearer-gated upload route. +5. Local verification per AGENTS.MD either way: `cargo test -p brain3 + --no-run`, then `cargo test`. If Option B ships (new gateway HTTP + surface), also run `uv run scripts/e2e_smoke.py` per AGENTS.MD's + explicit call-out for gateway/proxy changes. + +## Open questions for you before I implement anything + +- OK with doing the manual Claude.ai verification step (step 2) before + committing to either option's full implementation? It's the cheapest way + to avoid building the wrong one. +- If it comes to Option B: confirm you're on an individual Claude.ai plan + (Free/Pro/Max) — a filed GitHub issue (`anthropics/claude-code#63182`) + reports the Team-plan org-level allowlist not reaching the sandbox proxy, + so Option B would be unreliable on Team. +- If it comes to Option B: confirm you're willing to move to a named + Cloudflare tunnel, and sign off on the token-gated (not bearer-gated) + upload endpoint as a new ingress shape, given this repo's security + posture.